oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/opctl/utils.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import functools
+import logging
+import os
+import subprocess
+import sys
+import shlex
+import urllib.parse
+from subprocess import Popen, PIPE, STDOUT
+from typing import Union, List, Tuple, Dict
+import yaml
+import re
+
+import ads
+from ads.common.oci_client import OCIClientFactory
+from ads.opctl import logger
+from ads.opctl.constants import (
+    ML_JOB_IMAGE,
+    ML_JOB_GPU_IMAGE,
+)
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+
+
+CONTAINER_NETWORK = "CONTAINER_NETWORK"
+
+
+def get_service_pack_prefix() -> Dict:
+    curr_dir = os.path.dirname(os.path.abspath(__file__))
+    service_config_file = os.path.join(curr_dir, "conda", "config.yaml")
+    with open(service_config_file) as f:
+        service_config = yaml.safe_load(f.read())
+    if all(key in os.environ for key in ["CONDA_BUCKET_NS", "CONDA_BUCKET_NAME"]):
+        bucket_info = {
+            "name": os.environ["CONDA_BUCKET_NAME"],
+            "namespace": os.environ["CONDA_BUCKET_NS"],
+        }
+    else:
+        bucket_info = service_config["bucket_info"].get(
+            os.environ.get("NB_REGION", "default"),
+            service_config["bucket_info"]["default"],
+        )
+    return f"oci://{bucket_info['name']}@{bucket_info['namespace']}/{service_config['service_pack_prefix']}"
+
+
+def is_in_notebook_session():
+    return "CONDA_PREFIX" in os.environ and "NB_SESSION_OCID" in os.environ
+
+
+def parse_conda_uri(uri: str) -> Tuple[str, str, str, str]:
+    parsed = urllib.parse.urlparse(uri)
+    path = parsed.path
+    if path.startswith("/"):
+        path = path[1:]
+    bucket, ns = parsed.netloc.split("@")
+    slug = os.path.basename(path)
+    return ns, bucket, path, slug
+
+
+def list_ads_operators() -> dict:
+    curr_dir = os.path.dirname(os.path.abspath(__file__))
+    with open(os.path.join(curr_dir, "index.yaml"), "r") as f:
+        ads_operators = yaml.safe_load(f.read())
+    return ads_operators or []
+
+
+def get_oci_region(auth: dict) -> str:
+    if len(auth["config"]) > 0:
+        return auth["config"]["region"]
+    else:
+        return auth["signer"].region
+
+
+def get_namespace(auth: dict) -> str:
+    client = OCIClientFactory(**auth).object_storage
+    return client.get_namespace().data
+
+
+def get_region_key(auth: dict) -> str:
+    tenancy = auth["config"].get("tenancy")
+    if not tenancy:
+        tenancy = auth["signer"].tenancy_id
+    client = OCIClientFactory(**auth).identity
+    return client.get_tenancy(tenancy).data.home_region_key
+
+
+def publish_image(image: str, registry: str = None) -> None:  # pragma: no cover
+    """
+    Publish an image.
+
+    Parameters
+    ----------
+    image: str
+        image name
+    registry: str
+        registry to push to
+
+    Returns
+    -------
+    None
+    """
+    if not registry:
+        run_command(["docker", "push", image])
+        print(f"pushed {image}")
+        return image
+    else:
+        registry = registry.rstrip("/")
+        run_command(
+            ["docker", "tag", f"{image}", f"{registry}/{os.path.basename(image)}"]
+        )
+        run_command(["docker", "push", f"{registry}/{os.path.basename(image)}"])
+        print(f"pushed {registry}/{os.path.basename(image)}")
+        return f"{registry}/{os.path.basename(image)}"
+
+
+def build_image(image_type: str, gpu: bool = False) -> None:
+    """
+    Build an image for opctl.
+
+    Parameters
+    ----------
+    image_type: str
+        specify the image to build, can take 'job-local',
+        former for running job with conda pack locally,
+        latter for running operators
+    gpu: bool
+        whether to use gpu version of image
+    dst_image: str
+        image to save as when building custom operator
+
+    Returns
+    -------
+    None
+    """
+    curr_dir = os.path.dirname(os.path.abspath(__file__))
+    if image_type == "ads-ops-custom":
+        if not source_folder or not dst_image:
+            raise ValueError(
+                "Please provide both source_folder and image_name to build a image for custom operator."
+            )
+        proc = _build_custom_operator_image(gpu, source_folder, dst_image)
+    else:
+        # https://stackoverflow.com/questions/66842004/get-the-processor-type-using-python-for-apple-m1-processor-gives-me-an-intel-pro
+        import cpuinfo
+
+        # Just get the manufacturer of the processors
+        manufacturer = cpuinfo.get_cpu_info().get("brand_raw")
+        arch = (
+            "arm" if re.search("apple m\d", manufacturer, re.IGNORECASE) else "other"
+        )
+        print(f"The local machine's platform is {arch}.")
+        image, dockerfile, target = _get_image_name_dockerfile_target(
+            image_type, gpu, arch
+        )
+        print(f"dockerfile used is {dockerfile}")
+        command = [
+            "docker",
+            "build",
+            "-t",
+            image,
+            "-f",
+            os.path.join(curr_dir, "docker", dockerfile),
+        ]
+        if target:
+            command += ["--target", target]
+        if os.environ.get("no_proxy"):
+            command += ["--build-arg", f"no_proxy={os.environ['no_proxy']}"]
+        if os.environ.get("http_proxy"):
+            command += ["--build-arg", f"http_proxy={os.environ['http_proxy']}"]
+        if os.environ.get("https_proxy"):
+            command += ["--build-arg", f"https_proxy={os.environ['https_proxy']}"]
+        if os.environ.get(CONTAINER_NETWORK):
+            command += ["--network", os.environ[CONTAINER_NETWORK]]
+        command += [os.path.abspath(curr_dir)]
+        logger.info("Build image with command %s", command)
+        proc = run_command(command)
+    if proc.returncode != 0:
+        raise RuntimeError("Docker build failed.")
+
+
+def _get_image_name_dockerfile_target(type: str, gpu: bool, arch: str) -> str:
+    look_up = {
+        ("job-local", False, "arm"): (ML_JOB_IMAGE, "Dockerfile.job.arm", None),
+        ("job-local", False, "other"): (ML_JOB_IMAGE, "Dockerfile.job", None),
+        ("job-local", True, "other"): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None),
+    }
+    return look_up[(type, gpu, arch)]
+
+
+def run_command(
+    cmd: Union[str, List[str]], cwd: str = None, shell: bool = False
+) -> Popen:
+    proc = Popen(
+        cmd, cwd=cwd, stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=shell
+    )
+    for x in iter(proc.stdout.readline, ""):
+        print(x, file=sys.stdout, end="")
+    proc.wait()
+    return proc
+
+
+class _DebugTraceback:
+    def __init__(self, debug):
+        self.cur_logging_level = logger.getEffectiveLevel()
+        self.debug = debug
+
+    def __enter__(self):
+        if self.debug:
+            logger.setLevel(logging.DEBUG)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.debug and exc_type:
+            logger.setLevel(self.cur_logging_level)
+            return False
+        elif not self.debug and exc_type:
+            sys.stderr.write(f"{exc_type.__name__}: {str(exc_val)} \n")
+            sys.exit(1)
+        return True
+
+
+def suppress_traceback(debug: bool = True) -> None:
+    """
+    Decorator to suppress traceback when in debug mode.
+
+    Parameters
+    ----------
+    debug: bool
+        turn on debug mode or not
+
+    Returns
+    -------
+    None
+    """
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            with _DebugTraceback(debug) as tb:
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+@runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
+def get_docker_client() -> "docker.client.DockerClient":
+    import docker
+
+    process = subprocess.Popen(
+        ["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
+    )
+    process.communicate()
+    if process.returncode != 0:
+        raise RuntimeError("Docker is not started.")
+    return docker.from_env()
+
+
+@runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
+def run_container(
+    image: str,
+    bind_volumes: Dict,
+    env_vars: Dict,
+    command: str = None,
+    entrypoint: str = None,
+    verbose: bool = False,
+) -> int:
+    import docker
+
+    if env_vars is None:
+        env_vars = {}
+    # If Proxy variables are setup, pass it on to the docker run
+    if "http_proxy" in os.environ:
+        env_vars.update(
+            {
+                "http_proxy": os.environ.get("http_proxy"),
+                "https_proxy": os.environ.get("https_proxy"),
+            }
+        )
+    if verbose:
+        ads.opctl.logger.setLevel(logging.INFO)
+    logger.info(f"Running container with image: {image}")
+    logger.info(f"bind_volumes: {bind_volumes}")
+    logger.info(f"env_vars: {env_vars}")
+    logger.info(f"command: {command}")
+    logger.info(f"entrypoint: {entrypoint}")
+
+    # Print out the equivalent docker run command for debugging purpose
+    docker_run_cmd = [">>> docker run --rm"]
+    if entrypoint:
+        docker_run_cmd.append(f"--entrypoint {entrypoint}")
+    if env_vars:
+        docker_run_cmd.extend([f"-e {key}={val}" for key, val in env_vars.items()])
+    if bind_volumes:
+        docker_run_cmd.extend(
+            [f'-v {source}:{bind.get("bind")}' for source, bind in bind_volumes.items()]
+        )
+    docker_run_cmd.append(image)
+    if command:
+        docker_run_cmd.append(command)
+    logger.debug(" ".join(docker_run_cmd))
+
+    client = get_docker_client()
+    try:
+        client.api.inspect_image(image)
+    except docker.errors.ImageNotFound:
+        logger.warning(f"Image {image} not found. Try pulling it now....")
+        run_command(["docker", "pull", f"{image}"], None)
+    try:
+        kwargs = {}
+        if CONTAINER_NETWORK in os.environ:
+            kwargs["network_mode"] = os.environ[CONTAINER_NETWORK]
+        container = client.containers.run(
+            image=image,
+            volumes=bind_volumes,
+            command=shlex.split(command) if command else None,
+            environment=env_vars,
+            detach=True,
+            entrypoint=entrypoint,
+            user=0,
+            **kwargs
+            # auto_remove=True,
+        )
+        logger.info("Container ID: %s", container.id)
+        for line in container.logs(stream=True, follow=True):
+            print(line.decode("utf-8"), end="")
+
+        result = container.wait()
+        return result.get("StatusCode", -1)
+    except docker.errors.APIError as ex:
+        logger.error(ex.explanation)
+        return -1
+    finally:
+        # Remove the container
+        container.remove()

ads/oracledb/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/oracledb/oracle_db.py

@@ -0,0 +1,346 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2021, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+There are two potential candidates for oracle database driver -
+* cx_Oracle
+* oracledb
+
+Preference is to use oracledb if it is available in the environment else choose cx_Oracle
+
+If oracledb is loaded and user uses `wallet` to connect to database, prefer thick mode. Thin mode requires user to provide passphrase for PEM file which is not required in thick mode
+
+If user uses DSN string copied from OCI console with OCI database setup for TLS connection, oracledb driver is preferred. If cx_Oracle is the only driver available, warn user that oracledb is preferred driver.
+Note: We need to account for cx_Oracle though oracledb can operate in thick mode. The end user may be is using one of the old conda packs or an environment where cx_Oracle is the only available driver.
+"""
+
+import logging
+import os
+import tempfile
+import zipfile
+from time import time
+from typing import Dict, Iterator, List, Optional, Union
+
+import numpy as np
+import pandas as pd
+
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+)
+from ads.common.utils import ORACLE_DEFAULT_PORT
+
+logger = logging.getLogger("ads.oracle_connector")
+CX_ORACLE = "cx_Oracle"
+PYTHON_ORACLEDB = "PYTHON_ORACLEDB"
+PYTHON_DRIVER_NAME = None
+
+try:
+    import oracledb as oracle_driver  # Both the driver share same signature for the APIs that we are using.
+
+    PYTHON_DRIVER_NAME = PYTHON_ORACLEDB
+except ModuleNotFoundError:
+    logger.info("oracledb package not found. Trying to load cx_Oracle")
+    try:
+        import cx_Oracle as oracle_driver
+
+        PYTHON_DRIVER_NAME = CX_ORACLE
+    except ModuleNotFoundError as err2:
+        raise ModuleNotFoundError(
+            f"Neither `oracledb` nor `cx_Oracle` module was not found. Please run "
+            f"`pip install {OptionalDependency.DATA}`."
+        ) from err2
+
+
+class OracleRDBMSConnection(oracle_driver.Connection):
+    def __init__(
+        self,
+        user_name,
+        password,
+        service_name=None,
+        wallet_file=None,
+        sid=None,
+        dsn=None,
+        host=None,
+        port=ORACLE_DEFAULT_PORT,
+        **kwargs,
+    ):
+        logger.info(f"Using `{PYTHON_DRIVER_NAME}` for connection with Oracle database")
+        if "wallet_location" in kwargs:
+            wallet_file = kwargs.pop("wallet_location")
+        if PYTHON_DRIVER_NAME == PYTHON_ORACLEDB and wallet_file:
+            try:
+                oracle_driver.init_oracle_client()
+                logger.info(
+                    "Running oracledb driver in thick mode. For mTLS based connection, thick mode is default."
+                )
+            except Exception:
+                logger.info(
+                    "Could not use thick mode. The driver is running in thin mode. System might prompt for passphrase"
+                )
+
+        self.temp_dir = None
+        self.tns_entries = {}
+        kwargs["user"] = user_name
+        kwargs["password"] = password
+        if wallet_file:
+            self._setup_wallet_file(wallet_file)
+            if service_name:
+                dsn = service_name
+            if dsn in self.tns_entries:
+                dsn = self.tns_entries[dsn]
+                kwargs = kwargs.copy()
+                kwargs["dsn"] = dsn
+        elif dsn:
+            kwargs["dsn"] = dsn
+            logger.info("Using dsn for connection")
+            logger.debug(f"DSN string is {dsn}")
+            if PYTHON_DRIVER_NAME == CX_ORACLE and "protocol=tcps" in dsn:
+                logger.warning(
+                    "If you are connecting to Autonomous Database using TLS, install `oracledb` python package to use the connection string from OCI Autonomous Database Console."
+                )
+        elif service_name or sid:
+            logger.info(f"Connecting to {host}:{port}/{service_name or sid}")
+            if not host:
+                raise ValueError(
+                    "Missing `host` information or wallet file. To connect to the database without using wallet file, `host` infomration is required. Please set `host` or `wallet_file` with a valid value."
+                )
+            kwargs["dsn"] = self._construct_dsn_(host, port, service_name, sid)
+
+        super().__init__(**kwargs)
+
+        logger.info(
+            f"RDBMS version: {'.'.join([str(x) for x in self.version.split('.')[:2]])}"
+        )
+
+    def __del__(self):
+        if self.temp_dir is not None:
+            self.temp_dir.cleanup()
+
+    def _construct_dsn_(self, host, port, service_name=None, sid=None):
+        if PYTHON_DRIVER_NAME == CX_ORACLE:
+            return oracle_driver.makedsn(host, port, sid=sid, service_name=service_name)
+        else:
+            cp = oracle_driver.ConnectParams(
+                host=host, port=port, service_name=service_name, sid=sid
+            )
+            return cp.get_connect_string()
+
+    def _setup_wallet_file(self, wallet_file: str):
+        # extract files in wallet zip to a temporary directory
+        self.temp_dir = tempfile.TemporaryDirectory()
+        zipfile.ZipFile(wallet_file).extractall(self.temp_dir.name)
+
+        # parse tnsnames.ora to get list of entries and modify them to include the wallet location
+        fname = os.path.join(self.temp_dir.name, "tnsnames.ora")
+        for line in open(fname):
+            pos = line.find(" = ")
+            if pos < 0:
+                continue
+            name = line[:pos]
+            entry = line[pos + 3 :].strip()
+            key_phrase = "(security="
+            pos = entry.find(key_phrase) + len(key_phrase)
+            wallet_entry = f"(MY_WALLET_DIRECTORY={self.temp_dir.name})"
+            entry = entry[:pos] + wallet_entry + entry[pos:]
+            self.tns_entries[name] = entry
+
+    def insert(
+        self,
+        table_name: str,
+        df: pd.DataFrame,
+        if_exists: str,
+        batch_size=100000,
+        encoding="utf-8",
+    ):
+        if if_exists not in ["fail", "replace", "append"]:
+            raise ValueError(
+                f"Unknown option `if_exists`={if_exists}. Valid options are 'fail', 'replace', 'append'"
+            )
+        start_time = time()
+
+        df_orcl = df.copy()
+        # "object" type can be actual objects, or just plain strings, when inserting into the
+        # database we need to stringify these so they can be represented in a VARCHAR2 column
+
+        # object_columns = df_orcl.select_dtypes(include=["object"]).columns
+        # df_orcl = df_orcl.where(pd.notnull(df_orcl), None)
+        # df_orcl[object_columns] = df_orcl[object_columns].astype(str)
+
+        # prep column names for valid Oracle column names (alpha + # $ _)
+        df_orcl.columns = df_orcl.columns.str.replace(r"\W+", "_", regex=True)
+        table_exist = True
+        with self.cursor() as cursor:
+            if if_exists != "replace":
+                try:
+                    cursor.execute(f"SELECT 1 from {table_name} FETCH NEXT 1 ROWS ONLY")
+                except Exception:
+                    table_exist = False
+                if if_exists == "fail" and table_exist:
+                    raise ValueError(
+                        f"Table {table_name} already exists. Set `if_exists`='replace' or 'append' to replace or append to the existing table"
+                    )
+            # Oracle doesn't have boolean so convert to 1/0
+            df_orcl = df_orcl.where(
+                df.applymap(type) != bool, df_orcl.replace({True: 1, False: 0})
+            )
+
+            type_mappings = {
+                "bool": "NUMBER(1)",
+                "int16": "INTEGER",
+                "int32": "INTEGER",
+                "int64": "INTEGER",
+                "float16": "FLOAT",
+                "float32": "FLOAT",
+                "float64": "FLOAT",
+                "datetime64": "TIMESTAMP",
+            }
+
+            # add in any string types as Oracle's VARCHAR type setting length to accommodate longest
+            def get_max_str_len(df, column, encoding):
+                return df[column].dropna().str.encode(encoding).map(len).max()
+
+            longest_string_column = max(
+                (
+                    get_max_str_len(df_orcl, c, encoding)
+                    for c in df_orcl.select_dtypes(
+                        include=["object", "category"]
+                    ).columns
+                ),
+                default=0,
+            )
+
+            logger.debug(f"Max string column value: {longest_string_column}")
+
+            datatypes = {
+                c: f"VARCHAR2({get_max_str_len(df_orcl, c, encoding)})"
+                for c in df_orcl.select_dtypes(include=["object", "category"]).columns
+            }
+
+            for df_type, orcl_type in type_mappings.items():
+                datatypes.update(
+                    {
+                        column: orcl_type
+                        for column in df_orcl.select_dtypes(include=df_type).columns
+                    }
+                )
+
+            if set(datatypes.keys()) != set(df_orcl.columns):
+                raise Exception(
+                    f"unable to determine oracle data type to use for column(s): {', '.join(set(df_orcl.columns)-set(datatypes.keys()))}"
+                )
+
+            # create table
+            if if_exists == "replace":
+                try:
+                    cursor.execute(f"drop table {table_name}")
+                except:
+                    logger.info(f"Table {table_name} does not exist")
+            if if_exists == "replace" or not table_exist:
+                sql = (
+                    f"create table {table_name} ("
+                    + ", ".join([f"{col} {datatypes[col]}" for col in df_orcl.columns])
+                    + ")"
+                )
+                logger.info(sql)
+                try:
+                    cursor.execute(sql)
+                except Exception as e:
+                    raise Exception(
+                        f"'{e if e.args else 'Unexpected error encountered'}' with query: '{sql}'"
+                    ) from e
+
+            # insert
+            bind_columns = ", ".join([f"{col}" for col in df_orcl.columns])
+            bind_variables = ", ".join([f":{col}" for col in df_orcl.columns])
+            sql = f"insert into {table_name}({bind_columns}) values({bind_variables})"
+
+            logger.info(sql)
+
+            # prevent buffer reallocation by locking in the longest string value
+            # cursor.setinputsizes(None, longest_string_column)
+
+            # replace NaN with None before turning into database records, important - don't
+            # do this earlier in the logic because it can change the pandas
+            # data types
+
+            record_data = list(df_orcl.replace({np.nan: None}).itertuples(index=False))
+
+            def chunks(lst: List, batch_size: int):
+                """Yield successive batch_size chunks from lst."""
+                for i in range(0, len(lst), batch_size):
+                    yield lst[i : i + batch_size]
+
+            for batch in chunks(record_data, batch_size=batch_size):
+                cursor.executemany(sql, batch, batcherrors=True)
+
+                for error in cursor.getbatcherrors():
+                    logger.error(
+                        f"Error: {error.message}, at row offset: {error.offset}"
+                    )
+                    raise RuntimeError(
+                        f"Error: {error.message}, at row offset: {error.offset}"
+                    )
+
+            self.commit()
+
+            duration = time() - start_time
+            logger.info(
+                f"inserted {df_orcl.shape[0]} rows at {df_orcl.shape[0]/duration:.2f} rows/seconds"
+            )
+
+    def _fetch_by_batch(self, cursor, chunksize):
+        while True:
+            rows = cursor.fetchmany(chunksize)
+            if rows:
+                yield rows
+            else:
+                break
+
+    def query(
+        self, sql: str, bind_variables: Optional[Dict], chunksize=None
+    ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
+        start_time = time()
+
+        cursor = self.cursor()
+        cursor.arraysize = 50000
+
+        if chunksize:
+            logger.info(f"Chunksize is {chunksize}")
+            cursor.execute(sql, **bind_variables)
+            columns = [row[0] for row in cursor.description]
+            df = iter(
+                pd.DataFrame(data=rows, columns=columns)
+                for rows in self._fetch_by_batch(cursor, chunksize)
+            )
+
+        else:
+            df = pd.DataFrame(
+                cursor.execute(sql, **bind_variables),
+                columns=[row[0] for row in cursor.description],
+            )
+            duration = time() - start_time
+            logger.info(
+                f"fetched {df.shape[0]} rows at {df.shape[0]/duration:.2f} rows/seconds"
+            )
+
+        return df
+
+
+def get_adw_connection(vault_secret_id: str) -> "oracledb.Connection":
+    """Creates ADW connection from the credentials stored in the vault"""
+    import oracledb
+
+    from ads.secrets.adb import ADBSecretKeeper
+
+    secret = vault_secret_id
+
+    logging.getLogger().debug("A secret id was used to retrieve credentials.")
+    creds = ADBSecretKeeper.load_secret(secret).to_dict()
+    user = creds.pop("user_name", None)
+    password = creds.pop("password", None)
+    if not user or not password:
+        raise ValueError(f"The user or password is missing in {secret}")
+    logging.getLogger().debug("Downloaded secrets successfully.")
+    return oracledb.connect(user=user, password=password, **creds)