oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/opctl/spark/cmds.py

@@ -1,147 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-import os
-from xml.dom import minidom
-from xml.etree.ElementTree import Element
-from xml.etree import ElementTree as et
-
-from ads.common.auth import AuthType, create_signer
-from ads.opctl.config.utils import read_from_ini
-from ads.opctl.config.base import ConfigProcessor
-from ads.opctl.config.merger import ConfigMerger
-from ads.opctl.utils import get_oci_region, is_in_notebook_session
-from ads.opctl.constants import DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR
-
-
-def core_site(auth, oci_config, overwrite, oci_profile):
-
-    if is_in_notebook_session():
-        if not os.path.exists(
-            os.path.join(DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR, "spark-defaults.conf")
-        ):
-            raise RuntimeError("Pyspark conda pack is not installed.")
-        core_site_loc = os.path.join(
-            DEFAULT_NOTEBOOK_SESSION_SPARK_CONF_DIR, "core-site.xml"
-        )
-    else:
-        if "CONDA_PREFIX" not in os.environ or not os.path.exists(
-            os.path.join(os.environ["CONDA_PREFIX"], "spark-defaults.conf")
-        ):
-            raise RuntimeError("Please run this inside a Pyspark conda environment.")
-        core_site_loc = os.path.join(os.environ["CONDA_PREFIX"], "core-site.xml")
-
-    if os.path.exists(core_site_loc) and not overwrite:
-        print(f"{core_site_loc} already exists. Please use `--overwrite` option.")
-        return
-
-    p = ConfigProcessor().step(
-        ConfigMerger,
-        oci_config=oci_config,
-        oci_profile=oci_profile,
-        auth=auth,
-    )
-    exec_config = p.config["execution"]
-    oci_config = exec_config["oci_config"]
-    oci_profile = exec_config["oci_profile"]
-    auth = exec_config["auth"]
-
-    properties = generate_core_site_properties(auth, oci_config, oci_profile)
-    xmlstr = generate_core_site_properties_str(properties)
-
-    with open(core_site_loc, "w") as f:
-        f.write(xmlstr)
-
-    print(f"The core-site.xml is being written to {core_site_loc}")
-
-
-def generate_core_site_properties(
-    authentication,
-    oci_config=None,
-    oci_profile=None,
-):
-
-    region = os.getenv("NB_REGION", None)
-    if oci_config and oci_profile:
-        region = region or get_oci_region(
-            create_signer(AuthType.API_KEY, oci_config, oci_profile)
-        )
-
-    if authentication == "api_key":
-        if not oci_config or not oci_profile:
-            raise ValueError(
-                "oci-config and oci-profile must be provided when using api_key."
-            )
-        oci_config_path = os.path.abspath(os.path.expanduser(oci_config))
-
-        if not os.path.exists(oci_config_path):
-            raise RuntimeError(
-                f"The oci config path {oci_config_path} does not exist. Please refer to the getting-started.ipynb on generating oci API keys."
-            )
-
-        config = read_from_ini(oci_config_path)
-        if oci_profile not in config:
-            raise RuntimeError(
-                f"The configuration profile {oci_profile} does not exist in {oci_config_path}"
-            )
-        details_dict = config[oci_profile]
-
-        for key in ["user", "fingerprint", "key_file", "tenancy", "region"]:
-            if key not in details_dict.keys():
-                raise ValueError(
-                    f"{key} not found in oci config. Is {oci_config_path} is corrupt?"
-                )
-
-        # setup region
-        details_dict["region"] = region
-
-        return [
-            (
-                "fs.oci.client.hostname",
-                f"https://objectstorage.{details_dict['region']}.oraclecloud.com",
-            ),
-            ("fs.oci.client.auth.tenantId", details_dict["tenancy"]),
-            ("fs.oci.client.auth.userId", details_dict["user"]),
-            ("fs.oci.client.auth.fingerprint", details_dict["fingerprint"]),
-            (
-                "fs.oci.client.auth.pemfilepath",
-                os.path.expanduser(details_dict["key_file"]),
-            ),
-        ]
-
-    else:
-        return [
-            (
-                "fs.oci.client.hostname",
-                f"https://objectstorage.{region}.oraclecloud.com",
-            ),
-            (
-                "fs.oci.client.custom.authenticator",
-                "com.oracle.bmc.hdfs.auth.ResourcePrincipalsCustomAuthenticator",
-            ),
-        ]
-
-
-def generate_core_site_properties_str(properties):
-    def core_site_property(name, value):
-        elem = Element("property")
-        child_name = Element("name")
-        child_name.text = name
-        child_value = Element("value")
-        child_value.text = value
-        elem.append(child_name)
-        elem.append(child_value)
-        return elem
-
-    elem = Element("configuration")
-    for (name, value) in properties:
-        child = core_site_property(name, value)
-        elem.append(child)
-
-    return (
-        minidom.parseString(et.tostring(elem)).childNodes[0].toprettyxml(indent="   ")
-    )

ads/opctl/templates/diagnostic_report_template.jinja2

@@ -1,102 +0,0 @@
-
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
-    <meta name="description" content="">
-    <meta name="author" content="">
-    <link rel="canonical" href="https://getbootstrap.com/docs/3.4/examples/theme/">
-
-    <title>ODSC Distributed Training - Diagnostic Report</title>
-
-    <!-- Bootstrap core CSS -->
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
-
-
-
-  </head>
-
-  <body>
-
-
-    <div class="container theme-showcase" role="main">
-
-      <!-- Main jumbotron for a primary marketing message or call to action -->
-      <div class="jumbotron">
-        <h1>Diagnostic Report for {{cluster_type}}</h1>
-      </div>
-
-
-      <div class="row">
-        <div class="col-md-12">
-          <table class="table">
-            <thead>
-              <tr>
-                <th>#</th>
-                <th>Requirement</th>
-                <th>Status</th>
-              </tr>
-            </thead>
-            <tbody>
-            {% for requirement, status in report %}
-              <tr>
-                <td>{{loop.index}}</td>
-                <td>{{requirement}}</td>
-                <td>{{status}}</td>
-              </tr>
-            {% endfor %}
-            </tbody>
-          </table>
-        </div>
-      </div>
-      <div class="row">
-      {% if action_item|length == 0 %}
-        <div class="alert alert-success" role="alert">
-        {{success_message}}.
-        </div>
-        {% else -%}
-        <div  class="alert alert-danger" role="alert">
-        Following setup requirements are to be met to run {{cluster_type}} cluster
-        </div>
-        <div class="col-md-12">
-          <table class="table">
-            <thead>
-              <tr>
-                <th>#</th>
-                <th>Requirement</th>
-                <th>Resolution</th>
-              </tr>
-            </thead>
-            <tbody>
-            {% for requirement, action in action_item%}
-              <tr>
-                <td>{{loop.index}}</td>
-                <td>{{requirement}}</td>
-                <td>{{action}}</td>
-              </tr>
-            {% endfor %}
-            </tbody>
-          </table>
-        </div>
-        {% endif %}
-      </div>
-
-
-
-    </div> <!-- /container -->
-
-
-    <!-- Bootstrap core JavaScript
-    ================================================== -->
-    <!-- Placed at the end of the document so the pages load faster -->
-    <script src="https://code.jquery.com/jquery-1.12.4.min.js" integrity="sha384-nvAa0+6Qg9clwYCGGPpDQLVpLNn0fRaROjHqs13t4Ggj3Ez50XnGQqc/r8MhnRDZ" crossorigin="anonymous"></script>
-    <script>window.jQuery || document.write('<script src="../../assets/js/vendor/jquery.min.js"><\/script>')</script>
-    <script src="../../dist/js/bootstrap.min.js"></script>
-    <script src="../../assets/js/docs.min.js"></script>
-    <!-- IE10 viewport hack for Surface/desktop Windows 8 bug -->
-    <script src="../../assets/js/ie10-viewport-bug-workaround.js"></script>
-  </body>
-</html>

ads/opctl/utils.py

@@ -1,344 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-import functools
-import logging
-import os
-import subprocess
-import sys
-import shlex
-import urllib.parse
-from subprocess import Popen, PIPE, STDOUT
-from typing import Union, List, Tuple, Dict
-import yaml
-import re
-
-import ads
-from ads.common.oci_client import OCIClientFactory
-from ads.opctl import logger
-from ads.opctl.constants import (
-    ML_JOB_IMAGE,
-    ML_JOB_GPU_IMAGE,
-)
-from ads.common.decorator.runtime_dependency import (
-    runtime_dependency,
-    OptionalDependency,
-)
-
-
-CONTAINER_NETWORK = "CONTAINER_NETWORK"
-
-
-def get_service_pack_prefix() -> Dict:
-    curr_dir = os.path.dirname(os.path.abspath(__file__))
-    service_config_file = os.path.join(curr_dir, "conda", "config.yaml")
-    with open(service_config_file) as f:
-        service_config = yaml.safe_load(f.read())
-    if all(key in os.environ for key in ["CONDA_BUCKET_NS", "CONDA_BUCKET_NAME"]):
-        bucket_info = {
-            "name": os.environ["CONDA_BUCKET_NAME"],
-            "namespace": os.environ["CONDA_BUCKET_NS"],
-        }
-    else:
-        bucket_info = service_config["bucket_info"].get(
-            os.environ.get("NB_REGION", "default"),
-            service_config["bucket_info"]["default"],
-        )
-    return f"oci://{bucket_info['name']}@{bucket_info['namespace']}/{service_config['service_pack_prefix']}"
-
-
-def is_in_notebook_session():
-    return "CONDA_PREFIX" in os.environ and "NB_SESSION_OCID" in os.environ
-
-
-def parse_conda_uri(uri: str) -> Tuple[str, str, str, str]:
-    parsed = urllib.parse.urlparse(uri)
-    path = parsed.path
-    if path.startswith("/"):
-        path = path[1:]
-    bucket, ns = parsed.netloc.split("@")
-    slug = os.path.basename(path)
-    return ns, bucket, path, slug
-
-
-def list_ads_operators() -> dict:
-    curr_dir = os.path.dirname(os.path.abspath(__file__))
-    with open(os.path.join(curr_dir, "index.yaml"), "r") as f:
-        ads_operators = yaml.safe_load(f.read())
-    return ads_operators or []
-
-
-def get_oci_region(auth: dict) -> str:
-    if len(auth["config"]) > 0:
-        return auth["config"]["region"]
-    else:
-        return auth["signer"].region
-
-
-def get_namespace(auth: dict) -> str:
-    client = OCIClientFactory(**auth).object_storage
-    return client.get_namespace().data
-
-
-def get_region_key(auth: dict) -> str:
-    tenancy = auth["config"].get("tenancy")
-    if not tenancy:
-        tenancy = auth["signer"].tenancy_id
-    client = OCIClientFactory(**auth).identity
-    return client.get_tenancy(tenancy).data.home_region_key
-
-
-def publish_image(image: str, registry: str = None) -> None:  # pragma: no cover
-    """
-    Publish an image.
-
-    Parameters
-    ----------
-    image: str
-        image name
-    registry: str
-        registry to push to
-
-    Returns
-    -------
-    None
-    """
-    if not registry:
-        run_command(["docker", "push", image])
-        print(f"pushed {image}")
-        return image
-    else:
-        registry = registry.rstrip("/")
-        run_command(
-            ["docker", "tag", f"{image}", f"{registry}/{os.path.basename(image)}"]
-        )
-        run_command(["docker", "push", f"{registry}/{os.path.basename(image)}"])
-        print(f"pushed {registry}/{os.path.basename(image)}")
-        return f"{registry}/{os.path.basename(image)}"
-
-
-def build_image(image_type: str, gpu: bool = False) -> None:
-    """
-    Build an image for opctl.
-
-    Parameters
-    ----------
-    image_type: str
-        specify the image to build, can take 'job-local',
-        former for running job with conda pack locally,
-        latter for running operators
-    gpu: bool
-        whether to use gpu version of image
-    dst_image: str
-        image to save as when building custom operator
-
-    Returns
-    -------
-    None
-    """
-    curr_dir = os.path.dirname(os.path.abspath(__file__))
-    if image_type == "ads-ops-custom":
-        if not source_folder or not dst_image:
-            raise ValueError(
-                "Please provide both source_folder and image_name to build a image for custom operator."
-            )
-        proc = _build_custom_operator_image(gpu, source_folder, dst_image)
-    else:
-        # https://stackoverflow.com/questions/66842004/get-the-processor-type-using-python-for-apple-m1-processor-gives-me-an-intel-pro
-        import cpuinfo
-
-        # Just get the manufacturer of the processors
-        manufacturer = cpuinfo.get_cpu_info().get("brand_raw")
-        arch = (
-            "arm" if re.search("apple m\d", manufacturer, re.IGNORECASE) else "other"
-        )
-        print(f"The local machine's platform is {arch}.")
-        image, dockerfile, target = _get_image_name_dockerfile_target(
-            image_type, gpu, arch
-        )
-        print(f"dockerfile used is {dockerfile}")
-        command = [
-            "docker",
-            "build",
-            "-t",
-            image,
-            "-f",
-            os.path.join(curr_dir, "docker", dockerfile),
-        ]
-        if target:
-            command += ["--target", target]
-        if os.environ.get("no_proxy"):
-            command += ["--build-arg", f"no_proxy={os.environ['no_proxy']}"]
-        if os.environ.get("http_proxy"):
-            command += ["--build-arg", f"http_proxy={os.environ['http_proxy']}"]
-        if os.environ.get("https_proxy"):
-            command += ["--build-arg", f"https_proxy={os.environ['https_proxy']}"]
-        if os.environ.get(CONTAINER_NETWORK):
-            command += ["--network", os.environ[CONTAINER_NETWORK]]
-        command += [os.path.abspath(curr_dir)]
-        logger.info("Build image with command %s", command)
-        proc = run_command(command)
-    if proc.returncode != 0:
-        raise RuntimeError("Docker build failed.")
-
-
-def _get_image_name_dockerfile_target(type: str, gpu: bool, arch: str) -> str:
-    look_up = {
-        ("job-local", False, "arm"): (ML_JOB_IMAGE, "Dockerfile.job.arm", None),
-        ("job-local", False, "other"): (ML_JOB_IMAGE, "Dockerfile.job", None),
-        ("job-local", True, "other"): (ML_JOB_GPU_IMAGE, "Dockerfile.job.gpu", None),
-    }
-    return look_up[(type, gpu, arch)]
-
-
-def run_command(
-    cmd: Union[str, List[str]], cwd: str = None, shell: bool = False
-) -> Popen:
-    proc = Popen(
-        cmd, cwd=cwd, stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=shell
-    )
-    for x in iter(proc.stdout.readline, ""):
-        print(x, file=sys.stdout, end="")
-    proc.wait()
-    return proc
-
-
-class _DebugTraceback:
-    def __init__(self, debug):
-        self.cur_logging_level = logger.getEffectiveLevel()
-        self.debug = debug
-
-    def __enter__(self):
-        if self.debug:
-            logger.setLevel(logging.DEBUG)
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.debug and exc_type:
-            logger.setLevel(self.cur_logging_level)
-            return False
-        elif not self.debug and exc_type:
-            sys.stderr.write(f"{exc_type.__name__}: {str(exc_val)} \n")
-            sys.exit(1)
-        return True
-
-
-def suppress_traceback(debug: bool = True) -> None:
-    """
-    Decorator to suppress traceback when in debug mode.
-
-    Parameters
-    ----------
-    debug: bool
-        turn on debug mode or not
-
-    Returns
-    -------
-    None
-    """
-
-    def decorator(func):
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            with _DebugTraceback(debug) as tb:
-                return func(*args, **kwargs)
-
-        return wrapper
-
-    return decorator
-
-
-@runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
-def get_docker_client() -> "docker.client.DockerClient":
-    import docker
-
-    process = subprocess.Popen(
-        ["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT
-    )
-    process.communicate()
-    if process.returncode != 0:
-        raise RuntimeError("Docker is not started.")
-    return docker.from_env()
-
-
-@runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
-def run_container(
-    image: str,
-    bind_volumes: Dict,
-    env_vars: Dict,
-    command: str = None,
-    entrypoint: str = None,
-    verbose: bool = False,
-) -> int:
-    import docker
-
-    if env_vars is None:
-        env_vars = {}
-    # If Proxy variables are setup, pass it on to the docker run
-    if "http_proxy" in os.environ:
-        env_vars.update(
-            {
-                "http_proxy": os.environ.get("http_proxy"),
-                "https_proxy": os.environ.get("https_proxy"),
-            }
-        )
-    if verbose:
-        ads.opctl.logger.setLevel(logging.INFO)
-    logger.info(f"Running container with image: {image}")
-    logger.info(f"bind_volumes: {bind_volumes}")
-    logger.info(f"env_vars: {env_vars}")
-    logger.info(f"command: {command}")
-    logger.info(f"entrypoint: {entrypoint}")
-
-    # Print out the equivalent docker run command for debugging purpose
-    docker_run_cmd = [">>> docker run --rm"]
-    if entrypoint:
-        docker_run_cmd.append(f"--entrypoint {entrypoint}")
-    if env_vars:
-        docker_run_cmd.extend([f"-e {key}={val}" for key, val in env_vars.items()])
-    if bind_volumes:
-        docker_run_cmd.extend(
-            [f'-v {source}:{bind.get("bind")}' for source, bind in bind_volumes.items()]
-        )
-    docker_run_cmd.append(image)
-    if command:
-        docker_run_cmd.append(command)
-    logger.debug(" ".join(docker_run_cmd))
-
-    client = get_docker_client()
-    try:
-        client.api.inspect_image(image)
-    except docker.errors.ImageNotFound:
-        logger.warning(f"Image {image} not found. Try pulling it now....")
-        run_command(["docker", "pull", f"{image}"], None)
-    try:
-        kwargs = {}
-        if CONTAINER_NETWORK in os.environ:
-            kwargs["network_mode"] = os.environ[CONTAINER_NETWORK]
-        container = client.containers.run(
-            image=image,
-            volumes=bind_volumes,
-            command=shlex.split(command) if command else None,
-            environment=env_vars,
-            detach=True,
-            entrypoint=entrypoint,
-            user=0,
-            **kwargs
-            # auto_remove=True,
-        )
-        logger.info("Container ID: %s", container.id)
-        for line in container.logs(stream=True, follow=True):
-            print(line.decode("utf-8"), end="")
-
-        result = container.wait()
-        return result.get("StatusCode", -1)
-    except docker.errors.APIError as ex:
-        logger.error(ex.explanation)
-        return -1
-    finally:
-        # Remove the container
-        container.remove()

ads/oracledb/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/