oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/jobs/builders/runtimes/pytorch_runtime.py

@@ -1,204 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.jobs.builders.runtimes.artifact import PythonArtifact, GitPythonArtifact
-from ads.jobs.builders.runtimes.base import MultiNodeRuntime
-from ads.jobs.builders.runtimes.python_runtime import (
-    PythonRuntime,
-    GitPythonRuntime,
-)
-
-
-class PyTorchDistributedRuntime(PythonRuntime, MultiNodeRuntime):
-    """Represents runtime supporting PyTorch Distributed training."""
-    CONST_GIT = "git"
-    CONST_INPUT = "inputs"
-    CONST_DEP = "dependencies"
-    CONST_PIP_REQ = "pipRequirements"
-    CONST_PIP_PKG = "pipPackages"
-    CONST_COMMAND = "command"
-    CONST_DEEPSPEED = "deepspeed"
-
-    def with_git(
-        self, url: str, branch: str = None, commit: str = None, secret_ocid: str = None
-    ):
-        """Specifies the Git repository and branch/commit for the job source code.
-
-        Parameters
-        ----------
-        url : str
-            URL of the Git repository.
-        branch : str, optional
-            Git branch name, by default None, the default branch will be used.
-        commit : str, optional
-            Git commit ID (SHA1 hash), by default None, the most recent commit will be used.
-        secret_ocid : str
-            The secret OCID storing the SSH key content for checking out the Git repository.
-
-        Returns
-        -------
-        self
-            The runtime instance.
-        """
-        git_spec = {GitPythonRuntime.CONST_GIT_URL: url}
-        if branch:
-            git_spec[GitPythonRuntime.CONST_BRANCH] = branch
-        if commit:
-            git_spec[GitPythonRuntime.CONST_COMMIT] = commit
-        if secret_ocid:
-            git_spec[GitPythonRuntime.CONST_GIT_SSH_SECRET_ID] = secret_ocid
-        return self.set_spec(self.CONST_GIT, git_spec)
-
-    @property
-    def git(self) -> str:
-        """The specification for source code from Git repository."""
-        return self.get_spec(self.CONST_GIT)
-
-    def with_inputs(self, mappings: dict):
-        """Specifies the input files to be copied into the job run.
-
-        Parameters
-        ----------
-        mappings : dict
-            Each key is the source path (uri). It can be http/ftp link or OCI object storage URI.
-            The corresponding value is the destination path in the job run, relative to the working directory.
-
-        Returns
-        -------
-        self
-            The runtime instance.
-
-        Examples
-        --------
-        >>> pt_runtime.with_inputs({"oci://bucket@namespace/path/to/file.txt": "data/input.txt"})
-
-        """
-        return self.set_spec(self.CONST_INPUT, mappings)
-
-    @property
-    def inputs(self) -> dict:
-        """The input files to be copied into the job run."""
-        return self.get_spec(self.CONST_INPUT)
-
-    def with_dependency(self, pip_req=None, pip_pkg=None):
-        """Specifies additional dependencies to be installed using pip.
-
-        Parameters
-        ----------
-        pip_req : str, optional
-            Path of the requirements.txt file, relative to the working directory, by default None
-        pip_pkg : str, optional
-            Command line args for `pip install`, by default None.
-            Packages with version specification needs to be quoted.
-
-        Returns
-        -------
-        self
-            The runtime instance.
-
-        Examples
-        --------
-        >>> pt_runtime.with_dependency('"package>1.0"')
-        """
-        dep = {}
-        if pip_req:
-            dep[self.CONST_PIP_REQ] = pip_req
-        if pip_pkg:
-            dep[self.CONST_PIP_PKG] = pip_pkg
-        if dep:
-            self.set_spec(self.CONST_DEP, dep)
-        return self
-
-    @property
-    def dependencies(self) -> dict:
-        """Additional pip dependencies."""
-        return self.get_spec(self.CONST_DEP)
-
-    def with_command(self, command: str, use_deepspeed=False):
-        """Specifies the command for launching the workload.
-
-        Parameters
-        ----------
-        command : str
-            The command for launching the workload.
-            The command should start with `torchrun`, `deepspeed` or `accelerate launch`.
-
-            For `torchrun`,
-            ADS will set `--nnode`, `--nproc_per_node`, `--rdzv_backend` and `--rdzv_endpoint` automatically.
-            The default `rdzv_backend` will be `c10d`.
-            The default port for `rdzv_endpoint` is 29400
-
-            For `deepspeed`,
-            ADS will generate the hostfile automatically and setup the SSH configurations.
-
-            For `accelerate launch`
-            You can add your config YAML to the source code and specify it using `--config_file` argument.
-            In your config, please use `LOCAL_MACHINE` as the compute environment.
-            The same config file will be used by all nodes in multi-node workload.
-            ADS will set `--num_processes`, `--num_machines`, `--machine_rank`, `--main_process_ip`
-            and `--main_process_port` automatically. These values will override the ones from your config YAML.
-            The default `main_process_port` is 29400
-
-            If you don't want to use the options set by ADS automatically,
-            you can specify them explicitly in the command.
-
-        use_deepspeed : bool, optional
-            Indicate whether to configure deepspeed for multi-node workload, by default False.
-            If your command starts with "deepspeed" or contains the argument "--use_deepspeed",
-            your job runs will be configured for deepspeed regardless of this setting.
-            Make sure to set use_deepspeed to `True` here
-            if you are using `accelerate launch` with deepspeed setting in config YAML.
-
-        Returns
-        -------
-        self
-            The runtime instance.
-
-        Examples
-        --------
-        >>> pt_runtime.with_command("torchrun train.py")
-        """
-        if use_deepspeed:
-            self.set_spec(self.CONST_DEEPSPEED, True)
-        return self.set_spec(self.CONST_COMMAND, command)
-
-    @property
-    def command(self):
-        """The command for launching the workload."""
-        return self.get_spec(self.CONST_COMMAND)
-    
-    @property
-    def use_deepspeed(self):
-        """Indicate whether whether to configure deepspeed for multi-node workload"""
-        if self.get_spec(self.CONST_DEEPSPEED):
-            return True
-        return False
-
-
-class PyTorchDistributedArtifact(PythonArtifact):
-    CONST_DRIVER_SCRIPT = "driver_pytorch.py"
-    CONST_LIB_HOSTNAME = "hostname_from_env.c"
-    CONST_OCI_METRICS = "oci_metrics.py"
-
-    def __init__(self, source, runtime=None) -> None:
-        if not source:
-            source = ""
-        super().__init__(source, runtime)
-
-    def build(self):
-        """Prepares job artifact."""
-        self._copy_artifacts(
-            drivers=[
-                self.CONST_DRIVER_UTILS,
-                self.CONST_DRIVER_SCRIPT,
-                self.CONST_LIB_HOSTNAME,
-                self.CONST_OCI_METRICS,
-                GitPythonArtifact.CONST_DRIVER_SCRIPT,
-            ]
-        )
-
-        # Zip the job artifact
-        self.path = self._zip_artifacts()

ads/jobs/cli.py

@@ -1,104 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-import click
-
-import ads
-from ads.common.auth import AuthType, AuthContext
-from ads.jobs import Job, DataFlowRun, DataScienceJobRun
-from ads.opctl.utils import suppress_traceback
-
-
-@click.group("jobs")
-@click.help_option("--help", "-h")
-def commands():
-    pass
-
-
-@commands.command()
-@click.help_option("--help", "-h")
-@click.option("--file", "-f", help="path to YAML file", required=True)
-@click.option(
-    "--auth",
-    "-a",
-    help="authentication method",
-    type=click.Choice(AuthType.values()),
-    default="api_key",
-)
-@click.option(
-    "--oci-profile", help="oci config profile", required=False, default="DEFAULT"
-)
-@click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
-def run(file, auth, oci_profile, debug):
-    def _run(file, auth, oci_profile):
-        with AuthContext():
-            ads.set_auth(auth=auth, profile=oci_profile)
-            job = Job.from_yaml(uri=file)
-            job.create()
-            print(job.id)
-            job_run = job.run()
-            print(job_run.id)
-
-    suppress_traceback(debug)(_run)(file, auth, oci_profile)
-
-
-@commands.command()
-@click.help_option("--help", "-h")
-@click.argument("ocid")
-@click.option(
-    "--auth",
-    "-a",
-    help="authentication method",
-    type=click.Choice(AuthType.values()),
-    default="api_key",
-)
-@click.option(
-    "--oci-profile", help="oci config profile", required=False, default="DEFAULT"
-)
-@click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
-def watch(ocid, auth, oci_profile, debug):
-    def _watch(ocid, auth, oci_profile):
-        with AuthContext():
-            ads.set_auth(auth=auth, profile=oci_profile)
-            if "datasciencejobrun" in ocid:
-                run = DataScienceJobRun.from_ocid(ocid)
-            elif "dataflowrun" in ocid:
-                run = DataFlowRun.from_ocid(ocid)
-            else:
-                raise ValueError("Run OCID not recognized.")
-            run.watch()
-
-    suppress_traceback(debug)(_watch)(ocid, auth, oci_profile)
-
-
-@commands.command()
-@click.help_option("--help", "-h")
-@click.argument("ocid")
-@click.option(
-    "--auth",
-    "-a",
-    help="authentication method",
-    type=click.Choice(AuthType.values()),
-    default="api_key",
-)
-@click.option(
-    "--oci-profile", help="oci config profile", required=False, default="DEFAULT"
-)
-@click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
-def delete(ocid, auth, oci_profile, debug):
-    def _delete(ocid, auth, oci_profile):
-        with AuthContext():
-            ads.set_auth(auth=auth, profile=oci_profile)
-            if "datasciencejobrun" in ocid:
-                run = DataScienceJobRun.from_ocid(ocid)
-            elif "dataflowrun" in ocid:
-                run = DataFlowRun.from_ocid(ocid)
-            else:
-                raise ValueError("Run OCID not recognized.")
-            run.delete()
-
-    suppress_traceback(debug)(_delete)(ocid, auth, oci_profile)

ads/jobs/env_var_parser.py

@@ -1,131 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-import copy
-import json
-from typing import Union, List, Dict
-from configparser import ConfigParser, ExtendedInterpolation
-from configparser import (
-    InterpolationSyntaxError,
-    InterpolationDepthError,
-)
-from configparser import NoSectionError, NoOptionError, MAX_INTERPOLATION_DEPTH
-
-
-class EnvVarInterpolation(ExtendedInterpolation):
-    """Modified version of ExtendedInterpolation to ignore errors
-
-    https://github.com/python/cpython/blob/main/Lib/configparser.py
-    """
-
-    def before_set(self, parser, section: str, option: str, value: str) -> str:
-        return value
-
-    def _interpolate_some(self, parser, option, accum, rest, section, map, depth):
-        rawval = parser.get(section, option, raw=True, fallback=rest)
-        if depth > MAX_INTERPOLATION_DEPTH:
-            raise InterpolationDepthError(option, section, rawval)
-        while rest:
-            p = rest.find("$")
-            if p < 0:
-                accum.append(rest)
-                return
-            if p > 0:
-                accum.append(rest[:p])
-                rest = rest[p:]
-            # p is no longer used
-            c = rest[1:2]
-            if c == "$":
-                accum.append("$")
-                rest = rest[2:]
-            elif c == "{":
-                m = self._KEYCRE.match(rest)
-                if m is None:
-                    raise InterpolationSyntaxError(
-                        option,
-                        section,
-                        "Bad interpolation variable reference %r" % rest,
-                    )
-                path = m.group(1).split(":")
-                sect = section
-                opt = option
-                try:
-                    if len(path) == 1:
-                        opt = parser.optionxform(path[0])
-                        v = map[opt]
-                    elif len(path) == 2:
-                        sect = path[0]
-                        opt = parser.optionxform(path[1])
-                        v = parser.get(sect, opt, raw=True)
-                    else:
-                        raise InterpolationSyntaxError(
-                            option,
-                            section,
-                            "More than one ':' found: %r" % (rest[m.end() :],),
-                        )
-                except (KeyError, NoSectionError, NoOptionError):
-                    accum.append(rest)
-                    return
-
-                rest = rest[m.end() :]
-                if "$" in v:
-                    self._interpolate_some(
-                        parser,
-                        opt,
-                        accum,
-                        v,
-                        sect,
-                        dict(parser.items(sect, raw=True)),
-                        depth + 1,
-                    )
-                else:
-                    accum.append(v)
-            else:
-                accum.append("$")
-                rest = rest[1:]
-
-
-def parse(env_var: Union[Dict, List[dict]]) -> dict:
-    """Parse the environment variables and perform substitutions.
-    This will also converts kubernetes style environment variables from a list to a dictionary.
-
-    Parameters
-    ----------
-    env_var : dict or list
-        Environment variables specified as a list or a dictionary.
-        If evn_var is a list, it should be in the format of:
-            "[{"name": "ENV_NAME_1", "value": "ENV_VALUE_1"}, {"name": "ENV_NAME_2", "value": "ENV_VALUE_2"}]
-
-    Returns
-    -------
-    dict
-        Environment variable as a dictionary.
-    """
-    # Convert kubernetes style env to dict
-    if isinstance(env_var, list):
-        env_var = {ev["name"]: ev["value"] for ev in env_var}
-    else:
-        env_var = copy.deepcopy(env_var)
-    config = ConfigParser(interpolation=EnvVarInterpolation())
-    config.optionxform = str
-    for k in env_var.keys():
-        if env_var[k] is None:
-            # Convert None to empty string
-            env_var[k] = ""
-        elif not isinstance(env_var[k], str):
-            # If the value is not a string,
-            # try to dump it as json string
-            try:
-                env_var[k] = json.dumps(env_var[k])
-            except Exception:
-                # Cast the value to string if it is not json serializable
-                env_var[k] = str(env_var[k])
-
-    config["envs"] = env_var
-    return {k: config["envs"].get(k) for k in env_var.keys()}
-
-
-def escape(s: str) -> str:
-    return s.replace("$", "$$")

ads/jobs/extension.py

@@ -1,160 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-import json
-import os
-import shlex
-import tempfile
-import warnings
-from getopt import gnu_getopt
-
-from ads.common.decorator.runtime_dependency import (OptionalDependency,
-                                                     runtime_dependency)
-from ads.jobs import DataFlow, DataFlowRun, DataFlowRuntime, Job
-from ads.jobs.utils import get_dataflow_config
-from ads.opctl.constants import (ADS_DATAFLOW_CONFIG_FILE_NAME,
-                                 DEFAULT_ADS_CONFIG_FOLDER)
-
-
-def dataflow(line, cell=None):
-    opts, args = gnu_getopt(
-        shlex.split(line),
-        "f:a:c:t:n:hwo",
-        longopts=[
-            "filename=",
-            "archive=",
-            "config=",
-            "help=",
-            "watch=",
-            "log-type=",
-            "num-lines=",
-            "overwrite=",
-        ],
-    )
-    opts_dict = {k: v for k, v in opts}
-    if len(args) == 0 and ("-h" in opts_dict or "--help" in opts_dict):
-        print(
-            f"Run `dataflow run -h` or `dataflow log -h` to see options for subcommands."
-        )
-        return
-    if args[0] not in ("run", "log"):
-        print(
-            f"`dataflow` expects subcommand `run` or `log`. Use `dataflow log` or `dataflow run` with options."
-        )
-        return
-
-    if args[0] == "run":
-        dataflow_run(opts_dict, args, cell)
-    else:
-
-        dataflow_log(opts_dict, args)
-
-
-def dataflow_run(options, args, cell):
-    if "-h" in options or "--help" in options:
-        print(
-            f"""
-Usage: dataflow run [OPTIONS] -- [ARGS]
-Options:
-    -f, --filename, optional, filename to save the script to. default is "script.py".
-    -a, --archive, optional, uri to archive zip.
-    -c, --config, optional, configuration passed to dataflow. by default loads from {os.path.join(DEFAULT_ADS_CONFIG_FOLDER, ADS_DATAFLOW_CONFIG_FILE_NAME)}.
-    -w, --watch, optional, a flag indicating that dataflow run will be watched after submission.
-    -o, --overwrite, optional, overwrite file in object storage when uploading script or archive.zip.
-    -h, show this help message.
-Args:
-    arguments to pass to script.
-        """
-        )
-        return
-    if "-c" not in options and "--config" not in options:
-        dataflow_config = get_dataflow_config()
-    else:
-        if "-c" in options:
-            dataflow_config = json.loads(options["-c"])
-        if "--config" in options:
-            dataflow_config = json.loads(options["--config"])
-    if "-f" in options:
-        script_name = options["-f"]
-    elif "--file" in options:
-        script_name = options["--file"]
-    else:
-        script_name = "script.py"
-    if "-a" in options:
-        archive_name = options["-a"]
-    elif "--archive" in options:
-        archive_name = options["--archive"]
-    elif hasattr(dataflow_config, "archive_uri") and dataflow_config.archive_uri:
-        archive_name = dataflow_config.archive_uri
-    else:
-        archive_name = None
-    with tempfile.TemporaryDirectory() as td:
-        file = os.path.join(td, script_name)
-        with open(file, "w") as f:
-            f.write(cell)
-        rt_spec = {
-            "scriptPathURI": file,
-            "scriptBucket": dataflow_config.pop("script_bucket"),
-        }
-        if len(args) > 1:
-            rt_spec["args"] = args[1:]
-        if archive_name:
-            rt_spec["archiveUri"] = archive_name
-            rt_spec["archiveBucket"] = dataflow_config.pop("archive_bucket", None)
-            if not archive_name.startswith("oci://") and not rt_spec["archiveBucket"]:
-                raise ValueError(
-                    "`archiveBucket` has to be set in the config if `archive` is a local path."
-                )
-        rt = DataFlowRuntime(rt_spec)
-        infra = DataFlow(spec=dataflow_config)
-        if "-o" in options or "--overwrite" in options:
-            df = infra.create(rt, overwrite=True)
-        else:
-            df = infra.create(rt)
-        print("DataFlow App ID", df.id)
-        df_run = df.run()
-        print("DataFlow Run ID", df_run.id)
-        print("DataFlow Run Page", df_run.run_details_link)
-        if "-w" in options or "--watch" in options:
-            df_run.watch()
-            print(df_run.logs.application.stdout.tail())
-
-
-def dataflow_log(options, args):
-    if "-h" in options or "--help" in options:
-        print(
-            f"""
-Usage: dataflow log [OPTIONS] [RUN_ID]
-Options:
-    -t, --log-type, optional, should be one of application, driver, executor. default is "application".
-    -n, --num-lines, optional, show last `n` lines of the log
-    -h, show this help message.
-        """
-        )
-        return
-    log_type = (
-        options.get("-t", None) or options.get("--log-type", None) or "application"
-    )
-    if log_type not in ("application", "driver", "executor"):
-        print("Log type should be one of application, driver, executor.")
-        return
-    n = options.get("-n", None) or options.get("--num-lines", None)
-    n = int(n) if (n and len(n) > 0) else None
-    if len(args) < 2:
-        raise ValueError("DataFlow Run ID must be provided.")
-    df_run = DataFlowRun.from_ocid(args[1])
-    if log_type == "application":
-        print(df_run.logs.application.stdout.tail(n=n))
-    if log_type == "driver":
-        print(df_run.logs.driver.stdout.tail(n=n))
-    if log_type == "executor":
-        print(df_run.logs.executor.stdout.tail(n=n))
-
-
-@runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
-def load_ipython_extension(ipython):
-    ipython.register_magic_function(dataflow, "line_cell")

ads/jobs/schema/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/