oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/jobs/builders/runtimes/pytorch_runtime.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.jobs.builders.runtimes.artifact import PythonArtifact, GitPythonArtifact
+from ads.jobs.builders.runtimes.base import MultiNodeRuntime
+from ads.jobs.builders.runtimes.python_runtime import (
+    PythonRuntime,
+    GitPythonRuntime,
+)
+
+
+class PyTorchDistributedRuntime(PythonRuntime, MultiNodeRuntime):
+    """Represents runtime supporting PyTorch Distributed training."""
+    CONST_GIT = "git"
+    CONST_INPUT = "inputs"
+    CONST_DEP = "dependencies"
+    CONST_PIP_REQ = "pipRequirements"
+    CONST_PIP_PKG = "pipPackages"
+    CONST_COMMAND = "command"
+    CONST_DEEPSPEED = "deepspeed"
+
+    def with_git(
+        self, url: str, branch: str = None, commit: str = None, secret_ocid: str = None
+    ):
+        """Specifies the Git repository and branch/commit for the job source code.
+
+        Parameters
+        ----------
+        url : str
+            URL of the Git repository.
+        branch : str, optional
+            Git branch name, by default None, the default branch will be used.
+        commit : str, optional
+            Git commit ID (SHA1 hash), by default None, the most recent commit will be used.
+        secret_ocid : str
+            The secret OCID storing the SSH key content for checking out the Git repository.
+
+        Returns
+        -------
+        self
+            The runtime instance.
+        """
+        git_spec = {GitPythonRuntime.CONST_GIT_URL: url}
+        if branch:
+            git_spec[GitPythonRuntime.CONST_BRANCH] = branch
+        if commit:
+            git_spec[GitPythonRuntime.CONST_COMMIT] = commit
+        if secret_ocid:
+            git_spec[GitPythonRuntime.CONST_GIT_SSH_SECRET_ID] = secret_ocid
+        return self.set_spec(self.CONST_GIT, git_spec)
+
+    @property
+    def git(self) -> str:
+        """The specification for source code from Git repository."""
+        return self.get_spec(self.CONST_GIT)
+
+    def with_inputs(self, mappings: dict):
+        """Specifies the input files to be copied into the job run.
+
+        Parameters
+        ----------
+        mappings : dict
+            Each key is the source path (uri). It can be http/ftp link or OCI object storage URI.
+            The corresponding value is the destination path in the job run, relative to the working directory.
+
+        Returns
+        -------
+        self
+            The runtime instance.
+
+        Examples
+        --------
+        >>> pt_runtime.with_inputs({"oci://bucket@namespace/path/to/file.txt": "data/input.txt"})
+
+        """
+        return self.set_spec(self.CONST_INPUT, mappings)
+
+    @property
+    def inputs(self) -> dict:
+        """The input files to be copied into the job run."""
+        return self.get_spec(self.CONST_INPUT)
+
+    def with_dependency(self, pip_req=None, pip_pkg=None):
+        """Specifies additional dependencies to be installed using pip.
+
+        Parameters
+        ----------
+        pip_req : str, optional
+            Path of the requirements.txt file, relative to the working directory, by default None
+        pip_pkg : str, optional
+            Command line args for `pip install`, by default None.
+            Packages with version specification needs to be quoted.
+
+        Returns
+        -------
+        self
+            The runtime instance.
+
+        Examples
+        --------
+        >>> pt_runtime.with_dependency('"package>1.0"')
+        """
+        dep = {}
+        if pip_req:
+            dep[self.CONST_PIP_REQ] = pip_req
+        if pip_pkg:
+            dep[self.CONST_PIP_PKG] = pip_pkg
+        if dep:
+            self.set_spec(self.CONST_DEP, dep)
+        return self
+
+    @property
+    def dependencies(self) -> dict:
+        """Additional pip dependencies."""
+        return self.get_spec(self.CONST_DEP)
+
+    def with_command(self, command: str, use_deepspeed=False):
+        """Specifies the command for launching the workload.
+
+        Parameters
+        ----------
+        command : str
+            The command for launching the workload.
+            The command should start with `torchrun`, `deepspeed` or `accelerate launch`.
+
+            For `torchrun`,
+            ADS will set `--nnode`, `--nproc_per_node`, `--rdzv_backend` and `--rdzv_endpoint` automatically.
+            The default `rdzv_backend` will be `c10d`.
+            The default port for `rdzv_endpoint` is 29400
+
+            For `deepspeed`,
+            ADS will generate the hostfile automatically and setup the SSH configurations.
+
+            For `accelerate launch`
+            You can add your config YAML to the source code and specify it using `--config_file` argument.
+            In your config, please use `LOCAL_MACHINE` as the compute environment.
+            The same config file will be used by all nodes in multi-node workload.
+            ADS will set `--num_processes`, `--num_machines`, `--machine_rank`, `--main_process_ip`
+            and `--main_process_port` automatically. These values will override the ones from your config YAML.
+            The default `main_process_port` is 29400
+
+            If you don't want to use the options set by ADS automatically,
+            you can specify them explicitly in the command.
+
+        use_deepspeed : bool, optional
+            Indicate whether to configure deepspeed for multi-node workload, by default False.
+            If your command starts with "deepspeed" or contains the argument "--use_deepspeed",
+            your job runs will be configured for deepspeed regardless of this setting.
+            Make sure to set use_deepspeed to `True` here
+            if you are using `accelerate launch` with deepspeed setting in config YAML.
+
+        Returns
+        -------
+        self
+            The runtime instance.
+
+        Examples
+        --------
+        >>> pt_runtime.with_command("torchrun train.py")
+        """
+        if use_deepspeed:
+            self.set_spec(self.CONST_DEEPSPEED, True)
+        return self.set_spec(self.CONST_COMMAND, command)
+
+    @property
+    def command(self):
+        """The command for launching the workload."""
+        return self.get_spec(self.CONST_COMMAND)
+    
+    @property
+    def use_deepspeed(self):
+        """Indicate whether whether to configure deepspeed for multi-node workload"""
+        if self.get_spec(self.CONST_DEEPSPEED):
+            return True
+        return False
+
+
+class PyTorchDistributedArtifact(PythonArtifact):
+    CONST_DRIVER_SCRIPT = "driver_pytorch.py"
+    CONST_LIB_HOSTNAME = "hostname_from_env.c"
+    CONST_OCI_METRICS = "oci_metrics.py"
+
+    def __init__(self, source, runtime=None) -> None:
+        if not source:
+            source = ""
+        super().__init__(source, runtime)
+
+    def build(self):
+        """Prepares job artifact."""
+        self._copy_artifacts(
+            drivers=[
+                self.CONST_DRIVER_UTILS,
+                self.CONST_DRIVER_SCRIPT,
+                self.CONST_LIB_HOSTNAME,
+                self.CONST_OCI_METRICS,
+                GitPythonArtifact.CONST_DRIVER_SCRIPT,
+            ]
+        )
+
+        # Zip the job artifact
+        self.path = self._zip_artifacts()

ads/jobs/cli.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import click
+
+import ads
+from ads.common.auth import AuthType, AuthContext
+from ads.jobs import Job, DataFlowRun, DataScienceJobRun
+from ads.opctl.utils import suppress_traceback
+
+
+@click.group("jobs")
+@click.help_option("--help", "-h")
+def commands():
+    pass
+
+
+@commands.command()
+@click.help_option("--help", "-h")
+@click.option("--file", "-f", help="path to YAML file", required=True)
+@click.option(
+    "--auth",
+    "-a",
+    help="authentication method",
+    type=click.Choice(AuthType.values()),
+    default="api_key",
+)
+@click.option(
+    "--oci-profile", help="oci config profile", required=False, default="DEFAULT"
+)
+@click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
+def run(file, auth, oci_profile, debug):
+    def _run(file, auth, oci_profile):
+        with AuthContext():
+            ads.set_auth(auth=auth, profile=oci_profile)
+            job = Job.from_yaml(uri=file)
+            job.create()
+            print(job.id)
+            job_run = job.run()
+            print(job_run.id)
+
+    suppress_traceback(debug)(_run)(file, auth, oci_profile)
+
+
+@commands.command()
+@click.help_option("--help", "-h")
+@click.argument("ocid")
+@click.option(
+    "--auth",
+    "-a",
+    help="authentication method",
+    type=click.Choice(AuthType.values()),
+    default="api_key",
+)
+@click.option(
+    "--oci-profile", help="oci config profile", required=False, default="DEFAULT"
+)
+@click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
+def watch(ocid, auth, oci_profile, debug):
+    def _watch(ocid, auth, oci_profile):
+        with AuthContext():
+            ads.set_auth(auth=auth, profile=oci_profile)
+            if "datasciencejobrun" in ocid:
+                run = DataScienceJobRun.from_ocid(ocid)
+            elif "dataflowrun" in ocid:
+                run = DataFlowRun.from_ocid(ocid)
+            else:
+                raise ValueError("Run OCID not recognized.")
+            run.watch()
+
+    suppress_traceback(debug)(_watch)(ocid, auth, oci_profile)
+
+
+@commands.command()
+@click.help_option("--help", "-h")
+@click.argument("ocid")
+@click.option(
+    "--auth",
+    "-a",
+    help="authentication method",
+    type=click.Choice(AuthType.values()),
+    default="api_key",
+)
+@click.option(
+    "--oci-profile", help="oci config profile", required=False, default="DEFAULT"
+)
+@click.option("--debug", "-d", help="set debug mode", is_flag=True, default=False)
+def delete(ocid, auth, oci_profile, debug):
+    def _delete(ocid, auth, oci_profile):
+        with AuthContext():
+            ads.set_auth(auth=auth, profile=oci_profile)
+            if "datasciencejobrun" in ocid:
+                run = DataScienceJobRun.from_ocid(ocid)
+            elif "dataflowrun" in ocid:
+                run = DataFlowRun.from_ocid(ocid)
+            else:
+                raise ValueError("Run OCID not recognized.")
+            run.delete()
+
+    suppress_traceback(debug)(_delete)(ocid, auth, oci_profile)

ads/jobs/env_var_parser.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import copy
+import json
+from typing import Union, List, Dict
+from configparser import ConfigParser, ExtendedInterpolation
+from configparser import (
+    InterpolationSyntaxError,
+    InterpolationDepthError,
+)
+from configparser import NoSectionError, NoOptionError, MAX_INTERPOLATION_DEPTH
+
+
+class EnvVarInterpolation(ExtendedInterpolation):
+    """Modified version of ExtendedInterpolation to ignore errors
+
+    https://github.com/python/cpython/blob/main/Lib/configparser.py
+    """
+
+    def before_set(self, parser, section: str, option: str, value: str) -> str:
+        return value
+
+    def _interpolate_some(self, parser, option, accum, rest, section, map, depth):
+        rawval = parser.get(section, option, raw=True, fallback=rest)
+        if depth > MAX_INTERPOLATION_DEPTH:
+            raise InterpolationDepthError(option, section, rawval)
+        while rest:
+            p = rest.find("$")
+            if p < 0:
+                accum.append(rest)
+                return
+            if p > 0:
+                accum.append(rest[:p])
+                rest = rest[p:]
+            # p is no longer used
+            c = rest[1:2]
+            if c == "$":
+                accum.append("$")
+                rest = rest[2:]
+            elif c == "{":
+                m = self._KEYCRE.match(rest)
+                if m is None:
+                    raise InterpolationSyntaxError(
+                        option,
+                        section,
+                        "Bad interpolation variable reference %r" % rest,
+                    )
+                path = m.group(1).split(":")
+                sect = section
+                opt = option
+                try:
+                    if len(path) == 1:
+                        opt = parser.optionxform(path[0])
+                        v = map[opt]
+                    elif len(path) == 2:
+                        sect = path[0]
+                        opt = parser.optionxform(path[1])
+                        v = parser.get(sect, opt, raw=True)
+                    else:
+                        raise InterpolationSyntaxError(
+                            option,
+                            section,
+                            "More than one ':' found: %r" % (rest[m.end() :],),
+                        )
+                except (KeyError, NoSectionError, NoOptionError):
+                    accum.append(rest)
+                    return
+
+                rest = rest[m.end() :]
+                if "$" in v:
+                    self._interpolate_some(
+                        parser,
+                        opt,
+                        accum,
+                        v,
+                        sect,
+                        dict(parser.items(sect, raw=True)),
+                        depth + 1,
+                    )
+                else:
+                    accum.append(v)
+            else:
+                accum.append("$")
+                rest = rest[1:]
+
+
+def parse(env_var: Union[Dict, List[dict]]) -> dict:
+    """Parse the environment variables and perform substitutions.
+    This will also converts kubernetes style environment variables from a list to a dictionary.
+
+    Parameters
+    ----------
+    env_var : dict or list
+        Environment variables specified as a list or a dictionary.
+        If evn_var is a list, it should be in the format of:
+            "[{"name": "ENV_NAME_1", "value": "ENV_VALUE_1"}, {"name": "ENV_NAME_2", "value": "ENV_VALUE_2"}]
+
+    Returns
+    -------
+    dict
+        Environment variable as a dictionary.
+    """
+    # Convert kubernetes style env to dict
+    if isinstance(env_var, list):
+        env_var = {ev["name"]: ev["value"] for ev in env_var}
+    else:
+        env_var = copy.deepcopy(env_var)
+    config = ConfigParser(interpolation=EnvVarInterpolation())
+    config.optionxform = str
+    for k in env_var.keys():
+        if env_var[k] is None:
+            # Convert None to empty string
+            env_var[k] = ""
+        elif not isinstance(env_var[k], str):
+            # If the value is not a string,
+            # try to dump it as json string
+            try:
+                env_var[k] = json.dumps(env_var[k])
+            except Exception:
+                # Cast the value to string if it is not json serializable
+                env_var[k] = str(env_var[k])
+
+    config["envs"] = env_var
+    return {k: config["envs"].get(k) for k in env_var.keys()}
+
+
+def escape(s: str) -> str:
+    return s.replace("$", "$$")

ads/jobs/extension.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import json
+import os
+import shlex
+import tempfile
+import warnings
+from getopt import gnu_getopt
+
+from ads.common.decorator.runtime_dependency import (OptionalDependency,
+                                                     runtime_dependency)
+from ads.jobs import DataFlow, DataFlowRun, DataFlowRuntime, Job
+from ads.jobs.utils import get_dataflow_config
+from ads.opctl.constants import (ADS_DATAFLOW_CONFIG_FILE_NAME,
+                                 DEFAULT_ADS_CONFIG_FOLDER)
+
+
+def dataflow(line, cell=None):
+    opts, args = gnu_getopt(
+        shlex.split(line),
+        "f:a:c:t:n:hwo",
+        longopts=[
+            "filename=",
+            "archive=",
+            "config=",
+            "help=",
+            "watch=",
+            "log-type=",
+            "num-lines=",
+            "overwrite=",
+        ],
+    )
+    opts_dict = {k: v for k, v in opts}
+    if len(args) == 0 and ("-h" in opts_dict or "--help" in opts_dict):
+        print(
+            f"Run `dataflow run -h` or `dataflow log -h` to see options for subcommands."
+        )
+        return
+    if args[0] not in ("run", "log"):
+        print(
+            f"`dataflow` expects subcommand `run` or `log`. Use `dataflow log` or `dataflow run` with options."
+        )
+        return
+
+    if args[0] == "run":
+        dataflow_run(opts_dict, args, cell)
+    else:
+
+        dataflow_log(opts_dict, args)
+
+
+def dataflow_run(options, args, cell):
+    if "-h" in options or "--help" in options:
+        print(
+            f"""
+Usage: dataflow run [OPTIONS] -- [ARGS]
+Options:
+    -f, --filename, optional, filename to save the script to. default is "script.py".
+    -a, --archive, optional, uri to archive zip.
+    -c, --config, optional, configuration passed to dataflow. by default loads from {os.path.join(DEFAULT_ADS_CONFIG_FOLDER, ADS_DATAFLOW_CONFIG_FILE_NAME)}.
+    -w, --watch, optional, a flag indicating that dataflow run will be watched after submission.
+    -o, --overwrite, optional, overwrite file in object storage when uploading script or archive.zip.
+    -h, show this help message.
+Args:
+    arguments to pass to script.
+        """
+        )
+        return
+    if "-c" not in options and "--config" not in options:
+        dataflow_config = get_dataflow_config()
+    else:
+        if "-c" in options:
+            dataflow_config = json.loads(options["-c"])
+        if "--config" in options:
+            dataflow_config = json.loads(options["--config"])
+    if "-f" in options:
+        script_name = options["-f"]
+    elif "--file" in options:
+        script_name = options["--file"]
+    else:
+        script_name = "script.py"
+    if "-a" in options:
+        archive_name = options["-a"]
+    elif "--archive" in options:
+        archive_name = options["--archive"]
+    elif hasattr(dataflow_config, "archive_uri") and dataflow_config.archive_uri:
+        archive_name = dataflow_config.archive_uri
+    else:
+        archive_name = None
+    with tempfile.TemporaryDirectory() as td:
+        file = os.path.join(td, script_name)
+        with open(file, "w") as f:
+            f.write(cell)
+        rt_spec = {
+            "scriptPathURI": file,
+            "scriptBucket": dataflow_config.pop("script_bucket"),
+        }
+        if len(args) > 1:
+            rt_spec["args"] = args[1:]
+        if archive_name:
+            rt_spec["archiveUri"] = archive_name
+            rt_spec["archiveBucket"] = dataflow_config.pop("archive_bucket", None)
+            if not archive_name.startswith("oci://") and not rt_spec["archiveBucket"]:
+                raise ValueError(
+                    "`archiveBucket` has to be set in the config if `archive` is a local path."
+                )
+        rt = DataFlowRuntime(rt_spec)
+        infra = DataFlow(spec=dataflow_config)
+        if "-o" in options or "--overwrite" in options:
+            df = infra.create(rt, overwrite=True)
+        else:
+            df = infra.create(rt)
+        print("DataFlow App ID", df.id)
+        df_run = df.run()
+        print("DataFlow Run ID", df_run.id)
+        print("DataFlow Run Page", df_run.run_details_link)
+        if "-w" in options or "--watch" in options:
+            df_run.watch()
+            print(df_run.logs.application.stdout.tail())
+
+
+def dataflow_log(options, args):
+    if "-h" in options or "--help" in options:
+        print(
+            f"""
+Usage: dataflow log [OPTIONS] [RUN_ID]
+Options:
+    -t, --log-type, optional, should be one of application, driver, executor. default is "application".
+    -n, --num-lines, optional, show last `n` lines of the log
+    -h, show this help message.
+        """
+        )
+        return
+    log_type = (
+        options.get("-t", None) or options.get("--log-type", None) or "application"
+    )
+    if log_type not in ("application", "driver", "executor"):
+        print("Log type should be one of application, driver, executor.")
+        return
+    n = options.get("-n", None) or options.get("--num-lines", None)
+    n = int(n) if (n and len(n) > 0) else None
+    if len(args) < 2:
+        raise ValueError("DataFlow Run ID must be provided.")
+    df_run = DataFlowRun.from_ocid(args[1])
+    if log_type == "application":
+        print(df_run.logs.application.stdout.tail(n=n))
+    if log_type == "driver":
+        print(df_run.logs.driver.stdout.tail(n=n))
+    if log_type == "executor":
+        print(df_run.logs.executor.stdout.tail(n=n))
+
+
+@runtime_dependency(module="IPython", install_from=OptionalDependency.NOTEBOOK)
+def load_ipython_extension(ipython):
+    ipython.register_magic_function(dataflow, "line_cell")

ads/jobs/schema/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/