oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/opctl/diagnostics/check_distributed_job_requirements.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import ads
+from ads.common import oci_client as oc
+import fsspec
+import uuid
+import os
+from ads.jobs import Job
+import oci
+from collections import defaultdict
+import ipaddress
+from ads.opctl.distributed.common.abstract_cluster_provider import ClusterProvider
+from ads.opctl.diagnostics.requirement_exception import RequirementException
+import urllib.parse
+import requests
+
+cp = ClusterProvider(mode="MAIN")
+auth = cp.get_oci_auth()
+cf = oc.OCIClientFactory(**auth)
+
+
+class PortsChecker:
+    def __init__(self, subnet_id=None, auth=None, **kwargs):
+        if subnet_id:
+            self.subnet_id = subnet_id
+        else:
+            job_ocid = os.environ["JOB_OCID"]
+            jobDef = Job.from_datascience_job(job_ocid)
+            self.subnet_id = jobDef.infrastructure.subnet_id
+        self.authinfo = auth or ads.common.auth.default_signer()
+
+    def _fetch_security_list(self):
+        core_client = oci.core.VirtualNetworkClient(**self.authinfo)
+        subnet = core_client.get_subnet(self.subnet_id).data
+        cidr = subnet.cidr_block
+        security_list_ids = subnet.security_list_ids
+        print(f"{cidr=}")
+        # print(type(security_list_ids))
+        for id in security_list_ids:
+            print(f"{id=}")
+
+        port_open_details = defaultdict(set)
+        for id in security_list_ids:
+            security_list_info = core_client.get_security_list(id).data
+            # print(f"Egress: {security_list_info.egress_security_rules}")
+            ingress_security_rules = security_list_info.ingress_security_rules
+            for rule in ingress_security_rules:
+                tcp_options = (
+                    rule.tcp_options.destination_port_range if rule.tcp_options else {}
+                )
+                source = rule.source
+                if tcp_options:
+                    port_open_details[source].add((tcp_options.min, tcp_options.max))
+                if (
+                    rule.protocol == "all" or rule.protocol.lower() == "6"
+                ) and rule.tcp_options is None:
+                    port_open_details[source].add((0,))
+
+        for key in port_open_details:
+            port_open_details[key] = sorted(port_open_details[key])
+        return port_open_details
+
+    def _flatten_candidate_port_range(self, port_list):
+        """
+        Ports are opened by providing the CIDR block of the source host. We need to check only within the hosts that intersects the IP of the job instance
+        """
+        self_ip = cp.find_self_ip(self.authinfo)
+        flattened_port_list = []
+        for cidr in port_list:
+            if ipaddress.ip_address(self_ip) in ipaddress.ip_network(cidr):
+                flattened_port_list += port_list[cidr]
+        return sorted(flattened_port_list)
+
+    def check_ports(self, ports, *kwargs):
+        port_list = self._flatten_candidate_port_range(self._fetch_security_list())
+        unsatisfied_ports = []
+        print(f"Checking if ports are open", flush=True)
+        print(f"Open port information: {port_list}", flush=True)
+        if port_list[0] == (0,):
+            print(f"Subnet has all ports open.")
+            return True
+        for port in ports:
+            print(f"Checking if port {port} is open", flush=True)
+            if str(port).lower() == "all":
+                if port_list[0] != (0,):
+                    print(
+                        f"Requires ALL ports for this subnet to be open. Open ports are: {'None' if len(port_list) == 0 else port_list}"
+                    )
+                    unsatisfied_ports.append(port)
+                    break
+                else:
+                    return True
+            elif port in port_list:
+                print(f"found port in open port list: {port_list}", flush=True)
+                continue
+            else:
+                if len(port.split("-")) == 1:
+                    found = False
+                    for port_range in port_list:
+                        if int(port) >= port_range[0] and int(port) <= port_range[1]:
+                            print(
+                                f"found port: {port} in range {port_range}", flush=True
+                            )
+                            found = True
+                            break
+                    if not found:
+                        print(
+                            f"Not found port: {port} in {port_list}!!! Check security list",
+                            flush=True,
+                        )
+                        unsatisfied_ports.append(port)
+                else:
+                    minport = int(port.split("-")[0])
+                    maxport = int(port.split("-")[1])
+                    found = False
+                    curr_minport = minport
+                    # It might be better to normalize the range in port_list
+                    # to remove any possible overlaps between the ranges. That
+                    # will make search much easier than below.
+                    for port_range in port_list:
+                        if curr_minport >= port_range[1]:
+                            continue
+
+                        if curr_minport >= port_range[0]:
+                            if maxport <= port_range[1]:
+                                print(f"found port open in range {port_list}")
+                                found = True
+                                break
+                            else:
+                                curr_minport = (
+                                    port_range[1] + 1
+                                )  # Can fail if the next range starts at maxport or less than maxport
+                    if not found:
+                        print(
+                            f"Not found port: {port} in {port_list}!!! Check security list",
+                            flush=True,
+                        )
+                        unsatisfied_ports.append(port)
+        if len(unsatisfied_ports) == 0:
+            return True
+        else:
+            raise RequirementException(
+                f"Following ports are not opened: {unsatisfied_ports}. Update security list corresponding to the subnet: {self.subnet_id}",
+            )
+
+
+class Default:
+    def __init__(self, auth, *kwargs):
+        self.auth = auth
+
+    def job_info_access_policy(self, **kwargs):
+        print(f"Checking Job info access")
+        try:
+            job_ocid = os.environ["JOB_OCID"]
+            Job.from_datascience_job(job_ocid)
+            print(f"Passed Job info access")
+        except oci.exceptions.ServiceError as se:
+            print(se)
+            raise RequirementException(
+                f"Could not access Job Definition details. To fix this: \n \
+                Allow dynamic-group <distributed_training_job_runs> to use data-science-family in compartment <your_compartment_name>"
+            )
+
+    def job_vcn_info_access_policy(self, **kwargs):
+        print(f"Checking VCN info access")
+
+        job_ocid = os.environ["JOB_OCID"]
+        jobDef = Job.from_datascience_job(job_ocid)
+
+        subnet_id = jobDef.infrastructure.subnet_id
+        client = cf.create_client(oci.core.VirtualNetworkClient)
+        try:
+            client.get_subnet(subnet_id).data
+            print(f"Passed VCN info access")
+        except oci.exceptions.ServiceError as se:
+            print(se)
+            raise RequirementException(
+                f"Could not access Job Definition details. To fix this: \n \
+                Allow dynamic-group <distributed_training_job_runs> to use virtual-network-family in compartment <your_compartment_name>"
+            )
+
+    def work_dir_access(self):
+        test_file_name = f"{os.environ['OCI__WORK_DIR']}/{str(uuid.uuid4())}"
+        bucket_name = urllib.parse.urlparse(test_file_name).username
+        print(f"Checking Object storage access to {os.environ['OCI__WORK_DIR']}")
+        try:
+            with fsspec.open(test_file_name, "w", **self.auth) as tf:
+                tf.write("sample")
+            print(f"Passed object storage access")
+        except oci.exceptions.ServiceError as se:
+            print(se)
+            raise RequirementException(
+                "Could not access Job Definition details. To fix this: \n \
+                Allow dynamic-group <distributed_training_job_runs> to manage objects in compartment your_compartment_name where all {target.bucket.name="
+                + bucket_name
+                + "}"
+            )
+
+    def check_internet(self, **kwargs):
+        print(f"Checking for internet")
+        try:
+            res = requests.get("https://www.oracle.com", timeout=5)
+        except requests.exceptions.ConnectionError as ce:
+            print(ce)
+            raise RequirementException(
+                f"Could not access internet. To fix this: \n \
+                Add NAT gateway to your subnet. If you are behind proxy, add https_proxy and http_proxy to the environment variables"
+            )

ads/opctl/diagnostics/check_requirements.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import yaml
+import os
+from ads.opctl.distributed.common.abstract_cluster_provider import ClusterProvider
+from ads.opctl.diagnostics.check_distributed_job_requirements import (
+    PortsChecker,
+    Default,
+)
+from ads.opctl.diagnostics.requirement_exception import RequirementException
+from jinja2 import Environment, PackageLoader
+import fsspec
+import importlib
+import sys
+
+cp = ClusterProvider(mode="MAIN")
+auth = cp.get_oci_auth()
+
+
+class RequirementChecker:
+
+    providers = {}
+
+    CHECK = "\u2705"
+    CROSS = "\u274E"
+    TADA = "\u2728"
+    DIAGNOSTIC_REPORT_FILE = "diagnostic_report.html"
+
+    @classmethod
+    def register_provider(cls, key: str, value):
+        RequirementChecker.providers[key] = value
+
+    def load_requirement(self, framework):
+        req = {}
+        with open(f"{framework}_req.yaml") as rf:
+            req = yaml.load(rf, Loader=yaml.SafeLoader)
+        return req
+
+    def setup_syspath(self):
+        code_dir = os.environ.get("OCI__CODE_DIR", "/code/")
+        sys.path.append(code_dir)
+        for dirname in os.listdir(code_dir):
+            file = os.path.join(code_dir, dirname)
+            if os.path.isdir(file):
+                sys.path.append(file)
+
+    def load_external_modules(self, modules):
+        for module in modules:
+            importlib.import_module(module)
+        print(f"Loaded modules: {modules}")
+
+    def verify_requirements(self, kind):
+
+        self.kind = kind
+        self.setup_syspath()
+        configuration = {}
+        requirements = {}
+        cluster_type = os.environ["OCI__CLUSTER_TYPE"].lower()
+        DIAGNOSTIC_CONFIG = os.environ.get(
+            "OCI__DIAGNOSTIC_DEFAULT_CONFIG"
+        ) or os.path.join(
+            os.path.dirname(os.path.dirname(__file__)),
+            "config",
+            "diagnostics",
+            self.kind,
+            "default_requirements_config.yaml",
+        )
+        print(f"Loading requirements for diagnostics from {DIAGNOSTIC_CONFIG}")
+        with open(DIAGNOSTIC_CONFIG) as cf:
+            configuration = yaml.load(cf, Loader=yaml.SafeLoader)[cluster_type]
+            requirements = configuration.get("checkers")
+        if configuration.get("checker_provider"):
+            self.load_external_modules(configuration["checker_provider"])
+
+        failures = []
+        passed = []
+        for requirement in requirements:
+            print(f"Checking requirement: {requirement}")
+            provider = ".".join(requirement["checker"].split(".")[:-1])
+            method = requirement["checker"].split(".")[-1]
+            kwargs = requirement.get("args", {})
+            try:
+                getattr(RequirementChecker.providers[provider](auth=auth), method)(
+                    **kwargs
+                )
+                passed.append(requirement["description"])
+            except RequirementException as re:
+                failures.append((requirement["description"], str(re)))
+
+        for item in passed:
+            print(f"{item}: {RequirementChecker.CHECK}", flush=True)
+        for item in failures:
+            print(
+                f"{item[0]}: {RequirementChecker.CROSS} . Resolution: {item[1]}",
+                flush=True,
+            )
+        if len(failures) == 0:
+            print(
+                f"{RequirementChecker.TADA} You have satisfied all the requirements for cluster: {cluster_type} \u2728",
+                flush=True,
+            )
+        self.generate_report(cluster_type, passed, failures)
+
+    def generate_report(self, cluster_type, passed, failures):
+        _env = Environment(loader=PackageLoader("ads", "opctl/templates"))
+        # import jinja2
+
+        # loader = jinja2.FileSystemLoader(".")
+        # _env = jinja2.Environment(loader=loader)
+        report_template = _env.get_template(f"diagnostic_report_template.jinja2")
+        report = [(req, RequirementChecker.CHECK) for req in passed] + [
+            (req[0], RequirementChecker.CROSS) for req in failures
+        ]
+        success_message = f"{RequirementChecker.TADA} You have satisfied all the requirements for cluster: {cluster_type} {RequirementChecker.TADA}"
+        report_path = os.path.join(
+            os.path.join(os.environ.get("OCI__WORK_DIR", ""), cp.my_work_dir),
+            RequirementChecker.DIAGNOSTIC_REPORT_FILE,
+        )
+        with fsspec.open(
+            # "diagnostic_report.html",
+            report_path,
+            "w",
+            **auth,
+        ) as of:
+            of.write(
+                report_template.render(
+                    cluster_type=cluster_type,
+                    report=report,
+                    action_item=failures,
+                    success_message=success_message,
+                )
+            )
+            print(f"Finished writing to {report_path}")
+
+
+if __name__ == "__main__":
+    # fetch_security_list(subnet_id)
+    RequirementChecker.register_provider("cluster-ports-check", PortsChecker)
+    RequirementChecker.register_provider("cluster-generic", Default)
+    RequirementChecker().verify_requirements()

ads/opctl/diagnostics/requirement_exception.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class RequirementException(Exception):   # pragma: no cover
+    pass

ads/opctl/distributed/README.md

@@ -0,0 +1,109 @@
+In order to run your script on distributed jobs:
+
+
+Getting Started:
+
+0. Make sure you have ads opctl set up:
+
+1. Generate resources required for the distributed dask cluster:
+```
+ads opctl distributed init --framework dask --version v1
+ads opctl distributed init --framework horovod --type tensorflow --arch cpu --version v1
+```
+
+2.
+This should give you an output something like:
+```
+docker build -t iad.ocir.io/ociodscdev/my-distr-example:latest -f my-distr-example/Dockerfile .
+ads opctl publish-image iad.ocir.io/ociodscdev/my-distr-example:latest
+ads opctl run -f my-distr-example/dask_cluster.yaml
+```
+Before running the above, be sure to:
+- Put all of your code into `main.py`.
+- If you need more files, be sure to include them in the Dockerfile.
+- If you need more libraries, be sure to include them in the environment.yaml.
+- Anything you can factor out of main.py as an arg, should be put in the dask_cluster.yaml file.
+- Go through dask_cluster.yaml and fill in TODO's with valid parameters
+- You shouldn't need to touch any other file, but of course they're there if you need to.
+
+
+3. Then in separate terminals, watch the statuses:
+```
+ads opctl watch <main job run id>
+ads opctl watch <worker x job run id>
+```
+
+4. Next Steps:
+If you run into any errors that require changing the image, make those changes and go back to step 2.
+The best development approach is to make the script as configurable as possible through args, kwargs and env vars. This will allow for rapid development form the yaml, rather than re-building the image.
+
+## Yaml Example -
+
+```
+kind: distributed
+apiVersion: v1.0
+spec:
+  infrastructure: # This section maps to Job definition. Does not include environment variables
+    kind: infrastructure
+    type: dataScienceJob
+    apiVersion: v1.0
+    spec:
+      projectId: pjocid
+      compartmentId: ctocid
+      displayName: my_distributed_training
+      logGroupId: lgrpid
+      logId: logid
+      subnetId: subID
+      shapeName: VM.Standard2.1
+      blockStorageSizeGB: 50GB
+  cluster:
+    kind: dask
+    apiVersion: v1.0
+    spec:
+      image: default
+      workDir: "Working directory for the cluster"
+      ephemeral: True
+      name: cluster name
+      config:
+        env:
+          - name: death_timeout
+            value: 10
+          - name: death_timeout
+            value: 4
+        startOptions: # Only named args. Will construct OCI___CLUSTER_START_OPTIONS environment variable from list of args as " ".join(startOptions)
+      main:
+        image: optional
+        name: main-name
+        replicas: 1
+        config:
+          env:
+            - name: death_timeout
+              value: 20
+            - name: death_timeout
+              value: 5
+          startOptions: # Only named args. Will construct OCI___CLUSTER_START_OPTIONS environment variable from list of args as " ".join(startOptions)
+      worker:
+        name: worker-name
+        image: optional
+        replicas: 2 #Name is not decided
+        config:
+          env:
+            - name: death_timeout
+              value: 30
+            - name: death_timeout
+              value: 8
+          startOptions: # Only named args. Will construct OCI___CLUSTER_START_OPTIONS environment variable from list of args as " ".join(startOptions)
+            - --worker-port 8700:8800
+            - --nanny-port 3000:3100
+  runtime:
+    kind: python
+    apiVersion: v1.0
+    spec:
+      entryPoint: "printhello.py"
+      args:
+        - 500
+      kwargs:
+      env:
+        - name: TEST
+          value: "test"
+```

ads/opctl/distributed/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/distributed/certificates.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import oci
+import os
+
+
+def download_certificates(auth, ca_file_name, cert_file_name, key_file_name):
+    """ """
+    client = oci.certificates.CertificatesClient(**auth)
+    # Download Cert and Private Key
+    print("Fetching certificate details")
+    response = client.get_certificate_bundle(
+        certificate_id=os.environ["OCI__CERTIFICATE_OCID"],
+        certificate_bundle_type="CERTIFICATE_CONTENT_WITH_PRIVATE_KEY",
+    )
+    with open(cert_file_name, "w") as f:
+        print(f"writing {cert_file_name}")
+        f.write(response.data.certificate_pem)
+    with open(key_file_name, "w") as f:
+        print(f"writing {key_file_name}")
+        f.write(response.data.private_key_pem)
+    # Download CA Cert
+    response = client.get_certificate_authority_bundle(
+        certificate_authority_id=os.environ["OCI__CERTIFICATE_AUTHORITY_OCID"]
+    )
+    with open(ca_file_name, "w") as f:
+        print(f"writing {ca_file_name}")
+        f.write(response.data.cert_chain_pem)