oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/opctl/conda/config.yaml

@@ -1,34 +0,0 @@
-service_pack_prefix: "service_pack"
-bucket_info:
-  default:
-    name: "service-conda-packs"
-    namespace: "id19sfcrra6z"
-    index_path: "service_pack/index.json"
-  ap-chiyoda-1:
-    name: "service-conda-packs"
-    namespace: "axvucm64yaas"
-    index_path: "service_pack/index.json"
-  us-langley-1:
-    name: "service-conda-packs"
-    namespace: "ociodscprodoc2"
-    index_path: "service_pack/index.json"
-  us-luke-1:
-    name: "service-conda-packs"
-    namespace: "ociodscprodoc2"
-    index_path: "service_pack/index.json"
-  us-gov-ashburn-1:
-    name: "service-conda-packs"
-    namespace: "ociodscprodoc3"
-    index_path: "service_pack/index.json"
-  us-gov-phoenix-1:
-    name: "service-conda-packs"
-    namespace: "ociodscprodoc3"
-    index_path: "service_pack/index.json"
-  us-gov-chicago-1:
-    name: "service-conda-packs"
-    namespace: "ociodscprodoc3"
-    index_path: "index.json"
-service_pack_auth: "resource_principal"
-notebook_prefix: "notebooks"
-notebook_location: "/opt/notebooks"
-ip_packages: ["automl", "mlx"]

ads/opctl/conda/manifest_template.yaml

@@ -1,13 +0,0 @@
-manifest:
-  name: ""
-  description: ""
-  version: ""
-  slug: ""
-  created_by: ""
-  create_date: ""
-  source: ""
-  libraries: ""
-  size_mb: ""
-  python: ""
-  arch_type: ""
-  manifest_version: ""

ads/opctl/conda/multipart_uploader.py

@@ -1,188 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from typing import Dict, List
-
-from ads.common.oci_client import OCIClientFactory
-from ads.common.auth import create_signer
-from ads.opctl.utils import parse_conda_uri
-import oci
-import mmap
-import json
-import time
-from tqdm import tqdm
-from threading import Thread
-import os
-
-
-class MultiPartUploader:
-    """
-    Class that implements multipart uploading for conda packs.
-    """
-
-    def __init__(
-        self,
-        source_file: str,
-        dst_uri: str,
-        parts: int,
-        oci_config: str = None,
-        oci_profile: str = None,
-        auth_type: str = None,
-    ) -> None:
-        """Initialize the class.
-
-        Parameters
-        ----------
-        source_file : str
-            path to conda pack file
-        dst_uri : str
-            path to destination of object storage location
-        parts : int
-            number of parts
-        oci_config : str, optional
-            path to oci config file, by default None
-        oci_profile : str, optional
-            oci profile to use, by default None
-        auth_type : str
-            authentication method, by default None
-        """
-        self.src = source_file
-        self.dst = dst_uri
-        self.oci_auth = create_signer(auth_type, oci_config, oci_profile)
-        self.client = OCIClientFactory(**self.oci_auth).object_storage
-        self.ns, self.bucket, self.path, _ = parse_conda_uri(dst_uri)
-        self.file_size = os.path.getsize(self.src)
-        # mmap offset arg must be a multiple of the ALLOCATIONGRANULARITY, change chunk_size to control offset value
-        self.chunk_size = (
-            (self.file_size // parts)
-            // mmap.ALLOCATIONGRANULARITY
-            * mmap.ALLOCATIONGRANULARITY
-        )
-
-    def upload(self, opc_meta: Dict = None) -> bool:
-        """Uploading a conda pack to object storage.
-
-        Parameters
-        ----------
-        opc_meta : Dict, optional
-            metadata dictionary, by default None
-
-        Returns
-        -------
-        bool
-            whether uploading was successful
-
-        Raises
-        ------
-        RuntimeError
-            Uploading failed
-        """
-        multipart_upload_details = (
-            oci.object_storage.models.CreateMultipartUploadDetails()
-        )
-        multipart_upload_details.object = self.path
-        multipart_upload_details.metadata = {"opc-meta-manifest": json.dumps(opc_meta)}
-
-        upload_details = self.client.create_multipart_upload(
-            self.ns, self.bucket, multipart_upload_details
-        )
-        uploaded_parts = []
-        threads = []
-        mm_objects = []
-        responses = []
-        upload_id = upload_details.data.upload_id
-        print(f"The upload id is {upload_id}.")
-        with open(self.src, "rb") as pkf:
-            counter = 0
-            for offset, length in self._chunks():
-                counter += 1
-                mm = mmap.mmap(
-                    pkf.fileno(), length, access=mmap.ACCESS_READ, offset=offset
-                )
-                mm_objects.append((mm, length))
-                t = Thread(
-                    target=self._upload_chunk,
-                    args=(upload_id, counter, mm, uploaded_parts, responses),
-                )
-                t.start()
-                threads.append(t)
-            t = Thread(
-                target=self._track_progress, args=(mm_objects, counter, responses)
-            )
-            t.start()
-            threads.append(t)
-            for t in threads:
-                t.join()
-        # Successful responses HTTP status in range (200–299)
-        successful = all(200 <= r.status < 300 for r in responses)
-        if successful:
-            self.client.commit_multipart_upload(
-                self.ns,
-                self.bucket,
-                self.path,
-                upload_details.data.upload_id,
-                oci.object_storage.models.CommitMultipartUploadDetails(
-                    parts_to_commit=uploaded_parts
-                ),
-            )
-            print(f"{self.src} uploaded successfuly to {self.dst}.")
-        else:
-            for r in responses:
-                print(r.status, r.headers)
-            raise RuntimeError(f"{self.src} upload failed.")
-        return successful
-
-    def _chunks(self):
-        start_position = 0
-        while start_position < self.file_size:
-            yield start_position, min(self.chunk_size, self.file_size - start_position)
-            start_position += self.chunk_size
-
-    def _upload_chunk(
-        self,
-        upload_id: str,
-        counter: int,
-        mm: mmap.mmap,
-        uploaded_parts: List,
-        responses: List,
-    ) -> None:
-        response = self.client.upload_part(
-            self.ns, self.bucket, self.path, upload_id, counter, mm
-        )
-        responses.append(response)
-        uploaded_parts.append(
-            oci.object_storage.models.CommitMultipartUploadPartDetails(
-                etag=response.headers["etag"], part_num=counter
-            )
-        )
-
-    @staticmethod
-    def _track_progress(
-        mm_objects: List[mmap.mmap], counter: int, responses: List
-    ) -> None:
-        nresponses = 0
-        progress_objs = []
-        for index, (mm_object, length) in enumerate(mm_objects):
-            tqdm_obj = tqdm(
-                total=length,
-                unit="B",
-                unit_scale=True,
-                unit_divisor=2**10,
-                desc=f"Part {index + 1}",
-                position=index + 1,
-                leave=True,
-            )
-            progress_objs.append([mm_object, tqdm_obj, 0, length])
-        while nresponses < counter:
-            for obj in progress_objs:
-                current_pos = obj[0].tell()
-                obj[1].update(current_pos - obj[2])
-                obj[1].refresh()
-                obj[2] = current_pos
-            time.sleep(2)
-            nresponses = len(responses)
-        map(lambda x: x[1].refresh(), progress_objs)
-        map(lambda x: x[1].close(), progress_objs)

ads/opctl/conda/pack.py

@@ -1,89 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import subprocess
-from io import StringIO
-import tempfile
-import os
-import datetime
-import shutil
-import sys
-import glob
-import stat
-import conda_pack
-
-import yaml
-import argparse
-
-
-def main(pack_folder_path, manifest_file=None):
-    manifest_path = (
-        manifest_file or glob.glob(os.path.join(pack_folder_path, "*_manifest.yaml"))[0]
-    )
-    with open(manifest_path) as f:
-        env = yaml.safe_load(f.read())
-    
-    with tempfile.TemporaryDirectory() as td:
-        process = subprocess.Popen(
-            ["conda", "env", "export", "--prefix", pack_folder_path],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-        stdout, stderr = process.communicate()
-        
-        if process.returncode and stderr:
-            print(stderr)
-            raise Exception(
-                f"Error export environment information from {pack_folder_path}"
-            )
-        try:
-            new_env_info = yaml.safe_load(StringIO(stdout.decode("utf-8")))
-        except Exception as e:
-            print(f"Error reading dependency list from {stdout}")
-            raise e
-
-        manifest = env["manifest"]
-        slug = manifest.get("slug", os.path.basename(pack_folder_path))
-        if os.environ.get("CONDA_PUBLISH_TYPE") != "service":
-            manifest["type"] = "published"
-        new_env_info["manifest"] = manifest
-        with open(manifest_path, "w") as f:
-            yaml.safe_dump(new_env_info, f)
-        pack_file = os.path.join(td, f"{slug}.tar.gz")
-        conda_pack.pack(
-            prefix=pack_folder_path,
-            compress_level=7,
-            output=pack_file,
-            n_threads=-1,
-            ignore_missing_files=True,
-        )
-        if not os.path.exists(pack_file):
-            raise RuntimeError(
-                "Error creating the pack file using `conda_pack.pack()`."
-            )
-        print(f"Copy {pack_file} to {pack_folder_path}")
-        shutil.copy(pack_file, pack_folder_path)
-        file_path = os.path.join(pack_folder_path, os.path.basename(pack_file))
-        print(f"Pack built at {file_path}")
-        print(
-            f"changing permission for {file_path}",
-            flush=True,
-        )
-        os.chmod(file_path, 0o755)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        prog="Prepare conda archive",
-        description="Uses conda_pack library to pack the conda environment.",
-    )
-    parser.add_argument("--conda-path", type=str, help="Path to the conda environment")
-    parser.add_argument(
-        "--manifest-location", type=str, default=None, help="Path to manifest location"
-    )
-    args = parser.parse_args()
-
-    main(args.conda_path, args.manifest_location)

ads/opctl/config/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/config/base.py

@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from abc import abstractmethod
-from typing import Dict
-
-
-class ConfigProcessor:
-    def __init__(self, config: Dict = None) -> None:
-        """
-        Initializing a ConfigProcessor object given a configuration.
-
-        Parameters
-        ----------
-        config: dict
-            a dictionary of configurations
-        """
-        self.config = config if config else {}
-
-    @abstractmethod
-    def process(self, **kwargs) -> "ConfigProcessor":
-        """
-        Perform some processing on configuration
-        associated with this object.
-
-        Parameters
-        ----------
-        kwargs: dict
-            keyword arguments passed to the function
-
-        Returns
-        -------
-        ConfigProcessor
-            this instance itself
-        """
-        pass
-
-    def step(self, cls, **kwargs) -> "ConfigProcessor":
-        """
-        Perform some processing according to the cls given.
-
-        Parameters
-        ----------
-        cls: `ConfigProcessor`
-            a subclass of `ConfigProcessor`
-        kwargs: dict
-            keyword arguments passed to the `process` function of `cls`
-
-        Returns
-        -------
-        `ConfigProcessor`
-            this instance itself
-        """
-        return cls(self.config).process(**kwargs)

ads/opctl/config/diagnostics/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/config/diagnostics/distributed/default_requirements_config.yaml

@@ -1,62 +0,0 @@
-dask:
-  checkers:
-    - checker: cluster-generic.job_info_access_policy
-      description: "Check if job run has access to Job definition information"
-    - checker: cluster-generic.job_vcn_info_access_policy
-      description: "Check if job run has access to VCN information"
-    - checker: cluster-generic.work_dir_access
-      description: "Check if job run hass access to the WORK DIR (object storage) configured in the cluster definition file"
-    - checker: cluster-ports-check.check_ports
-      args:
-        ports:
-          - '8787'
-          - '8786'
-          - '3000-3100'
-      description: "Check Ports for Ingress"
-    - checker: cluster-generic.check_internet
-      description: "Check if internet access is available"
-pytorch:
-  checkers:
-    - checker: cluster-generic.job_info_access_policy
-      description: "Check if job run has access to Job definition information"
-    - checker: cluster-generic.job_vcn_info_access_policy
-      description: "Check if job run has access to VCN information"
-    - checker: cluster-generic.work_dir_access
-      description: "Check if job run hass access to the WORK DIR (object storage) configured in the cluster definition file"
-    - checker: cluster-ports-check.check_ports
-      args:
-        ports:
-          - '29400'
-      description: "Check Ports for Ingress"
-    - checker: cluster-generic.check_internet
-      description: "Check if internet access is available"
-horovod:
-  checkers:
-    - checker: cluster-generic.job_info_access_policy
-      description: "Check if job run has access to Job definition information"
-    - checker: cluster-generic.job_vcn_info_access_policy
-      description: "Check if job run has access to VCN information"
-    - checker: cluster-generic.work_dir_access
-      description: "Check if job run hass access to the WORK DIR (object storage) configured in the cluster definition file"
-    - checker: cluster-ports-check.check_ports
-      args:
-        ports:
-          - 'all'
-      description: "Check Ports for Ingress"
-    - checker: cluster-generic.check_internet
-      description: "Check if internet access is available"
-tensorflow:
-  checkers:
-    - checker: cluster-generic.job_info_access_policy
-      description: "Check if job run has access to Job definition information"
-    - checker: cluster-generic.job_vcn_info_access_policy
-      description: "Check if job run has access to VCN information"
-    - checker: cluster-generic.work_dir_access
-      description: "Check if job run hass access to the WORK DIR (object storage) configured in the cluster definition file"
-    - checker: cluster-ports-check.check_ports
-      args:
-        ports:
-          - 'all'
-      description: "Check Ports for Ingress"
-    - checker: cluster-generic.check_internet
-      description: "Check if internet access is available"