oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/jobs/serializer.py

@@ -1,296 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-import json
-from abc import ABC, abstractmethod
-from typing import Type, TypeVar, Union, Dict
-from urllib.parse import urlparse
-
-import fsspec
-import yaml
-from ads.common.auth import default_signer
-
-# Special type to represent the current enclosed class.
-# This type is used by factory class method or when a method returns ``self``.
-Self = TypeVar("Self", bound="Serializable")
-
-
-class Serializable(ABC):
-    """Base class that represents a serializable item."""
-
-    @abstractmethod
-    def to_dict(self, **kwargs: Dict) -> Dict:
-        """Serializes an instance of class into a dictionary.
-
-        Parameters
-        ----------
-        **kwargs: Dict
-            Additional arguments.
-
-        Returns
-        -------
-        Dict
-            The result dictionary.
-        """
-
-    @classmethod
-    @abstractmethod
-    def from_dict(cls, obj_dict: dict):
-        """Initialize an instance of the class from a dictionary
-
-        Parameters
-        ----------
-        obj_dict : dict
-            Dictionary representation of the object
-        """
-
-    @staticmethod
-    def _write_to_file(s: str, uri: str, **kwargs) -> None:
-        """Write string content into a file specified by uri
-
-        Parameters
-        ----------
-        s : str
-            The content to be written.
-        uri : str
-            URI of the file to save the content.
-        kwargs : dict
-            keyword arguments to be passed into fsspec.open().
-            For OCI object storage, this can be config="path/to/.oci/config".
-        """
-
-        overwrite = kwargs.pop("overwrite", True)
-        if not overwrite:
-            dst_path_scheme = urlparse(uri).scheme or "file"
-            if fsspec.filesystem(dst_path_scheme, **kwargs).exists(uri):
-                raise FileExistsError(
-                    f"The `{uri}` is already exists. Set `overwrite` to True "
-                    "if you wish to overwrite."
-                )
-
-        # Add default signer if the uri is an object storage uri, and
-        # the user does not specify config or signer.
-        if (
-            uri.startswith("oci://")
-            and "config" not in kwargs
-            and "signer" not in kwargs
-        ):
-            kwargs.update(default_signer())
-        with fsspec.open(uri, "w", **kwargs) as f:
-            f.write(s)
-
-    @staticmethod
-    def _read_from_file(uri: str, **kwargs) -> str:
-        """Returns contents from a file specified by URI
-
-        Parameters
-        ----------
-        uri : str
-            The URI of the file.
-
-        Returns
-        -------
-        str
-            The content of the file as a string.
-        """
-        # Add default signer if the uri is an object storage uri, and
-        # the user does not specify config or signer.
-        if (
-            uri.startswith("oci://")
-            and "config" not in kwargs
-            and "signer" not in kwargs
-        ):
-            kwargs.update(default_signer())
-        with fsspec.open(uri, "r", **kwargs) as f:
-            return f.read()
-
-    def to_json(
-        self, uri: str = None, encoder: callable = json.JSONEncoder, **kwargs
-    ) -> str:
-        """Returns the object serialized as a JSON string
-
-        Parameters
-        ----------
-        uri : str, optional
-            URI location to save the JSON string, by default None
-        encoder : callable, optional
-            Encoder for custom data structures, by default json.JSONEncoder
-        kwargs : dict
-            overwrite: (bool, optional). Defaults to True.
-                Whether to overwrite existing file or not.
-
-            The other keyword arguments to be passed into fsspec.open().
-            For OCI object storage, this could be config="path/to/.oci/config".
-
-        Returns
-        -------
-        Union[str, None]
-            Serialized version of object.
-            `None` in case when `uri` provided.
-        """
-        json_string = json.dumps(self.to_dict(**kwargs), cls=encoder)
-        if uri:
-            self._write_to_file(s=json_string, uri=uri, **kwargs)
-            return None
-        return json_string
-
-    @classmethod
-    def from_json(
-        cls: Type[Self],
-        json_string: str = None,
-        uri: str = None,
-        decoder: callable = json.JSONDecoder,
-        **kwargs,
-    ) -> Self:
-        """Creates an object from JSON string provided or from URI location containing JSON string
-
-        Args:
-            json_string (string, optional): JSON string. Defaults to None.
-            uri (string, optional): URI location of file containing JSON string. Defaults to None.
-            decoder (callable, optional): Custom decoder. Defaults to simple JSON decoder.
-            kwargs (dict): keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config".
-                           For other storage connections consider e.g. host, port, username, password, etc.
-
-        Raises:
-            ValueError: Raised if neither string nor uri is provided
-
-        Returns:
-            cls: Returns instance of the class
-
-        Parameters
-        ----------
-        json_string : str, optional
-            JSON string, by default None
-        uri : str, optional
-            URI location of file containing JSON string, by default None
-        decoder : callable, optional
-            Decoder for custom data structures, by default json.JSONDecoder
-        kwargs : dict
-            keyword arguments to be passed into fsspec.open().
-            For OCI object storage, this can be config="path/to/.oci/config".
-
-        Returns
-        -------
-        Type[Self]
-            Object initialized from JSON data.
-
-        Raises
-        ------
-        ValueError
-            Both json_string and uri are empty, or
-            The input is not a valid JSON.
-        """
-        if json_string:
-            return cls.from_dict(json.loads(json_string, cls=decoder))
-        if uri:
-            json_dict = json.loads(cls._read_from_file(uri, **kwargs), cls=decoder)
-            return cls.from_dict(json_dict)
-        raise ValueError("Must provide either JSON string or URI location")
-
-    def to_yaml(
-        self, uri: str = None, dumper: callable = yaml.SafeDumper, **kwargs
-    ) -> Union[str, None]:
-        """Returns object serialized as a YAML string
-
-        Parameters
-        ----------
-        uri : str, optional
-            URI location to save the YAML string, by default None
-        dumper : callable, optional
-            Custom YAML Dumper, by default yaml.SafeDumper
-        kwargs : dict
-            overwrite: (bool, optional). Defaults to True.
-                Whether to overwrite existing file or not.
-            note: (str, optional)
-                The note that needs to be added in the beginning of the YAML.
-                It will be added as is without any formatting.
-
-            The other keyword arguments to be passed into fsspec.open().
-            For OCI object storage, this could be config="path/to/.oci/config".
-
-        Returns
-        -------
-        Union[str, None]
-            Serialized version of object.
-            `None` in case when `uri` provided.
-        """
-        note = kwargs.pop("note", "")
-
-        yaml_string = f"{note}\n" + yaml.dump(self.to_dict(**kwargs), Dumper=dumper)
-        if uri:
-            self._write_to_file(s=yaml_string, uri=uri, **kwargs)
-            return None
-
-        return yaml_string
-
-    @classmethod
-    def from_yaml(
-        cls: Type[Self],
-        yaml_string: str = None,
-        uri: str = None,
-        loader: callable = yaml.SafeLoader,
-        **kwargs,
-    ) -> Self:
-        """Initializes an object from YAML string or URI location containing the YAML
-
-        Parameters
-        ----------
-        yaml_string : str, optional
-            YAML string, by default None
-        uri : str, optional
-            URI location of file containing YAML, by default None
-        loader : callable, optional
-            Custom YAML loader, by default yaml.SafeLoader
-
-        Returns
-        -------
-        Self
-            Object initialized from the YAML.
-
-        Raises
-        ------
-        ValueError
-            Raised if neither string nor uri is provided
-        """
-        if yaml_string:
-            return cls.from_dict(yaml.load(yaml_string, Loader=loader))
-        if uri:
-            yaml_dict = yaml.load(cls._read_from_file(uri=uri, **kwargs), Loader=loader)
-            return cls.from_dict(yaml_dict)
-        raise ValueError("Must provide either YAML string or URI location")
-
-    @classmethod
-    def from_string(
-        cls: Type[Self],
-        obj_string: str = None,
-        uri: str = None,
-        loader: callable = yaml.SafeLoader,
-        **kwargs,
-    ) -> Self:
-        """Initializes an object from YAML/JSON string or URI location containing the YAML/JSON
-
-        Parameters
-        ----------
-        obj_string : str, optional
-            YAML/JSON string, by default None
-        uri : str, optional
-            URI location of file containing YAML/JSON, by default None
-        loader : callable, optional
-            Custom YAML loader, by default yaml.SafeLoader
-
-        Returns
-        -------
-        Self
-            Object initialized from the YAML.
-        """
-        return cls.from_yaml(yaml_string=obj_string, uri=uri, loader=loader, **kwargs)
-
-    def __repr__(self) -> str:
-        """Returns printable version of object
-
-        Returns:
-            string: Serialized version of object as a YAML string
-        """
-        return self.to_yaml()

ads/jobs/templates/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/jobs/templates/container.py

@@ -1,6 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-# This is a placeholder file for container job.

ads/jobs/templates/driver_notebook.py

@@ -1,177 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-"""This module runs a Jupyter Python notebook with nbconvert and print the outputs.
-This is a driver script auto-generated by Oracle ADS.
-
-The following environment variables are used:
-JOB_RUN_NOTEBOOK:
-    The relative path of the jupyter Python notebook to be executed.
-NOTEBOOK_EXCLUDE_TAGS:
-    Optional, a list of tags serialized to JSON string.
-    Notebook cells with one of the tags will be excluded from running.
-NOTEBOOK_ENCODING:
-    Optional, the encoding for opening the notebook.
-OUTPUT_URI:
-    Optional, object storage URI for saving files from the output directory.
-"""
-import logging
-import json
-import os
-from typing import Optional
-
-import nbformat
-from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
-
-try:
-    # This is used by ADS and testing
-    from .driver_utils import OCIHelper, JobRunner, set_log_level
-except ImportError:
-    # This is used when the script is in a job run.
-    from driver_utils import OCIHelper, JobRunner, set_log_level
-
-
-logger = logging.getLogger(__name__)
-logger = set_log_level(logger)
-
-# The directory for storing the user code/notebook.
-# The basename of the directory should match the ADS PythonArtifact.USER_CODE_DIR
-CODE_DIR = os.path.join(os.path.dirname(__file__), "code")
-
-
-class ADSExecutePreprocessor(ExecutePreprocessor):
-    """Customized Execute Preprocessor for running notebook."""
-
-    def __init__(self, exclude_tags=None, **kw):
-        """Initialize the preprocessor
-
-        Parameters
-        ----------
-        exclude_tags : list, optional
-            A list of cell tags, notebook cells with any of these cell tag will be skipped.
-            Defaults to None.
-        """
-        self.exclude_tags = exclude_tags
-        super().__init__(**kw)
-
-    @staticmethod
-    def _print_cell_outputs(cell):
-        """Prints the outputs of a notebook cell."""
-        for output in cell.outputs:
-            output_type = output.get("output_type")
-            if output_type == "stream":
-                # stream outputs includes line break already
-                print(output.text, end="")
-            elif output_type == "execute_result":
-                # execute_result may contain text/plain
-                text = output.get("data", {}).get("text/plain", [])
-                # The value could be str or list of str
-                if isinstance(text, list):
-                    for line in text:
-                        print(line)
-                else:
-                    print(text)
-
-    def preprocess_cell(self, cell, resources, *args, **kwargs):
-        """Runs the notebook cell and print out the outputs"""
-        # Skip the cell if any of the cell tags matching an exclude tag.
-        if self.exclude_tags:
-            # Log an error message if there is an error reading the cell tags,
-            # and continue to run the cell.
-            try:
-                cell_tags = cell.get("metadata", {}).get("tags", [])
-                for tag in cell_tags:
-                    if tag in self.exclude_tags:
-                        return cell, resources
-            except Exception as ex:
-                logger.exception("An error occurred when reading cell tags.")
-        # Run the cell
-        cell, resources = super().preprocess_cell(cell, resources, *args, **kwargs)
-        # Print cell output
-        if hasattr(cell, "outputs"):
-            # Log a message if there is an error getting the cell output,
-            # and continue to run the next cell.
-            try:
-                self._print_cell_outputs(cell)
-            except Exception as ex:
-                logger.exception("An error occurred when reading cell outputs.")
-        return cell, resources
-
-
-def run_notebook(
-    notebook_path: str,
-    working_dir: Optional[str] = None,
-    exclude_tags: Optional[list] = None,
-) -> Optional[CellExecutionError]:
-    """Runs a notebook
-
-    Parameters
-    ----------
-    notebook_path : str
-        The path of the notebook
-    working_dir : str, optional
-        The working directory for running the notebook, by default None.
-        If this is None, the same directory of the notebook_path will be used.
-    exclude_tags : list, optional
-        Tags for excluding cells, by default None
-
-    Returns
-    -------
-    CellExecutionError or None
-        Exception object when there is an error in a notebook cell.
-        Otherwise, None.
-    """
-    # Read the notebook
-    encoding = os.environ.get("NOTEBOOK_ENCODING", "utf-8")
-    with open(notebook_path, encoding=encoding) as f:
-        nb = nbformat.read(f, as_version=4)
-
-    # Working/Output directory
-    if not working_dir:
-        working_dir = os.path.dirname(notebook_path)
-
-    # The path of the output notebook with results/plots
-    notebook_filename_out = os.path.join(working_dir, os.path.basename(notebook_path))
-
-    ep = ADSExecutePreprocessor(exclude_tags=exclude_tags, kernel_name="python")
-
-    try:
-        ep.preprocess(nb, {"metadata": {"path": working_dir}})
-        ex = None
-    except CellExecutionError as exc:
-        msg = "Error executing the notebook.\n\n"
-        logger.error(msg)
-        ex = exc
-    finally:
-        with open(notebook_filename_out, mode="w", encoding=encoding) as f:
-            nbformat.write(nb, f)
-    return ex
-
-
-def main() -> None:
-    """Runs the driver to execute a notebook."""
-    JobRunner().conda_unpack()
-
-    notebook_file_path = os.path.join(CODE_DIR, os.environ.get("JOB_RUN_NOTEBOOK"))
-    # By default, the output directory will be the one containing the notebook
-    output_dir = os.environ.get("OUTPUT_DIR", os.path.dirname(notebook_file_path))
-
-    # Exclude tags
-    tags = os.environ.get("NOTEBOOK_EXCLUDE_TAGS")
-    if tags:
-        tags = json.loads(tags)
-        logger.info("Excluding cells with any of the following tags: %s", tags)
-    # Run the notebook
-    ex = run_notebook(notebook_file_path, working_dir=output_dir, exclude_tags=tags)
-
-    # Save the outputs
-    OCIHelper.copy_outputs(output_dir)
-
-    if ex:
-        raise ex
-
-
-if __name__ == "__main__":
-    main()