oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/jobs/serializer.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import json
+from abc import ABC, abstractmethod
+from typing import Type, TypeVar, Union, Dict
+from urllib.parse import urlparse
+
+import fsspec
+import yaml
+from ads.common.auth import default_signer
+
+# Special type to represent the current enclosed class.
+# This type is used by factory class method or when a method returns ``self``.
+Self = TypeVar("Self", bound="Serializable")
+
+
+class Serializable(ABC):
+    """Base class that represents a serializable item."""
+
+    @abstractmethod
+    def to_dict(self, **kwargs: Dict) -> Dict:
+        """Serializes an instance of class into a dictionary.
+
+        Parameters
+        ----------
+        **kwargs: Dict
+            Additional arguments.
+
+        Returns
+        -------
+        Dict
+            The result dictionary.
+        """
+
+    @classmethod
+    @abstractmethod
+    def from_dict(cls, obj_dict: dict):
+        """Initialize an instance of the class from a dictionary
+
+        Parameters
+        ----------
+        obj_dict : dict
+            Dictionary representation of the object
+        """
+
+    @staticmethod
+    def _write_to_file(s: str, uri: str, **kwargs) -> None:
+        """Write string content into a file specified by uri
+
+        Parameters
+        ----------
+        s : str
+            The content to be written.
+        uri : str
+            URI of the file to save the content.
+        kwargs : dict
+            keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this can be config="path/to/.oci/config".
+        """
+
+        overwrite = kwargs.pop("overwrite", True)
+        if not overwrite:
+            dst_path_scheme = urlparse(uri).scheme or "file"
+            if fsspec.filesystem(dst_path_scheme, **kwargs).exists(uri):
+                raise FileExistsError(
+                    f"The `{uri}` is already exists. Set `overwrite` to True "
+                    "if you wish to overwrite."
+                )
+
+        # Add default signer if the uri is an object storage uri, and
+        # the user does not specify config or signer.
+        if (
+            uri.startswith("oci://")
+            and "config" not in kwargs
+            and "signer" not in kwargs
+        ):
+            kwargs.update(default_signer())
+        with fsspec.open(uri, "w", **kwargs) as f:
+            f.write(s)
+
+    @staticmethod
+    def _read_from_file(uri: str, **kwargs) -> str:
+        """Returns contents from a file specified by URI
+
+        Parameters
+        ----------
+        uri : str
+            The URI of the file.
+
+        Returns
+        -------
+        str
+            The content of the file as a string.
+        """
+        # Add default signer if the uri is an object storage uri, and
+        # the user does not specify config or signer.
+        if (
+            uri.startswith("oci://")
+            and "config" not in kwargs
+            and "signer" not in kwargs
+        ):
+            kwargs.update(default_signer())
+        with fsspec.open(uri, "r", **kwargs) as f:
+            return f.read()
+
+    def to_json(
+        self, uri: str = None, encoder: callable = json.JSONEncoder, **kwargs
+    ) -> str:
+        """Returns the object serialized as a JSON string
+
+        Parameters
+        ----------
+        uri : str, optional
+            URI location to save the JSON string, by default None
+        encoder : callable, optional
+            Encoder for custom data structures, by default json.JSONEncoder
+        kwargs : dict
+            overwrite: (bool, optional). Defaults to True.
+                Whether to overwrite existing file or not.
+
+            The other keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this could be config="path/to/.oci/config".
+
+        Returns
+        -------
+        Union[str, None]
+            Serialized version of object.
+            `None` in case when `uri` provided.
+        """
+        json_string = json.dumps(self.to_dict(**kwargs), cls=encoder)
+        if uri:
+            self._write_to_file(s=json_string, uri=uri, **kwargs)
+            return None
+        return json_string
+
+    @classmethod
+    def from_json(
+        cls: Type[Self],
+        json_string: str = None,
+        uri: str = None,
+        decoder: callable = json.JSONDecoder,
+        **kwargs,
+    ) -> Self:
+        """Creates an object from JSON string provided or from URI location containing JSON string
+
+        Args:
+            json_string (string, optional): JSON string. Defaults to None.
+            uri (string, optional): URI location of file containing JSON string. Defaults to None.
+            decoder (callable, optional): Custom decoder. Defaults to simple JSON decoder.
+            kwargs (dict): keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config".
+                           For other storage connections consider e.g. host, port, username, password, etc.
+
+        Raises:
+            ValueError: Raised if neither string nor uri is provided
+
+        Returns:
+            cls: Returns instance of the class
+
+        Parameters
+        ----------
+        json_string : str, optional
+            JSON string, by default None
+        uri : str, optional
+            URI location of file containing JSON string, by default None
+        decoder : callable, optional
+            Decoder for custom data structures, by default json.JSONDecoder
+        kwargs : dict
+            keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this can be config="path/to/.oci/config".
+
+        Returns
+        -------
+        Type[Self]
+            Object initialized from JSON data.
+
+        Raises
+        ------
+        ValueError
+            Both json_string and uri are empty, or
+            The input is not a valid JSON.
+        """
+        if json_string:
+            return cls.from_dict(json.loads(json_string, cls=decoder))
+        if uri:
+            json_dict = json.loads(cls._read_from_file(uri, **kwargs), cls=decoder)
+            return cls.from_dict(json_dict)
+        raise ValueError("Must provide either JSON string or URI location")
+
+    def to_yaml(
+        self, uri: str = None, dumper: callable = yaml.SafeDumper, **kwargs
+    ) -> Union[str, None]:
+        """Returns object serialized as a YAML string
+
+        Parameters
+        ----------
+        uri : str, optional
+            URI location to save the YAML string, by default None
+        dumper : callable, optional
+            Custom YAML Dumper, by default yaml.SafeDumper
+        kwargs : dict
+            overwrite: (bool, optional). Defaults to True.
+                Whether to overwrite existing file or not.
+            note: (str, optional)
+                The note that needs to be added in the beginning of the YAML.
+                It will be added as is without any formatting.
+
+            The other keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this could be config="path/to/.oci/config".
+
+        Returns
+        -------
+        Union[str, None]
+            Serialized version of object.
+            `None` in case when `uri` provided.
+        """
+        note = kwargs.pop("note", "")
+
+        yaml_string = f"{note}\n" + yaml.dump(self.to_dict(**kwargs), Dumper=dumper)
+        if uri:
+            self._write_to_file(s=yaml_string, uri=uri, **kwargs)
+            return None
+
+        return yaml_string
+
+    @classmethod
+    def from_yaml(
+        cls: Type[Self],
+        yaml_string: str = None,
+        uri: str = None,
+        loader: callable = yaml.SafeLoader,
+        **kwargs,
+    ) -> Self:
+        """Initializes an object from YAML string or URI location containing the YAML
+
+        Parameters
+        ----------
+        yaml_string : str, optional
+            YAML string, by default None
+        uri : str, optional
+            URI location of file containing YAML, by default None
+        loader : callable, optional
+            Custom YAML loader, by default yaml.SafeLoader
+
+        Returns
+        -------
+        Self
+            Object initialized from the YAML.
+
+        Raises
+        ------
+        ValueError
+            Raised if neither string nor uri is provided
+        """
+        if yaml_string:
+            return cls.from_dict(yaml.load(yaml_string, Loader=loader))
+        if uri:
+            yaml_dict = yaml.load(cls._read_from_file(uri=uri, **kwargs), Loader=loader)
+            return cls.from_dict(yaml_dict)
+        raise ValueError("Must provide either YAML string or URI location")
+
+    @classmethod
+    def from_string(
+        cls: Type[Self],
+        obj_string: str = None,
+        uri: str = None,
+        loader: callable = yaml.SafeLoader,
+        **kwargs,
+    ) -> Self:
+        """Initializes an object from YAML/JSON string or URI location containing the YAML/JSON
+
+        Parameters
+        ----------
+        obj_string : str, optional
+            YAML/JSON string, by default None
+        uri : str, optional
+            URI location of file containing YAML/JSON, by default None
+        loader : callable, optional
+            Custom YAML loader, by default yaml.SafeLoader
+
+        Returns
+        -------
+        Self
+            Object initialized from the YAML.
+        """
+        return cls.from_yaml(yaml_string=obj_string, uri=uri, loader=loader, **kwargs)
+
+    def __repr__(self) -> str:
+        """Returns printable version of object
+
+        Returns:
+            string: Serialized version of object as a YAML string
+        """
+        return self.to_yaml()

ads/jobs/templates/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/jobs/templates/container.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+# This is a placeholder file for container job.

ads/jobs/templates/driver_notebook.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+"""This module runs a Jupyter Python notebook with nbconvert and print the outputs.
+This is a driver script auto-generated by Oracle ADS.
+
+The following environment variables are used:
+JOB_RUN_NOTEBOOK:
+    The relative path of the jupyter Python notebook to be executed.
+NOTEBOOK_EXCLUDE_TAGS:
+    Optional, a list of tags serialized to JSON string.
+    Notebook cells with one of the tags will be excluded from running.
+NOTEBOOK_ENCODING:
+    Optional, the encoding for opening the notebook.
+OUTPUT_URI:
+    Optional, object storage URI for saving files from the output directory.
+"""
+import logging
+import json
+import os
+from typing import Optional
+
+import nbformat
+from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
+
+try:
+    # This is used by ADS and testing
+    from .driver_utils import OCIHelper, JobRunner, set_log_level
+except ImportError:
+    # This is used when the script is in a job run.
+    from driver_utils import OCIHelper, JobRunner, set_log_level
+
+
+logger = logging.getLogger(__name__)
+logger = set_log_level(logger)
+
+# The directory for storing the user code/notebook.
+# The basename of the directory should match the ADS PythonArtifact.USER_CODE_DIR
+CODE_DIR = os.path.join(os.path.dirname(__file__), "code")
+
+
+class ADSExecutePreprocessor(ExecutePreprocessor):
+    """Customized Execute Preprocessor for running notebook."""
+
+    def __init__(self, exclude_tags=None, **kw):
+        """Initialize the preprocessor
+
+        Parameters
+        ----------
+        exclude_tags : list, optional
+            A list of cell tags, notebook cells with any of these cell tag will be skipped.
+            Defaults to None.
+        """
+        self.exclude_tags = exclude_tags
+        super().__init__(**kw)
+
+    @staticmethod
+    def _print_cell_outputs(cell):
+        """Prints the outputs of a notebook cell."""
+        for output in cell.outputs:
+            output_type = output.get("output_type")
+            if output_type == "stream":
+                # stream outputs includes line break already
+                print(output.text, end="")
+            elif output_type == "execute_result":
+                # execute_result may contain text/plain
+                text = output.get("data", {}).get("text/plain", [])
+                # The value could be str or list of str
+                if isinstance(text, list):
+                    for line in text:
+                        print(line)
+                else:
+                    print(text)
+
+    def preprocess_cell(self, cell, resources, *args, **kwargs):
+        """Runs the notebook cell and print out the outputs"""
+        # Skip the cell if any of the cell tags matching an exclude tag.
+        if self.exclude_tags:
+            # Log an error message if there is an error reading the cell tags,
+            # and continue to run the cell.
+            try:
+                cell_tags = cell.get("metadata", {}).get("tags", [])
+                for tag in cell_tags:
+                    if tag in self.exclude_tags:
+                        return cell, resources
+            except Exception as ex:
+                logger.exception("An error occurred when reading cell tags.")
+        # Run the cell
+        cell, resources = super().preprocess_cell(cell, resources, *args, **kwargs)
+        # Print cell output
+        if hasattr(cell, "outputs"):
+            # Log a message if there is an error getting the cell output,
+            # and continue to run the next cell.
+            try:
+                self._print_cell_outputs(cell)
+            except Exception as ex:
+                logger.exception("An error occurred when reading cell outputs.")
+        return cell, resources
+
+
+def run_notebook(
+    notebook_path: str,
+    working_dir: Optional[str] = None,
+    exclude_tags: Optional[list] = None,
+) -> Optional[CellExecutionError]:
+    """Runs a notebook
+
+    Parameters
+    ----------
+    notebook_path : str
+        The path of the notebook
+    working_dir : str, optional
+        The working directory for running the notebook, by default None.
+        If this is None, the same directory of the notebook_path will be used.
+    exclude_tags : list, optional
+        Tags for excluding cells, by default None
+
+    Returns
+    -------
+    CellExecutionError or None
+        Exception object when there is an error in a notebook cell.
+        Otherwise, None.
+    """
+    # Read the notebook
+    encoding = os.environ.get("NOTEBOOK_ENCODING", "utf-8")
+    with open(notebook_path, encoding=encoding) as f:
+        nb = nbformat.read(f, as_version=4)
+
+    # Working/Output directory
+    if not working_dir:
+        working_dir = os.path.dirname(notebook_path)
+
+    # The path of the output notebook with results/plots
+    notebook_filename_out = os.path.join(working_dir, os.path.basename(notebook_path))
+
+    ep = ADSExecutePreprocessor(exclude_tags=exclude_tags, kernel_name="python")
+
+    try:
+        ep.preprocess(nb, {"metadata": {"path": working_dir}})
+        ex = None
+    except CellExecutionError as exc:
+        msg = "Error executing the notebook.\n\n"
+        logger.error(msg)
+        ex = exc
+    finally:
+        with open(notebook_filename_out, mode="w", encoding=encoding) as f:
+            nbformat.write(nb, f)
+    return ex
+
+
+def main() -> None:
+    """Runs the driver to execute a notebook."""
+    JobRunner().conda_unpack()
+
+    notebook_file_path = os.path.join(CODE_DIR, os.environ.get("JOB_RUN_NOTEBOOK"))
+    # By default, the output directory will be the one containing the notebook
+    output_dir = os.environ.get("OUTPUT_DIR", os.path.dirname(notebook_file_path))
+
+    # Exclude tags
+    tags = os.environ.get("NOTEBOOK_EXCLUDE_TAGS")
+    if tags:
+        tags = json.loads(tags)
+        logger.info("Excluding cells with any of the following tags: %s", tags)
+    # Run the notebook
+    ex = run_notebook(notebook_file_path, working_dir=output_dir, exclude_tags=tags)
+
+    # Save the outputs
+    OCIHelper.copy_outputs(output_dir)
+
+    if ex:
+        raise ex
+
+
+if __name__ == "__main__":
+    main()