oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/templates/score_tensorflow.jinja2

@@ -0,0 +1,184 @@
+# score.py {{SCORE_VERSION}} generated by ADS {{ADS_VERSION}} on {{time_created}}
+import os
+import sys
+import logging
+import pandas as pd
+import numpy as np
+import json
+import tensorflow as tf
+import base64
+from io import BytesIO
+from functools import lru_cache
+
+model_name = '{{model_file_name}}'
+
+
+"""
+   Inference script. This script is used for prediction by scoring server when schema is known.
+"""
+
+
+@lru_cache(maxsize=10)
+def load_model(model_file_name=model_name):
+    """
+    Loads model from the serialized format
+
+    Returns
+    -------
+    model:  a model instance on which predict API can be invoked
+    """
+    model_dir = os.path.dirname(os.path.realpath(__file__))
+    if model_dir not in sys.path:
+        sys.path.insert(0, model_dir)
+    contents = os.listdir(model_dir)
+    if model_file_name not in contents:
+        raise Exception(f'{model_file_name} is not found in model directory {model_dir}')
+
+{% if model_serializer == "tf" %}
+    print(f'Start loading {model_file_name} from model directory {model_dir} ...')
+    loaded_model = tf.keras.models.load_model(os.path.join(model_dir, model_file_name))
+    print("Model is successfully loaded.")
+    return loaded_model
+
+{% else %}
+    raise NotImplementedError("`load_model()` needs to be implemented.")
+{% endif %}
+
+@lru_cache(maxsize=1)
+def fetch_data_type_from_schema(input_schema_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), "input_schema.json")):
+    """
+    Returns data type information fetch from input_schema.json.
+
+    Parameters
+    ----------
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data_type: data type fetch from input_schema.json.
+
+    """
+    data_type = {}
+    if os.path.exists(input_schema_path):
+        schema = json.load(open(input_schema_path))
+        for col in schema['schema']:
+            data_type[col['name']] = col['dtype']
+    else:
+        print("input_schema has to be passed in in order to recover the same data type. pass `X_sample` in `ads.model.framework.tensorflow_model.TensorFlowModel.prepare` function to generate the input_schema. Otherwise, the data type might be changed after serialization/deserialization.")
+    return data_type
+
+
+def deserialize(data, input_schema_path):
+    """
+    Deserialize json-serialized data to data in original type when sent to
+predict.
+
+    Parameters
+    ----------
+    data: serialized input data.
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data: deserialized input data.
+
+    """
+{% if data_deserializer == "json" %}
+    if isinstance(data, bytes):
+        logging.warning(
+            "bytes are passed directly to the model. If the model expects a specific data format, you need to write the conversion logic in `deserialize()` yourself."
+        )
+        return data
+
+    data_type = data.get('data_type', '') if isinstance(data, dict) else ''
+    json_data = data.get('data', data) if isinstance(data, dict) else data
+
+    if "numpy.ndarray" in data_type:
+        load_bytes = BytesIO(base64.b64decode(json_data.encode('utf-8')))
+        return np.load(load_bytes, allow_pickle=True)
+    if "tensorflow.python.framework.ops.EagerTensor" in data_type:
+        load_bytes = BytesIO(base64.b64decode(json_data.encode('utf-8')))
+        return tf.convert_to_tensor(np.load(load_bytes, allow_pickle=True))
+    if "pandas.core.series.Series" in data_type:
+        return pd.Series(json_data)
+    if "pandas.core.frame.DataFrame" in data_type or isinstance(json_data, str):
+        return pd.read_json(json_data, dtype=fetch_data_type_from_schema(input_schema_path))
+
+    return json_data
+
+{% elif data_deserializer == "cloudpickle" %}
+    import cloudpickle
+    from pickle import UnpicklingError
+    deserialized_data = data
+    try:
+        deserialized_data = cloudpickle.loads(data)
+    except TypeError:
+        pass
+    except UnpicklingError:
+        logger.warning(
+            "bytes are passed directly to the model. If the model expects a specific data format, you need to write the conversion logic in `deserialize()` yourself."
+        )
+
+    return deserialized_data
+
+{% else %}
+    # Add further data deserialization if needed
+    return data
+{% endif %}
+
+def pre_inference(data, input_schema_path):
+    """
+    Preprocess json-serialized data to feed into predict function.
+
+    Parameters
+    ----------
+    data: Data format as expected by the predict API of the core estimator.
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data: Data format after any processing.
+    """
+    data = deserialize(data, input_schema_path)
+
+    # Add further data preprocessing if needed
+    return data
+
+def post_inference(yhat):
+    """
+    Post-process the model results.
+
+    Parameters
+    ----------
+    yhat: Data format after calling model.predict.
+
+    Returns
+    -------
+    yhat: Data format after any processing.
+
+    """
+
+    return yhat.numpy().tolist()
+
+def predict(data, model=load_model(), input_schema_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), "input_schema.json")):
+    """
+    Returns prediction given the model and data to predict.
+
+    Parameters
+    ----------
+    model: Model instance returned by load_model API
+    data: Data format as expected by the predict API of the core estimator.
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    predictions: Output from scoring server
+        Format: {'prediction': output from model.predict method}
+
+    """
+    inputs = pre_inference(data, input_schema_path)
+
+    yhat = post_inference(
+        model(inputs)
+    )
+    return {'prediction': yhat}

ads/templates/score_xgboost.jinja2

@@ -0,0 +1,178 @@
+# score.py {{SCORE_VERSION}} generated by ADS {{ADS_VERSION}} on {{time_created}}
+import json
+import os
+import sys
+import xgboost as xgb
+import pandas as pd
+import numpy as np
+from functools import lru_cache
+from io import BytesIO
+import base64
+import logging
+
+model_name = '{{model_file_name}}'
+
+
+"""
+   Inference script. This script is used for prediction by scoring server when schema is known.
+"""
+
+
+@lru_cache(maxsize=10)
+def load_model(model_file_name=model_name):
+    """
+    Loads model from the serialized format
+
+    Returns
+    -------
+    model:  a model instance on which predict API can be invoked
+    """
+    model_xgb = xgb.Booster()
+    model_dir = os.path.dirname(os.path.realpath(__file__))
+    if model_dir not in sys.path:
+        sys.path.insert(0, model_dir)
+    contents = os.listdir(model_dir)
+    if model_file_name not in contents:
+        raise Exception(f'{model_file_name} is not found in model directory {model_dir}')
+
+{% if model_serializer == "xgboost" %}
+    print(f'Start loading {model_file_name} from model directory {model_dir} ...')
+    model_xgb.load_model(os.path.join(os.path.dirname(os.path.realpath(__file__)), model_file_name))
+    print("Model is successfully loaded.")
+    return model_xgb
+
+{% else %}
+    raise NotImplementedError("`load_model()` needs to be implemented.")
+{% endif %}
+
+@lru_cache(maxsize=1)
+def fetch_data_type_from_schema(input_schema_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), "input_schema.json")):
+    """
+    Returns data type information fetch from input_schema.json.
+
+    Parameters
+    ----------
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data_type: data type fetch from input_schema.json.
+
+    """
+    data_type = {}
+    if os.path.exists(input_schema_path):
+        schema = json.load(open(input_schema_path))
+        for col in schema['schema']:
+            data_type[col['name']] = col['dtype']
+    else:
+        print("input_schema has to be passed in in order to recover the same data type. pass `X_sample` in `ads.model.framework.xgboost_model.XGBoostModel.prepare` function to generate the input_schema. Otherwise, the data type might be changed after serialization/deserialization.")
+    return data_type
+
+
+def deserialize(data, input_schema_path):
+    """
+    Deserialize json serialization data to data in original type when sent to predict.
+
+    Parameters
+    ----------
+    data: serialized input data.
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data: deserialized input data.
+
+    """
+{% if data_deserializer == "json" %}
+    if isinstance(data, bytes):
+        logging.warning(
+            "bytes are passed directly to the model. If the model expects a specific data format, you need to write the conversion logic in `deserialize()` yourself."
+        )
+        return data
+
+    data_type = data.get('data_type', '') if isinstance(data, dict) else ''
+    json_data = data.get('data', data) if isinstance(data, dict) else data
+
+    if "numpy.ndarray" in data_type:
+        load_bytes = BytesIO(base64.b64decode(json_data.encode('utf-8')))
+        return np.load(load_bytes, allow_pickle=True)
+    if "pandas.core.series.Series" in data_type:
+        return pd.Series(json_data)
+    if "pandas.core.frame.DataFrame" in data_type or isinstance(json_data, str):
+        return pd.read_json(json_data, dtype=fetch_data_type_from_schema(input_schema_path))
+    if isinstance(json_data, dict):
+        return pd.DataFrame.from_dict(json_data)
+
+    return json_data
+
+{% elif data_deserializer == "cloudpickle" %}
+    import cloudpickle
+    from pickle import UnpicklingError
+    deserialized_data = data
+    try:
+        deserialized_data = cloudpickle.loads(data)
+    except TypeError:
+        pass
+    except UnpicklingError:
+        logger.warning(
+            "bytes are passed directly to the model. If the model expects a specific data format, you need to write the conversion logic in `deserialize()` yourself."
+        )
+
+    return deserialized_data
+
+{% else %}
+    # Add further data deserialization if needed
+    return data
+{% endif %}
+
+def pre_inference(data, input_schema_path):
+    """
+    Preprocess data
+
+    Parameters
+    ----------
+    data: Data format as expected by the predict API of the core estimator.
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data: Data format after any processing.
+
+    """
+    data = deserialize(data, input_schema_path)
+    return data
+
+def post_inference(yhat):
+    """
+    Post-process the model results
+
+    Parameters
+    ----------
+    yhat: Data format after calling model.predict.
+
+    Returns
+    -------
+    yhat: Data format after any processing.
+
+    """
+    return yhat.tolist()
+
+def predict(data, model=load_model(), input_schema_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), "input_schema.json")):
+    """
+    Returns prediction given the model and data to predict
+
+    Parameters
+    ----------
+    model: Model instance returned by load_model API
+    data: Data format as expected by the predict API of the core estimator. For eg. in case xgboost models it could be numpy array/List of list/Pandas DataFrame
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    predictions: Output from scoring server
+        Format: {'prediction': output from model.predict method}
+
+    """
+    data = pre_inference(data, input_schema_path)
+    data = xgb.DMatrix(data)
+    return {'prediction': post_inference(model.predict(data))}

ads/text_dataset/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/text_dataset/backends.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import fsspec
+import io
+import json
+import os
+from typing import Dict, Generator, List, Union
+
+from ads.common.decorator.runtime_dependency import runtime_dependency
+from ads.text_dataset.utils import PY4JGateway, experimental
+from fsspec.core import OpenFile
+
+
+class Base:
+    """Base class for backends."""
+
+    def read_line(
+        self, fhandler: OpenFile
+    ) -> Generator[Union[str, List[str]], None, None]:
+        """Read lines from a file.
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            a file handler returned by `fsspec`
+
+        Yields
+        -------
+        Generator
+            a generator that yields lines
+        """
+        with fhandler as f:
+            for line in f:
+                yield line.decode(fhandler.encoding)
+
+    def read_text(
+        self, fhandler: OpenFile
+    ) -> Generator[Union[str, List[str]], None, None]:
+        """Read entire file into a string.
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            a file handler returned by `fsspec`
+
+        Yields
+        -------
+        Generator
+            a generator that yields text in the file
+        """
+        with fhandler as f:
+            yield f.read().decode(fhandler.encoding)
+
+    @staticmethod
+    def _validate_dest(src, dst_folder: str, fname: str = None) -> str:
+        if fname is None:
+            fname = os.path.splitext(os.path.basename(src))[0] + ".txt"
+        return os.path.join(dst_folder, fname)
+
+    def convert_to_text(
+        self,
+        fhandler: OpenFile,
+        dst_path: str,
+        fname: str = None,
+        storage_options: Dict = None,
+    ) -> str:
+        """Convert input file to a text file
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            a file handler returned by `fsspec`
+        dst_path: str
+            local folder or cloud storage prefix to save converted text files
+        fname: str, optional
+            filename for converted output, relative to dirname or prefix, by default None
+        storage_options: dict, optional
+            storage options for cloud storage
+
+        Returns
+        -------
+        str
+            path to saved output
+        """
+        dest = self._validate_dest(fhandler.path, dst_path, fname)
+        storage_options = {} if storage_options is None else storage_options
+        with fsspec.open(dest, mode="wb", **storage_options) as fout:
+            with fhandler as fin:
+                fout.write(fin.read())
+        return dest
+
+    def get_metadata(self, fhandler: OpenFile) -> Dict:
+        """Get metadata of a file.
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            a file handler returned by fsspec
+
+        Returns
+        -------
+        dict
+            dictionary of metadata
+        """
+        return {}
+
+
+class Tika(Base):
+    def read_line(self, fhandler):
+        with PY4JGateway() as gateway:
+            parser = gateway.jvm.parsers.TikaParser()
+            with fhandler as f:
+                reader = parser.reader(f.read())
+            while True:
+                line = reader.readLine()
+                if line is not None:
+                    yield line
+                else:
+                    break
+
+    def read_text(self, fhandler):
+        with PY4JGateway() as gateway:
+            parser = gateway.jvm.parsers.TikaParser()
+            with fhandler as f:
+                yield parser.readText(f.read()).decode()
+
+    def convert_to_text(self, fhandler, dst_path, fname=None, storage_options=None):
+        dest = self._validate_dest(fhandler.path, dst_path, fname)
+        storage_options = {} if storage_options is None else storage_options
+        with fsspec.open(dest, mode="w", **storage_options) as f:
+            f.write(next(self.read_text(fhandler)))
+        return dest
+
+    def get_metadata(self, fhandler):
+        with PY4JGateway() as gateway:
+            parser = gateway.jvm.parsers.TikaParser()
+            meta = gateway.jvm.org.apache.tika.metadata.Metadata()
+            with fhandler as f:
+                parser.readText(f.read(), meta)
+            return json.loads(parser.getMetadata(meta))
+
+    def detect_encoding(self, fhandler: OpenFile):
+        with PY4JGateway() as gateway:
+            with fhandler as f:
+                return gateway.jvm.parsers.TikaParser.detectEncoding(f.read())
+
+
+class PDFPlumber(Base):
+    @runtime_dependency(
+        module="pdfplumber", err_msg="pdfplumber must be installed first."
+    )
+    def __init__(self):
+        super().__init__()
+
+    def read_line(self, fhandler):
+        import pdfplumber
+
+        with fhandler as f:
+            pdf = pdfplumber.PDF(f)
+            for page in pdf.pages:
+                reader = io.StringIO(page.extract_text())
+                for line in reader:
+                    yield line
+
+    def read_text(self, fhandler):
+        import pdfplumber
+
+        with fhandler as f:
+            pdf = pdfplumber.PDF(f)
+            texts = (page.extract_text() for page in pdf.pages)
+            yield "\n".join([text for text in texts if text is not None])
+
+    def convert_to_text(
+        self,
+        fhandler,
+        dst_path,
+        fname=None,
+        storage_options=None,
+    ):
+        import pdfplumber
+
+        dest = self._validate_dest(fhandler.path, dst_path, fname)
+        storage_options = {} if storage_options is None else storage_options
+        with fhandler as fin:
+            pdf = pdfplumber.PDF(fin)
+            with fsspec.open(dest, mode="w", **storage_options) as fout:
+                for page in pdf.pages:
+                    text = page.extract_text()
+                    if text is not None:
+                        fout.write(text)
+        return dest
+
+    def get_metadata(self, fhandler):
+        import pdfplumber
+
+        with fhandler as f:
+            pdf = pdfplumber.PDF(f)
+            return pdf.metadata
+
+
+@experimental
+class OITCC(Base):  # pragma: no cover
+    def convert_to_text(self, fhandler, output_path, fname=None, storage_options=None):
+        dest = self._validate_dest(fhandler.path, output_path, fname)
+        storage_options = {} if storage_options is None else storage_options
+        with fsspec.open(dest, mode="w", **storage_options) as f:
+            f.write(next(self.read_text(fhandler)))