oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/opctl/operator/lowcode/forecast/whatifserve/score.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+import os
+from joblib import load
+import pandas as pd
+import numpy as np
+from functools import lru_cache
+import logging
+import ads
+from ads.opctl.operator.lowcode.common.utils import load_data
+from ads.opctl.operator.common.operator_config import InputData
+from ads.opctl.operator.lowcode.forecast.const import SupportedModels
+
+ads.set_auth("resource_principal")
+
+logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s', level=logging.INFO)
+logger_pred = logging.getLogger('model-prediction')
+logger_pred.setLevel(logging.INFO)
+logger_feat = logging.getLogger('input-features')
+logger_feat.setLevel(logging.INFO)
+
+"""
+   Inference script. This script is used for prediction by scoring server when schema is known.
+"""
+
+
+@lru_cache(maxsize=10)
+def load_model():
+    """
+    Loads model from the serialized format
+
+    Returns
+    -------
+    model:  a model instance on which predict API can be invoked
+    """
+    model_dir = os.path.dirname(os.path.realpath(__file__))
+    contents = os.listdir(model_dir)
+    model_file_name = "models.pickle"
+    if model_file_name in contents:
+        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), model_file_name), "rb") as file:
+            model = load(file)
+    else:
+        raise Exception('{0} is not found in model directory {1}'.format(model_file_name, model_dir))
+    return model
+
+
+@lru_cache(maxsize=1)
+def fetch_data_type_from_schema(
+        input_schema_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), "input_schema.json")):
+    """
+    Returns data type information fetch from input_schema.json.
+
+    Parameters
+    ----------
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data_type: data type fetch from input_schema.json.
+
+    """
+    data_type = {}
+    if os.path.exists(input_schema_path):
+        schema = json.load(open(input_schema_path))
+        for col in schema['schema']:
+            data_type[col['name']] = col['dtype']
+    else:
+        print(
+            "input_schema has to be passed in in order to recover the same data type. pass `X_sample` in `ads.model.framework.sklearn_model.SklearnModel.prepare` function to generate the input_schema. Otherwise, the data type might be changed after serialization/deserialization.")
+    return data_type
+
+
+def deserialize(data, input_schema_path):
+    """
+    Deserialize json serialization data to data in original type when sent to predict.
+
+    Parameters
+    ----------
+    data: serialized input data.
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data: deserialized input data.
+
+    """
+
+    # Add further data deserialization if needed
+    return data
+
+
+def pre_inference(data, input_schema_path):
+    """
+    Preprocess data
+
+    Parameters
+    ----------
+    data: Data format as expected by the predict API of the core estimator.
+    input_schema_path: path of input schema.
+
+    Returns
+    -------
+    data: Data format after any processing.
+
+    """
+    return deserialize(data, input_schema_path)
+
+
+def post_inference(yhat):
+    """
+    Post-process the model results
+
+    Parameters
+    ----------
+    yhat: Data format after calling model.predict.
+
+    Returns
+    -------
+    yhat: Data format after any processing.
+
+    """
+    if isinstance(yhat, pd.core.frame.DataFrame):
+        yhat = yhat.values
+    if isinstance(yhat, np.ndarray):
+        yhat = yhat.tolist()
+    return yhat
+
+
+def get_forecast(future_df, model_name, series_id, model_object, date_col, target_column, target_cat_col, horizon):
+    date_col_name = date_col["name"]
+    date_col_format = date_col["format"]
+    future_df[target_cat_col] = future_df[target_cat_col].astype("str")
+    future_df[date_col_name] = pd.to_datetime(
+        future_df[date_col_name], format=date_col_format
+    )
+    if model_name == SupportedModels.AutoTS:
+        series_id_col = "Series"
+        full_data_indexed = future_df.rename(columns={target_cat_col: series_id_col})
+        additional_regressors = list(
+            set(full_data_indexed.columns) - {target_column, series_id_col, date_col_name}
+        )
+        future_reg = full_data_indexed.reset_index().pivot(
+            index=date_col_name,
+            columns=series_id_col,
+            values=additional_regressors,
+        )
+        pred_obj = model_object.predict(future_regressor=future_reg)
+        return pred_obj.forecast[series_id].tolist()
+    elif model_name == SupportedModels.Prophet and series_id in model_object:
+        model = model_object[series_id]['model']
+        label_encoder = model_object[series_id]['le']
+        processed = future_df.rename(columns={date_col_name: 'ds', target_column: 'y'})
+        encoded_df = label_encoder.transform(processed)
+        forecast = model.predict(encoded_df)
+        return forecast['yhat'].tolist()
+    elif model_name == SupportedModels.NeuralProphet and series_id in model_object:
+        model = model_object[series_id]['model']
+        label_encoder = model_object[series_id]['le']
+        model.restore_trainer()
+        accepted_regressors = list(model.config_regressors.regressors.keys())
+        data = future_df.rename(columns={date_col_name: 'ds', target_column: 'y'})
+        encoded_df = label_encoder.transform(data)
+        future = encoded_df[accepted_regressors + ["ds"]].reset_index(drop=True)
+        future["y"] = None
+        forecast = model.predict(future)
+        return forecast['yhat1'].tolist()
+    elif model_name == SupportedModels.Arima and series_id in model_object:
+        model = model_object[series_id]['model']
+        label_encoder = model_object[series_id]['le']
+        predict_cols = model_object[series_id]["predict_component_cols"]
+        encoded_df = label_encoder.transform(future_df)
+        x_pred = encoded_df.set_index(date_col_name)
+        x_pred = x_pred.drop(target_cat_col, axis=1)
+        yhat, conf_int = model.predict(
+            n_periods=horizon,
+            X=x_pred[predict_cols],
+            return_conf_int=True
+        )
+        yhat_clean = pd.DataFrame(yhat, index=yhat.index, columns=["yhat"])
+        return yhat_clean['yhat'].tolist()
+    elif model_name == SupportedModels.AutoMLX and series_id in model_object:
+        model = model_object[series_id]['model']
+        label_encoder = model_object[series_id]['le']
+        encoded_df = label_encoder.transform(future_df)
+        x_pred = encoded_df.drop(target_cat_col, axis=1)
+        x_pred = x_pred.set_index(date_col_name)
+        forecast = model.forecast(
+            X=x_pred,
+            periods=horizon
+        )
+        return forecast[target_column].tolist()
+    else:
+        raise Exception(f"Invalid model object type: {type(model_object).__name__}.")
+
+
+def predict(data, model=load_model()) -> dict:
+    """
+    Returns prediction given the model and data to predict
+
+    Parameters
+    ----------
+    model: Model instance returned by load_model API
+    data: Data format as expected by the predict API of the core estimator. For eg. in case of sckit models it could be numpy array/List of list/Panda DataFrame
+
+    Returns
+    -------
+    predictions: Output from scoring server
+        Format: { 'prediction': output from `model.predict` method }
+
+    """
+    assert model is not None, "Model is not loaded"
+
+    models = model["models"]
+    specs = model["spec"]
+    horizon = specs["horizon"]
+    model_name = specs["model"]
+    date_col = specs["datetime_column"]
+    target_column = specs["target_column"]
+    target_category_column = specs["target_category_columns"][0]
+
+    try:
+        input_data = InputData(**data["additional_data"])
+    except TypeError as e:
+        raise ValueError(f"Validation error: {e}")
+    additional_data = load_data(input_data)
+
+    unique_values = additional_data[target_category_column].unique()
+    forecasts = {}
+    for key in unique_values:
+        try:
+            s_id = str(key)
+            filtered = additional_data[additional_data[target_category_column] == key]
+            future = filtered.tail(horizon)
+            forecast = get_forecast(future, model_name, s_id, models, date_col,
+                                    target_column, target_category_column, horizon)
+            forecasts[s_id] = json.dumps(forecast)
+        except Exception as e:
+            raise RuntimeError(
+                f"An error occurred during prediction: {e}."
+                f" Please ensure the input data matches the format and structure of the data used during training.")
+
+    return {'prediction': json.dumps(forecasts)}

ads/opctl/operator/lowcode/pii/MLoperator

@@ -0,0 +1,17 @@
+type: pii
+version: v1
+name: PII Operator
+conda_type: published
+conda: pii_v1
+gpu: no
+keywords:
+  - PII
+  - Spacy
+backends:
+  - job
+  - operator.local
+description: |
+  PII operator, that detects detect and redact Personally Identifiable Information
+  (PII) data in datasets by combining pattern match and machine learning solution.
+  Use `ads operator info -t pii` to get more details about the pii operator."
+logo: data:image/svg+xml;base64,PHN2ZyBpZD0ic3ZnIiB2ZXJzaW9uPSIxLjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHdpZHRoPSI1MCIKICBoZWlnaHQ9IjUwLjk0ODQ1MzYwODI0NzQ2IiB2aWV3Qm94PSIwLCAwLCA0MDAsMzY0Ljk0ODQ1MzYwODI0NzQ2Ij4KICA8ZyBpZD0ic3ZnZyI+CiAgICA8cGF0aCBpZD0icGF0aDAiCiAgICAgIGQ9Ik0xODIuNDc0IDQuMTk0IEMgMTI2LjI1NyAxMC4yODIsNjkuOTExIDQ0LjA5MCwyNy4zNzggOTcuMjUxIEMgMjIuMjM3IDEwMy42NzcsMTUuMDY1IDExMi4yOTgsMTEuNDQwIDExNi40MDkgQyAtMy4wMjcgMTMyLjgxOSwtMi41MTYgMTM2Ljk2MywxNi43OTIgMTU5Ljc5NCBDIDc5LjAzMCAyMzMuMzg3LDEzNS43MjMgMjY2LjI4NiwyMDAuMzQ0IDI2Ni4zMDcgQyAyNjIuNTYyIDI2Ni4zMjcsMzI0LjI1NSAyMzEuNjc1LDM3NS45NDkgMTY3LjY3MyBDIDM3OS4xNDQgMTYzLjcxOCwzODQuNDc5IDE1Ny4zODgsMzg3LjgwNCAxNTMuNjA4IEMgNDA0LjEwNSAxMzUuMDc3LDQwMi41OTggMTMxLjEwMSwzNjIuMTUzIDg1LjkxMSBDIDMwOS4zNjEgMjYuOTI1LDI0NC41OTAgLTIuNTMyLDE4Mi40NzQgNC4xOTQgTTIxNC40MzcgMzQuMDA4IEMgMjM0LjkwOSAzNi40MzYsMjYzLjkxOCA0NC43MDcsMjYzLjkxOCA0OC4xMTUgQyAyNjMuOTE4IDQ4LjY3MiwyNDQuNDg1IDcwLjI0OCwyMzIuNjYxIDgyLjgxOCBDIDIyNy42ODMgODguMTEwLDIxOS45MjEgOTYuNDUwLDIxNS40MTMgMTAxLjM1MiBDIDIxMC45MDUgMTA2LjI1MywyMDQuNTg4IDExMy4wNjIsMjAxLjM3NSAxMTYuNDgzIEMgMTk4LjE2MiAxMTkuOTAzLDE5Mi4yODUgMTI2LjI1MSwxODguMzE2IDEzMC41ODggQyAxODQuMzQ3IDEzNC45MjYsMTc3LjA3OSAxNDIuNzU4LDE3Mi4xNjUgMTQ3Ljk5NCBDIDE2MS4zMDcgMTU5LjU2MiwxNDEuOTEwIDE4MC40NDMsMTMyLjQ3NCAxOTAuNzIyIEMgMTI4LjY1NyAxOTQuODgwLDEyMi44MzMgMjAxLjA5NywxMTkuNTMxIDIwNC41MzcgTCAxMTMuNTI5IDIxMC43OTMgMTA0LjcwMiAyMDQuMjYzIEMgODcuMDMxIDE5MS4xODksNjYuNzk0IDE3Mi4wMDksNDguNzE4IDE1MS4yMDMgQyAzNC43NjMgMTM1LjE0MCwzNC42MjggMTM0Ljk1NCwzNS44NTggMTMzLjUxMCBDIDM2LjQyNCAxMzIuODQ2LDQwLjg3OSAxMjcuNTA5LDQ1Ljc1NyAxMjEuNjQ5IEMgNjAuMDIzIDEwNC41MTUsNzUuOTI3IDg3Ljk0Myw4Ni45NDIgNzguNzM0IEMgOTUuNjI4IDcxLjQ3MiwxMDkuMTU0IDYxLjE2OCwxMTAuMDAwIDYxLjE2OCBDIDExMC4zMTYgNjEuMTY4LDExNi45MDkgNjcuNTM5LDEyNC42NTMgNzUuMzI2IEwgMTM4LjczMiA4OS40ODMgMTM2LjM3OCA5Mi43NTkgQyAxMjQuMzU5IDEwOS40ODksMTIwLjExOSAxMzguMzMyLDEyNi41ODYgMTU5LjM3MiBDIDEyOC4xNzcgMTY0LjU1MCwxMzIuNjA1IDE3NC41NzAsMTMzLjMwMSAxNzQuNTcwIEMgMTMzLjU1NiAxNzQuNTcwLDEzOS4wMDIgMTY4Ljk0MCwxNDUuNDA1IDE2Mi4wNTkgQyAxNTEuODA3IDE1NS4xNzgsMTU5LjEzMiAxNDcuMzQ1LDE2MS42ODQgMTQ0LjY1MyBDIDE2NS42MTUgMTQwLjUwNiwxNjYuMzIzIDEzOS4zODcsMTY2LjMyMyAxMzcuMzI3IEMgMTY2LjMyMyAxMjEuOTgwLDE3Ny41NTkgMTA4LjYzOCwxOTIuNDYwIDEwNi4yOTIgTCAxOTcuMzY3IDEwNS41MTkgMjA1LjcyOCA5Ni44OTggQyAyMTkuOTEzIDgyLjI3MywyMzEuNjE1IDY5LjYyOCwyMzEuNjE1IDY4LjkyNyBDIDIzMS42MTUgNjguNTU5LDIyOC45MDkgNjcuNDQwLDIyNS42MDEgNjYuNDQwIEMgMjA4LjExNyA2MS4xNTMsMTgyLjgzMyA2Mi4zMzUsMTY3LjY1MSA2OS4xNDggTCAxNjMuNDcwIDcxLjAyNSAxNTEuMzIyIDU4Ljg2NCBDIDE0MS41NzkgNDkuMTEwLDEzOS4zNzkgNDYuNTcyLDE0MC4yMDYgNDYuMDM1IEMgMTUzLjc5NSAzNy4yMjcsMTkwLjg5NCAzMS4yMTYsMjE0LjQzNyAzNC4wMDggTTI5Ny40NTIgNjYuMDk4IEMgMzEyLjY0MSA3Ny4yMDEsMzM3LjU1MCAxMDEuOTgxLDM1Ny42MzYgMTI1Ljk2OSBMIDM2NC45OTQgMTM0Ljc1NiAzNjEuMTA5IDEzOS4zNzEgQyAzNTguOTcyIDE0MS45MDksMzU2LjA5NiAxNDUuMjIzLDM1NC43MTcgMTQ2LjczNSBDIDM1My4zMzkgMTQ4LjI0NywzNDguNjMwIDE1My41MDUsMzQ0LjI1NCAxNTguNDE5IEMgMzIyLjA1MSAxODMuMzUxLDI5MC4wNjkgMjExLjQ4MCwyODcuMzI2IDIwOC40ODggQyAyODYuNjg1IDIwNy43ODksMjgwLjY4MSAyMDEuNjE1LDI3My45ODUgMTk0Ljc2OCBMIDI2MS44MTAgMTgyLjMyMCAyNjQuMjcwIDE3OC42MTcgQyAyNzguMjU3IDE1Ny41NjYsMjgwLjg1MiAxMjUuNDI1LDI3MC4yMzYgMTA0LjczMyBMIDI2OS4wMjkgMTAyLjM4MCAyNTEuNTU4IDEyMS4zNDcgQyAyMzQuNDI1IDEzOS45NDgsMjM0LjA2NiAxNDAuMzk4LDIzMy4wMTAgMTQ0LjU5MSBDIDIzMC42MDMgMTU0LjE1NywyMjEuMDI0IDE2My41NDAsMjEzLjY0MSAxNjMuNTY3IEMgMjEyLjQxMiAxNjMuNTcyLDIxMS4xNzkgMTY0LjMzOCwyMDkuODYwIDE2NS45MTYgQyAyMDguNzg0IDE2Ny4yMDQsMjAyLjgwMSAxNzMuODkwLDE5Ni41NjQgMTgwLjc3NCBDIDE3My42MjAgMjA2LjA5OCwxNzQuNzQ5IDIwNC4yNTAsMTgxLjI3MSAyMDUuODA1IEMgMTk3Ljg4OSAyMDkuNzY4LDIxNi4wMTYgMjA4LjMxNSwyMzMuMjQxIDIwMS42NDAgTCAyMzYuNTg2IDIwMC4zNDQgMjQ3Ljg3OSAyMTEuOTI5IEMgMjYwLjU2NyAyMjQuOTQ1LDI2MC4zMTYgMjI0LjM0MSwyNTQuMDg3IDIyNi44OTQgQyAyMjYuODU5IDIzOC4wNTMsMTkxLjU5NyAyMzkuOTkzLDE2My4yMzAgMjMxLjg5MiBDIDE1Ni4xNzQgMjI5Ljg3NywxNDQuMjk0IDIyNS41NzUsMTQzLjM2MCAyMjQuNjk3IEMgMTQyLjk5NyAyMjQuMzU1LDE0Ni43NDAgMjE5Ljg4MCwxNTIuMjk1IDIxNC4wMTggQyAxNjEuODI5IDIwMy45NTYsMTYzLjQyMCAyMDIuMjM0LDE4MS43MjcgMTgyLjE3NiBDIDIwMC45MzUgMTYxLjEzMSwyMTUuNTg3IDE0NS4yMzQsMjIxLjk5NCAxMzguNDkwIEMgMjI1LjU4NiAxMzQuNzA5LDIzMC4wMzggMTI5LjkxNCwyMzEuODg4IDEyNy44MzUgQyAyMzMuNzM4IDEyNS43NTYsMjQwLjEwMiAxMTguOTUyLDI0Ni4wMzEgMTEyLjcxNSBDIDI1MS45NjAgMTA2LjQ3OCwyNTguNzk3IDk5LjIxNiwyNjEuMjIzIDk2LjU3OCBDIDI2My42NTAgOTMuOTQwLDI2OS4wMzggODguMjE4LDI3My4xOTYgODMuODYzIEMgMjc3LjM1NCA3OS41MDgsMjgzLjM2NCA3My4wODQsMjg2LjU1MSA2OS41ODggQyAyOTMuMjk4IDYyLjE4NSwyOTIuNDY1IDYyLjQ1MiwyOTcuNDUyIDY2LjA5OCBNMjAyLjQwNSAyOTYuMDI1IEMgMjAxLjY0OSAyOTYuMjA4LDIwMC4zMDYgMjk3LjAyMiwxOTkuNDE5IDI5Ny44MzUgQyAxOTUuOTMzIDMwMS4wMzEsMTk1LjkxNyAzMDEuMTc1LDE5NS43MzUgMzI5LjczNyBMIDE5NS41NjQgMzU2LjUwNiAxOTcuMzc3IDM1OS4yNDUgQyAyMDEuNzkxIDM2NS45MTUsMjA5Ljk5NSAzNjUuNjQzLDIxNC4wODkgMzU4LjY5MSBDIDIxNS4zOTkgMzU2LjQ2OCwyMTUuNDY0IDM1NS4xMjEsMjE1LjQ2NCAzMzAuMjQxIEMgMjE1LjQ2NCAzMDUuNDk1LDIxNS4zOTIgMzAzLjk4MCwyMTQuMDg5IDMwMS4zODQgQyAyMTEuNzYzIDI5Ni43NDcsMjA3LjUwMSAyOTQuNzkyLDIwMi40MDUgMjk2LjAyNSBNMjM5Ljg4NiAyOTYuNzM2IEMgMjM0Ljc3NSAyOTkuMzE1LDIzNC43MzQgMjk5LjU1OCwyMzQuNDkyIDMyOC45NTUgQyAyMzQuMjUyIDM1OC4xNDgsMjM0LjI5MCAzNTguNDM4LDIzOC43MzMgMzYxLjgyNyBDIDI0My42ODUgMzY1LjYwNCwyNDkuNjU0IDM2NC40MTQsMjUyLjczOCAzNTkuMDM1IEMgMjU1LjE0NyAzNTQuODMyLDI1NS4xNTcgMzA1Ljc1MSwyNTIuNzQ5IDMwMS4wNTYgQyAyNTAuMTU4IDI5Ni4wMDIsMjQ0Ljg2NSAyOTQuMjI1LDIzOS44ODYgMjk2LjczNiBNMTQxLjA1OSAyOTcuMTMxIEMgMTM3LjA5NSAyOTkuMzMyLDEzNy4xMzIgMjk5LjA1MSwxMzYuODkyIDMyOC4zMTIgQyAxMzYuNjQzIDM1OC42NjksMTM2LjgxMiAzNTkuODgyLDE0MS43MDYgMzYyLjg2NiBDIDE0OS43OTMgMzY3Ljc5NiwxNTUuMzI2IDM2Mi4xNDEsMTU1LjMyNiAzNDguOTQ2IEwgMTU1LjMyNiAzNDEuMDU1IDE2MC42NTMgMzQwLjU5NCBDIDE4NC40NzIgMzM4LjUzMSwxOTAuMjYxIDMwOC4zNTgsMTY4Ljc4NyAyOTguMTk5IEMgMTY0LjAyNyAyOTUuOTQ4LDE0NC41NDIgMjk1LjE5NywxNDEuMDU5IDI5Ny4xMzEgTTE2My4yNjEgMzE1LjA4OSBDIDE2Ni44NjcgMzE4LjY5NSwxNjQuNTM1IDMyMy4yMzcsMTU5LjAzNiAzMjMuMzE4IEwgMTU1LjY3MCAzMjMuMzY4IDE1NS40NjEgMzE5LjMyMSBDIDE1NS4xMzEgMzEyLjkzMCwxNTkuMDAyIDMxMC44MzAsMTYzLjI2MSAzMTUuMDg5ICIKICAgICAgc3Ryb2tlPSJub25lIiBmaWxsPSIjMjk0NDY1IiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPjwvcGF0aD4KICA8L2c+Cjwvc3ZnPg==

ads/opctl/operator/lowcode/pii/README.md

@@ -0,0 +1,208 @@
+# PII Operator
+
+
+The PII Operator aims to detect and redact Personally Identifiable Information (PII) in datasets. PII data includes information such as names, addresses, and social security numbers, which can be used to identify individuals. This operator combine pattern matching and machine learning solution to identify PII, and then redacts or anonymizes it to protect the privacy of individuals.
+
+Below are the steps to configure and run the PII Operator on different resources.
+
+## 1. Prerequisites
+
+Follow the [CLI Configuration](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/opctl/configure.html) steps from the ADS documentation. This step is mandatory as it sets up default values for different options while running the PII Operator on OCI Data Science jobs.
+
+## 2. Generating configs
+
+To generate starter configs, run the command below. This will create a list of YAML configs and place them in the `output` folder.
+
+```bash
+ads operator init -t pii --overwrite --output ~/pii/
+```
+
+The most important files expected to be generated are:
+
+- `pii.yaml`: Contains pii-related configuration.
+- `pii_operator_local_python.yaml`: This includes a local backend configuration for running pii operator in a local environment. The environment should be set up manually before running the operator.
+- `pii_operator_local_container.yaml`: This includes a local backend configuration for running pii operator within a local container. The container should be built before running the operator. Please refer to the instructions below for details on how to accomplish this.
+- `pii_job_container.yaml`: Contains Data Science job-related config to run pii operator in a Data Science job within a container (BYOC) runtime. The container should be built and published before running the operator. Please refer to the instructions below for details on how to accomplish this.
+- `pii_job_python.yaml`: Contains Data Science job-related config to run pii operator in a Data Science job within a conda runtime. The conda should be built and published before running the operator.
+
+All generated configurations should be ready to use without the need for any additional adjustments. However, they are provided as starter kit configurations that can be customized as needed.
+
+## 3. Running Pii on the local conda environment
+
+To run pii operator locally, create and activate a new conda environment (`ads-pii`). Install all the required libraries listed in the `environment.yaml` file.
+
+```yaml
+- aiohttp
+- report-creator
+- gender_guesser
+- nameparser
+- oracle_ads[opctl]
+- plotly
+- scrubadub
+- scrubadub_spacy
+- spacy-transformers==1.2.5
+- spacy==3.6.1
+```
+
+Please review the previously generated `pii.yaml` file using the `init` command, and make any necessary adjustments to the input and output file locations. By default, it assumes that the files should be located in the same folder from which the `init` command was executed.
+
+Use the command below to verify the pii config.
+
+```bash
+ads operator verify -f ~/pii/pii.yaml
+```
+
+Use the following command to run the pii operator within the `ads-pii` conda environment.
+
+```bash
+ads operator run -f ~/pii/pii.yaml -b local
+```
+
+The operator will run in your local environment without requiring any additional modifications.
+
+## 4. Running pii on the local container
+
+To run the pii operator within a local container, follow these steps:
+
+Use the command below to build the pii container.
+
+```bash
+ads operator build-image -t pii
+```
+
+This will create a new `pii:v1` image, with `/etc/operator` as the designated working directory within the container.
+
+
+Check the `pii_operator_local_container.yaml` config file. By default, it should have a `volume` section with the `.oci` configs folder mounted.
+
+```yaml
+volume:
+  - "/Users/<user>/.oci:/root/.oci"
+```
+
+Mounting the OCI configs folder is only required if an OCI Object Storage bucket will be used to store the input data or output result. The input/output folders can also be mounted to the container.
+
+```yaml
+volume:
+  - /Users/<user>/.oci:/root/.oci
+  - /Users/<user>/pii/data:/etc/operator/data
+  - /Users/<user>/pii/result:/etc/operator/result
+```
+
+The full config can look like:
+```yaml
+kind: operator.local
+spec:
+  image: pii:v1
+  volume:
+  - /Users/<user>/.oci:/root/.oci
+  - /Users/<user>/pii/data:/etc/operator/data
+  - /Users/<user>/pii/result:/etc/operator/result
+type: container
+version: v1
+```
+
+Run the pii operator within a container using the command below:
+
+```bash
+ads operator run -f ~/pii/pii.yaml --backend-config ~/pii/pii_operator_local_container.yaml
+```
+
+## 5. Running pii in the Data Science job within container runtime
+
+To execute the pii operator within a Data Science job using container runtime, please follow the steps outlined below:
+
+You can use the following command to build the forecast container. This step can be skipped if you have already done this for running the operator within a local container.
+
+```bash
+ads operator build-image -t pii
+```
+
+This will create a new `pii:v1` image, with `/etc/operator` as the designated working directory within the container.
+
+Publish the `pii:v1` container to the [Oracle Container Registry](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/home.htm). To become familiar with OCI, read the documentation links posted below.
+
+- [Access Container Registry](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/Concepts/registryoverview.htm#access)
+- [Create repositories](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/Tasks/registrycreatingarepository.htm#top)
+- [Push images](https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/Content/Registry/Tasks/registrypushingimagesusingthedockercli.htm#Pushing_Images_Using_the_Docker_CLI)
+
+To publish `pii:v1` to OCR, use the command posted below:
+
+```bash
+ads operator publish-image pii:v1 --registry <iad.ocir.io/tenancy/>
+```
+
+After the container is published to OCR, it can be used within Data Science jobs service. Check the `backend_job_container_config.yaml` config file. It should contain pre-populated infrastructure and runtime sections. The runtime section should contain an image property, something like `image: iad.ocir.io/<tenancy>/pii:v1`. More details about supported options can be found in the ADS Jobs documentation - [Run a Container](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/jobs/run_container.html).
+
+Adjust the `pii.yaml` config with proper input/output folders. When the operator is run in the Data Science job, it will not have access to local folders. Therefore, input data and output folders should be placed in the Object Storage bucket. Open the `pii.yaml` and adjust the following fields:
+
+```yaml
+input_data:
+  url: oci://bucket@namespace/pii/input_data/data.csv
+output_directory:
+  url: oci://bucket@namespace/pii/result/
+```
+
+Run the pii operator on the Data Science jobs using the command posted below:
+
+```bash
+ads operator run -f ~/pii/pii.yaml --backend-config ~/pii/pii_job_container.yaml
+```
+
+The logs can be monitored using the `ads opctl watch` command.
+
+```bash
+ads opctl watch <OCID>
+```
+
+
+## 6. Running pii in the Data Science job within conda runtime
+
+To execute the pii operator within a Data Science job using conda runtime, please follow the steps outlined below:
+
+You can use the following command to build the pii conda environment.
+
+```bash
+ads operator build-conda -t pii
+```
+
+This will create a new `pii_v1` conda environment and place it in the folder specified within `ads opctl configure` command.
+
+Use the command below to Publish the `pii_v1` conda environment to the Object Storage bucket.
+
+```bash
+ads opctl conda publish pii_v1
+```
+More details about configuring CLI can be found here - [Configuring CLI](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/opctl/configure.html)
+
+
+After the conda environment is published to Object Storage, it can be used within Data Science jobs service. Check the `pii_job_python.yaml` config file. It should contain pre-populated infrastructure and runtime sections. The runtime section should contain a `conda` section.
+
+```yaml
+conda:
+  type: published
+  uri: oci://bucket@namespace/conda_environments/cpu/pii/1/pii_v1
+```
+
+More details about supported options can be found in the ADS Jobs documentation - [Run a Python Workload](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/jobs/run_python.html).
+
+Adjust the `pii.yaml` config with proper input/output folders. When the pii is run in the Data Science job, it will not have access to local folders. Therefore, input data and output folders should be placed in the Object Storage bucket. Open the `pii.yaml` and adjust the following fields:
+
+```yaml
+input_data:
+  url: oci://bucket@namespace/pii/input_data/data.csv
+output_directory:
+  url: oci://bucket@namespace/pii/result/
+```
+
+Run the pii on the Data Science jobs using the command posted below:
+
+```bash
+ads operator run -f ~/pii/pii.yaml --backend-config ~/pii/pii_job_python.yaml
+```
+
+The logs can be monitored using the `ads opctl watch` command.
+
+```bash
+ads opctl watch <OCID>
+```

ads/opctl/operator/lowcode/pii/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/operator/lowcode/pii/__main__.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+import os
+import sys
+from typing import Dict, List
+
+import yaml
+
+from ads.opctl import logger
+from ads.opctl.operator.common.const import ENV_OPERATOR_ARGS
+from ads.opctl.operator.common.utils import _parse_input_args
+
+from .model.guardrails import PIIGuardrail
+from .operator_config import PiiOperatorConfig
+
+
+def operate(operator_config: PiiOperatorConfig) -> None:
+    """Runs the PII operator."""
+    guard = PIIGuardrail(config=operator_config)
+    guard.process()
+
+
+def verify(spec: Dict, **kwargs: Dict) -> bool:
+    """Verifies the PII operator config."""
+    operator = PiiOperatorConfig.from_dict(spec)
+    msg_header = (
+        f"{'*' * 30} The operator config has been successfully verified {'*' * 30}"
+    )
+    print(msg_header)
+    print(operator.to_yaml())
+    print("*" * len(msg_header))
+
+
+def main(raw_args: List[str]):
+    """The entry point of the PII the operator."""
+    args, _ = _parse_input_args(raw_args)
+    if not args.file and not args.spec and not os.environ.get(ENV_OPERATOR_ARGS):
+        logger.info(
+            "Please specify -f[--file] or -s[--spec] or "
+            f"pass operator's arguments via {ENV_OPERATOR_ARGS} environment variable."
+        )
+        return
+
+    logger.info("-" * 100)
+    logger.info(f"{'Running' if not args.verify else 'Verifying'} the operator...")
+
+    # if spec provided as input string, then convert the string into YAML
+    yaml_string = ""
+    if args.spec or os.environ.get(ENV_OPERATOR_ARGS):
+        operator_spec_str = args.spec or os.environ.get(ENV_OPERATOR_ARGS)
+        try:
+            yaml_string = yaml.safe_dump(json.loads(operator_spec_str))
+        except json.JSONDecodeError:
+            yaml_string = yaml.safe_dump(yaml.safe_load(operator_spec_str))
+        except:
+            yaml_string = operator_spec_str
+
+    operator_config = PiiOperatorConfig.from_yaml(
+        uri=args.file,
+        yaml_string=yaml_string,
+    )
+
+    logger.info(operator_config.to_yaml())
+
+    # run operator
+    if args.verify:
+        verify(operator_config)
+    else:
+        operate(operator_config)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

ads/opctl/operator/lowcode/pii/cmd.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Dict
+
+from ads.opctl import logger
+from ads.opctl.operator.common.operator_yaml_generator import YamlGenerator
+from ads.opctl.operator.common.utils import _load_yaml_from_uri
+
+
+def init(**kwargs: Dict) -> str:
+    """
+    Generates operator config by the schema.
+
+    Properties
+    ----------
+    kwargs: (Dict, optional).
+        Additional key value arguments.
+
+        - type: str
+            The type of the operator.
+
+    Returns
+    -------
+    str
+        The YAML specification generated based on the schema.
+    """
+    logger.info("==== PII related options ====")
+
+    default_detector = [{"name": "<type>.<entity>", "action": "mask"}]
+
+    return YamlGenerator(
+        schema=_load_yaml_from_uri(__file__.replace("cmd.py", "schema.yaml"))
+    ).generate_example_dict(
+        values={"type": kwargs.get("type"), "detectors": default_detector}
+    )