oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/feature_store/docs/source/transformation.rst

@@ -0,0 +1,199 @@
+Transformation
+**************
+
+Transformations in a Feature Store refer to the operations and processes applied to raw data to create, modify, or derive new features that can be used as inputs for ML Models. These transformations are crucial for improving the quality, relevance, and usefulness of features which in turn can enhance the performance of ML models. It is an object that represents a transformation applied on the feature group and can be a pandas transformation or Spark SQL transformation.
+
+.. image:: figures/transformation.png
+
+* ``TransformationMode.PANDAS``: Pandas Transformation lets users to do the transformation using native pandas functionality. The interface expects ``PANDAS`` dataframe as input and ``PANDAS`` dataframe as output.
+* ``TransformationMode.SQL``: Spark SQL brings native support for SQL to Spark. Users generally can give the spark transformation that they wish to do using spark SQL. The interface expects ``SPARK`` dataframe as input and ``SQL STRING`` as output.
+* ``TransformationMode.SPARK``: Spark Transformation lets users to do the transformation using native spark functionality. The interface expects ``SPARK`` dataframe as input and ``SPARK`` dataframe as output.
+
+.. tabs::
+
+  .. code-tab:: Python3
+    :caption: TransformationMode.SQL
+
+    from ads.feature_store.transformation import Transformation,TransformationMode
+
+    def transactions_df(transactions_batch):
+        sql_query = f"select id, cc_num, amount from {transactions_batch}"
+        return sql_query
+
+    transformation = (
+        Transformation()
+         .with_description("Feature store description")
+         .with_compartment_id(os.environ["PROJECT_COMPARTMENT_OCID"])
+         .with_name("transformation_name")
+         .with_feature_store_id(feature_store.id)
+         .with_transformation_mode(TransformationMode.SQL)
+         .with_source_code_function(transactions_df)
+    )
+    transformation.create()
+
+  .. code-tab:: Python3
+    :caption: TransformationMode.PANDAS
+
+    def chained_transformation(patient_result_df, **transformation_args):
+        def label_encoder_transformation(patient_result_df, **transformation_args):
+            from sklearn.preprocessing import LabelEncoder
+            # creating instance of labelencoder
+            labelencoder = LabelEncoder()
+            result_df = patient_result_df.copy()
+            column_labels= transformation_args.get("label_encode_column")
+            if isinstance(column_labels,list):
+                for col in column_labels:
+                    result_df[col] = labelencoder.fit_transform(result_df[col])
+            elif isinstance(column_labels, str):
+                 result_df[column_labels] = labelencoder.fit_transform(result_df[column_labels])
+            else:
+                return None
+            return result_df
+
+        def min_max_scaler(patient_result_df, **transformation_args):
+            from sklearn.preprocessing import MinMaxScaler
+            final_result_df = patient_result_df.copy()
+            scaler = MinMaxScaler(feature_range=(0, 1))
+            column_labels= transformation_args.get("scaling_column_labels")
+            final_result_df[column_labels] = scaler.fit_transform(final_result_df[column_labels])
+            return patient_result_df
+
+        def feature_removal(input_df, **transformation_args):
+            output_df = input_df.copy()
+            output_df.drop(transformation_args.get("redundant_feature_label"), axis=1, inplace=True)
+            return output_df
+
+        out1 = label_encoder_transformation(patient_result_df, **transformation_args)
+        out2 = min_max_scaler(out1, **transformation_args)
+        return feature_removal(out2, **transformation_args)
+
+    transformation_args = {
+        "label_encode_column": ["SOURCE"],
+        "scaling_column_labels": [],
+        "redundant_feature_label": ["MCH", "MCHC", "MCV"]
+    }
+
+    from ads.feature_store.transformation import Transformation,TransformationMode
+
+    transformation = (
+        Transformation()
+        .with_name("chained_transformation")
+        .with_feature_store_id(feature_store.id)
+        .with_source_code_function(chained_transformation)
+        .with_transformation_mode(TransformationMode.PANDAS)
+        .with_description("transformation to perform feature engineering")
+        .with_compartment_id(compartment_id)
+    )
+
+    transformation.create()
+
+
+  .. code-tab:: Python3
+    :caption: TransformationMode.SPARK
+
+    def credit_score_transformation(credit_score):
+        import pyspark.sql.functions as F
+
+        # Create a new Spark DataFrame that contains the transformed credit score.
+        transformed_credit_score = credit_score.select(
+            "user_id",
+            "date",
+            F.when(F.col("credit_score").cast("int") > 500, 1).otherwise(0).alias("credit_score")
+        )
+
+        # Return the new Spark DataFrame.
+        return transformed_credit_score
+
+    from ads.feature_store.transformation import Transformation,TransformationMode
+
+    transformation = (
+        Transformation()
+        .with_name("spark_transformation")
+        .with_feature_store_id(feature_store.id)
+        .with_source_code_function(credit_score_transformation)
+        .with_transformation_mode(TransformationMode.SPARK)
+        .with_description("transformation to perform feature engineering")
+        .with_compartment_id(compartment_id)
+    )
+
+    transformation.create()
+
+
+Define
+======
+
+In an ADS Feature Store module, you can use the Python API or YAML file to define a transformation.
+
+
+With the following way specified, you can define a transformation and give it a name.
+A ``Transformation`` instance is created.
+
+.. tabs::
+
+  .. code-tab:: Python3
+    :caption: Python
+
+    from ads.feature_store.transformation import Transformation
+
+    transformation = (
+        Transformation
+        .with_name("<transformation_name>")
+        .with_feature_store_id("<feature_store_id>")
+        .with_source_code("<source_code>")
+        .with_transformation_mode("<transformation_mode>")
+        .with_description("<transformation_description>")
+        .with_compartment_id("<compartment_id>")
+    )
+
+  .. code-tab:: Python3
+    :caption: YAML
+
+    from ads.feature_store.transformation import Transformation
+
+    yaml_string = """
+    kind: transformation
+    spec:
+      compartmentId: ocid1.compartment..<unique_id>
+      description: <transformation_description>
+      name: <transformation_name>
+      featureStoreId: <feature_store_id>
+      sourceCode: <source_code>
+      transformationMode: <transformation_mode>
+    type: transformation
+    """
+
+    transformation = Transformation.from_yaml(yaml_string)
+
+
+Create
+======
+
+You can call the ``create()`` method of the ``Transformation`` instance to create a transformation.
+
+.. code-block:: python3
+
+  # Create an transformation
+  transformation.create()
+
+
+Load
+====
+
+Use the ``from_id()`` method from the ``Transformation`` class to load an existing transformation with its OCID provided. It returns a ``Transformation`` instance.
+
+.. code-block:: python3
+
+  from ads.feature_store.transformation import Transformation
+
+  transformation = Transformation.from_id("ocid1.transformation..<unique_id>")
+
+Delete
+======
+
+Use the ``.delete()`` method on the ``Transformation`` instance to delete a transformation.
+
+A transformation can only be deleted when its associated entities are all deleted.
+
+.. code-block:: python3
+
+  transformation.delete()

ads/feature_store/docs/source/ui.rst

@@ -0,0 +1,65 @@
+User Interface
+****************
+You can use the user-friendly UI for feature discovery by using notebook extensions. The UI helps you to identify which features belong to specific datasets and feature groups, and establish the process of exploring available features in a particular dataset or feature group.
+
+Groups in the organization must share the pipelines existing in the ecosystem to promote reuse of features if the pipelines already exists. The work happens in silos that make discovery impossible, which results in teams being unable to use each others work. The following are a few examples of the interfaces that could help you discover features as part of a particular feature group, discover features part of a particular dataset, and how transformations are applied to see the source and destination of the data.
+
+Feature Store
+=============
+Feature Store is the top-level entity for a Feature Store.
+
+
+Feature Group
+=============
+Feature groups are instrumental in feature discovery as they offer a structured way to organize and manage features based on their semantic meaning or context. By grouping related features together, data scientists can efficiently locate and use the relevant attributes for model development. The versioning capability of feature groups ensures that changes in feature definitions are tracked. Tracking enables reproducibility and aids in evaluating the impact of feature modifications on model performance. The collaborative aspect of feature groups fosters knowledge sharing and reuse across data driven projects, which promots efficiency and consistency in the feature discovery processes.
+
+.. image:: figures/featuregroup.gif
+
+Validation
+###########
+Feature validation is assessing the quality, relevance, and effectiveness of features through techniques like cross-validation. Validaion helps to prevent overfitting, and improves model generalization.
+
+.. image:: figures/validation_fg.png
+
+Lineage
+###########
+Lineage tracking provides a historical record of data transformations and processing, which ensures transparency and reproducibility in feature engineering and model development.
+
+.. image:: figures/lineage_fg.png
+
+Statistics
+###########
+Statistical analysis of features helps uncover insights about their distributions, central tendencies, and variations, which aids in feature selection and understanding data characteristics.
+
+.. image:: figures/stats_fg.png
+
+Dataset
+=======
+Datasets also support iterative experimentation, letting data scientists create various dataset configurations with different feature combinations and transformations, which facilites the discovery of the most valuable features for model training.
+
+.. image:: figures/dataset.gif
+
+
+Lineage
+###########
+Lineage tracking provides a historical record of data transformations and processing, which ensures transparency and reproducibility in feature engineering and model development.
+
+.. image:: figures/lineage_d1.png
+
+.. image:: figures/lineage_d2.png
+
+Statistics
+###########
+Statistical analysis of features helps uncover insights about their distributions, central tendencies, and variations, which aids in feature selection and understanding data characteristics.
+
+.. image:: figures/stats_d.png
+
+Entity
+======
+An entity is a group of semantically related features. The first step a consumer of features would typically do when accessing the feature store service, is to list the entities and the entities associated with features. Another way to look at it is that an entity is an object or concept that's described by its features. Examples of entities are customer, product, transaction, review, image, and document.
+
+Transformation
+==============
+Transformation constructs are a pivotal component of feature engineering, and enable data scientists to adapt and enhance features to improve model performance. The flexibility to perform mathematical operations, scaling, normalization, handling missing data, and encoding categorical variables, transformation constructs empowers data professionals to craft features that align with specific modeling requirements.
+
+.. image:: figures/transformations.gif

ads/feature_store/docs/source/user_guides.setup.feature_store_operator.rst

@@ -0,0 +1,66 @@
+=================================
+Feature Store Deployment Operator (Recommended)
+=================================
+
+The Feature Store marketplace operator uses your current infrastructure to set up a Feature Store. It helps in setting up the Feature Store API server in your exisiting OKE cluster and MySQL database.
+
+
+Installing the Feature Store Marketplace Operator
+-------------------------------------------------
+
+The Feature Store Marketplace Operator can be installed from PyPi using the following command.
+
+.. code-block:: bash
+
+    python3 -m pip install oracle-ads[feature-store-marketplace]
+
+
+After that, the Operator is ready to go!
+
+Configuration
+-------------
+
+After having set up `ads opctl` on your desired machine using ``ads opctl configure``, you are ready to begin setting up Feature Store. At a minimum, you need to provide the following details about your infrastructure:
+
+- The path to the OCIR repository where Feature Store container images are cloned.
+- The compartment ID where Feature Store is set up.
+- :ref:`Database configuration details <Database configuration>`
+- The app name to use for Helm.
+- The namespace to use in the Kubernetes cluster.
+- The version of the Feature Store stack to install.
+
+Optionally you can specify details for the  API Gateway setup for Feature Store to enable authentication and authorization via OCI IAM:
+ - Tenancy OCID
+ - Region of deployment
+ - User group OCIDs authorized to use Feature Store
+ - OCID of resource manager stack to use for API Gateway deployment
+
+These details can be easily configured in an interactive manner by running the command
+
+.. code-block:: bash
+
+    ads operator init --type feature_store_marketplace
+
+Prerequisites for running the operator
+----------------------------------------
+
+Before running the operator you need to configure the following requirements:
+
+1. Helm: Helm is required to be installed on the machine for deploying Feature Store helm chart to the Kubernetes cluster. Ref: `Installing Helm   <https://helm.sh/docs/intro/install/>`_
+2. Kubectl: Kubectl is required to be installed to deploy the helm chart to the cluster. Ref: `Installing Kubectl <https://kubernetes.io/docs/tasks/tools/>`_
+3. :ref:`Policies`: Required policies for API server and user running the operator.
+4. `Setup cluster access locally <https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengdownloadkubeconfigfile.htm#:~:text=Under%20Containers%20%26%20Artifacts%2C%20click%20Kubernetes,shows%20details%20of%20the%20cluster>`_
+
+
+Run
+----
+
+After the ``feature_store_marketplace.yaml`` is written using the init step above, you can start the deployment using:
+
+.. important::
+
+ The installation will happen by default in ``default`` namespace. Ensure that NLB (Network Load Balancer) limit does not exceed the maximum limit.
+
+.. code-block:: bash
+
+    ads operator run -f feature_store_marketplace.yaml -b marketplace

ads/feature_store/docs/source/user_guides.setup.helm_chart.rst

@@ -0,0 +1,192 @@
+=================================
+Setup using Helm Charts
+=================================
+
+We always suggest to deploy feature store via the :doc:`Feature Store Operator <./user_guides.setup.feature_store_operator>` to setup Feature Store API server in OKE Cluster. This method should preferably be used only when the operator can not satisfy your requirements
+as it is much simpler to do the setup via the operator.
+
+
+Prerequisites
+_____________
+
+- Helm: Helm is required to be installed on the machine for deploying Feature Store helm chart to the Kubernetes cluster. Ref: `Installing Helm   <https://helm.sh/docs/intro/install/>`_
+- Kubectl: Kubectl is required to be installed to deploy the helm chart to the cluster. Ref: `Installing Kubectl <https://kubernetes.io/docs/tasks/tools/>`_
+- Setup `MySQL Database <https://docs.public.oneportal.content.oci.oraclecloud.com/en-us/iaas/mysql-database/doc/overview-mysql-database-service.html>`_  so that it is reachable from the API server
+.. seealso::
+   :ref:`Database configuration`
+- :ref:`Helm Setup`
+- :ref:`Policies`
+- `Setup cluster access locally <https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengdownloadkubeconfigfile.htm#:~:text=Under%20Containers%20%26%20Artifacts%2C%20click%20Kubernetes,shows%20details%20of%20the%20cluster>`_
+
+Steps to deploy API server image
+________________________________
+
+- Export the package to OCIR using the `Feature store marketplace listing <https://cloud.oracle.com/marketplace/application/ocid1.mktpublisting.oc1.iad.amaaaaaabiudgxya26lzh2dsyvg7cfzgllvdl6xo5phz4mnsoktxeutecrvq>`_
+- Wait for export work request to complete
+- Identify the Helm chart and API images exported to OCIR:
+   - Helm chart image would be of format: ``<ocir-image>:<version>``
+   - API image would be of format: ``<ocir-image>:<export-number>-<image-id>-<tenancy-namespace>-feature-store-api-<version>``
+
+- :ref:`Create kubernetes docker secret <Kubernetes secret>`
+
+- :ref:`Generate custom values.yaml for deployment <Helm values>`
+
+- Install the helm chart
+
+.. code-block:: bash
+
+   helm upgrade <app-name> oci://<helm-chart-image-path> --namespace <kubernetes-namespace> --values <path-to-values-yaml> --timeout 300s --wait -i --version <marketplace-version>
+- (Optional) `Setup Feature Store API Gateway <https://github.com/oracle-samples/oci-data-science-ai-samples/tree/main/feature_store/apigw_terraform>`_
+
+
+Appendix
+________
+
+.. _Helm Setup:
+
+Setup Helm to use OCIR
+______________________
+
+To login to Container Registry using the Helm CLI:
+
+- If you already have an auth token, go to the next step. Otherwise:
+   - In the top-right corner of the Console, open the Profile menu and then click User settings to view the details.
+   - On the Auth Tokens page, click Generate Token.
+   - Enter a friendly description for the auth token. Avoid entering confidential information.
+   - Click Generate Token. The new auth token is displayed.
+   - Copy the auth token immediately to a secure location from where you can retrieve it later, because you won't see the auth token again in the Console.
+   - Close the Generate Token dialog.
+
+- In a terminal window on the client machine running Docker, log in to Container Registry by entering  ``helm registry login <region-key>.ocir.io``, where <region-key> corresponds to the key for the Container Registry region you're using. For example, ``helm registry login iad.ocir.io``. See `Availability by Region <https://docs.oracle.com/en-us/iaas/Content/Registry/Concepts/registryprerequisites.htm#regional-availability>`_.
+- When prompted for a username, enter your username in the format <tenancy-namespace>/<username>, where <tenancy-namespace> is the auto-generated Object Storage namespace string of your tenancy (as shown on the Tenancy Information page). For example, ansh81vru1zp/jdoe@acme.com. If your tenancy is federated with Oracle Identity Cloud Service, use the format <tenancy-namespace>/oracleidentitycloudservice/<username>.
+- When prompted for a password, enter the auth token you copied earlier.
+
+.. _Kubernetes secret:
+
+Kubernetes Docker Secret Configuration
+__________________________________________________________
+- If you don't already have an auth token refer :ref:`Helm configuration  <Helm Setup>`
+- `Login to Kubernetes cluster <https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengdownloadkubeconfigfile.htm#:~:text=Under%20Containers%20%26%20Artifacts%2C%20click%20Kubernetes,shows%20details%20of%20the%20cluster>`_
+- Run command
+
+.. code-block:: bash
+
+   kubectl create secret docker-registry <secret-name> --docker-server=<region-key>.ocir.io --docker-username=<tenancy-namespace>/<username> --docker-password=<auth token>
+
+.. _Helm values:
+
+Helm values configuration:
+__________________________________________________________
+
+- Minimal Helm values example for getting started:
+
+.. code-block:: yaml
+
+    db:
+      configuredDB: MYSQL
+      mysql:
+         authType: BASIC
+         basic:
+            password: #enter-db-password-here
+         jdbcURL: jdbc:mysql://<db-ip>:3306/FeatureStore?createDatabaseIfNotExist=true
+         username: #enter-db-username-here
+    imagePullSecrets:
+    - name: #enter secret name containing docker secret here
+    oci_meta:
+      images:
+         api:
+            image: #ocir image: The name of image
+            tag: #API Image tag
+         authoriser:
+            image: na
+            tag: na
+      repo: #ocir repo: <region-key>.ocir.io/<tenancy-namespace>/repository
+
+
+
+- All available Helm values
+
+.. code-block:: yaml
+
+    oci_meta:
+        repo: #ocir repo: <region-key>.ocir.io/<tenancy-namespace>/repository
+        images:
+          api:
+             image: #ocir image: The name of image
+             tag: #API Image tag
+          authoriser: # We don't want to deploy this image. This image will be deployed with OCI functions
+                image: na
+                tag: na
+
+    imagePullSecrets:
+    - name:  #name-of-docker-secret-with-credentials
+
+    db:
+        configuredDB: #Type of DB configured. Possible values: "MYSQL"
+        mysql:
+          authType: #Type of authentication to use for connecting to database.
+                    # Possible values: 'BASIC', 'VAULT'
+          jdbcURL: #JDBC URL of the MySQL server
+          username: #Name of the user on MySQL server
+          basic:
+             password: #Password to mysql server in plain-text format
+          vault:
+             vaultOcid: #OCID of the vault where the secret is kept
+             secretName: #Name of the secret used for connecting to vault
+
+    resources: #https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+
+    nameOverride: #Value for label app.kubernetes.io/name
+
+    podSecurityContext: #Pod security #https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+
+    securityContext: #Container Security context #https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+
+    deploymentStrategy: #This block is directly inserted into pod spec
+                      #https://kubernetes.io/docs/concepts/workloads/controllers/deployment/
+
+
+    nodeSelector: {} #Pod node selector
+                   #https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
+
+    tolerations: []  #Pod tolerations
+                    #https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
+
+    affinity: {}  #Pod affinity
+                 #https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
+
+    replicaCount: #Pod replicas
+                 #https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
+
+    autoscaling: #Horizontal pod scaling details
+                #https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/
+      enabled:
+      minReplicas:
+      maxReplicas:
+      targetCPUUtilizationPercentage:
+      targetMemoryUtilizationPercentage:
+    scaleUp:
+      stabilizationWindowSeconds:
+      periodSeconds:
+      podCount:
+      percentage:
+    scaleDown:
+      stabilizationWindowSeconds:
+      periodSeconds:
+      podCount:
+      percentage:
+
+    applicationEnv:
+    containerName: #Container name
+
+    livenessProbe: # Liveness probe details
+      initialDelaySeconds:
+      periodSeconds:
+      timeoutSeconds:
+      failureThreshold:
+
+    readinessProbe: # Readiness probe details
+      initialDelaySeconds:
+      periodSeconds:
+      timeoutSeconds:
+      failureThreshold: