oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/feature_store/dataset_job.py

@@ -0,0 +1,530 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+from copy import deepcopy
+from typing import Dict, List, Any, Union
+
+import pandas
+
+from ads.common import utils
+from ads.feature_store.common.enums import (
+    JobConfigurationType,
+    BatchIngestionMode,
+    StreamingIngestionMode,
+)
+from ads.feature_store.feature_option_details import FeatureOptionDetails
+from ads.feature_store.service.oci_dataset_job import OCIDatasetJob
+from ads.jobs.builders.base import Builder
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetJob(Builder):
+    """Represents an DatasetJob Resource.
+
+    Methods
+    --------
+
+    create(self, **kwargs) -> "DatasetJob"
+        Creates dataset_run resource.
+    from_id(cls, id: str) -> "DatasetJob"
+        Gets an existing dataset_run resource by id.
+    list(cls, compartment_id: str = None, **kwargs) -> List["DatasetJob"]
+        Lists dataset_run resources in a given compartment.
+    list_df(cls, compartment_id: str = None, **kwargs) -> "pandas.DataFrame"
+        Lists dataset_run resources as a pandas dataframe.
+    with_description(self, description: str) -> "DatasetJob"
+        Sets the description.
+    with_compartment_id(self, compartment_id: str) -> "DatasetJob"
+        Sets the compartment ID.
+    with_dataset_id(self, dataset_id: str) -> "DatasetJob"
+        Sets the dataset ID.
+    with_display_name(self, name: str) -> "DatasetJob"
+        Sets the name.
+    with_ingestion_mode(self, ingestion_mode: IngestionMode) -> "DatasetJob"
+        Sets the ingestion mode.
+    Examples
+    --------
+    >>> from ads.feature_store import dataset_job
+    >>> import oci
+    >>> import os
+    >>> dataset_run = dataset_run.DatasetJob()
+    >>>     .with_compartment_id(os.environ["PROJECT_COMPARTMENT_OCID"])
+    >>>     .with_dataset_id("dataset_id")
+    >>>     .with_ingestion_mode(BatchIngestionMode.SQL)
+    >>> dataset_run.create()
+    """
+
+    _PREFIX = "dataset_run_resource"
+
+    CONST_ID = "id"
+    CONST_COMPARTMENT_ID = "compartmentId"
+    CONST_DATASET_ID = "datasetId"
+    CONST_INGESTION_MODE = "ingestionMode"
+    CONST_JOB_CONFIGURATION_DETAILS = "jobConfigurationDetails"
+    CONST_JOB_CONFIGURATION_TYPE = "jobConfigurationType"
+    CONST_FREEFORM_TAG = "freeformTags"
+    CONST_DEFINED_TAG = "definedTags"
+    CONST_TIME_FROM = "timeFrom"
+    CONST_TIME_TO = "timeTo"
+    CONST_FEATURE_OPTION_DETAILS = "featureOptionsDetails"
+    CONST_ERROR_DETAILS = "errorDetails"
+    CONST_LIFECYCLE_STATE = "lifecycleState"
+    CONST_DATA_FLOW_EXECUTION_OUTPUT = "dataFlowBatchExecutionOutput"
+    CONST_VALIDATION_OUTPUT = "validation_output"
+    CONST_DATA_FLOW_READ_WRITE_DETAIL = "data_flow_read_write_detail"
+    CONST_DATA_READ_IN_BYTES = "data_read_in_bytes"
+    CONST_DATA_WRITTEN_BYTES = "data_written_in_bytes"
+    CONST_JOB_OUTPUT_DETAILS = "jobOutputDetails"
+
+    attribute_map = {
+        CONST_ID: "id",
+        CONST_COMPARTMENT_ID: "compartment_id",
+        CONST_DATASET_ID: "dataset_id",
+        CONST_INGESTION_MODE: "ingestion_mode",
+        CONST_JOB_CONFIGURATION_DETAILS: "job_configuration_details",
+        CONST_JOB_CONFIGURATION_TYPE: "job_configuration_type",
+        CONST_FEATURE_OPTION_DETAILS: "feature_option_details",
+        CONST_TIME_FROM: "time_from",
+        CONST_TIME_TO: "time_to",
+        CONST_FREEFORM_TAG: "freeform_tags",
+        CONST_DEFINED_TAG: "defined_tags",
+        CONST_JOB_OUTPUT_DETAILS: "job_output_details",
+    }
+
+    def __init__(self, spec: Dict = None, **kwargs) -> None:
+        """Initializes DatasetJob Resource.
+
+        Parameters
+        ----------
+        spec: (Dict, optional). Defaults to None.
+            Object specification.
+
+        kwargs: Dict
+            Specification as keyword arguments.
+            If 'spec' contains the same key as the one in kwargs,
+            the value from kwargs will be used.
+        """
+        super().__init__(spec=spec, **deepcopy(kwargs))
+        # Specify oci DatasetJob instance
+        self.oci_fs_dataset_run = self._to_oci_fs_dataset_run(**kwargs)
+
+    def _to_oci_fs_dataset_run(self, **kwargs):
+        """Creates an `OCIDatasetJob` instance from the  `DatasetJob`.
+
+        kwargs
+            Additional kwargs arguments.
+            Can be any attribute that `feature_store.models.DatasetJob` accepts.
+
+        Returns
+        -------
+        OCIDatasetJob
+            The instance of the OCIDatasetJob.
+        """
+        fs_spec = {}
+        for infra_attr, dsc_attr in self.attribute_map.items():
+            value = self.get_spec(infra_attr)
+            fs_spec[dsc_attr] = value
+        fs_spec.update(**kwargs)
+        return OCIDatasetJob(**fs_spec)
+
+    @property
+    def kind(self) -> str:
+        """The kind of the object as showing in a YAML."""
+        return "dataset_job"
+
+    @property
+    def compartment_id(self) -> str:
+        return self.get_spec(self.CONST_COMPARTMENT_ID)
+
+    @compartment_id.setter
+    def compartment_id(self, value: str):
+        self.with_compartment_id(value)
+
+    def with_compartment_id(self, compartment_id: str) -> "DatasetJob":
+        """Sets the compartment_id.
+
+        Parameters
+        ----------
+        compartment_id: str
+            The compartment_id.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self)
+        """
+        return self.set_spec(self.CONST_COMPARTMENT_ID, compartment_id)
+
+    @property
+    def dataset_id(self) -> str:
+        return self.get_spec(self.CONST_DATASET_ID)
+
+    @dataset_id.setter
+    def dataset_id(self, value: str):
+        self.with_dataset_id(value)
+
+    def with_dataset_id(self, dataset_id: str) -> "DatasetJob":
+        """Sets the dataset_id.
+
+        Parameters
+        ----------
+        dataset_id: str
+            The dataset id.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self)
+        """
+        return self.set_spec(self.CONST_DATASET_ID, dataset_id)
+
+    @property
+    def id(self) -> str:
+        return self.get_spec(self.CONST_ID)
+
+    def with_id(self, id: str) -> "DatasetJob":
+        return self.set_spec(self.CONST_ID, id)
+
+    @property
+    def job_configuration_details(self) -> str:
+        return self.get_spec(self.CONST_JOB_CONFIGURATION_DETAILS)
+
+    @job_configuration_details.setter
+    def job_configuration_details(
+        self, job_configuration_type: JobConfigurationType, **kwargs: Dict[str, Any]
+    ):
+        self.with_job_configuration_details(job_configuration_type, **kwargs)
+
+    def with_job_configuration_details(
+        self, job_configuration_type: JobConfigurationType, **kwargs: Dict[str, Any]
+    ) -> "DatasetJob":
+        """Sets the job configuration details.
+
+        Parameters
+        ----------
+        job_configuration_type: JobConfigurationType
+            The job_configuration_type of job
+        kwargs: Dict[str, Any]
+            Additional key value arguments
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self)
+        """
+        return self.set_spec(
+            self.CONST_JOB_CONFIGURATION_DETAILS,
+            {
+                self.CONST_JOB_CONFIGURATION_TYPE: job_configuration_type.value,
+                **kwargs,
+            },
+        )
+
+    @property
+    def ingestion_mode(self) -> str:
+        return self.get_spec(self.CONST_INGESTION_MODE)
+
+    @ingestion_mode.setter
+    def ingestion_mode(
+        self, ingestion_mode: Union[BatchIngestionMode, StreamingIngestionMode]
+    ) -> "DatasetJob":
+        return self.with_ingestion_mode(ingestion_mode)
+
+    def with_ingestion_mode(
+        self, ingestion_mode: Union[BatchIngestionMode, StreamingIngestionMode]
+    ) -> "DatasetJob":
+        """Sets the mode of the dataset ingestion mode.
+
+        Parameters
+        ----------
+        ingestion_mode: IngestionMode
+            The mode of the dataset ingestion mode.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self)
+        """
+        return self.set_spec(self.CONST_INGESTION_MODE, ingestion_mode.value)
+
+    def with_error_details(self, error_details: str) -> "DatasetJob":
+        """Sets the error details.
+
+        Parameters
+        ----------
+        error_details: str
+            The error_details.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self)
+        """
+        return self.set_spec(self.CONST_ERROR_DETAILS, error_details)
+
+    def with_lifecycle_state(self, lifecycle_state: str) -> "DatasetJob":
+        """Sets the lifecycle_state.
+
+        Parameters
+        ----------
+        lifecycle_state: str
+            The lifecycle_state.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self)
+        """
+        return self.set_spec(self.CONST_LIFECYCLE_STATE, lifecycle_state)
+
+    @property
+    def feature_option_details(self):
+        return self.get_spec(self.CONST_FEATURE_OPTION_DETAILS)
+
+    def with_feature_option_details(
+        self, feature_option_details: FeatureOptionDetails
+    ) -> "DatasetJob":
+        """Sets the feature_option_details.
+
+        Parameters
+        ----------
+        feature_option_details: FeatureOptionDetails
+
+        Returns
+        -------
+        DatasetJob
+            The FeatureGroupJob instance (self)
+        """
+        return self.set_spec(
+            self.CONST_FEATURE_OPTION_DETAILS, feature_option_details.to_dict()
+        )
+
+    @property
+    def job_output_details(self) -> Dict:
+        return self.get_spec(self.CONST_JOB_OUTPUT_DETAILS)
+
+    def with_job_output_details(self, job_output_details: Dict) -> "DatasetJob":
+        """Sets the job output details.
+
+        Parameters
+        ----------
+        job_output_details: Dict
+            The job output details which contains error_details, validation_output and commit id.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self)
+        """
+        return self.set_spec(self.CONST_JOB_OUTPUT_DETAILS, job_output_details)
+
+    @classmethod
+    def from_id(cls, id: str) -> "DatasetJob":
+        """Gets an existing dataset_run resource by Id.
+
+        Parameters
+        ----------
+        id: str
+            The dataset_run id.
+
+        Returns
+        -------
+        FeatureStore
+            An instance of DatasetJob resource.
+        """
+        return cls()._update_from_oci_fs_model(OCIDatasetJob.from_id(id))
+
+    def create(self, **kwargs) -> "DatasetJob":
+        """Creates dataset_run  resource.
+
+        Parameters
+        ----------
+        kwargs
+            Additional kwargs arguments.
+            Can be any attribute that `feature_store.models.DatasetJob` accepts.
+
+        Returns
+        -------
+        FeatureStore
+            The DatasetJob instance (self)
+
+        Raises
+        ------
+        ValueError
+            If compartment id not provided.
+        """
+
+        if not self.compartment_id:
+            raise ValueError("Compartment id must be provided.")
+
+        payload = deepcopy(self._spec)
+        payload.pop("id", None)
+        logger.debug(f"Creating a dataset_run resource with payload {payload}")
+
+        # Create entity
+        logger.info("Saving entity.")
+        self.oci_fs_dataset_run = self._to_oci_fs_dataset_run(**kwargs).create()
+        self.with_id(self.oci_fs_dataset_run.id)
+        return self
+
+    def update(self, **kwargs) -> "DatasetJob":
+        """Updates DatasetJob in the feature store.
+
+        Parameters
+        ----------
+        kwargs
+            Additional kwargs arguments.
+            Can be any attribute that `feature_store.models.DatasetJob` accepts.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self).
+        """
+
+        if not self.id:
+            raise ValueError(
+                "DatasetJob needs to be saved to the feature store before it can be updated."
+            )
+
+        self.oci_fs_dataset_run = self._to_oci_fs_dataset_run(**kwargs).update()
+        return self
+
+    def _mark_job_complete(self, job_output_details) -> "DatasetJob":
+        """
+        Completes the current dataset job and returns a new `DatasetJob` object that represents it.
+
+        :param job_output_details: The output details of the completed job.
+        :return: A new `FeatureGroupJob` object representing the completed job.
+        :raises ValueError: If the current `FeatureGroupJob` object has not been saved to the feature store yet.
+        """
+        # Check if the current FeatureGroupJob has been saved to the feature store
+        if not self.id:
+            raise ValueError(
+                "DatasetJob needs to be saved to the feature store before it can be marked as completed."
+            )
+
+        # Update the job's output details with the given ones
+        self.with_job_output_details(job_output_details)
+
+        # Complete the job in the OCI feature store and return a new FeatureGroupJob object
+        self.oci_fs_dataset_run = self._to_oci_fs_dataset_run().complete_dataset_job()
+        return self
+
+    def _update_from_oci_fs_model(
+        self, oci_fs_dataset_run: OCIDatasetJob
+    ) -> "DatasetJob":
+        """Update the properties from an OCIDatasetJob object.
+
+        Parameters
+        ----------
+        oci_fs_dataset_run: OCIDatasetJob
+            An instance of OCIDatasetJob.
+
+        Returns
+        -------
+        DatasetJob
+            The DatasetJob instance (self).
+        """
+
+        # Update the main properties
+        self.oci_fs_dataset_run = oci_fs_dataset_run
+        dataset_run_details = oci_fs_dataset_run.to_dict()
+
+        for infra_attr, dsc_attr in self.attribute_map.items():
+            if infra_attr in dataset_run_details:
+                self.set_spec(infra_attr, dataset_run_details[infra_attr])
+
+        return self
+
+    @classmethod
+    def list_df(cls, compartment_id: str = None, **kwargs) -> "pandas.DataFrame":
+        """Lists dataset_run resources in a given compartment.
+
+        Parameters
+        ----------
+        compartment_id: (str, optional). Defaults to `None`.
+            The compartment OCID.
+        kwargs
+            Additional keyword arguments for filtering models.
+
+        Returns
+        -------
+        pandas.DataFrame
+            The list of the dataset_run resources in a pandas dataframe format.
+        """
+        records = []
+        for oci_fs_dataset_run in OCIDatasetJob.list_resource(compartment_id, **kwargs):
+            records.append(
+                {
+                    "id": oci_fs_dataset_run.id,
+                    "display_name": oci_fs_dataset_run.display_name,
+                    "time_created": oci_fs_dataset_run.time_created.strftime(
+                        utils.date_format
+                    ),
+                    "time_updated": oci_fs_dataset_run.time_updated.strftime(
+                        utils.date_format
+                    ),
+                    "lifecycle_state": oci_fs_dataset_run.lifecycle_state,
+                    "created_by": f"...{oci_fs_dataset_run.created_by[-6:]}",
+                    "compartment_id": f"...{oci_fs_dataset_run.compartment_id[-6:]}",
+                    "dataset_id": oci_fs_dataset_run.dataset_id,
+                }
+            )
+        return pandas.DataFrame.from_records(records)
+
+    @classmethod
+    def list(cls, compartment_id: str = None, **kwargs) -> List["DatasetJob"]:
+        """Lists DatasetJob Resources in a given compartment.
+
+        Parameters
+        ----------
+        compartment_id: (str, optional). Defaults to `None`.
+            The compartment OCID.
+        kwargs
+            Additional keyword arguments for filtering DatasetJob.
+
+        Returns
+        -------
+        List[DatasetJob]
+            The list of the DatasetJob Resources.
+        """
+        return [
+            cls()._update_from_oci_fs_model(oci_fs_dataset_run)
+            for oci_fs_dataset_run in OCIDatasetJob.list_resource(
+                compartment_id, **kwargs
+            )
+        ]
+
+    def _random_display_name(self):
+        """Generates a random display name."""
+        return f"{self._PREFIX}-{utils.get_random_name_for_resource()}"
+
+    def to_dict(self) -> Dict:
+        """Serializes dataset_run  to a dictionary.
+
+        Returns
+        -------
+        dict
+            The dataset_run resource serialized as a dictionary.
+        """
+
+        spec = deepcopy(self._spec)
+        for key, value in spec.items():
+            if hasattr(value, "to_dict"):
+                value = value.to_dict()
+            spec[key] = value
+
+        return {
+            "kind": self.kind,
+            "type": self.type,
+            "spec": utils.batch_convert_case(spec, "camel"),
+        }
+
+    def __repr__(self) -> str:
+        """Displays the object as YAML."""
+        return self.to_yaml()

ads/feature_store/docs/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.8
+USER root
+COPY build/html /root/docs/.
+WORKDIR /root
+EXPOSE 8000
+ENTRYPOINT ["python3"]
+CMD ["-m", "http.server", "8000"]

ads/feature_store/docs/Makefile

@@ -0,0 +1,44 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+IMAGE_NAME:=oci-featurestore-docs
+TAG:=preview
+REGISTRY:=iad.ocir.io/ociodscdev
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is me`ant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+livehtml:
+	sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+clean:
+	rm -rf $(BUILDDIR)/*
+
+build-image: clean html
+	docker build --network host -t $(IMAGE_NAME):$(TAG) .
+
+push: build-image
+	docker tag $(IMAGE_NAME):$(TAG) $(REGISTRY)/$(IMAGE_NAME):$(TAG) && \
+	docker push $(REGISTRY)/$(IMAGE_NAME):$(TAG)
+
+# https://objectstorage.us-ashburn-1.oraclecloud.com/p/vv1bNIvcMnbvbM5TUW5NDVSoisL4BAS54l9bGFDmyiNkTPZ4q6NtAmOdemNr8lTv/n/ociodscdev/b/oci-feature-store/o/beta/index.html#document-quickstart
+publish: clean
+	sphinx-build "$(SOURCEDIR)" "$(BUILDDIR)" -b singlehtml && \
+			oci os object put --name beta/index.html --file build/index.html -bn oci-feature-store -ns ociodscdev --force --content-type "text/html"
+
+launch:
+	docker run -it --network host $(IMAGE_NAME):$(TAG)

ads/feature_store/docs/conf.py

@@ -0,0 +1,28 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "oci_feature_store"
+copyright = "2023, Oracle"
+author = "Oracle"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = []
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+root_doc = "index2"
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "alabaster"
+html_static_path = ["_static"]

ads/feature_store/docs/requirements.txt

@@ -0,0 +1,14 @@
+autodoc nbsphinx
+sphinx
+sphinxcontrib-napoleon
+sphinx_copybutton
+sphinx_code_tabs
+sphinx-autobuild
+sphinx-autorun
+oracle_ads
+furo
+IPython
+pandoc
+rstcheck
+restructuredtext_lint
+doc8

ads/feature_store/docs/source/ads.feature_store.query.rst

@@ -0,0 +1,20 @@
+ads.feature_store.query package
+===============================
+
+ads.feature_store.query package
+-------------------------------
+
+.. automodule:: ads.feature_store.query.filter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+.. automodule:: ads.feature_store.query.join
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+.. automodule:: ads.feature_store.query.query
+   :members:
+   :undoc-members:
+   :show-inheritance: