oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/feature_store/statistics/charts/probability_distribution.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from typing import List
+
+from ads.common.decorator.runtime_dependency import OptionalDependency
+from ads.feature_store.statistics.abs_feature_value import AbsFeatureValue
+from ads.feature_store.statistics.charts.abstract_feature_plot import AbsFeaturePlot
+
+try:
+    from plotly.graph_objs import Figure
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        f"The `plotly` module was not found. Please run `pip install "
+        f"{OptionalDependency.FEATURE_STORE}`."
+    )
+
+
+class ProbabilityDistribution(AbsFeaturePlot):
+    CONST_DENSITY = "density"
+    CONST_BINS = "bins"
+    CONST_PROBABILITY_DISTRIBUTION_TITLE = "Probability Distribution"
+
+    def __init__(self, density: List, bins: List):
+        self.density = density
+        self.bins = bins
+        super().__init__()
+
+    def __validate__(self):
+        assert type(self.density) == list
+        assert type(self.bins) == list
+        # assert 0 < len(self.density) == len(self.bins) > 0
+
+    @classmethod
+    def __from_json__(
+        cls, json_dict: dict, version: int = 1
+    ) -> "ProbabilityDistribution":
+        if version == 2:
+            return cls.__from_json_v2__(json_dict)
+        return cls(
+            density=json_dict.get(ProbabilityDistribution.CONST_DENSITY),
+            bins=json_dict.get(ProbabilityDistribution.CONST_BINS),
+        )
+
+    @classmethod
+    def __from_json_v2__(cls, json_dict: dict) -> "ProbabilityDistribution":
+        metric_data = json_dict.get(AbsFeatureValue.CONST_METRIC_DATA)
+        return cls(bins=metric_data[0], density=metric_data[1])
+
+    def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
+        xaxis_str, yaxis_str, x_str, y_str = self.get_x_y_str_axes(xaxis, yaxis)
+        if (
+            type(self.density) == list
+            and type(self.bins) == list
+            and 0 < len(self.density)
+            and 0 < len(self.bins)
+        ):
+            fig.add_bar(
+                x=self.bins,
+                y=self.density,
+                xaxis=x_str,
+                yaxis=y_str,
+                name="",
+            )
+        fig.layout.annotations[xaxis].text = self.CONST_PROBABILITY_DISTRIBUTION_TITLE
+        fig.layout[xaxis_str]["title"] = "Bins"
+        fig.layout[yaxis_str]["title"] = "Density"

ads/feature_store/statistics/charts/top_k_frequent_elements.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from typing import List
+
+from ads.feature_store.statistics.abs_feature_value import AbsFeatureValue
+
+from ads.common.decorator.runtime_dependency import OptionalDependency
+
+from ads.feature_store.statistics.charts.abstract_feature_plot import AbsFeaturePlot
+
+try:
+    from plotly.graph_objs import Figure
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        f"The `plotly` module was not found. Please run `pip install "
+        f"{OptionalDependency.FEATURE_STORE}`."
+    )
+
+
+class TopKFrequentElements(AbsFeaturePlot):
+    CONST_VALUE = "value"
+    CONST_TOP_K_FREQUENT_TITLE = "Top K Frequent Elements"
+
+    class TopKFrequentElement(AbsFeatureValue):
+        CONST_VALUE = "value"
+        CONST_ESTIMATE = "estimate"
+        CONST_LOWER_BOUND = "lower_bound"
+        CONST_UPPER_BOUND = "upper_bound"
+
+        def __init__(
+            self, value: str, estimate: int, lower_bound: int, upper_bound: int
+        ):
+            self.value = value
+            self.estimate = estimate
+            self.lower_bound = lower_bound
+            self.upper_bound = upper_bound
+            super().__init__()
+
+        def __validate__(self):
+            assert type(self.value) == str and len(self.value) > 0
+            assert type(self.estimate) == int and self.estimate >= 0
+
+        @classmethod
+        def __from_json__(
+            cls, json_dict: dict, version: int = 1
+        ) -> "TopKFrequentElements.TopKFrequentElement":
+            return cls(
+                value=json_dict.get(cls.CONST_VALUE),
+                estimate=json_dict.get(cls.CONST_ESTIMATE),
+                lower_bound=json_dict.get(cls.CONST_LOWER_BOUND),
+                upper_bound=json_dict.get(cls.CONST_UPPER_BOUND),
+            )
+
+    def __init__(self, values: List, estimates: List):
+        self.values = values
+        self.estimates = estimates
+        super().__init__()
+
+    def __validate__(self):
+        assert type(self.values) == list
+        assert type(self.estimates) == list
+        assert 0 < len(self.values) == len(self.estimates) > 0
+
+    @classmethod
+    def __from_json__(cls, json_dict: dict, version: int = 1) -> "TopKFrequentElements":
+        if version == 2:
+            return cls.__from_json_v2__(json_dict)
+        elements = json_dict.get(cls.CONST_VALUE)
+        top_k_frequent_elements = [
+            cls.TopKFrequentElement.__from_json__(element) for element in elements
+        ]
+        values = [element.value for element in top_k_frequent_elements]
+        estimates = [element.estimate for element in top_k_frequent_elements]
+        return cls(values, estimates)
+
+    @classmethod
+    def __from_json_v2__(cls, json_dict: dict) -> "TopKFrequentElements":
+        metric_data = json_dict.get(AbsFeatureValue.CONST_METRIC_DATA)
+        return cls(values=metric_data[0], estimates=metric_data[1])
+
+    def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
+        xaxis_str, yaxis_str, x_str, y_str = self.get_x_y_str_axes(xaxis, yaxis)
+        if (
+            type(self.values) == list
+            and len(self.values) > 0
+            and type(self.estimates) == list
+            and len(self.estimates) > 0
+        ):
+            y_axis = [value for value in self.values]
+            x_axis = [estimate for estimate in self.estimates]
+            fig.add_bar(
+                x=x_axis, y=y_axis, xaxis=x_str, yaxis=y_str, name="", orientation="h"
+            )
+        fig.layout.annotations[xaxis].text = self.CONST_TOP_K_FREQUENT_TITLE
+        fig.layout[yaxis_str]["title"] = "Element"
+        fig.layout[xaxis_str]["title"] = "Count"

ads/feature_store/statistics/feature_stat.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.decorator.runtime_dependency import OptionalDependency
+from typing import List
+from ads.feature_store.statistics.charts.abstract_feature_plot import AbsFeaturePlot
+from ads.feature_store.statistics.charts.box_plot import BoxPlot
+from ads.feature_store.statistics.charts.frequency_distribution import (
+    FrequencyDistribution,
+)
+from ads.feature_store.statistics.charts.probability_distribution import (
+    ProbabilityDistribution,
+)
+from ads.feature_store.statistics.charts.top_k_frequent_elements import (
+    TopKFrequentElements,
+)
+
+try:
+    import plotly
+    from plotly.graph_objs import Figure
+    import plotly.graph_objects as go
+    from plotly.subplots import make_subplots
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        f"The `plotly` module was not found. Please run `pip install "
+        f"{OptionalDependency.FEATURE_STORE}`."
+    )
+
+
+class FeatureStatistics:
+    CONST_FREQUENCY_DISTRIBUTION = "FrequencyDistribution"
+    CONST_TITLE_FORMAT = "<b>{}</b>"
+    CONST_PLOT_FORMAT = "{}_plot"
+    CONST_PROBABILITY_DISTRIBUTION = "ProbabilityDistribution"
+    CONST_TOP_K_FREQUENT = "TopKFrequentElements"
+
+    def __init__(
+        self,
+        feature_name: str,
+        top_k_frequent_elements: TopKFrequentElements = None,
+        frequency_distribution: FrequencyDistribution = None,
+        probability_distribution: ProbabilityDistribution = None,
+        box_plot: BoxPlot = None,
+    ):
+        self.feature_name: str = feature_name
+        self.top_k_frequent_elements = top_k_frequent_elements
+        self.frequency_distribution = frequency_distribution
+        self.probability_distribution = probability_distribution
+        self.box_plot = box_plot
+
+    @classmethod
+    def from_json(
+        cls, feature_name: str, json_dict: dict, version: int
+    ) -> "FeatureStatistics":
+        if json_dict is not None:
+            frequency_distribution = FrequencyDistribution.from_json(
+                json_dict.get(cls.CONST_FREQUENCY_DISTRIBUTION),
+                version,
+                ignore_errors=True,
+            )
+
+            # inject box points for boxplot creation
+            json_dict[
+                BoxPlot.CONST_BOX_POINTS
+            ] = BoxPlot.get_box_points_from_frequency_distribution(
+                frequency_distribution
+            )
+            return cls(
+                feature_name,
+                TopKFrequentElements.from_json(
+                    json_dict.get(cls.CONST_TOP_K_FREQUENT), version, ignore_errors=True
+                ),
+                frequency_distribution,
+                ProbabilityDistribution.from_json(
+                    json_dict.get(cls.CONST_PROBABILITY_DISTRIBUTION),
+                    version,
+                    ignore_errors=True,
+                ),
+                BoxPlot.from_json(json_dict, version, ignore_errors=True),
+            )
+        else:
+            return cls(feature_name)
+
+    @property
+    def __feature_stat_objects__(self) -> List[AbsFeaturePlot]:
+        return [
+            stat
+            for stat in [
+                self.box_plot,
+                self.top_k_frequent_elements,
+                self.frequency_distribution,
+                self.probability_distribution,
+            ]
+            if stat is not None
+        ]
+
+    def to_viz(self):
+        # TODO: make it generic
+        def next_graph_position_generator():
+            yield 1
+            yield 0
+            yield 2
+
+        if len(self.__feature_stat_objects__) > 0:
+            fig = make_subplots(cols=3, column_titles=[" "] * 3)
+            for idx, stat in zip(
+                next_graph_position_generator(), self.__feature_stat_objects__
+            ):
+                stat.add_to_figure(fig, idx, idx)
+
+            fig.layout.title = self.CONST_TITLE_FORMAT.format(self.feature_name)
+            fig.update_layout(title_font_size=20)
+            # Center align the title
+            fig.update_layout(title_x=0.5)
+            # Disable legend for unrelated plots
+            fig.update_layout(showlegend=False)
+            plotly.offline.iplot(
+                fig,
+                filename=self.CONST_PLOT_FORMAT.format(self.feature_name),
+            )
+        else:
+            print(
+                f"No statistical information for feature {self.feature_name} can be visualised"
+            )

ads/feature_store/statistics/generic_feature_value.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+from ads.feature_store.statistics.abs_feature_value import AbsFeatureValue
+
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+class GenericFeatureValue(AbsFeatureValue):
+    CONST_VALUE = "value"
+
+    def __init__(self, val: any):
+        self.val = val
+        super().__init__()
+
+    def __validate__(self):
+        pass
+
+    @classmethod
+    def __from_json_v2__(cls, json_dict: dict) -> "GenericFeatureValue":
+        val = None
+        if type(json_dict) == dict:
+            metric_data = json_dict.get(AbsFeatureValue.CONST_METRIC_DATA)
+            val = metric_data[0]
+        return GenericFeatureValue(val=val)
+
+    @classmethod
+    def __from_json__(cls, json_dict: dict, version: int = 1) -> "GenericFeatureValue":
+        if version == 2:
+            return cls.__from_json_v2__(json_dict)
+        val = None
+        if type(json_dict) == dict:
+            val = json_dict.get(cls.CONST_VALUE)
+        return GenericFeatureValue(val=val)

ads/feature_store/statistics/statistics.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import List
+from ads.feature_store.statistics.feature_stat import FeatureStatistics
+from ads.feature_store.response.response_builder import ResponseBuilder
+import json
+
+
+class Statistics(ResponseBuilder):
+    """
+    Represents statistical information.
+    """
+
+    @property
+    def kind(self) -> str:
+        """
+        Gets the kind of the statistics object.
+
+        Returns
+        -------
+        str
+            The kind of the statistics object, which is always "statistics".
+        """
+        return "statistics"
+
+    def to_viz(self, feature_list: List[str] = None):
+        """Visualises statistics inside notebook
+        Parameters
+        ----------
+        feature_list: (str, optional). Defaults to `None`.
+            The specific features of the FeatureGroup or Dataset we want to visualise
+        """
+        if self.content is not None:
+            [
+                FeatureStatistics.from_json(feature, stat, self.version).to_viz()
+                for feature, stat in json.loads(self.content).items()
+                if (feature_list is None or feature in feature_list)
+            ]

ads/feature_store/statistics_config.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from copy import deepcopy
+from typing import Dict, List
+
+from ads.jobs.builders.base import Builder
+
+
+class StatisticsConfig(Builder):
+    """Sets the Statistics Config Details.
+    Methods
+    -------
+        with_enabled(self, enabled: bool) -> "StatisticsConfig"
+        Sets True/False for enabled
+        with_columns(self, columns: List[str]) -> "StatisticsConfig"
+        Sets the column names for the statistics config
+    """
+
+    CONST_ENABLED = "isEnabled"
+    CONST_COLUMNS = "columns"
+
+    attribute_map = {
+        CONST_ENABLED: "is_enabled",
+        CONST_COLUMNS: "columns",
+    }
+
+    def __init__(self, is_enabled: bool = True, columns: List[str] = None) -> None:
+        super().__init__()
+
+        if columns is None:
+            columns = []
+        self.with_is_enabled(is_enabled)
+        if columns:
+            self.with_columns(columns)
+
+    @property
+    def is_enabled(self) -> bool:
+        return self.get_spec(self.CONST_ENABLED)
+
+    @is_enabled.setter
+    def is_enabled(self, is_enabled: bool):
+        self.with_is_enabled(is_enabled)
+
+    def with_is_enabled(self, is_enabled: bool) -> "StatisticsConfig":
+        """Sets True/False for enabled
+
+        Parameters
+        ----------
+        is_enabled: bool
+            enable or disable the statistics computation
+
+        Returns
+        -------
+        StatisticsConfig
+            The StatisticsConfig instance (self)
+        """
+        return self.set_spec(self.CONST_ENABLED, is_enabled)
+
+    @property
+    def columns(self) -> List[str]:
+        return self.get_spec(self.CONST_COLUMNS)
+
+    @columns.setter
+    def columns(self, columns: List[str]):
+        self.with_columns(columns)
+
+    def with_columns(self, columns: List[str]) -> "StatisticsConfig":
+        """Sets the columns for which the stats to be calculated .
+
+        Parameters
+        ----------
+        columns: List[str]
+            columns for which the stats to be calculated.
+
+        Returns
+        -------
+        StatisticsConfig
+            The StatisticsConfig instance (self)
+        """
+        return self.set_spec(self.CONST_COLUMNS, columns)
+
+    @property
+    def kind(self) -> str:
+        """The kind of the object as showing in a YAML."""
+        return "statistics_config"
+
+    def to_dict(self) -> Dict:
+        """Serializes rule to a dictionary.
+
+        Returns
+        -------
+        dict
+            The rule resource serialized as a dictionary.
+        """
+
+        spec = deepcopy(self._spec)
+        return spec

ads/feature_store/templates/feature_store_template.yaml

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+apiVersion: 20230101
+kind: featureStore
+spec:
+  displayName: '{feature_store_name}'
+  offlineConfig:
+    metastoreId: '{metastore_id}'
+
+  entity: &entity
+    - kind: entity
+      spec:
+        displayName: '{entity_name}'
+
+  # we can pass sql query in case of spark based sql transformations by passing query inline in transformationQueryInput
+  transformation: &transformation
+    - kind: transformation
+      spec:
+        displayName: '{transformation_name}'
+        transformationMode: '{transformation_mode}'
+        sourceCode: '{source_code}'
+
+  featureGroup:
+    - kind: featureGroup
+      spec:
+        displayName: '{feature_group_name}'
+        transformation: *transformation
+        entity: *entity
+        primaryKeys: '{fg_primary_key}'
+        inputFeatureDetails:
+          - name: '{feature_name}'
+            featureType: '{feature_type}'
+            orderNumber: 1
+
+  dataset:
+    - kind: dataset
+      spec:
+        displayName: '{dataset_name}'
+        entity: *entity
+        datasetIngestionMode: '{ingestion_mode}'
+        query: '{query_statement}'