oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/type_discovery/typed_feature.py

@@ -0,0 +1,594 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+import copy
+import json
+import re
+from collections import defaultdict
+from time import time
+
+import numpy as np
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.utils import Bunch
+
+from ads.common import utils
+from ads.common.card_identifier import card_identify
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.common.utils import JsonConverter
+
+
+class TypedFeature(Bunch):
+    def __init__(self, name, meta_data):
+        assert "type" in meta_data
+        self.meta_data = meta_data
+        self.meta_data["feature_name"] = name
+        super().__init__(**meta_data)
+
+    @staticmethod
+    def build(name, series):
+        pass
+
+    def __repr__(self):
+        d = copy.deepcopy(self.meta_data)
+        if "internal" in d:
+            d.pop("internal")
+        return json.dumps(d, indent=2, cls=JsonConverter)
+
+
+class ContinuousTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    @runtime_dependency(module="scipy", install_from=OptionalDependency.VIZ)
+    def build(name, series):
+        series = series.astype("float")
+
+        non_null_series = series.loc[~series.isnull()]
+        desc = non_null_series.describe()
+        stats = {
+            "mode": series.mode().iloc[0],
+            "median": desc["50%"],
+            "kurtosis": series.kurt(),
+            "variance": series.var(),
+            "skewness": series.skew(),
+            "outlier_percentage": 100
+            * np.count_nonzero(np.abs(scipy.stats.zscore(non_null_series) >= 2.999))
+            / series.size,
+        }
+        stats.update({k: v for k, v in desc.items()})
+
+        return ContinuousTypedFeature(
+            name,
+            {
+                "type": "continuous",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": stats,
+                "internal": {"sample": series.head(5)},
+            },
+        )
+
+
+class DiscreteTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+
+class OrdinalTypedFeature(DiscreteTypedFeature):
+    def __init__(self, name, meta_data):
+        DiscreteTypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        nulls_removed = series.astype("category").loc[~series.isnull()]
+        desc = nulls_removed.describe(include="all")
+        value_counts = series.value_counts(ascending=False)
+        x_min, x_max = np.nanmin(nulls_removed), np.nanmax(nulls_removed)
+
+        stats = {
+            "unique percentage": 100 * series.nunique() / series.size,
+            "x_min": x_min,
+            "x_max": x_max,
+            "mode": series.mode().iloc[0],
+        }
+        stats.update({k: v for k, v in desc.items()})
+
+        return OrdinalTypedFeature(
+            name,
+            {
+                "type": "ordinal",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": stats,
+                "internal": {
+                    "sample": series.head(5),
+                    "unique": series.nunique(),
+                    "counts": utils.truncate_series_top_n(
+                        value_counts, n=min(16, len(value_counts))
+                    ),
+                    "high_cardinality": bool(series.nunique() > 30),
+                    "very_high_cardinality": bool(
+                        series.nunique() >= 0.95 * series.size
+                    ),
+                },
+            },
+        )
+
+
+class CategoricalTypedFeature(DiscreteTypedFeature):
+    def __init__(self, name, meta_data):
+        DiscreteTypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        desc = series.astype("category").loc[~series.isnull()].describe(include="all")
+        value_counts = series.value_counts(ascending=False)
+        top = desc["top"] if "top" in desc else None
+        if isinstance(top, str):
+            mode = top if len(top) < 30 else top[:24] + "..."
+        else:
+            mode = top
+
+        stats = {
+            "unique percentage": 100 * series.nunique() / series.size,
+            "mode": mode,
+        }
+        stats.update({k: v for k, v in desc.items()})
+
+        return CategoricalTypedFeature(
+            name,
+            {
+                "type": "categorical",
+                "low_level_type": series.dtype.name,
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "stats": stats,
+                "internal": {
+                    "sample": series.sample(n=min(100, series.size)),
+                    "unique": series.nunique(),
+                    "counts": utils.truncate_series_top_n(
+                        value_counts, n=min(16, len(value_counts))
+                    ),
+                    "high_cardinality": bool(series.nunique() > 30),
+                    "very_high_cardinality": bool(
+                        series.nunique() >= 0.95 * series.size
+                    ),
+                },
+            },
+        )
+
+
+class IPAddressTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        desc = series.astype("category").loc[~series.isnull()].describe()
+
+        nulls_removed = series.loc[~series.isnull()]
+        value_counts = nulls_removed.value_counts()
+
+        return IPAddressTypedFeature(
+            name,
+            {
+                "type": "ipaddress",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": {
+                    "unique percentage": 100 * series.nunique() / series.size,
+                    "mode": series.mode().iloc[0],
+                },
+                "internal": {
+                    "sample": nulls_removed.sample(n=min(100, nulls_removed.size)),
+                    "counts": utils.truncate_series_top_n(
+                        value_counts, n=min(16, len(value_counts))
+                    ),
+                    "unique": series.nunique(),
+                },
+            },
+        )
+
+
+class PhoneNumberTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        desc = series.astype("category").loc[~series.isnull()].describe()
+
+        nulls_removed = list(series.loc[~series.isnull()])
+        pat = re.compile(r"^(\+?\d{1,2}[\s-])?\(?(\d{3})\)?[\s.-]?\d{3}[\s.-]?\d{4}$")
+        area_codes = [
+            g.group(2)
+            for g in filter(None, [pat.match(x) for x in nulls_removed])
+            if g and len(g.groups()) >= 2
+        ]
+        value_counts = pd.Series(area_codes).value_counts()
+
+        # resolve area codes
+        return PhoneNumberTypedFeature(
+            name,
+            {
+                "type": "phone",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": {
+                    "unique percentage": 100 * series.nunique() / series.size,
+                    "mode": series.mode().iloc[0],
+                },
+                "internal": {
+                    "sample": series.sample(n=min(100, series.size)),
+                    "counts": utils.truncate_series_top_n(
+                        value_counts, n=min(16, len(value_counts))
+                    ),
+                    "unique": series.nunique(),
+                },
+            },
+        )
+
+
+class GISTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series, samples):
+        desc = series.astype("category").loc[~series.isnull()].describe()
+        value_counts = series.value_counts(ascending=False)
+        return GISTypedFeature(
+            name,
+            {
+                "type": "gis",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": {
+                    "observations": desc["count"],
+                    "unique percentage": 100 * series.nunique() / series.size,
+                    # TODO mid point
+                },
+                "internal": {"sample": samples, "unique": series.nunique()},
+            },
+        )
+
+
+class AddressTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        nulls_removed = series.loc[~series.isnull()]
+        mean_document_length = nulls_removed.str.len().mean()
+        sampled_nulls_removed = nulls_removed.sample(n=min(len(nulls_removed), 1000))
+
+        return AddressTypedFeature(
+            name,
+            {
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "type": "address",
+                "low_level_type": series.dtype.name,
+                "stats": {
+                    "mean_character_count": mean_document_length,
+                    "mean_word_length": nulls_removed.str.split().str.len().mean(),
+                },
+                "internal": {
+                    "sample": series.sample(n=min(100, series.size)),
+                    "word_frequencies": DocumentTypedFeature.vectorization(
+                        name, sampled_nulls_removed, mean_document_length
+                    ),
+                },
+            },
+        )
+
+
+class DocumentTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    @runtime_dependency(module="wordcloud", install_from=OptionalDependency.TEXT)
+    def corpus_processor(series):
+        pat_punct = re.compile(r"[^a-zA-Z]", re.UNICODE + re.MULTILINE)
+        pat_tags = re.compile(r"<.*?>", re.UNICODE + re.MULTILINE)
+        pat_digits = re.compile(r"(\d|\W)+", re.UNICODE + re.MULTILINE)
+        pat_splitter = re.compile(r"\s+", re.UNICODE + re.MULTILINE)
+        for doc in series:
+            # Remove punctuations, tags,  special characters and digits
+            text = pat_punct.sub(" ", doc.lower())
+
+            # remove tags
+            text = pat_tags.sub("", text)
+
+            # remove special characters and digits
+            text = pat_digits.sub(" ", text)
+
+            from wordcloud import STOPWORDS
+
+            stop_words = STOPWORDS
+
+            # Convert to list from string and return a generator expression
+            yield " ".join(
+                [
+                    x
+                    for x in pat_splitter.split(text)
+                    if len(x) >= 3 and x not in stop_words
+                ]
+            )
+
+    @staticmethod
+    def sub_vectorization(
+        feature_name,
+        series,
+        min_df=0.0,
+        max_df=1.0,
+        min_tf=2,
+    ):
+        start = time()
+
+        v1 = CountVectorizer(
+            ngram_range=(1, 1),
+            max_features=2000,
+            decode_error="ignore",
+            strip_accents="unicode",
+            stop_words=None,
+            min_df=min_df,
+            max_df=max_df,
+        )
+        X1 = v1.fit_transform(DocumentTypedFeature.corpus_processor(series))
+        unigrams = {
+            k: int(v)
+            for k, v in dict(
+                zip(v1.get_feature_names_out(), np.asarray(X1.sum(axis=0)).ravel())
+            ).items()
+        }
+
+        v2 = CountVectorizer(
+            ngram_range=(2, 2),
+            max_features=2000,
+            decode_error="ignore",
+            strip_accents="unicode",
+            stop_words=None,
+            min_df=min_df,
+            max_df=max_df,
+        )
+        X2 = v2.fit_transform(DocumentTypedFeature.corpus_processor(series))
+        bigrams = {
+            k: int(v)
+            for k, v in dict(
+                zip(v2.get_feature_names_out(), np.asarray(X2.sum(axis=0)).ravel())
+            ).items()
+        }
+
+        # drop unigrams that are prefixes of bigrams
+        for k, v in bigrams.items():
+            # take the key (k), split it to get first word and look for that word in unigrams
+            for word in k.split():
+                if word in unigrams:
+                    # boost the bigram
+                    bigrams[k] = v + unigrams[word]
+                    del unigrams[word]
+
+        unigrams.update(bigrams)
+
+        ret = {k: v for k, v in unigrams.items() if v >= min_tf}
+        return ret if ret else unigrams.items()
+
+    @staticmethod
+    def vectorization(feature_name, series, mean_document_length):
+        # set the min_df and max_df as functions of the mean_document_length
+        min_df = 0.1
+        max_df = 0.8
+
+        if mean_document_length > 5000:
+            min_df = 0.2
+            max_df = 0.7
+        try:
+            return DocumentTypedFeature.sub_vectorization(
+                feature_name, series, min_df=min_df, max_df=max_df
+            )
+        except ValueError:
+            return DocumentTypedFeature.sub_vectorization(
+                feature_name, series, min_df=0.0, max_df=1.0
+            )
+
+    @staticmethod
+    def build(name, series, is_cjk, is_html):
+        internal = {"cjk": is_cjk, "html": is_html}
+
+        if is_cjk:
+            stats = {"mean_document_character_count": int(series.str.len().mean())}
+
+            internal["sample"] = series.sample(n=min(100, series.size))
+
+        else:
+            nulls_removed = series.loc[~series.isnull()]
+            if is_html:
+                nulls_removed = nulls_removed.replace(
+                    to_replace="<[^<]+?>", value=" ", regex=True
+                ).replace("\n", " ", regex=True)
+
+            internal["sample"] = nulls_removed.sample(n=min(100, nulls_removed.size))
+
+            mean_document_length = nulls_removed.str.len().mean()
+            sample_size = int(
+                2e6 // mean_document_length
+            )  # vectorize at most 2MB of text
+
+            sampled_nulls_removed = nulls_removed.sample(
+                n=min(len(nulls_removed), sample_size)
+            )
+
+            stats = {
+                "mean_document_character_count": int(series.str.len().mean()),
+                "mean_word_length": nulls_removed.str.split().str.len().mean(),
+            }
+
+            internal["word_frequencies"] = DocumentTypedFeature.vectorization(
+                name, sampled_nulls_removed, mean_document_length
+            )
+
+        return DocumentTypedFeature(
+            name,
+            {
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "type": "document",
+                "low_level_type": series.dtype.name,
+                "stats": stats,
+                "internal": internal,
+            },
+        )
+
+
+class ZipcodeTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        max_threshold = 24
+        suffix = ""
+
+        series = series.loc[~series.isnull()].astype(str).str.slice(0, 5)
+
+        for i in [5, 4, 3, 2, 1]:
+            if series.nunique() <= max_threshold:
+                break
+            series = series.str[:-1]
+            suffix += "*"
+
+        series += suffix
+
+        return ZipcodeTypedFeature(
+            name,
+            {
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "type": "zipcode",
+                "low_level_type": series.dtype.name,
+                "stats": {"mode": series.mode().iloc[0]},
+                "internal": {
+                    "sample": series.sample(n=min(100, series.size)),
+                    "histogram": series.value_counts(),
+                },
+            },
+        )
+
+
+class UnknownTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        return UnknownTypedFeature(
+            name,
+            {
+                "type": "unknown",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": {"mode": series.mode().iloc[0]},
+                "internal": {"sample": series.sample(n=min(100, series.size))},
+            },
+        )
+
+
+class ConstantTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        desc = series.astype("category").loc[~series.isnull()].describe()
+        if "top" in desc:
+            value = desc["top"]
+            value_counts = utils.truncate_series_top_n(series.value_counts())
+        else:
+            value = "NaN"
+            value_counts = {value: len(series)}
+        return ConstantTypedFeature(
+            name,
+            {
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "type": "constant",
+                "stats": {"value": value},
+                "internal": {
+                    "sample": series.sample(n=min(100, series.size)),
+                    "counts": value_counts,
+                },
+            },
+        )
+
+
+class CreditCardTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        null_removed_series = series.loc[~series.isnull()]
+        desc = null_removed_series.astype("category").describe()
+        sampled_series = null_removed_series.sample(
+            n=min(1000, len(null_removed_series))
+        )
+
+        d_scheme = defaultdict(int)
+
+        # use counting method to build dict of credit card data
+        for s in sampled_series:
+            scheme = card_identify().identify_issue_network(s)
+            d_scheme[scheme] += 1
+
+        return CreditCardTypedFeature(
+            name,
+            {
+                "type": "creditcard",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": {
+                    "unique percentage": 100 * series.nunique() / series.size,
+                    "mode": desc["top"] if "top" in desc else None,
+                },
+                "internal": {
+                    "sample": series.sample(n=min(100, series.size)),
+                    "counts": dict(d_scheme),
+                    "unique": series.nunique(),
+                },
+            },
+        )
+
+
+class DateTimeTypedFeature(TypedFeature):
+    def __init__(self, name, meta_data):
+        TypedFeature.__init__(self, name, meta_data)
+
+    @staticmethod
+    def build(name, series):
+        desc = series.loc[~series.isnull()].describe()
+
+        return DateTimeTypedFeature(
+            name,
+            {
+                "type": "datetime",
+                "missing_percentage": 100 * series.isna().sum() / series.size,
+                "low_level_type": series.dtype.name,
+                "stats": {
+                    "unique percentage": 100 * series.nunique() / series.size,
+                    "first": desc["first"] if "first" in desc else None,
+                    "last": desc["last"] if "last" in desc else None,
+                    "mode": desc["top"] if "top" in desc else None,
+                },
+                "internal": {
+                    "sample": series.sample(n=min(100, series.size)),
+                    "unique": series.nunique(),
+                },
+            },
+        )

ads/type_discovery/unknown_detector.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from __future__ import print_function, absolute_import, division
+
+import pandas as pd
+
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
+from ads.type_discovery.typed_feature import (
+    UnknownTypedFeature,
+    CategoricalTypedFeature,
+)
+
+
+class UnknownDetector(AbstractTypeDiscoveryDetector):
+    def discover(self, name, series):
+        candidate = series.loc[~series.isnull()].iloc[0]
+
+        if series.dtype == "object":
+            #
+            # if we got all through all the other detectors and it's a string type of feature then we
+            # just call it a high dimensional categorical
+            #
+            return CategoricalTypedFeature.build(name, series)
+        else:
+            logger.debug(
+                "type discovery on column [{}]/[{}] result is Unknown".format(
+                    name, series.dtype
+                )
+            )
+            return UnknownTypedFeature.build(name, series)
+
+
+if __name__ == "__main__":
+    dd = UnknownDetector()
+    print(dd.discover("unknown", pd.Series([None, "94065", "90210", None])))

ads/type_discovery/zipcode_detector.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from __future__ import print_function, absolute_import, division
+
+import re
+
+import pandas as pd
+
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
+from ads.type_discovery.typed_feature import ZipcodeTypedFeature
+
+
+class ZipCodeDetector(AbstractTypeDiscoveryDetector):
+    def _is_zip_code(self, values):
+        return all(
+            [re.match("^[0-9]{5}(?:-[0-9]{4})?$", str(v)) for v in values.head(10)]
+        )
+
+    def discover(self, name, series):
+        candidates = series.loc[~series.isnull()]
+        if (
+            self._is_zip_code(candidates.head(1000))
+            if candidates.dtype == "object"
+            else self._is_zip_code(candidates.astype("object"))
+        ):
+            logger.debug(
+                "type discovery on column [{}]/[{}] found to be a zipcode".format(
+                    name, series.dtype
+                )
+            )
+            return ZipcodeTypedFeature.build(name, series)
+
+        return False
+
+
+if __name__ == "__main__":
+    dd = ZipCodeDetector()
+    print(
+        dd.discover(
+            "zip", pd.Series([None, "00501", "94065", "94065-1107", "90210", None])
+        )
+    )

ads/vault/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.vault.vault import Vault