oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/text_dataset/udfs.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import re
+from typing import Callable
+
+
+class UDF:
+    @staticmethod
+    def from_regex(regex: str) -> Callable:
+        def function(content):
+            match = re.match(regex, content)
+            if match:
+                if len(match.groups()) == 0:
+                    return [match.group(0)]
+                else:
+                    return match.groups()
+
+        return function

ads/text_dataset/utils.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import functools
+import os
+
+
+class NotSupportedError(Exception):   # pragma: no cover
+    pass
+
+
+def experimental(cls):
+    @functools.wraps(cls)
+    def wrapper(*args, **kwargs):
+        instance = cls(*args, **kwargs)
+        print(f"{cls.__name__} is experimental and may be removed in the future.")
+        return instance
+
+    return wrapper
+
+
+class PY4JGateway:
+    def __init__(self) -> None:
+        try:
+            from py4j.java_gateway import GatewayParameters, JavaGateway, launch_gateway
+        except ModuleNotFoundError:
+            raise ModuleNotFoundError("py4j is not installed.")
+        if "CONDA_PREFIX" not in os.environ or not os.path.exists(
+            os.path.join(os.environ.get("CONDA_PREFIX"), "text-extraction-tools.jar")
+        ):
+            raise NotSupportedError(
+                "Tika is not supported in this distribution. Use alternatives such as pdfplumber."
+            )
+        port = launch_gateway(
+            java_path="/usr/bin/java",
+            classpath=os.path.join(
+                os.environ.get("CONDA_PREFIX"), "text-extraction-tools.jar"
+            ),
+        )
+        self.gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port))
+
+    def __enter__(self) -> None:
+        return self.gateway
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self.gateway.shutdown()

ads/type_discovery/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+
+logger = logging.getLogger(__name__)

ads/type_discovery/abstract_detector.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from __future__ import print_function, absolute_import
+
+import abc
+
+
+class AbstractTypeDiscoveryDetector(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def discover(self, name, series):
+        return
+
+
+class DiscreteDiscoveryDetector(AbstractTypeDiscoveryDetector, metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def discover(self, name, series):
+        return

ads/type_discovery/constant_detector.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from __future__ import print_function, absolute_import, division
+
+import pandas as pd
+
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
+from ads.type_discovery.typed_feature import ConstantTypedFeature
+
+
+class ConstantDetector(AbstractTypeDiscoveryDetector):
+    def is_constant(self, name, values):
+        #
+        # if all the values are null we treat this as a const feature
+        #
+
+        return values.size == 0 or values.nunique() == 1
+
+    def discover(self, name, series):
+        candidates = series.loc[~series.isnull()]
+
+        if self.is_constant(name, candidates):
+            logger.debug("column [{}]/[{}] Constant".format(name, series.dtype))
+            return ConstantTypedFeature.build(name, series)
+
+        return False
+
+
+if __name__ == "__main__":
+    dd = ConstantDetector()
+    print(dd.discover("zipcodes", pd.Series([None, "94065", "94065", "94065", None])))
+    print(dd.discover("years", pd.Series([2008, 2008, 2008, 2008, 2008])))
+    df = pd.read_csv(
+        "https://artifacthub.oraclecorp.com/dsc-generic/advanced-ds/datasets/flights.csv"
+    )
+    print(dd.discover("flights_years", df["Year"]))

ads/type_discovery/continuous_detector.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from __future__ import print_function, absolute_import, division
+
+import pandas as pd
+
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
+from ads.type_discovery.typed_feature import ContinuousTypedFeature
+from ads.common import utils
+
+
+class ContinuousDetector(AbstractTypeDiscoveryDetector):
+    @staticmethod
+    def _target_is_continuous(series):
+        if str(series.dtype) in ["float16", "float32", "float64"]:
+            return True  # treat target variable as continuous
+        elif str(series.dtype) in ["int16", "int32", "int64"]:
+            if series.nunique() >= 20:
+                return True  # treat target variable as continuous
+
+        return False
+
+    def _is_continuous(self, series):
+        if series.dtype.name in ["object"]:
+            try:
+                series.astype("float")
+                return True
+            except:
+                pass
+
+        if series.dtype.name in utils.numeric_pandas_dtypes():
+            #
+            # if the type is float we simply beleive pandas and go with continuous
+            #
+            return True
+
+    def discover(self, name, series):
+
+        if self._is_continuous(series):
+            logger.debug("column [{}]/[{}] continuous".format(name, series.dtype))
+            return ContinuousTypedFeature.build(name, series)
+
+        return False
+
+
+if __name__ == "__main__":
+    dd = ContinuousDetector()
+    print(dd.discover("continuous", pd.Series([None, 3.14, 12.0, 1, 2, 3, None])))

ads/type_discovery/credit_card_detector.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+"""
+NOTE:
+
+    There's an opportunity here to generate a new feature, credict card numbers are not preditive because they
+    don't generalize, however, if the feature is replaced by the type of card that might be predictive.
+
+    - Visa: ^4[0-9]{12}(?:[0-9]{3})?$ All Visa card numbers start with a 4. New cards have 16 digits. Old cards have 13.
+    - MasterCard: ^(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}$ MasterCard numbers
+        either start with the numbers 51 through 55 or with the numbers 2221 through 2720. All have 16 digits.
+    - American Express: ^3[47][0-9]{13}$ American Express card numbers start with 34 or 37 and have 15 digits.
+    - Diners Club: ^3(?:0[0-5]|[68][0-9])[0-9]{11}$ Diners Club card numbers begin with 300 through 305, 36 or 38.
+        All have 14 digits. There are Diners Club cards that begin with 5 and have 16 digits. These are a joint
+        venture between Diners Club and MasterCard, and should be processed like a MasterCard.
+    - Discover: ^6(?:011|5[0-9]{2})[0-9]{12}$ Discover card numbers begin with 6011 or 65. All have 16 digits.
+    - JCB: ^(?:2131|1800|35\d{3})\d{11}$ JCB cards beginning with 2131 or 1800 have 15 digits.
+        JCB cards beginning with 35 have 16 digits.
+
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import re
+
+import pandas as pd
+
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
+from ads.type_discovery.typed_feature import CreditCardTypedFeature
+
+
+class CreditCardDetector(AbstractTypeDiscoveryDetector):
+
+    _max_sample_size_to_luhn_check = 1000
+    _pattern_string = r"""^(?:4[0-9]{12}(?:[0-9]{3})?            # Visa
+                             |  (?:5[1-5][0-9]{2}                # MasterCard
+                                 | 222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}
+                             |  3[47][0-9]{13}                   # American Express
+                             |  3(?:0[0-5]|[68][0-9])[0-9]{11}   # Diners Club
+                             |  6(?:011|5[0-9]{2})[0-9]{12}      # Discover
+                             |  (?:2131|1800|35\d{3})\d{11}      # JCB
+                             |  (5018|5020|5038|5612|5893|6304|6759|6761|6762|6763|0604|6390)\d+$   # Maestro
+                             |  ^(5[06789]|6)[0-9]{0,}$          # Maestro
+                             |  ^4[0-9]{12}(?:[0-9]{6})?$        #Visa 19 digit
+                            )$"""
+
+    def luhn_checksum(self, card_number):
+        def digits_of(n):
+            return [int(d) for d in str(n)]
+
+        digits = digits_of(card_number)
+        odd_digits = digits[-1::-2]
+        even_digits = digits[-2::-2]
+        checksum = 0
+        checksum += sum(odd_digits)
+        for d in even_digits:
+            checksum += sum(digits_of(d * 2))
+        return checksum % 10
+
+    def is_luhn_valid(self, card_number):
+        return self.luhn_checksum(card_number) == 0
+
+    def is_credit_card(self, name, values):
+        cc = re.compile(CreditCardDetector._pattern_string, re.VERBOSE)
+        # since the nulls have been previously filtered we can safely do "all"
+        samp = (
+            values
+            if values.size <= CreditCardDetector._max_sample_size_to_luhn_check
+            else values.sample(n=CreditCardDetector._max_sample_size_to_luhn_check)
+        )
+
+        if samp.dtype.name in ["float16", "float32", "float64"]:
+            if samp.apply(float.is_integer).all():
+                samp = samp.fillna(0.0).astype(int)
+
+        if samp.dtype.name in ["int16", "int32", "int64"]:
+            samp = samp.astype(str)
+
+        if all([cc.match(str(x)) for x in samp]):
+            #
+            # iff the pattern matching succeeds do we try the luhn algorithm on a sample
+            #
+            return all([self.is_luhn_valid(x) for x in samp])
+
+        return False
+
+    def discover(self, name, series):
+        candidates = series.loc[~series.isnull()]
+
+        if self.is_credit_card(name, candidates.head(1000)):
+            logger.debug("column [{}]/[{}] credit card".format(name, series.dtype))
+            return CreditCardTypedFeature.build(name, series)
+
+        return False

ads/type_discovery/datetime_detector.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from __future__ import print_function, absolute_import, division
+
+import pandas as pd
+
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
+from ads.type_discovery.typed_feature import DateTimeTypedFeature
+
+
+class DateTimeDetector(AbstractTypeDiscoveryDetector):
+    @runtime_dependency(module="datefinder", install_from=OptionalDependency.DATA)
+    def _is_date_time(self, name, values, low_level_type_name):
+        if low_level_type_name.startswith("datetime64"):
+            return lambda x: x
+        else:
+            #
+            # if the column/feature contains the word "timestamp" then
+            #
+            if low_level_type_name.startswith("int") and "timestamp" in name.lower():
+                # either s (max len 10) on ns (max len) 19
+                unit = "s" if values.astype("str").str.len().max() <= 10 else "ns"
+                try:
+                    pd.to_datetime(values, unit=unit)
+                    return lambda x: pd.to_datetime(x, unit=unit)
+                except:
+                    pass
+            if values.dtype == "object":
+                try:
+                    pd.to_datetime(values, infer_datetime_format=True)
+                    datefinder_result = all(
+                        [bool(list(datefinder.find_dates(str(x)))) for x in values]
+                    )
+                    if datefinder_result:
+                        return lambda x: pd.to_datetime(x, infer_datetime_format=True)
+                except:
+                    pass
+
+        return None
+
+    def discover(self, name, series):
+        candidates = series.loc[~series.isnull()]
+        fn = self._is_date_time(name, candidates.head(500), series.dtype.name)
+        if fn:
+            logger.debug("column [{}]/[{}] datetime".format(name, series.dtype))
+            return DateTimeTypedFeature.build(name, fn(series))
+
+        return False
+
+
+if __name__ == "__main__":
+    dd = DateTimeDetector()
+    print(
+        dd.discover(
+            "date-range",
+            pd.Series(pd.date_range(start="1/1/2018", end="1/08/2018", freq="H")),
+        )
+    )
+    print(dd.discover("dates", pd.Series(["12/12/12", "12/12/13", None, "12/12/14"])))
+    print(
+        dd.discover(
+            "dates-with-other-values",
+            pd.Series(["12/12/12", "Monday", None, "12/12/14"]),
+        )
+    )
+    print(
+        dd.discover(
+            "timestamp s",
+            pd.Series(
+                [
+                    978300760,
+                    978302109,
+                    978301968,
+                    978300275,
+                    978824291,
+                    978302268,
+                    978302039,
+                ]
+            ),
+        )
+    )
+    print(dd.discover("timestamp ns", pd.Series([1490195805433502912])))

ads/type_discovery/discrete_detector.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from __future__ import print_function, absolute_import, division
+
+import pandas as pd
+from sklearn.utils.multiclass import type_of_target
+
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import DiscreteDiscoveryDetector
+from ads.type_discovery.typed_feature import (
+    OrdinalTypedFeature,
+    CategoricalTypedFeature,
+)
+from ads.common import utils
+
+
+class DiscreteDetector(DiscreteDiscoveryDetector):
+
+    _max_categorical_values = 100
+
+    def _get_categorical_or_ordinal(self, name, series):
+        #
+        # categoricals are unordered discreet types
+        # ordinals are ordered discreet (int) types
+        #
+
+        low_level_type_name = series.dtype.name
+
+        if low_level_type_name == "category" or low_level_type_name == "bool":
+            return "categorical"
+
+        else:
+            #
+            # after removing nulls the new Series might already be categorical
+            #
+            nulls_removed = pd.Series(list(series.loc[~series.isna()]))
+            if (
+                nulls_removed.dtype.name == "category"
+                or nulls_removed.dtype.name == "bool"
+            ):
+                return "categorical"
+
+            count_distinct = series.nunique()
+            observations = series.size
+
+            tot = type_of_target(
+                list(nulls_removed.head(min(nulls_removed.size, 2000)))
+            )
+
+            if tot == "binary":
+                return "categorical"
+
+            elif tot == "multiclass":
+                if count_distinct <= DiscreteDetector._max_categorical_values:
+                    if low_level_type_name in utils.numeric_pandas_dtypes():
+                        return "ordinal"
+                    else:
+                        return "categorical"
+
+                if low_level_type_name.startswith(
+                    "int"
+                ) or low_level_type_name.startswith("float"):
+                    if nulls_removed.min() >= 0:
+                        if (
+                            low_level_type_name.startswith("int")
+                            or nulls_removed.sum()
+                            == nulls_removed.astype("int64").sum()
+                        ):
+                            return "ordinal"
+                    # by summing all the values and summing all the int values we can know all the values are integers
+
+        return False
+
+    def discover(self, name, series):
+
+        guessed_type = self._get_categorical_or_ordinal(
+            name, series.loc[~series.isnull()]
+        )
+
+        if guessed_type == "categorical":
+            logger.debug("column [{}]/[{}] categorical".format(name, series.dtype))
+            return CategoricalTypedFeature.build(name, series)
+        elif guessed_type == "ordinal":
+            logger.debug("column [{}]/[{}] ordinal".format(name, series.dtype))
+            return OrdinalTypedFeature.build(name, series)
+        else:
+            return False
+
+
+if __name__ == "__main__":
+    dd = DiscreteDetector()
+
+    print(
+        dd.discover(
+            "str-categorical",
+            pd.Series(["a", "a", "a", "b", "c", "a"], dtype="category"),
+        )
+    )
+    print(
+        dd.discover(
+            "bool-categorical", pd.Series([True, False, True, True, True, None, True])
+        )
+    )
+    print(dd.discover("continuous", pd.Series([None, 3.14, 12.0, 1, 2, 3, None])))
+    print(
+        dd.discover("int-1-categorical", pd.Series([1, 1, 1, 5, 5, 5, 5, 5, 5, 5, 9]))
+    )
+    print(dd.discover("int-2-categorical", pd.Series([1, 1, 1, 5, 9])))
+    print(dd.discover("real-3-categorical", pd.Series([1.0, 2.0, 3.0, 1.0, 4.0, 5.0])))
+    print(
+        dd.discover(
+            "bool-categorical", pd.Series([True, False, True, True, True, None, True])
+        )
+    )

ads/type_discovery/document_detector.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from __future__ import print_function, absolute_import, division
+
+import re
+
+import pandas as pd
+
+from ads.type_discovery import logger
+from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
+from ads.type_discovery.typed_feature import DocumentTypedFeature, AddressTypedFeature
+
+
+class DocumentDetector(AbstractTypeDiscoveryDetector):
+
+    _min_cjk_chars_for_document = 100
+    _min_words = 10
+    _min_html_tags = 5
+
+    _html_pattern = re.compile("<.*?>")
+
+    _unicode_ranges = [
+        {"from": ord(u"\u3300"), "to": ord(u"\u33ff")},  # compatibility ideographs
+        {"from": ord(u"\ufe30"), "to": ord(u"\ufe4f")},  # compatibility ideographs
+        {"from": ord(u"\uf900"), "to": ord(u"\ufaff")},  # compatibility ideographs
+        {
+            "from": ord(u"\U0002F800"),
+            "to": ord(u"\U0002fa1f"),
+        },  # compatibility ideographs
+        {"from": ord(u"\u3040"), "to": ord(u"\u309f")},  # Japanese Hiragana
+        {"from": ord(u"\u30a0"), "to": ord(u"\u30ff")},  # Japanese Katakana
+        {"from": ord(u"\u2e80"), "to": ord(u"\u2eff")},  # cjk radicals supplement
+        {"from": ord(u"\u4e00"), "to": ord(u"\u9fff")},
+        {"from": ord(u"\u3400"), "to": ord(u"\u4dbf")},
+        {"from": ord(u"\U00020000"), "to": ord(u"\U0002a6df")},
+        {"from": ord(u"\U0002a700"), "to": ord(u"\U0002b73f")},
+        {"from": ord(u"\U0002b740"), "to": ord(u"\U0002b81f")},
+        {
+            "from": ord(u"\U0002b820"),
+            "to": ord(u"\U0002ceaf"),
+        },  # included as of Unicode 8.0
+    ]
+
+    def _is_cjk_char(self, char):
+        return any(
+            [
+                range["from"] <= ord(char) <= range["to"]
+                for range in DocumentDetector._unicode_ranges
+            ]
+        )
+
+    def cjk_string(self, document):
+        cjk_char_count = sum([1 if self._is_cjk_char(c) else 0 for c in document])
+        return cjk_char_count / len(document) >= 0.2
+
+    def html_document(self, document):
+        return (
+            len(list(re.finditer(DocumentDetector._html_pattern, document)))
+            > DocumentDetector._min_html_tags
+        )
+
+    def discover(self, name, series):
+        #
+        # very basic detection of a document. If the document is CJK then we use only the document length
+        # otherwise we split on whitespace and confirm that there are word-like strings
+        #
+        if series.dtype == "object":
+            null_series = series.loc[~series.isnull()]
+            first_non_null_document = null_series.iloc[0]
+
+            if isinstance(first_non_null_document, str):
+
+                is_html = self.html_document(first_non_null_document)
+
+                if self.cjk_string(first_non_null_document):
+                    if (
+                        len(first_non_null_document)
+                        >= DocumentDetector._min_cjk_chars_for_document
+                    ):
+                        tf = DocumentTypedFeature.build(
+                            name, series, is_cjk=True, is_html=is_html
+                        )
+                        logger.debug(
+                            "type discovery on CJK column [{}]/[{}] found to be a document".format(
+                                name, series.dtype
+                            )
+                        )
+                        return tf
+                else:
+                    # find rows with above average length
+                    above_avg_series = null_series.loc[
+                        null_series.str.len() >= null_series.str.len().mean()
+                    ]
+
+                    # take a sample. max 500 docs
+                    above_avg_series_sample = above_avg_series.sample(
+                        n=min(500, len(above_avg_series))
+                    )
+
+                    # if all of the samples have more than min_words tokens..
+                    mean_number_of_words = (
+                        above_avg_series_sample.str.split().str.len().mean()
+                    )
+                    if mean_number_of_words > DocumentDetector._min_words:
+                        if (
+                            mean_number_of_words < 15
+                            and above_avg_series_sample.str.count(",").mean()
+                            / mean_number_of_words
+                            > 0.1
+                        ):
+                            # many commas probably means address type
+                            logger.debug(
+                                "type discovery on column [{}]/[{}] looks like an address type".format(
+                                    name, series.dtype
+                                )
+                            )
+                            return AddressTypedFeature.build(name, series)
+                        else:
+                            logger.debug(
+                                "type discovery on non-CJK column [{}]/[{}] found to be a document".format(
+                                    name, series.dtype
+                                )
+                            )
+
+                            # previous check of first document for HTML is now refined using longer documents
+                            is_html = all(
+                                [
+                                    self.html_document(doc)
+                                    for doc in above_avg_series_sample
+                                ]
+                            )
+
+                            return DocumentTypedFeature.build(
+                                name, series, is_cjk=False, is_html=is_html
+                            )
+
+        return False
+
+
+if __name__ == "__main__":
+    dd = DocumentDetector()