oracle-ads 2.13.7__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/text_dataset/udfs.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import re
-from typing import Callable
-
-
-class UDF:
-    @staticmethod
-    def from_regex(regex: str) -> Callable:
-        def function(content):
-            match = re.match(regex, content)
-            if match:
-                if len(match.groups()) == 0:
-                    return [match.group(0)]
-                else:
-                    return match.groups()
-
-        return function

ads/text_dataset/utils.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import functools
-import os
-
-
-class NotSupportedError(Exception):   # pragma: no cover
-    pass
-
-
-def experimental(cls):
-    @functools.wraps(cls)
-    def wrapper(*args, **kwargs):
-        instance = cls(*args, **kwargs)
-        print(f"{cls.__name__} is experimental and may be removed in the future.")
-        return instance
-
-    return wrapper
-
-
-class PY4JGateway:
-    def __init__(self) -> None:
-        try:
-            from py4j.java_gateway import GatewayParameters, JavaGateway, launch_gateway
-        except ModuleNotFoundError:
-            raise ModuleNotFoundError("py4j is not installed.")
-        if "CONDA_PREFIX" not in os.environ or not os.path.exists(
-            os.path.join(os.environ.get("CONDA_PREFIX"), "text-extraction-tools.jar")
-        ):
-            raise NotSupportedError(
-                "Tika is not supported in this distribution. Use alternatives such as pdfplumber."
-            )
-        port = launch_gateway(
-            java_path="/usr/bin/java",
-            classpath=os.path.join(
-                os.environ.get("CONDA_PREFIX"), "text-extraction-tools.jar"
-            ),
-        )
-        self.gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port))
-
-    def __enter__(self) -> None:
-        return self.gateway
-
-    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
-        self.gateway.shutdown()

ads/type_discovery/__init__.py

@@ -1,9 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import logging
-
-logger = logging.getLogger(__name__)

ads/type_discovery/abstract_detector.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from __future__ import print_function, absolute_import
-
-import abc
-
-
-class AbstractTypeDiscoveryDetector(metaclass=abc.ABCMeta):
-    @abc.abstractmethod
-    def discover(self, name, series):
-        return
-
-
-class DiscreteDiscoveryDetector(AbstractTypeDiscoveryDetector, metaclass=abc.ABCMeta):
-    @abc.abstractmethod
-    def discover(self, name, series):
-        return

ads/type_discovery/constant_detector.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from __future__ import print_function, absolute_import, division
-
-import pandas as pd
-
-from ads.type_discovery import logger
-from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
-from ads.type_discovery.typed_feature import ConstantTypedFeature
-
-
-class ConstantDetector(AbstractTypeDiscoveryDetector):
-    def is_constant(self, name, values):
-        #
-        # if all the values are null we treat this as a const feature
-        #
-
-        return values.size == 0 or values.nunique() == 1
-
-    def discover(self, name, series):
-        candidates = series.loc[~series.isnull()]
-
-        if self.is_constant(name, candidates):
-            logger.debug("column [{}]/[{}] Constant".format(name, series.dtype))
-            return ConstantTypedFeature.build(name, series)
-
-        return False
-
-
-if __name__ == "__main__":
-    dd = ConstantDetector()
-    print(dd.discover("zipcodes", pd.Series([None, "94065", "94065", "94065", None])))
-    print(dd.discover("years", pd.Series([2008, 2008, 2008, 2008, 2008])))
-    df = pd.read_csv(
-        "https://artifacthub.oraclecorp.com/dsc-generic/advanced-ds/datasets/flights.csv"
-    )
-    print(dd.discover("flights_years", df["Year"]))

ads/type_discovery/continuous_detector.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-from __future__ import print_function, absolute_import, division
-
-import pandas as pd
-
-from ads.type_discovery import logger
-from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
-from ads.type_discovery.typed_feature import ContinuousTypedFeature
-from ads.common import utils
-
-
-class ContinuousDetector(AbstractTypeDiscoveryDetector):
-    @staticmethod
-    def _target_is_continuous(series):
-        if str(series.dtype) in ["float16", "float32", "float64"]:
-            return True  # treat target variable as continuous
-        elif str(series.dtype) in ["int16", "int32", "int64"]:
-            if series.nunique() >= 20:
-                return True  # treat target variable as continuous
-
-        return False
-
-    def _is_continuous(self, series):
-        if series.dtype.name in ["object"]:
-            try:
-                series.astype("float")
-                return True
-            except:
-                pass
-
-        if series.dtype.name in utils.numeric_pandas_dtypes():
-            #
-            # if the type is float we simply beleive pandas and go with continuous
-            #
-            return True
-
-    def discover(self, name, series):
-
-        if self._is_continuous(series):
-            logger.debug("column [{}]/[{}] continuous".format(name, series.dtype))
-            return ContinuousTypedFeature.build(name, series)
-
-        return False
-
-
-if __name__ == "__main__":
-    dd = ContinuousDetector()
-    print(dd.discover("continuous", pd.Series([None, 3.14, 12.0, 1, 2, 3, None])))

ads/type_discovery/credit_card_detector.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-"""
-NOTE:
-
-    There's an opportunity here to generate a new feature, credict card numbers are not preditive because they
-    don't generalize, however, if the feature is replaced by the type of card that might be predictive.
-
-    - Visa: ^4[0-9]{12}(?:[0-9]{3})?$ All Visa card numbers start with a 4. New cards have 16 digits. Old cards have 13.
-    - MasterCard: ^(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}$ MasterCard numbers
-        either start with the numbers 51 through 55 or with the numbers 2221 through 2720. All have 16 digits.
-    - American Express: ^3[47][0-9]{13}$ American Express card numbers start with 34 or 37 and have 15 digits.
-    - Diners Club: ^3(?:0[0-5]|[68][0-9])[0-9]{11}$ Diners Club card numbers begin with 300 through 305, 36 or 38.
-        All have 14 digits. There are Diners Club cards that begin with 5 and have 16 digits. These are a joint
-        venture between Diners Club and MasterCard, and should be processed like a MasterCard.
-    - Discover: ^6(?:011|5[0-9]{2})[0-9]{12}$ Discover card numbers begin with 6011 or 65. All have 16 digits.
-    - JCB: ^(?:2131|1800|35\d{3})\d{11}$ JCB cards beginning with 2131 or 1800 have 15 digits.
-        JCB cards beginning with 35 have 16 digits.
-
-"""
-
-from __future__ import print_function, absolute_import, division
-
-import re
-
-import pandas as pd
-
-from ads.type_discovery import logger
-from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
-from ads.type_discovery.typed_feature import CreditCardTypedFeature
-
-
-class CreditCardDetector(AbstractTypeDiscoveryDetector):
-
-    _max_sample_size_to_luhn_check = 1000
-    _pattern_string = r"""^(?:4[0-9]{12}(?:[0-9]{3})?            # Visa
-                             |  (?:5[1-5][0-9]{2}                # MasterCard
-                                 | 222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}
-                             |  3[47][0-9]{13}                   # American Express
-                             |  3(?:0[0-5]|[68][0-9])[0-9]{11}   # Diners Club
-                             |  6(?:011|5[0-9]{2})[0-9]{12}      # Discover
-                             |  (?:2131|1800|35\d{3})\d{11}      # JCB
-                             |  (5018|5020|5038|5612|5893|6304|6759|6761|6762|6763|0604|6390)\d+$   # Maestro
-                             |  ^(5[06789]|6)[0-9]{0,}$          # Maestro
-                             |  ^4[0-9]{12}(?:[0-9]{6})?$        #Visa 19 digit
-                            )$"""
-
-    def luhn_checksum(self, card_number):
-        def digits_of(n):
-            return [int(d) for d in str(n)]
-
-        digits = digits_of(card_number)
-        odd_digits = digits[-1::-2]
-        even_digits = digits[-2::-2]
-        checksum = 0
-        checksum += sum(odd_digits)
-        for d in even_digits:
-            checksum += sum(digits_of(d * 2))
-        return checksum % 10
-
-    def is_luhn_valid(self, card_number):
-        return self.luhn_checksum(card_number) == 0
-
-    def is_credit_card(self, name, values):
-        cc = re.compile(CreditCardDetector._pattern_string, re.VERBOSE)
-        # since the nulls have been previously filtered we can safely do "all"
-        samp = (
-            values
-            if values.size <= CreditCardDetector._max_sample_size_to_luhn_check
-            else values.sample(n=CreditCardDetector._max_sample_size_to_luhn_check)
-        )
-
-        if samp.dtype.name in ["float16", "float32", "float64"]:
-            if samp.apply(float.is_integer).all():
-                samp = samp.fillna(0.0).astype(int)
-
-        if samp.dtype.name in ["int16", "int32", "int64"]:
-            samp = samp.astype(str)
-
-        if all([cc.match(str(x)) for x in samp]):
-            #
-            # iff the pattern matching succeeds do we try the luhn algorithm on a sample
-            #
-            return all([self.is_luhn_valid(x) for x in samp])
-
-        return False
-
-    def discover(self, name, series):
-        candidates = series.loc[~series.isnull()]
-
-        if self.is_credit_card(name, candidates.head(1000)):
-            logger.debug("column [{}]/[{}] credit card".format(name, series.dtype))
-            return CreditCardTypedFeature.build(name, series)
-
-        return False

ads/type_discovery/datetime_detector.py

@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-from __future__ import print_function, absolute_import, division
-
-import pandas as pd
-
-from ads.common.decorator.runtime_dependency import (
-    runtime_dependency,
-    OptionalDependency,
-)
-from ads.type_discovery import logger
-from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
-from ads.type_discovery.typed_feature import DateTimeTypedFeature
-
-
-class DateTimeDetector(AbstractTypeDiscoveryDetector):
-    @runtime_dependency(module="datefinder", install_from=OptionalDependency.DATA)
-    def _is_date_time(self, name, values, low_level_type_name):
-        if low_level_type_name.startswith("datetime64"):
-            return lambda x: x
-        else:
-            #
-            # if the column/feature contains the word "timestamp" then
-            #
-            if low_level_type_name.startswith("int") and "timestamp" in name.lower():
-                # either s (max len 10) on ns (max len) 19
-                unit = "s" if values.astype("str").str.len().max() <= 10 else "ns"
-                try:
-                    pd.to_datetime(values, unit=unit)
-                    return lambda x: pd.to_datetime(x, unit=unit)
-                except:
-                    pass
-            if values.dtype == "object":
-                try:
-                    pd.to_datetime(values, infer_datetime_format=True)
-                    datefinder_result = all(
-                        [bool(list(datefinder.find_dates(str(x)))) for x in values]
-                    )
-                    if datefinder_result:
-                        return lambda x: pd.to_datetime(x, infer_datetime_format=True)
-                except:
-                    pass
-
-        return None
-
-    def discover(self, name, series):
-        candidates = series.loc[~series.isnull()]
-        fn = self._is_date_time(name, candidates.head(500), series.dtype.name)
-        if fn:
-            logger.debug("column [{}]/[{}] datetime".format(name, series.dtype))
-            return DateTimeTypedFeature.build(name, fn(series))
-
-        return False
-
-
-if __name__ == "__main__":
-    dd = DateTimeDetector()
-    print(
-        dd.discover(
-            "date-range",
-            pd.Series(pd.date_range(start="1/1/2018", end="1/08/2018", freq="H")),
-        )
-    )
-    print(dd.discover("dates", pd.Series(["12/12/12", "12/12/13", None, "12/12/14"])))
-    print(
-        dd.discover(
-            "dates-with-other-values",
-            pd.Series(["12/12/12", "Monday", None, "12/12/14"]),
-        )
-    )
-    print(
-        dd.discover(
-            "timestamp s",
-            pd.Series(
-                [
-                    978300760,
-                    978302109,
-                    978301968,
-                    978300275,
-                    978824291,
-                    978302268,
-                    978302039,
-                ]
-            ),
-        )
-    )
-    print(dd.discover("timestamp ns", pd.Series([1490195805433502912])))

ads/type_discovery/discrete_detector.py

@@ -1,118 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from __future__ import print_function, absolute_import, division
-
-import pandas as pd
-from sklearn.utils.multiclass import type_of_target
-
-from ads.type_discovery import logger
-from ads.type_discovery.abstract_detector import DiscreteDiscoveryDetector
-from ads.type_discovery.typed_feature import (
-    OrdinalTypedFeature,
-    CategoricalTypedFeature,
-)
-from ads.common import utils
-
-
-class DiscreteDetector(DiscreteDiscoveryDetector):
-
-    _max_categorical_values = 100
-
-    def _get_categorical_or_ordinal(self, name, series):
-        #
-        # categoricals are unordered discreet types
-        # ordinals are ordered discreet (int) types
-        #
-
-        low_level_type_name = series.dtype.name
-
-        if low_level_type_name == "category" or low_level_type_name == "bool":
-            return "categorical"
-
-        else:
-            #
-            # after removing nulls the new Series might already be categorical
-            #
-            nulls_removed = pd.Series(list(series.loc[~series.isna()]))
-            if (
-                nulls_removed.dtype.name == "category"
-                or nulls_removed.dtype.name == "bool"
-            ):
-                return "categorical"
-
-            count_distinct = series.nunique()
-            observations = series.size
-
-            tot = type_of_target(
-                list(nulls_removed.head(min(nulls_removed.size, 2000)))
-            )
-
-            if tot == "binary":
-                return "categorical"
-
-            elif tot == "multiclass":
-                if count_distinct <= DiscreteDetector._max_categorical_values:
-                    if low_level_type_name in utils.numeric_pandas_dtypes():
-                        return "ordinal"
-                    else:
-                        return "categorical"
-
-                if low_level_type_name.startswith(
-                    "int"
-                ) or low_level_type_name.startswith("float"):
-                    if nulls_removed.min() >= 0:
-                        if (
-                            low_level_type_name.startswith("int")
-                            or nulls_removed.sum()
-                            == nulls_removed.astype("int64").sum()
-                        ):
-                            return "ordinal"
-                    # by summing all the values and summing all the int values we can know all the values are integers
-
-        return False
-
-    def discover(self, name, series):
-
-        guessed_type = self._get_categorical_or_ordinal(
-            name, series.loc[~series.isnull()]
-        )
-
-        if guessed_type == "categorical":
-            logger.debug("column [{}]/[{}] categorical".format(name, series.dtype))
-            return CategoricalTypedFeature.build(name, series)
-        elif guessed_type == "ordinal":
-            logger.debug("column [{}]/[{}] ordinal".format(name, series.dtype))
-            return OrdinalTypedFeature.build(name, series)
-        else:
-            return False
-
-
-if __name__ == "__main__":
-    dd = DiscreteDetector()
-
-    print(
-        dd.discover(
-            "str-categorical",
-            pd.Series(["a", "a", "a", "b", "c", "a"], dtype="category"),
-        )
-    )
-    print(
-        dd.discover(
-            "bool-categorical", pd.Series([True, False, True, True, True, None, True])
-        )
-    )
-    print(dd.discover("continuous", pd.Series([None, 3.14, 12.0, 1, 2, 3, None])))
-    print(
-        dd.discover("int-1-categorical", pd.Series([1, 1, 1, 5, 5, 5, 5, 5, 5, 5, 9]))
-    )
-    print(dd.discover("int-2-categorical", pd.Series([1, 1, 1, 5, 9])))
-    print(dd.discover("real-3-categorical", pd.Series([1.0, 2.0, 3.0, 1.0, 4.0, 5.0])))
-    print(
-        dd.discover(
-            "bool-categorical", pd.Series([True, False, True, True, True, None, True])
-        )
-    )

ads/type_discovery/document_detector.py

@@ -1,146 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8; -*-
-
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-from __future__ import print_function, absolute_import, division
-
-import re
-
-import pandas as pd
-
-from ads.type_discovery import logger
-from ads.type_discovery.abstract_detector import AbstractTypeDiscoveryDetector
-from ads.type_discovery.typed_feature import DocumentTypedFeature, AddressTypedFeature
-
-
-class DocumentDetector(AbstractTypeDiscoveryDetector):
-
-    _min_cjk_chars_for_document = 100
-    _min_words = 10
-    _min_html_tags = 5
-
-    _html_pattern = re.compile("<.*?>")
-
-    _unicode_ranges = [
-        {"from": ord(u"\u3300"), "to": ord(u"\u33ff")},  # compatibility ideographs
-        {"from": ord(u"\ufe30"), "to": ord(u"\ufe4f")},  # compatibility ideographs
-        {"from": ord(u"\uf900"), "to": ord(u"\ufaff")},  # compatibility ideographs
-        {
-            "from": ord(u"\U0002F800"),
-            "to": ord(u"\U0002fa1f"),
-        },  # compatibility ideographs
-        {"from": ord(u"\u3040"), "to": ord(u"\u309f")},  # Japanese Hiragana
-        {"from": ord(u"\u30a0"), "to": ord(u"\u30ff")},  # Japanese Katakana
-        {"from": ord(u"\u2e80"), "to": ord(u"\u2eff")},  # cjk radicals supplement
-        {"from": ord(u"\u4e00"), "to": ord(u"\u9fff")},
-        {"from": ord(u"\u3400"), "to": ord(u"\u4dbf")},
-        {"from": ord(u"\U00020000"), "to": ord(u"\U0002a6df")},
-        {"from": ord(u"\U0002a700"), "to": ord(u"\U0002b73f")},
-        {"from": ord(u"\U0002b740"), "to": ord(u"\U0002b81f")},
-        {
-            "from": ord(u"\U0002b820"),
-            "to": ord(u"\U0002ceaf"),
-        },  # included as of Unicode 8.0
-    ]
-
-    def _is_cjk_char(self, char):
-        return any(
-            [
-                range["from"] <= ord(char) <= range["to"]
-                for range in DocumentDetector._unicode_ranges
-            ]
-        )
-
-    def cjk_string(self, document):
-        cjk_char_count = sum([1 if self._is_cjk_char(c) else 0 for c in document])
-        return cjk_char_count / len(document) >= 0.2
-
-    def html_document(self, document):
-        return (
-            len(list(re.finditer(DocumentDetector._html_pattern, document)))
-            > DocumentDetector._min_html_tags
-        )
-
-    def discover(self, name, series):
-        #
-        # very basic detection of a document. If the document is CJK then we use only the document length
-        # otherwise we split on whitespace and confirm that there are word-like strings
-        #
-        if series.dtype == "object":
-            null_series = series.loc[~series.isnull()]
-            first_non_null_document = null_series.iloc[0]
-
-            if isinstance(first_non_null_document, str):
-
-                is_html = self.html_document(first_non_null_document)
-
-                if self.cjk_string(first_non_null_document):
-                    if (
-                        len(first_non_null_document)
-                        >= DocumentDetector._min_cjk_chars_for_document
-                    ):
-                        tf = DocumentTypedFeature.build(
-                            name, series, is_cjk=True, is_html=is_html
-                        )
-                        logger.debug(
-                            "type discovery on CJK column [{}]/[{}] found to be a document".format(
-                                name, series.dtype
-                            )
-                        )
-                        return tf
-                else:
-                    # find rows with above average length
-                    above_avg_series = null_series.loc[
-                        null_series.str.len() >= null_series.str.len().mean()
-                    ]
-
-                    # take a sample. max 500 docs
-                    above_avg_series_sample = above_avg_series.sample(
-                        n=min(500, len(above_avg_series))
-                    )
-
-                    # if all of the samples have more than min_words tokens..
-                    mean_number_of_words = (
-                        above_avg_series_sample.str.split().str.len().mean()
-                    )
-                    if mean_number_of_words > DocumentDetector._min_words:
-                        if (
-                            mean_number_of_words < 15
-                            and above_avg_series_sample.str.count(",").mean()
-                            / mean_number_of_words
-                            > 0.1
-                        ):
-                            # many commas probably means address type
-                            logger.debug(
-                                "type discovery on column [{}]/[{}] looks like an address type".format(
-                                    name, series.dtype
-                                )
-                            )
-                            return AddressTypedFeature.build(name, series)
-                        else:
-                            logger.debug(
-                                "type discovery on non-CJK column [{}]/[{}] found to be a document".format(
-                                    name, series.dtype
-                                )
-                            )
-
-                            # previous check of first document for HTML is now refined using longer documents
-                            is_html = all(
-                                [
-                                    self.html_document(doc)
-                                    for doc in above_avg_series_sample
-                                ]
-                            )
-
-                            return DocumentTypedFeature.build(
-                                name, series, is_cjk=False, is_html=is_html
-                            )
-
-        return False
-
-
-if __name__ == "__main__":
-    dd = DocumentDetector()