oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/feature_engineering/feature_type/adsstring/parsers/nltk_parser.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+import functools
+import nltk
+import pandas as pd
+from nltk import FreqDist, ngrams, pos_tag
+from nltk.stem import SnowballStemmer
+from nltk.tokenize import sent_tokenize, word_tokenize
+from ads.feature_engineering.feature_type.adsstring.parsers.base import Parser
+from typing import List
+
+if "CONDA_PREFIX" in os.environ:
+    nltk.data.path.append(f"{os.environ['CONDA_PREFIX']}/nltk")
+
+
+class NLTKParser(Parser):
+
+    stemmers = {}  # language -> stemmer
+
+    def __init__(self):
+        if not self.language in NLTKParser.stemmers:
+            NLTKParser.stemmers[self.language] = SnowballStemmer(self.language)
+
+    @property
+    @functools.lru_cache()
+    def pos(self) -> pd.DataFrame:
+        return pd.DataFrame(data=self._pos(), columns=["Word", "Label"])
+
+    @functools.lru_cache()
+    def _pos(self):
+        return pos_tag(word_tokenize(self.string))
+
+    @property
+    @functools.lru_cache()
+    def adjective(self) -> List[str]:
+        return [word for (word, pos) in self._pos() if pos.startswith("JJ")]
+
+    @property
+    @functools.lru_cache()
+    def noun(self) -> List[str]:
+        return [word for (word, pos) in self._pos() if pos.startswith("NN")]
+
+    @property
+    @functools.lru_cache()
+    def adverb(self) -> List[str]:
+        return [word for (word, pos) in self._pos() if pos.startswith("RB")]
+
+    @property
+    @functools.lru_cache()
+    def verb(self) -> List[str]:
+        return [word for (word, pos) in self._pos() if pos.startswith("VB")]
+
+    @property
+    @functools.lru_cache()
+    def sentence(self) -> List[str]:
+        return sent_tokenize(self.string)
+
+    @property
+    @functools.lru_cache()
+    def token(self) -> List[str]:
+        return word_tokenize(self.string)
+
+    @property
+    def word(self) -> List[str]:
+        return [word.lower() for word in self.token if word.isalpha()]
+
+    @property
+    @functools.lru_cache()
+    def word_count(self) -> pd.DataFrame:
+        word_count = list(FreqDist(self.word).items())
+        df = pd.DataFrame(data=word_count, columns=["Word", "Count"])
+        return df.sort_values(
+            ["Count", "Word"], ascending=[False, True], ignore_index=True
+        )
+
+    @property
+    @functools.lru_cache()
+    def bigram(self) -> pd.DataFrame:
+        res = list(ngrams(self.word, 2))
+        return pd.DataFrame(data=res, columns=["Word 1", "Word 2"])
+
+    @property
+    @functools.lru_cache()
+    def trigram(self) -> pd.DataFrame:
+        res = list(ngrams(self.word, 3))
+        return pd.DataFrame(data=res, columns=["Word 1", "Word 2", "Word 3"])
+
+    @property
+    @functools.lru_cache()
+    def stem(self) -> List:
+        return [NLTKParser.stemmers[self.language].stem(w) for w in self.word]

ads/feature_engineering/feature_type/adsstring/parsers/spacy_parser.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import itertools
+from collections import Counter
+from typing import List, Sequence, Tuple
+import os
+import functools
+import pandas as pd
+
+from ads.feature_engineering.feature_type.adsstring.parsers.base import Parser
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+
+
+# This module is only used when user installed spacy. spacy has conflicts with oci-cli.
+class SpacyParser(Parser):  # pragma: no cover
+    @runtime_dependency(module="spacy", install_from=OptionalDependency.TEXT)
+    def __init__(self):
+        try:
+            # for ADS conda packs, spacy files are located under $CONDA_PREFIX/spacy
+            self.nlp = spacy.load(f"{os.environ['CONDA_PREFIX']}/spacy")
+        except OSError:
+            try:
+                # if not inside a conda pack, trying loading first
+                # if failed download to default path
+                self.nlp = spacy.load("en_core_web_sm")
+            except:
+                from spacy.cli import download
+
+                download("en_core_web_sm")
+                self.nlp = spacy.load("en_core_web_sm")
+
+    @property
+    @functools.lru_cache()
+    def _parsed_nlp_obj(self):
+        return self.nlp(self.string)
+
+    @property
+    @functools.lru_cache()
+    def pos(self) -> pd.DataFrame:
+        pos = (
+            (token.text, token.pos_)
+            for token in self._parsed_nlp_obj
+            if not token.is_space
+        )
+        return pd.DataFrame(data=pos, columns=["Word", "Label"])
+
+    def _pos_tokens(self, pos: str) -> List[str]:
+        return _f7(
+            [
+                token.text
+                for token in self._parsed_nlp_obj
+                if (
+                    not token.is_stop
+                    and not token.is_punct
+                    and not token.is_space
+                    and token.pos_ == pos
+                )
+            ]
+        )
+
+    @property
+    @functools.lru_cache()
+    def noun(self) -> List[str]:
+        return self._pos_tokens("NOUN")
+
+    @property
+    @functools.lru_cache()
+    def adjective(self) -> List[str]:
+        return self._pos_tokens("ADJ")
+
+    @property
+    @functools.lru_cache()
+    def adverb(self) -> List[str]:
+        return self._pos_tokens("ADV")
+
+    @property
+    @functools.lru_cache()
+    def verb(self) -> List[str]:
+        return self._pos_tokens("VERB")
+
+    @property
+    @functools.lru_cache()
+    def noun_phrase(self) -> List[str]:
+        np = []
+        for sentence in self.sentence:
+            for chunk in self.nlp(sentence).noun_chunks:
+                if chunk.text.strip():
+                    np.append(chunk.text.strip())
+
+        return np
+
+    @property
+    @functools.lru_cache()
+    def entity_extract(self) -> pd.DataFrame:
+        return pd.DataFrame(data=self._entity_extract(), columns=["Entity", "Label"])
+
+    @functools.lru_cache()
+    def _entity_extract(self) -> List[Tuple[str, str]]:
+        entities = []
+        for sentence in self.sentence:
+            entities.extend([(ent.text, ent.label_) for ent in self.nlp(sentence).ents])
+
+        return _f7(entities)
+
+    def _entity_tokens(self, entity_label: str) -> List[str]:
+        return [
+            token for (token, label) in self._entity_extract() if label == entity_label
+        ]
+
+    @property
+    @functools.lru_cache()
+    def entity_people(self) -> List[str]:
+        # People, including fictional.
+        return self._entity_tokens("PERSON")
+
+    @property
+    @functools.lru_cache()
+    def entity_location(self) -> List[str]:
+        # Location entity, i.e. countries, cities, states, mountain ranges, bodies of water, buildings, airports, highways, bridges
+        return (
+            self._entity_tokens("GPE")
+            + self._entity_tokens("LOC")
+            + self._entity_tokens("FAC")
+        )
+
+    @property
+    @functools.lru_cache()
+    def entity_organization(self) -> List[str]:
+        # Companies, agencies, institutions.
+        return self._entity_tokens("ORG")
+
+    @property
+    @functools.lru_cache()
+    def entity_artwork(self) -> List[str]:
+        # title of books, songs, etc
+        return self._entity_tokens("WORK_OF_ART")
+
+    @property
+    @functools.lru_cache()
+    def entity_product(self) -> List[str]:
+        # product names, etc
+        return self._entity_tokens("PRODUCT")
+
+    @property
+    @functools.lru_cache()
+    def sentence(self) -> List[str]:
+        return [sent.text.strip() for sent in self._parsed_nlp_obj.sents]
+
+    @property
+    @functools.lru_cache()
+    def token(self) -> List[str]:
+        return [token.text for token in self._parsed_nlp_obj]
+
+    def _words_by_sentence(self) -> List[str]:
+        x = []
+        for sentence in self.sentence:
+            x.append([token.text for token in self.nlp(sentence) if token.is_alpha])
+
+        return x
+
+    @property
+    @functools.lru_cache()
+    def word(self) -> List[str]:
+        return list(itertools.chain(*self._words_by_sentence()))
+
+    def _ngrams(self, tokens, n):
+        return [tokens[i : i + n] for i in range(len(tokens) - n + 1)]
+
+    @property
+    @functools.lru_cache()
+    def bigram(self) -> pd.DataFrame:
+        bigram = list(
+            itertools.chain(
+                *[
+                    self._ngrams(sentence_words, 2)
+                    for sentence_words in self._words_by_sentence()
+                ]
+            )
+        )
+        return pd.DataFrame(data=bigram, columns=["Word 1", "Word 2"])
+
+    @property
+    @functools.lru_cache()
+    def trigram(self) -> pd.DataFrame:
+        trigram = list(
+            itertools.chain(
+                *[
+                    self._ngrams(sentence_words, 3)
+                    for sentence_words in self._words_by_sentence()
+                ]
+            )
+        )
+        return pd.DataFrame(data=trigram, columns=["Word 1", "Word 2", "Word 3"])
+
+    @property
+    @functools.lru_cache()
+    def lemma(self) -> List[str]:
+        return [token.lemma_ for token in self._parsed_nlp_obj]
+
+    @property
+    @functools.lru_cache()
+    def word_count(self) -> pd.DataFrame:
+        word_count = Counter([word.lower() for word in self.word]).most_common()
+        df = pd.DataFrame(data=word_count, columns=["Word", "Count"])
+        return df.sort_values(
+            ["Count", "Word"], ascending=[False, True], ignore_index=True
+        )
+
+
+def _f7(seq: Sequence) -> List:
+    """order preserving de-duplicate sequence"""
+    seen = set()
+    seen_add = seen.add
+    return [x for x in seq if not (x in seen or seen_add(x))]

ads/feature_engineering/feature_type/adsstring/string.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import functools
+
+from typing import Any, Callable
+from ads.feature_engineering.feature_type.base import FeatureType
+from ads.feature_engineering.feature_type.adsstring.common_regex_mixin import (
+    CommonRegexMixin,
+)
+from ads.feature_engineering.feature_type.adsstring.oci_language import OCILanguage
+from ads.common.decorator.runtime_dependency import OptionalDependency
+from ads.feature_engineering.feature_type.adsstring.oci_language import OCILanguage
+
+
+def to_adsstring(func: Callable) -> Callable:
+    """Decorator that converts output of a function to `ADSString` if it returns a string.
+
+    Parameters
+    ----------
+    func : Callable
+        function to decorate
+
+    Returns
+    -------
+    Callable
+        decorated function
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        out = func(*args, **kwargs)
+        if isinstance(out, str):
+            return ADSString(out)
+        else:
+            return out
+
+    return wrapper
+
+
+def wrap_output_string(decorator: Callable) -> Callable:
+    """Class decorator that applies a decorator to all methods of a class.
+
+    Parameters
+    ----------
+    decorator : Callable
+        decorator to apply
+
+    Returns
+    -------
+    Callable
+        class decorator
+    """
+
+    def decorate(cls):
+        for attr in dir(cls):
+            if (
+                not attr.startswith("__")
+                and not attr.startswith("_")
+                and callable(getattr(cls, attr))
+            ):
+                setattr(cls, attr, decorator(getattr(cls, attr)))
+        return cls
+
+    return decorate
+
+
+@wrap_output_string(to_adsstring)
+class ADSString(str, FeatureType, CommonRegexMixin):
+    """Defines an enhanced string class for the purpose of performing NLP tasks.
+    Its functionalities can be extended by registering plugins.
+
+    Attributes
+    ----------
+    plugins: List
+        list of plugins that add functionalities to the class.
+    string: str
+        plain string
+
+    Example
+    -------
+    >>> ADSString.nlp_backend('nltk')
+    >>> s = ADSString("Walking my dog on a breezy day is the best.")
+    >>> s.lower() # regular string methods still work
+    >>> s.replace("a", "e")
+    >>> s.noun
+    >>> s.pos #parts of speech
+    >>> s = ADSString("get in touch with my associate at john.smith@gmail.com to schedule")
+    >>> s.email
+    >>> ADSString.plugin_register(OCILanguage)
+    >>> s = ADSString("This movie is awesome.")
+    >>> s.absa
+    """
+
+    plugins = []
+    language_model_cache = dict()
+    description = "Type representing enhanced string class."
+
+    def __init__(self, text: str, language="english") -> None:
+        """Initialze the class and register plugins.
+
+        Parameters
+        ----------
+        text : str
+            input text
+        language : str, optional
+            language of the text, by default "english".
+
+        Raises
+        ------
+        TypeError
+            input text is not a string.
+        """
+        if not isinstance(text, str):
+            raise TypeError("Text must be a string.")
+        if isinstance(text, ADSString):
+            self.raw = text.string
+        else:
+            self.raw = text
+        self._string = self.raw.strip()
+        self.language = language
+
+        if not hasattr(ADSString, "bases"):
+            # remember the bases before adding any plugins
+            setattr(ADSString, "bases", ADSString.__bases__)
+        else:
+            # reset bases before installing plugins
+            ADSString.__bases__ = ADSString.bases
+
+        # adding default nlp plugin
+        ADSString.plugins.insert(0, OCILanguage)
+
+        # remove duplicates and preserve order
+        _plugins = []
+        for plg in ADSString.plugins[::-1]:
+            if plg not in _plugins:
+                _plugins.insert(0, plg)
+
+        for cls in _plugins:
+            self.__class__.__bases__ = (cls,) + self.__class__.__bases__
+            super(cls, self).__init__()
+        super(ADSString, self).__init__()
+
+    @property
+    def string(self):
+        return self._string
+
+    @classmethod
+    def nlp_backend(cls, backend: str = "nltk") -> None:
+        """Set backend for extracting NLP related properties.
+
+        Parameters
+        ----------
+        backend : str, optional
+            name of backend, by default 'nltk'.
+
+        Raises
+        ------
+        ModuleNotFoundError
+            module corresponding to backend is not found.
+        ValueError
+            input backend is invalid.
+
+        Returns
+        -------
+        None
+        """
+        if backend == "spacy":
+            try:
+                import spacy
+            except:
+                raise ModuleNotFoundError(
+                    f"The `spacy` module was not found. Please run "
+                    f"`pip install {OptionalDependency.TEXT}`."
+                )
+            from ads.feature_engineering.feature_type.adsstring.parsers.spacy_parser import (
+                SpacyParser,
+            )
+
+            cls.plugin_register(SpacyParser)
+        elif backend == "nltk":
+            try:
+                import nltk
+            except:
+                raise ModuleNotFoundError("nltk must be installed.")
+            from ads.feature_engineering.feature_type.adsstring.parsers.nltk_parser import (
+                NLTKParser,
+            )
+
+            cls.plugin_register(NLTKParser)
+        else:
+            raise ValueError(
+                "Currently only `nltk` and `spacy` are supported. Default uses `nltk`."
+            )
+
+    @classmethod
+    def plugin_clear(cls) -> None:
+        """Clears plugins."""
+        cls.plugins.clear()
+
+    @classmethod
+    def plugin_register(cls, plugin: Any) -> None:
+        """Register a plugin
+
+        Parameters
+        ----------
+        plugin : Any
+            plugin to register
+
+        Returns
+        -------
+        None
+        """
+        cls.plugins.append(plugin)
+
+    @classmethod
+    def plugin_list(cls) -> None:
+        """List registered plugins."""
+        if not cls.plugins:
+            print("No plugin registered.")
+            return []
+
+        return [plugin.__name__ for plugin in cls.plugins]
+
+    @classmethod
+    def help(cls, plugin: Any = None) -> None:
+        """List available properties.
+
+        Parameters
+        ----------
+        plugin : Any
+            registered plugin
+
+        Returns
+        -------
+        None
+        """
+        if not plugin:
+            props = [
+                attr
+                for attr in dir(cls)
+                if not attr.startswith("__") and not attr.startswith("_")
+            ]
+            print(f"{cls.__name__}::Available properties: {', '.join(props)}")
+
+        else:
+            props = [
+                attr
+                for attr in dir(plugin)
+                if not attr.startswith("__") and not attr.startswith("_")
+            ]
+            print(
+                f"{cls.__name__}::Available properties from plugin, {plugin.__name__}:"
+            )
+            print(props)

ads/feature_engineering/feature_type/base.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from abc import ABC, ABCMeta
+
+from ads.feature_engineering.feature_type.handler.feature_validator import (
+    FeatureValidator,
+)
+from ads.feature_engineering.feature_type.handler.feature_warning import (
+    FeatureWarning,
+)
+from ads.common.utils import camel_to_snake
+
+
+class Name:
+    def __get__(self, instance, owner):
+        return camel_to_snake(owner.__name__)
+
+
+class FeatureBaseType(type):
+    """The helper metaclass to extend fucntionality of FeatureType class."""
+
+    def __new__(cls, classname, bases, dictionary):
+        dictionary["validator"] = FeatureValidator()
+        dictionary["warning"] = FeatureWarning()
+        return type.__new__(cls, classname, bases, dictionary)
+
+
+class FeatureBaseTypeMeta(FeatureBaseType, ABCMeta):
+    """The class to provide compatibility between ABC and FeatureBaseType metaclass."""
+
+
+class FeatureType(ABC, metaclass=FeatureBaseTypeMeta):
+    """
+    Abstract case for feature types. Default class attribute include name and description.
+    Name is auto generated using camel to snake conversion unless specified.
+    """
+
+    name = Name()
+    description = "Base feature type."
+
+
+class Tag:
+    """Class for free form tags. Name must be specified."""
+
+    def __init__(self, name: str) -> None:
+        """
+        Initialize a tag instance.
+
+        Parameters
+        ----------
+        name: str
+            The name of the tag.
+        """
+        self.name = name