spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/stemmer.py

@@ -0,0 +1,79 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the Stemmer."""
+from sparknlp.common import *
+
+
+class Stemmer(AnnotatorModel):
+    """Returns hard-stems out of words with the objective of retrieving the
+    meaningful part of the word.
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/stemmer/Word_Stemming_with_Stemmer.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``TOKEN``              ``TOKEN``
+    ====================== ======================
+
+    Parameters
+    ----------
+    None
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("token")
+    >>> stemmer = Stemmer() \\
+    ...     .setInputCols(["token"]) \\
+    ...     .setOutputCol("stem")
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     tokenizer,
+    ...     stemmer
+    ... ])
+    >>> data = spark.createDataFrame([["Peter Pipers employees are picking pecks of pickled peppers."]]) \\
+    ...     .toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("stem.result").show(truncate = False)
+    +-------------------------------------------------------------+
+    |result                                                       |
+    +-------------------------------------------------------------+
+    |[peter, piper, employe, ar, pick, peck, of, pickl, pepper, .]|
+    +-------------------------------------------------------------+
+    """
+
+    inputAnnotatorTypes = [AnnotatorType.TOKEN]
+
+    outputAnnotatorType = AnnotatorType.TOKEN
+
+    language = Param(Params._dummy(), "language", "stemmer algorithm", typeConverter=TypeConverters.toString)
+
+    name = "Stemmer"
+
+    @keyword_only
+    def __init__(self):
+        super(Stemmer, self).__init__(classname="com.johnsnowlabs.nlp.annotators.Stemmer")
+        self._setDefault(
+            language="english"
+        )

sparknlp/annotator/stop_words_cleaner.py

@@ -0,0 +1,190 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the StopWordsCleaner."""
+from sparknlp.common import *
+
+
+class StopWordsCleaner(AnnotatorModel):
+    """This annotator takes a sequence of strings (e.g. the output of a
+    Tokenizer, Normalizer, Lemmatizer, and Stemmer) and drops all the stop words
+    from the input sequences.
+
+    By default, it uses stop words from MLlibs `StopWordsRemover
+    <https://spark.apache.org/docs/latest/ml-features#stopwordsremover>`__. Stop
+    words can also be defined by explicitly setting them with
+    :meth:`.setStopWords` or loaded from pretrained models using ``pretrained``
+    of its companion object.
+
+
+    >>> stopWords = StopWordsCleaner.pretrained() \\
+    ...     .setInputCols(["token"]) \\
+    ...     .setOutputCol("cleanTokens")
+
+    This will load the default pretrained model ``"stopwords_en"``.
+
+    For available pretrained models please see the `Models Hub
+    <https://sparknlp.orgtask=Stop+Words+Removal>`__.
+
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/stop-words/StopWordsCleaner.ipynb>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``TOKEN``              ``TOKEN``
+    ====================== ======================
+
+    Parameters
+    ----------
+    stopWords
+        The words to be filtered out, by default english stopwords from Spark ML
+    caseSensitive
+        Whether to consider case, by default False
+    locale
+        Locale of the input. ignored when case sensitive, by default locale of
+        the JVM
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> sentenceDetector = SentenceDetector() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("sentence")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["sentence"]) \\
+    ...     .setOutputCol("token")
+    >>> stopWords = StopWordsCleaner() \\
+    ...     .setInputCols(["token"]) \\
+    ...     .setOutputCol("cleanTokens") \\
+    ...     .setCaseSensitive(False)
+    >>> pipeline = Pipeline().setStages([
+    ...       documentAssembler,
+    ...       sentenceDetector,
+    ...       tokenizer,
+    ...       stopWords
+    ...     ])
+    >>> data = spark.createDataFrame([
+    ...     ["This is my first sentence. This is my second."],
+    ...     ["This is my third sentence. This is my forth."]
+    ... ]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("cleanTokens.result").show(truncate=False)
+    +-------------------------------+
+    |result                         |
+    +-------------------------------+
+    |[first, sentence, ., second, .]|
+    |[third, sentence, ., forth, .] |
+    +-------------------------------+
+    """
+
+    name = "StopWordsCleaner"
+
+    inputAnnotatorTypes = [AnnotatorType.TOKEN]
+
+    outputAnnotatorType = AnnotatorType.TOKEN
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.StopWordsCleaner", java_model=None):
+        super(StopWordsCleaner, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        self._setDefault(
+            stopWords=StopWordsCleaner.loadDefaultStopWords("english"),
+            caseSensitive=False,
+            locale=self._java_obj.getLocale()
+        )
+
+    stopWords = Param(Params._dummy(), "stopWords", "The words to be filtered out",
+                      typeConverter=TypeConverters.toListString)
+    caseSensitive = Param(Params._dummy(), "caseSensitive", "whether to do a case sensitive " +
+                          "comparison over the stop words", typeConverter=TypeConverters.toBoolean)
+    locale = Param(Params._dummy(), "locale", "locale of the input. ignored when case sensitive " +
+                   "is true", typeConverter=TypeConverters.toString)
+
+    def setStopWords(self, value):
+        """Sets the words to be filtered out, by default english stopwords from
+        Spark ML.
+
+        Parameters
+        ----------
+        value : List[str]
+            The words to be filtered out
+        """
+        return self._set(stopWords=value)
+
+    def setCaseSensitive(self, value):
+        """Sets whether to do a case sensitive comparison, by default False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to do a case sensitive comparison
+        """
+        return self._set(caseSensitive=value)
+
+    def setLocale(self, value):
+        """Sets locale of the input. Ignored when case sensitive, by default
+        locale of the JVM.
+
+        Parameters
+        ----------
+        value : str
+            Locale of the input
+        """
+        return self._set(locale=value)
+
+    def loadDefaultStopWords(language="english"):
+        """Loads the default stop words for the given language.
+
+        Supported languages: danish, dutch, english, finnish, french, german,
+        hungarian, italian, norwegian, portuguese, russian, spanish, swedish,
+        turkish
+
+        Parameters
+        ----------
+        language : str, optional
+            Language stopwords to load, by default "english"
+        """
+        from pyspark.ml.wrapper import _jvm
+        stopWordsObj = _jvm().org.apache.spark.ml.feature.StopWordsRemover
+        return list(stopWordsObj.loadDefaultStopWords(language))
+
+    @staticmethod
+    def pretrained(name="stopwords_en", lang="en", remote_loc=None):
+        """Downloads and loads a pretrained model.
+
+        Parameters
+        ----------
+        name : str, optional
+            Name of the pretrained model, by default "stopwords_en"
+        lang : str, optional
+            Language of the pretrained model, by default "en"
+        remote_loc : str, optional
+            Optional remote address of the resource, by default None. Will use
+            Spark NLPs repositories otherwise.
+
+        Returns
+        -------
+        StopWordsCleaner
+            The restored model
+        """
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(StopWordsCleaner, name, lang, remote_loc)

sparknlp/annotator/tf_ner_dl_graph_builder.py

@@ -0,0 +1,179 @@
+from pyspark.ml import Model, Estimator
+from pyspark.ml.util import DefaultParamsWritable, DefaultParamsReadable
+from sparknlp.common import *
+
+
+class TFNerDLGraphBuilderModel(Model, DefaultParamsWritable, DefaultParamsReadable):
+    def _transform(self, dataset):
+        return dataset
+
+
+class TFNerDLGraphBuilder(Estimator, DefaultParamsWritable, DefaultParamsReadable):
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT, AnnotatorType.TOKEN, AnnotatorType.WORD_EMBEDDINGS]
+
+    labelColumn = Param(Params._dummy(),
+                        "labelColumn",
+                        "Labels",
+                        typeConverter=TypeConverters.toString)
+
+    inputCols = Param(Params._dummy(),
+                      "inputCols",
+                      "Input columns",
+                      typeConverter=TypeConverters.toListString)
+
+    graphFolder = Param(Params._dummy(), "graphFolder", "Folder path that contain external graph files",
+                        TypeConverters.toString)
+
+    graphFile = Param(Params._dummy(), "graphFile", "Graph file name. If empty, default name is generated.",
+                      TypeConverters.toString)
+
+    hiddenUnitsNumber = Param(Params._dummy(),
+                              "hiddenUnitsNumber",
+                              "Number of hidden units",
+                              typeConverter=TypeConverters.toInt)
+
+    def setHiddenUnitsNumber(self, value):
+        """Sets the number of hidden units for AssertionDLApproach and MedicalNerApproach
+
+        Parameters
+        ----------
+        value : int
+            Number of hidden units for AssertionDLApproach and MedicalNerApproach
+        """
+        return self._set(hiddenUnitsNumber=value)
+
+    def getHiddenUnitsNumber(self):
+        """Gets the number of hidden units for AssertionDLApproach and MedicalNerApproach."""
+        return self.getOrDefault(self.hiddenUnitsNumber)
+
+    def setLabelColumn(self, value):
+        """Sets the name of the column for data labels.
+
+        Parameters
+        ----------
+        value : str
+            Column for data labels
+        """
+        return self._set(labelColumn=value)
+
+    def getLabelColumn(self):
+        """Gets the name of the label column."""
+        return self.getOrDefault(self.labelColumn)
+
+    def setInputCols(self, *value):
+        """Sets column names of input annotations.
+
+        Parameters
+        ----------
+        *value : List[str]
+            Input columns for the annotator
+        """
+        if type(value[0]) == str or type(value[0]) == list:
+            self.inputColsValidation(value)
+            if len(value) == 1 and type(value[0]) == list:
+                return self._set(inputCols=value[0])
+            else:
+                return self._set(inputCols=list(value))
+        else:
+            raise TypeError("InputCols datatype not supported. It must be either str or list")
+
+    def inputColsValidation(self, value):
+        actual_columns = len(value)
+        if type(value[0]) == list:
+            actual_columns = len(value[0])
+
+        expected_columns = len(self.inputAnnotatorTypes)
+
+        if actual_columns != expected_columns:
+            raise TypeError(
+                f"setInputCols in {self.uid} expecting {expected_columns} columns. "
+                f"Provided column amount: {actual_columns}. "
+                f"Which should be columns from the following annotators: {self.inputAnnotatorTypes}")
+
+    def getInputCols(self):
+        """Gets current column names of input annotations."""
+        return self.getOrDefault(self.inputCols)
+
+    def setGraphFolder(self, value):
+        """Sets folder path that contain external graph files.
+
+        Parameters
+        ----------
+        value : srt
+            Folder path that contain external graph files.
+        """
+        return self._set(graphFolder=value)
+
+    def getGraphFolder(self):
+        """Gets the graph folder."""
+        return self.getOrDefault(self.graphFolder)
+
+    def setGraphFile(self, value):
+        """Sets the graph file name.
+
+        Parameters
+        ----------
+        value : srt
+            Greaph file name. If set to "auto", then the graph builder will use the model specific default graph
+            file name.
+        """
+        return self._set(graphFile=value)
+
+    def getGraphFile(self):
+        """Gets the graph file name."""
+        return self.getOrDefault(self.graphFile)
+
+    def _fit(self, dataset):
+        from ..training.tfgraphs import tf_graph, tf_graph_1x
+
+        build_params = {}
+
+        from sparknlp.internal import _NerDLGraphBuilder
+
+        params_java = _NerDLGraphBuilder(
+            dataset,
+            self.getInputCols(),
+            self.getLabelColumn())._java_obj
+        params = list(map(int, params_java.toString().replace("(", "").replace(")", "").split(",")))
+        build_params["ntags"] = params[0]
+        build_params["embeddings_dim"] = params[1]
+        build_params["nchars"] = params[2]
+        if self.getHiddenUnitsNumber() is not None:
+            build_params["lstm_size"] = self.getHiddenUnitsNumber()
+
+        graph_file = "auto"
+        if self.getGraphFile() is not None:
+            graph_file = self.getGraphFile()
+
+        graph_folder = ""
+        if self.getGraphFolder() is not None:
+            graph_folder = self.getGraphFolder()
+
+        print("Ner DL Graph Builder configuration:")
+        print("Graph folder: {}".format(graph_folder))
+        print("Graph file name: {}".format(graph_file))
+        print("Build params: ", end="")
+        print(build_params)
+
+        try:
+            tf_graph.build("ner_dl", build_params=build_params, model_location=self.getGraphFolder(),
+                           model_filename=graph_file)
+        except Exception:
+            print("Can't build the tensorflow graph with TF 2 graph factory, attempting TF 1.15 factory")
+            try:
+                tf_graph_1x.build("ner_dl", build_params=build_params, model_location=self.getGraphFolder())
+            except Exception:
+                raise Exception("The tensorflow graphs can't be build.")
+
+        return TFNerDLGraphBuilderModel()
+
+    def __init__(self):
+        super(TFNerDLGraphBuilder, self).__init__()
+        self._setDefault(
+            labelColumn=None,
+            inputCols=None,
+            graphFolder=None,
+            graphFile=None,
+            hiddenUnitsNumber=None
+        )

sparknlp/annotator/token/__init__.py

@@ -0,0 +1,19 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Module of annotators for text tokenization."""
+from sparknlp.annotator.token.chunk_tokenizer import *
+from sparknlp.annotator.token.recursive_tokenizer import *
+from sparknlp.annotator.token.regex_tokenizer import *
+from sparknlp.annotator.token.tokenizer import *

sparknlp/annotator/token/chunk_tokenizer.py

@@ -0,0 +1,118 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the ChunkTokenizer."""
+
+from sparknlp.common import *
+from sparknlp.annotator.token.tokenizer import Tokenizer, TokenizerModel
+
+
+class ChunkTokenizer(Tokenizer):
+    """Tokenizes and flattens extracted NER chunks.
+
+    The ChunkTokenizer will split the extracted NER ``CHUNK`` type Annotations
+    and will create ``TOKEN`` type Annotations.
+    The result is then flattened, resulting in a single array.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``CHUNK``              ``TOKEN``
+    ====================== ======================
+
+    Parameters
+    ----------
+    None
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> sparknlp.common import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> sentenceDetector = SentenceDetector() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("sentence")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["sentence"]) \\
+    ...     .setOutputCol("token")
+    >>> entityExtractor = TextMatcher() \\
+    ...     .setInputCols(["sentence", "token"]) \\
+    ...     .setEntities("src/test/resources/entity-extractor/test-chunks.txt", ReadAs.TEXT) \\
+    ...     .setOutputCol("entity")
+    >>> chunkTokenizer = ChunkTokenizer() \\
+    ...     .setInputCols(["entity"]) \\
+    ...     .setOutputCol("chunk_token")
+    >>> pipeline = Pipeline().setStages([
+    ...         documentAssembler,
+    ...         sentenceDetector,
+    ...         tokenizer,
+    ...         entityExtractor,
+    ...         chunkTokenizer
+    ... ])
+    >>> data = spark.createDataFrame([
+    ...     ["Hello world, my name is Michael, I am an artist and I work at Benezar"],
+    ...     ["Robert, an engineer from Farendell, graduated last year. The other one, Lucas, graduated last week."]
+    >>> ]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("entity.result as entity" , "chunk_token.result as chunk_token").show(truncate=False)
+    +-----------------------------------------------+---------------------------------------------------+
+    |entity                                         |chunk_token                                        |
+    +-----------------------------------------------+---------------------------------------------------+
+    |[world, Michael, work at Benezar]              |[world, Michael, work, at, Benezar]                |
+    |[engineer from Farendell, last year, last week]|[engineer, from, Farendell, last, year, last, week]|
+    +-----------------------------------------------+---------------------------------------------------+
+    """
+    name = 'ChunkTokenizer'
+
+    inputAnnotatorTypes = [AnnotatorType.CHUNK]
+
+    @keyword_only
+    def __init__(self):
+        super(Tokenizer, self).__init__(classname="com.johnsnowlabs.nlp.annotators.ChunkTokenizer")
+
+    def _create_model(self, java_model):
+        return ChunkTokenizerModel(java_model=java_model)
+
+
+class ChunkTokenizerModel(TokenizerModel):
+    """Instantiated model of the ChunkTokenizer.
+
+    This is the instantiated model of the :class:`.ChunkTokenizer`.
+    For training your own model, please see the documentation of that class.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``CHUNK``              ``TOKEN``
+    ====================== ======================
+
+    Parameters
+    ----------
+    None
+    """
+    name = 'ChunkTokenizerModel'
+
+    inputAnnotatorTypes = [AnnotatorType.CHUNK]
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.ChunkTokenizerModel", java_model=None):
+        super(TokenizerModel, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+