spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/document_normalizer.py

@@ -0,0 +1,235 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the DocumentNormalizer"""
+from sparknlp.common import *
+
+
+class DocumentNormalizer(AnnotatorModel):
+    """Annotator which normalizes raw text from tagged text, e.g. scraped web
+    pages or xml documents, from document type columns into Sentence.
+
+    Removes all dirty characters from text following one or more input regex
+    patterns. Can apply not wanted character removal with a specific policy.
+    Can apply lower case normalization.
+
+    For extended examples of usage, see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/document-normalizer/document_normalizer_notebook.ipynb
+>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+    action
+        action to perform before applying regex patterns on text, by default
+        "clean"
+    patterns
+        normalization regex patterns which match will be removed from document,
+        by default ['<[^>]*>']
+    replacement
+        replacement string to apply when regexes match, by default " "
+    lowercase
+        whether to convert strings to lowercase, by default False
+    policy
+        policy to remove pattern from text, by default "pretty_all"
+    encoding
+        file encoding to apply on normalized documents, by default "UTF-8"
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> cleanUpPatterns = ["<[^>]>"]
+    >>> documentNormalizer = DocumentNormalizer() \\
+    ...     .setInputCols("document") \\
+    ...     .setOutputCol("normalizedDocument") \\
+    ...     .setAction("clean") \\
+    ...     .setPatterns(cleanUpPatterns) \\
+    ...     .setReplacement(" ") \\
+    ...     .setPolicy("pretty_all") \\
+    ...     .setLowercase(True)
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     documentNormalizer
+    ... ])
+    >>> text = \"\"\"
+    ... <div id="theworldsgreatest" class='my-right my-hide-small my-wide toptext' style="font-family:'Segoe UI',Arial,sans-serif">
+    ...     THE WORLD'S LARGEST WEB DEVELOPER SITE
+    ...     <h1 style="font-size:300%;">THE WORLD'S LARGEST WEB DEVELOPER SITE</h1>
+    ...     <p style="font-size:160%;">Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum..</p>
+    ... </div>
+    ... </div>\"\"\"
+    >>> data = spark.createDataFrame([[text]]).toDF("text")
+    >>> pipelineModel = pipeline.fit(data)
+    >>> result = pipelineModel.transform(data)
+    >>> result.selectExpr("normalizedDocument.result").show(truncate=False)
+    +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |result                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+    +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |[ the world's largest web developer site the world's largest web developer site lorem ipsum is simply dummy text of the printing and typesetting industry. lorem ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. it has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. it was popularised in the 1960s with the release of letraset sheets containing lorem ipsum passages, and more recently with desktop publishing software like aldus pagemaker including versions of lorem ipsum..]|
+    +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    """
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    action = Param(Params._dummy(),
+                   "action",
+                   "action to perform applying regex patterns on text",
+                   typeConverter=TypeConverters.toString)
+
+    patterns = Param(Params._dummy(),
+                     "patterns",
+                     "normalization regex patterns which match will be removed from document. Defaults is <[^>]*>",
+                     typeConverter=TypeConverters.toListString)
+
+    replacement = Param(Params._dummy(),
+                        "replacement",
+                        "replacement string to apply when regexes match",
+                        typeConverter=TypeConverters.toString)
+
+    lowercase = Param(Params._dummy(),
+                      "lowercase",
+                      "whether to convert strings to lowercase",
+                      typeConverter=TypeConverters.toBoolean)
+
+    policy = Param(Params._dummy(),
+                   "policy",
+                   "policy to remove pattern from text",
+                   typeConverter=TypeConverters.toString)
+
+    encoding = Param(Params._dummy(),
+                     "encoding",
+                     "file encoding to apply on normalized documents",
+                     typeConverter=TypeConverters.toString)
+
+    presetPattern = Param(
+        Params._dummy(),
+        "presetPattern",
+        "Selects a single text cleaning function from the functional presets (e.g., 'CLEAN_BULLETS', 'CLEAN_DASHES', etc.).",
+        typeConverter=TypeConverters.toString
+    )
+
+    autoMode = Param(
+        Params._dummy(),
+        "autoMode",
+        "Enables a predefined cleaning mode combining multiple text cleaner functions (e.g., 'light_clean', 'document_clean', 'html_clean', 'full_auto').",
+        typeConverter=TypeConverters.toString
+    )
+
+
+    @keyword_only
+    def __init__(self):
+        super(DocumentNormalizer, self).__init__(classname="com.johnsnowlabs.nlp.annotators.DocumentNormalizer")
+        self._setDefault(
+            action="clean",
+            patterns=["<[^>]*>"],
+            replacement=" ",
+            lowercase=False,
+            policy="pretty_all",
+            encoding="UTF-8"
+        )
+
+    def setAction(self, value):
+        """Sets action to perform before applying regex patterns on text, by
+        default "clean".
+
+        Parameters
+        ----------
+        value : str
+            Action to perform before applying regex patterns
+        """
+        return self._set(action=value)
+
+    def setPatterns(self, value):
+        """Sets normalization regex patterns which match will be removed from
+        document, by default ['<[^>]*>'].
+
+        Parameters
+        ----------
+        value : List[str]
+            Normalization regex patterns which match will be removed from
+            document
+        """
+        return self._set(patterns=value)
+
+    def setReplacement(self, value):
+        """Sets replacement string to apply when regexes match, by default " ".
+
+        Parameters
+        ----------
+        value : str
+            Replacement string to apply when regexes match
+        """
+        return self._set(replacement=value)
+
+    def setLowercase(self, value):
+        """Sets whether to convert strings to lowercase, by default False.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to convert strings to lowercase, by default False
+        """
+        return self._set(lowercase=value)
+
+    def setPolicy(self, value):
+        """Sets policy to remove pattern from text, by default "pretty_all".
+
+        Parameters
+        ----------
+        value : str
+            Policy to remove pattern from text, by default "pretty_all"
+        """
+        return self._set(policy=value)
+
+    def setEncoding(self, value):
+        """Sets file encoding to apply on normalized documents, by default
+        "UTF-8".
+
+        Parameters
+        ----------
+        value : str
+            File encoding to apply on normalized documents, by default "UTF-8"
+        """
+        return self._set(encoding=value)
+
+    def setPresetPattern(self, value):
+        """Sets a single text cleaning preset pattern.
+
+        Parameters
+        ----------
+        value : str
+            Preset cleaning pattern name, e.g., 'CLEAN_BULLETS', 'CLEAN_DASHES'.
+        """
+        return self._set(presetPattern=value)
+
+
+    def setAutoMode(self, value):
+        """Sets an automatic text cleaning mode using predefined groups of cleaning functions.
+
+        Parameters
+        ----------
+        value : str
+            Auto cleaning mode, e.g., 'light_clean', 'document_clean', 'social_clean', 'html_clean', 'full_auto'.
+        """
+        return self._set(autoMode=value)

sparknlp/annotator/document_token_splitter.py

@@ -0,0 +1,175 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the DocumentNormalizer"""
+from sparknlp.common import *
+
+
+class DocumentTokenSplitter(AnnotatorModel):
+    """Annotator that splits large documents into smaller documents based on the number of tokens in
+    the text.
+
+    Currently, DocumentTokenSplitter splits the text by whitespaces to create the tokens. The
+    number of these tokens will then be used as a measure of the text length. In the future, other
+    tokenization techniques will be supported.
+
+    For example, given 3 tokens and overlap 1:
+
+    .. code-block:: python
+
+        He was, I take it, the most perfect reasoning and observing machine that the world has seen.
+
+        ["He was, I", "I take it,", "it, the most", "most perfect reasoning", "reasoning and observing", "observing machine that", "that the world", "world has seen."]
+
+
+    Additionally, you can set
+
+      - whether to trim whitespaces with setTrimWhitespace
+      - whether to explode the splits to individual rows with setExplodeSplits
+
+    For extended examples of usage, see the
+    `DocumentTokenSplitterTest <https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/DocumentTokenSplitterTest.scala>`__.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT``           ``DOCUMENT``
+    ====================== ======================
+
+    Parameters
+    ----------
+
+    numTokens
+        Limit of the number of tokens in a text
+    tokenOverlap
+        Length of the token overlap between text chunks, by default `0`.
+    explodeSplits
+        Whether to explode split chunks to separate rows, by default `False`.
+    trimWhitespace
+        Whether to trim whitespaces of extracted chunks, by default `True`.
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> textDF = spark.read.text(
+    ...    "sherlockholmes.txt",
+    ...    wholetext=True
+    ... ).toDF("text")
+    >>> documentAssembler = DocumentAssembler().setInputCol("text")
+    >>> textSplitter = DocumentTokenSplitter() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("splits") \\
+    ...     .setNumTokens(512) \\
+    ...     .setTokenOverlap(10) \\
+    ...     .setExplodeSplits(True)
+    >>> pipeline = Pipeline().setStages([documentAssembler, textSplitter])
+    >>> result = pipeline.fit(textDF).transform(textDF)
+    >>> result.selectExpr(
+    ...       "splits.result as result",
+    ...       "splits[0].begin as begin",
+    ...       "splits[0].end as end",
+    ...       "splits[0].end - splits[0].begin as length",
+    ...       "splits[0].metadata.numTokens as tokens") \\
+    ...     .show(8, truncate = 80)
+    +--------------------------------------------------------------------------------+-----+-----+------+------+
+    |                                                                          result|begin|  end|length|tokens|
+    +--------------------------------------------------------------------------------+-----+-----+------+------+
+    |[ Project Gutenberg's The Adventures of Sherlock Holmes, by Arthur Conan Doyl...|    0| 3018|  3018|   512|
+    |[study of crime, and occupied his\\nimmense faculties and extraordinary powers...| 2950| 5707|  2757|   512|
+    |[but as I have changed my clothes I can't imagine how you\\ndeduce it. As to M...| 5659| 8483|  2824|   512|
+    |[quarters received. Be in your chamber then at that hour, and do\\nnot take it...| 8427|11241|  2814|   512|
+    |[a pity\\nto miss it."\\n\\n"But your client--"\\n\\n"Never mind him. I may want y...|11188|13970|  2782|   512|
+    |[person who employs me wishes his agent to be unknown to\\nyou, and I may conf...|13918|16898|  2980|   512|
+    |[letters back."\\n\\n"Precisely so. But how--"\\n\\n"Was there a secret marriage?...|16836|19744|  2908|   512|
+    |[seven hundred in\\nnotes," he said.\\n\\nHolmes scribbled a receipt upon a shee...|19683|22551|  2868|   512|
+    +--------------------------------------------------------------------------------+-----+-----+------+------+
+
+    """
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+
+    outputAnnotatorType = AnnotatorType.DOCUMENT
+
+    numTokens = Param(Params._dummy(),
+                      "numTokens",
+                      "Limit of the number of tokens in a text",
+                      typeConverter=TypeConverters.toInt)
+    tokenOverlap = Param(Params._dummy(),
+                         "tokenOverlap",
+                         "Length of the token overlap between text chunks",
+                         typeConverter=TypeConverters.toInt)
+    explodeSplits = Param(Params._dummy(),
+                          "explodeSplits",
+                          "Whether to explode split chunks to separate rows",
+                          typeConverter=TypeConverters.toBoolean)
+    trimWhitespace = Param(Params._dummy(),
+                           "trimWhitespace",
+                           "Whether to trim whitespaces of extracted chunks",
+                           typeConverter=TypeConverters.toBoolean)
+
+    @keyword_only
+    def __init__(self):
+        super(DocumentTokenSplitter, self).__init__(
+            classname="com.johnsnowlabs.nlp.annotators.DocumentTokenSplitter")
+        self._setDefault(
+            tokenOverlap=0,
+            explodeSplits=False,
+            trimWhitespace=True
+        )
+
+    def setNumTokens(self, value):
+        """Sets the limit of the number of tokens in a text
+
+        Parameters
+        ----------
+        value : int
+            Number of tokens in a text
+        """
+        if value < 1:
+            raise ValueError("Number of tokens should be larger than 0.")
+        return self._set(numTokens=value)
+
+    def setTokenOverlap(self, value):
+        """Length of the token overlap between text chunks, by default `0`.
+
+        Parameters
+        ----------
+        value : int
+            Length of the token overlap between text chunks
+        """
+        if value > self.getOrDefault(self.numTokens):
+            raise ValueError("Token overlap can't be larger than number of tokens.")
+        return self._set(tokenOverlap=value)
+
+    def setExplodeSplits(self, value):
+        """Sets whether to explode split chunks to separate rows, by default `False`.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to explode split chunks to separate rows
+        """
+        return self._set(explodeSplits=value)
+
+    def setTrimWhitespace(self, value):
+        """Sets whether to trim whitespaces of extracted chunks, by default `True`.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to trim whitespaces of extracted chunks
+        """
+        return self._set(trimWhitespace=value)

sparknlp/annotator/document_token_splitter_test.py

@@ -0,0 +1,85 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import unittest
+
+import pytest
+
+from sparknlp.annotator import *
+from sparknlp.base import *
+from test.util import SparkSessionForTest
+
+
+@pytest.mark.fast
+class DocumentTokenSplitterTestSpec(unittest.TestCase):
+    def setUp(self):
+        self.data = SparkSessionForTest.spark.createDataFrame(
+            [
+                [
+                    (
+                        "All emotions, and that\none particularly, were abhorrent to his cold, precise"
+                        " but\nadmirably balanced mind.\n\nHe was, I take it, the most perfect\nreasoning"
+                        " and observing machine that the world has seen."
+                    )
+                ]
+            ]
+        ).toDF("text")
+
+    def test_run(self):
+        df = self.data
+
+        document_assembler = (
+            DocumentAssembler().setInputCol("text").setOutputCol("document")
+        )
+
+        document_token_splitter = (
+            DocumentTokenSplitter()
+            .setInputCols("document")
+            .setOutputCol("splits")
+            .setNumTokens(3)
+            .setTokenOverlap(1)
+            .setExplodeSplits(True)
+            .setTrimWhitespace(True)
+        )
+
+        pipeline = Pipeline().setStages([document_assembler, document_token_splitter])
+
+        pipeline_df = pipeline.fit(df).transform(df)
+
+        results = pipeline_df.select("splits").collect()
+
+        splits = [
+            row["splits"][0].result.replace("\n\n", " ").replace("\n", " ")
+            for row in results
+        ]
+
+        expected = [
+            "All emotions, and",
+            "and that one",
+            "one particularly, were",
+            "were abhorrent to",
+            "to his cold,",
+            "cold, precise but",
+            "but admirably balanced",
+            "balanced mind. He",
+            "He was, I",
+            "I take it,",
+            "it, the most",
+            "most perfect reasoning",
+            "reasoning and observing",
+            "observing machine that",
+            "that the world",
+            "world has seen.",
+        ]
+
+        assert splits == expected

sparknlp/annotator/embeddings/__init__.py

@@ -0,0 +1,45 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Module of annotators for text embeddings."""
+from sparknlp.annotator.embeddings.albert_embeddings import *
+from sparknlp.annotator.embeddings.bert_embeddings import *
+from sparknlp.annotator.embeddings.bert_sentence_embeddings import *
+from sparknlp.annotator.embeddings.camembert_embeddings import *
+from sparknlp.annotator.embeddings.chunk_embeddings import *
+from sparknlp.annotator.embeddings.deberta_embeddings import *
+from sparknlp.annotator.embeddings.distil_bert_embeddings import *
+from sparknlp.annotator.embeddings.doc2vec import *
+from sparknlp.annotator.embeddings.elmo_embeddings import *
+from sparknlp.annotator.embeddings.e5_embeddings import *
+from sparknlp.annotator.embeddings.instructor_embeddings import *
+from sparknlp.annotator.embeddings.longformer_embeddings import *
+from sparknlp.annotator.embeddings.minilm_embeddings import *
+from sparknlp.annotator.embeddings.mpnet_embeddings import *
+from sparknlp.annotator.embeddings.roberta_embeddings import *
+from sparknlp.annotator.embeddings.roberta_sentence_embeddings import *
+from sparknlp.annotator.embeddings.sentence_embeddings import *
+from sparknlp.annotator.embeddings.universal_sentence_encoder import *
+from sparknlp.annotator.embeddings.word2vec import *
+from sparknlp.annotator.embeddings.word_embeddings import *
+from sparknlp.annotator.embeddings.xlm_roberta_embeddings import *
+from sparknlp.annotator.embeddings.xlm_roberta_sentence_embeddings import *
+from sparknlp.annotator.embeddings.xlnet_embeddings import *
+from sparknlp.annotator.embeddings.bge_embeddings import *
+from sparknlp.annotator.embeddings.uae_embeddings import *
+from sparknlp.annotator.embeddings.mxbai_embeddings import *
+from sparknlp.annotator.embeddings.snowflake_embeddings import *
+from sparknlp.annotator.embeddings.nomic_embeddings import *
+from sparknlp.annotator.embeddings.auto_gguf_embeddings import *
+from sparknlp.annotator.embeddings.e5v_embeddings import *