spark-nlp 4.2.6__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/er/entity_ruler.py

@@ -27,9 +27,6 @@ class EntityRulerApproach(AnnotatorApproach, HasStorage):
     to be set as the "format" field in the ``option`` parameter map and
     depending on the file type, additional parameters might need to be set.
 
-    To enable regex extraction, ``setEnablePatternRegex(True)`` needs to be
-    called.
-
     If the file is in a JSON format, then the rule definitions need to be given
     in a list with the fields "id", "label" and "patterns"::
 
@@ -71,8 +68,6 @@ class EntityRulerApproach(AnnotatorApproach, HasStorage):
     ----------
     patternsResource
         Resource in JSON or CSV format to map entities to patterns
-    enablePatternRegex
-        Enables regex pattern match
     useStorage
         Whether to use RocksDB storage to serialize patterns
 
@@ -106,8 +101,7 @@ class EntityRulerApproach(AnnotatorApproach, HasStorage):
     ...       "patterns.csv",
     ...       ReadAs.TEXT,
     ...       {"format": "csv", "delimiter": "\\\\|"}
-    ...     ) \\
-    ...     .setEnablePatternRegex(True)
+    ...     )
     >>> pipeline = Pipeline().setStages([
     ...     documentAssembler,
     ...     tokenizer,
@@ -135,11 +129,6 @@ class EntityRulerApproach(AnnotatorApproach, HasStorage):
                              "Resource in JSON or CSV format to map entities to patterns",
                              typeConverter=TypeConverters.identity)
 
-    enablePatternRegex = Param(Params._dummy(),
-                               "enablePatternRegex",
-                               "Enables regex pattern match",
-                               typeConverter=TypeConverters.toBoolean)
-
     useStorage = Param(Params._dummy(),
                        "useStorage",
                        "Whether to use RocksDB storage to serialize patterns",
@@ -174,16 +163,6 @@ class EntityRulerApproach(AnnotatorApproach, HasStorage):
         """
         return self._set(patternsResource=ExternalResource(path, read_as, options))
 
-    def setEnablePatternRegex(self, value):
-        """Sets whether to enable regex pattern matching.
-
-        Parameters
-        ----------
-        value : bool
-            Whether to enable regex pattern matching.
-        """
-        return self._set(enablePatternRegex=value)
-
     def setUseStorage(self, value):
         """Sets whether to use RocksDB storage to serialize patterns.
 
@@ -236,6 +215,20 @@ class EntityRulerModel(AnnotatorModel, HasStorageModel):
 
     outputAnnotatorType = AnnotatorType.CHUNK
 
+    autoMode = Param(
+        Params._dummy(),
+        "autoMode",
+        "Enable built-in regex presets that combine related entity patterns (e.g., 'communication_entities', 'network_entities', 'media_entities', etc.).",
+        typeConverter=TypeConverters.toString
+    )
+
+    extractEntities = Param(
+        Params._dummy(),
+        "extractEntities",
+        "List of entity types to extract. If not set, all entities in the active autoMode or from regexPatterns are used.",
+        typeConverter=TypeConverters.toListString
+    )
+
     def __init__(self, classname="com.johnsnowlabs.nlp.annotators.er.EntityRulerModel", java_model=None):
         super(EntityRulerModel, self).__init__(
             classname=classname,
@@ -249,5 +242,26 @@ class EntityRulerModel(AnnotatorModel, HasStorageModel):
 
     @staticmethod
     def loadStorage(path, spark, storage_ref):
-        HasStorageModel.loadStorages(path, spark, storage_ref, EntityRulerModel.databases)
+        HasStorageModel.loadStorages(path, spark, storage_ref, EntityRulerModel.database)
+
 
+    def setAutoMode(self, value):
+        """Sets the auto mode for predefined regex entity groups.
+
+        Parameters
+        ----------
+        value : str
+            Name of the auto mode to activate (e.g., 'communication_entities', 'network_entities', etc.)
+        """
+        return self._set(autoMode=value)
+
+
+    def setExtractEntities(self, value):
+        """Sets specific entities to extract, filtering only those defined in regexPatterns or autoMode.
+
+        Parameters
+        ----------
+        value : list[str]
+            List of entity names to extract, e.g., ['EMAIL_ADDRESS_PATTERN', 'IPV4_PATTERN'].
+        """
+        return self._set(extractEntities=value)

sparknlp/annotator/keyword_extraction/yake_keyword_extraction.py

@@ -44,8 +44,8 @@ class YakeKeywordExtraction(AnnotatorModel):
     lower the score better the keyword). Therefore to filter the keywords, an
     upper bound for the score can be set with :meth:`.setThreshold`.
 
-    For extended examples of usage, see the `Spark NLP Workshop
-    <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/8.Keyword_Extraction_YAKE.ipynb>`__.
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/keyword-extraction/Keyword_Extraction_YAKE.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -268,4 +268,3 @@ class YakeKeywordExtraction(AnnotatorModel):
         from pyspark.ml.wrapper import _jvm
         stopWordsObj = _jvm().org.apache.spark.ml.feature.StopWordsRemover
         return list(stopWordsObj.loadDefaultStopWords(language))
-

sparknlp/annotator/ld_dl/language_detector_dl.py

@@ -37,9 +37,9 @@ class LanguageDetectorDL(AnnotatorModel, HasStorageRef, HasEngine):
     The default model is ``"ld_wiki_tatoeba_cnn_21"``, default language is
     ``"xx"`` (meaning multi-lingual), if no values are provided.
 
-    For available pretrained models please see the `Models Hub <https://nlp.johnsnowlabs.com/models?task=Language+Detection>`__.
+    For available pretrained models please see the `Models Hub <https://sparknlp.org/models?task=Language+Detection>`__.
 
-    For extended examples of usage, see the `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/language-detection/Language_Detection_and_Indentification.ipynb>`__.
+    For extended examples of usage, see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/language-detection/Language_Detection_and_Indentification.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type

sparknlp/annotator/lemmatizer.py

@@ -24,8 +24,8 @@ class Lemmatizer(AnnotatorApproach):
 
     For instantiated/pretrained models, see :class:`.LemmatizerModel`.
 
-    For available pretrained models please see the `Models Hub <https://nlp.johnsnowlabs.com/models?task=Lemmatization>`__.
-    For extended examples of usage, see the `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb>`__.
+    For available pretrained models please see the `Models Hub <https://sparknlp.org/models?task=Lemmatization>`__.
+    For extended examples of usage, see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/training/italian/Train-Lemmatizer-Italian.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -194,7 +194,7 @@ class LemmatizerModel(AnnotatorModel):
     ...     .setInputCols(["token"]) \\
     ...     .setOutputCol("lemma")
 
-    For available pretrained models please see the `Models Hub <https://nlp.johnsnowlabs.com/models?task=Lemmatization>`__.
+    For available pretrained models please see the `Models Hub <https://sparknlp.org/models?task=Lemmatization>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -248,4 +248,3 @@ class LemmatizerModel(AnnotatorModel):
         """
         from sparknlp.pretrained import ResourceDownloader
         return ResourceDownloader.downloadModel(LemmatizerModel, name, lang, remote_loc)
-

sparknlp/annotator/matcher/date_matcher.py

@@ -67,6 +67,16 @@ class DateMatcherUtils(Params):
                            "source language for explicit translation",
                            typeConverter=TypeConverters.toString)
 
+    relaxedFactoryStrategy = Param(Params._dummy(),
+                                   "relaxedFactoryStrategy",
+                                   "Matched Strategy to searches relaxed dates",
+                                   typeConverter=TypeConverters.toString)
+
+    aggressiveMatching = Param(Params._dummy(),
+                               "aggressiveMatching",
+                               "Whether to aggressively attempt to find date matches, even in ambiguous or less common formats",
+                               typeConverter=TypeConverters.toBoolean)
+
     def setInputFormats(self, value):
         """Sets input formats patterns to match in the documents.
 
@@ -159,6 +169,29 @@ class DateMatcherUtils(Params):
         """
         return self._set(anchorDateDay=value)
 
+    def setRelaxedFactoryStrategy(self, matchStrategy=MatchStrategy.MATCH_FIRST):
+        """ Sets matched strategy to search relaxed dates by ordered rules by more exhaustive to less Strategy.
+
+        Not all of the date information needs to be included. For example
+        ``"YYYY"`` is also a valid input.
+
+        Parameters
+        ----------
+        matchStrategy : MatchStrategy
+            Matched strategy to search relaxed dates by ordered rules by more exhaustive to less Strategy
+        """
+        return self._set(relaxedFactoryStrategy=matchStrategy)
+
+    def setAggressiveMatching(self, value):
+        """ Sets whether to aggressively attempt to find date matches, even in ambiguous or less common formats
+
+        Parameters
+        ----------
+        aggressiveMatching : Boolean
+            Whether to aggressively attempt to find date matches, even in ambiguous or less common formats
+        """
+        return self._set(aggressiveMatching=value)
+
 
 class DateMatcher(AnnotatorModel, DateMatcherUtils):
     """Matches standard date formats into a provided format
@@ -184,10 +217,10 @@ class DateMatcher(AnnotatorModel, DateMatcherUtils):
     ``2008/04/31``.
 
     Pretrained pipelines are available for this module, see
-    `Pipelines <https://nlp.johnsnowlabs.com/docs/en/pipelines>`__.
+    `Pipelines <https://sparknlp.org/docs/en/pipelines>`__.
 
     For extended examples of usage, see the
-    `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb>`__.
+    `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/MultiDateMatcherMultiLanguage_en.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -268,4 +301,3 @@ class DateMatcher(AnnotatorModel, DateMatcherUtils):
             anchorDateMonth=-1,
             anchorDateDay=-1
         )
-

sparknlp/annotator/matcher/multi_date_matcher.py

@@ -33,7 +33,7 @@ class MultiDateMatcher(AnnotatorModel, DateMatcherUtils):
     For example ``"The 31st of April in the year 2008"`` will be converted into
     ``2008/04/31``.
 
-    For extended examples of usage, see the `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb>`__.
+    For extended examples of usage, see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/MultiDateMatcherMultiLanguage_en.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -107,4 +107,3 @@ class MultiDateMatcher(AnnotatorModel, DateMatcherUtils):
             readMonthFirst=True,
             defaultDayWhenMissing=1
         )
-

sparknlp/annotator/matcher/regex_matcher.py

@@ -32,10 +32,10 @@ class RegexMatcher(AnnotatorApproach):
     delimited text file.
 
     Pretrained pipelines are available for this module, see `Pipelines
-    <https://nlp.johnsnowlabs.com/docs/en/pipelines>`__.
+    <https://sparknlp.org/docs/en/pipelines>`__.
 
-    For extended examples of usage, see the `Spark NLP Workshop
-    <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb>`__.
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/regex-matcher/Matching_Text_with_RegexMatcher.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type

sparknlp/annotator/matcher/text_matcher.py

@@ -24,8 +24,8 @@ class TextMatcher(AnnotatorApproach):
     A text file of predefined phrases must be provided with
     :meth:`.setEntities`.
 
-    For extended examples of usage, see the `Spark NLP Workshop
-    <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb>`__.
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/text-matcher-pipeline/extractor.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -288,4 +288,3 @@ class TextMatcherModel(AnnotatorModel):
         """
         from sparknlp.pretrained import ResourceDownloader
         return ResourceDownloader.downloadModel(TextMatcherModel, name, lang, remote_loc)
-

sparknlp/annotator/n_gram_generator.py

@@ -27,7 +27,7 @@ class NGramGenerator(AnnotatorModel):
     length is less than n (number of elements per n-gram), no n-grams are
     returned.
 
-    For more extended examples see the `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/chunking/NgramGenerator.ipynb>`__.
+    For more extended examples see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/chunking/NgramGenerator.ipynb>`__.
 
     ====================== ======================
     Input Annotation types Output Annotation type
@@ -139,4 +139,3 @@ class NGramGenerator(AnnotatorModel):
         if len(value) > 1:
             raise Exception("Delimiter should have length == 1")
         return self._set(delimiter=value)
-

sparknlp/annotator/ner/__init__.py

@@ -1,4 +1,4 @@
-#  Copyright 2017-2022 John Snow Labs
+#  Copyright 2017-2023 John Snow Labs
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -16,4 +16,6 @@ from sparknlp.annotator.ner.ner_approach import *
 from sparknlp.annotator.ner.ner_converter import *
 from sparknlp.annotator.ner.ner_crf import *
 from sparknlp.annotator.ner.ner_dl import *
+from sparknlp.annotator.ner.ner_dl_graph_checker import *
 from sparknlp.annotator.ner.ner_overwriter import *
+from sparknlp.annotator.ner.zero_shot_ner_model import *

sparknlp/annotator/ner/ner_converter.py

@@ -98,6 +98,13 @@ class NerConverter(AnnotatorModel):
         typeConverter=TypeConverters.toBoolean
     )
 
+    nerHasNoSchema = Param(
+        Params._dummy(),
+        "nerHasNoSchema",
+        "set this to true if your NER tags coming from a model that does not have a IOB/IOB2 schema",
+        typeConverter=TypeConverters.toBoolean
+    )
+
     def setWhiteList(self, entities):
         """Sets list of entities to process. The rest will be ignored.
 
@@ -124,6 +131,17 @@ class NerConverter(AnnotatorModel):
         """
         return self._set(preservePosition=value)
 
+    def setNerHasNoSchema(self, value):
+        """
+        set this to true if your NER tags coming from a model that does not have a IOB/IOB2 schema
+
+        Parameters
+        ----------
+        value : bool
+            set this to true if your NER tags coming from a model that does not have a IOB/IOB2 schema
+        """
+        return self._set(nerHasNoSchema=value)
+
     @keyword_only
     def __init__(self):
         super(NerConverter, self).__init__(

sparknlp/annotator/ner/ner_crf.py

@@ -39,7 +39,7 @@ class NerCrfApproach(AnnotatorApproach, NerApproach):
     Optionally the user can provide an entity dictionary file with
     :meth:`.setExternalFeatures` for better accuracy.
 
-    For extended examples of usage, see the `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/training/english/crf-ner/ner_dl_crf.ipynb>`__.
+    For extended examples of usage, see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/training/english/crf-ner/ner_dl_crf.ipynb>`__.
 
     ========================================= ======================
     Input Annotation types                    Output Annotation type
@@ -278,10 +278,10 @@ class NerCrfModel(AnnotatorModel):
 
     The default model is ``"ner_crf"``, if no name is provided. For available
     pretrained models please see the `Models Hub
-    <https://nlp.johnsnowlabs.com/models?task=Named+Entity+Recognition>`__.
+    <https://sparknlp.org/models?task=Named+Entity+Recognition>`__.
 
-    For extended examples of usage, see the `Spark NLP Workshop
-    <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/jupyter/annotation/english/model-downloader/Running_Pretrained_pipelines.ipynb>`__.
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/training/english/crf-ner/ner_dl_crf.ipynb>`__.
 
     ========================================= ======================
     Input Annotation types                    Output Annotation type
@@ -395,4 +395,3 @@ class NerCrfModel(AnnotatorModel):
         """
         from sparknlp.pretrained import ResourceDownloader
         return ResourceDownloader.downloadModel(NerCrfModel, name, lang, remote_loc)
-

sparknlp/annotator/ner/ner_dl.py

@@ -41,6 +41,11 @@ class NerDLApproach(AnnotatorApproach, NerApproach, EvaluationDLParams):
     - a WordEmbeddingsModel (any embeddings can be chosen, e.g. BertEmbeddings
       for BERT based embeddings).
 
+    By default, collects all data points into memory for training. For larger datasets, use
+    ``setEnableMemoryOptimizer(true)``. This will optimize memory usage during training at the cost
+    of speed. Note that this annotator will use as much memory as the largest partition of the
+    input dataset, so we recommend repartitioning to batch sizes.
+
     Setting a test dataset to monitor model metrics can be done with
     ``.setTestDataset``. The method expects a path to a parquet file containing a
     dataframe that has the same required columns as the training dataframe. The
@@ -72,7 +77,7 @@ class NerDLApproach(AnnotatorApproach, NerApproach, EvaluationDLParams):
     ...     .setOutputCol("ner") \\
     ...     .setTestDataset("test_data")
 
-    For extended examples of usage, see the `Spark NLP Workshop <https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/jupyter/training/english/dl-ner>`__.
+    For extended examples of usage, see the `Examples <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/training/english/dl-ner>`__.
 
     ==================================== ======================
     Input Annotation types               Output Annotation type
@@ -420,16 +425,16 @@ class NerDLModel(AnnotatorModel, HasStorageRef, HasBatchedAnnotate, HasEngine):
     The default model is ``"ner_dl"``, if no name is provided.
 
     For available pretrained models please see the `Models Hub
-    <https://nlp.johnsnowlabs.com/models?task=Named+Entity+Recognition>`__.
+    <https://sparknlp.org/models?task=Named+Entity+Recognition>`__.
     Additionally, pretrained pipelines are available for this module, see
-    `Pipelines <https://nlp.johnsnowlabs.com/docs/en/pipelines>`__.
+    `Pipelines <https://sparknlp.org/docs/en/pipelines>`__.
 
     Note that some pretrained models require specific types of embeddings,
     depending on which they were trained on. For example, the default model
     ``"ner_dl"`` requires the WordEmbeddings ``"glove_100d"``.
 
-    For extended examples of usage, see the `Spark NLP Workshop
-    <https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/3.SparkNLP_Pretrained_Models.ipynb>`__.
+    For extended examples of usage, see the `Examples
+    <https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/model-downloader/Create%20custom%20pipeline%20-%20NerDL.ipynb>`__.
 
     ==================================== ======================
     Input Annotation types               Output Annotation type