spark-nlp 2.6.3rc1__py2.py3-none-any.whl → 6.2.1__py2.py3-none-any.whl

sparknlp/annotator/er/entity_ruler.py

@@ -0,0 +1,267 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for the EntityRuler."""
+
+from sparknlp.common import *
+
+
+class EntityRulerApproach(AnnotatorApproach, HasStorage):
+    """Fits an Annotator to match exact strings or regex patterns provided in a
+    file against a Document and assigns them an named entity. The definitions
+    can contain any number of named entities.
+
+    There are multiple ways and formats to set the extraction resource. It is
+    possible to set it either as a "JSON", "JSONL" or "CSV" file. A path to the
+    file needs to be provided to ``setPatternsResource``. The file format needs
+    to be set as the "format" field in the ``option`` parameter map and
+    depending on the file type, additional parameters might need to be set.
+
+    If the file is in a JSON format, then the rule definitions need to be given
+    in a list with the fields "id", "label" and "patterns"::
+
+         [
+            {
+              "id": "person-regex",
+              "label": "PERSON",
+              "patterns": ["\\w+\\s\\w+", "\\w+-\\w+"]
+            },
+            {
+              "id": "locations-words",
+              "label": "LOCATION",
+              "patterns": ["Winterfell"]
+            }
+        ]
+
+    The same fields also apply to a file in the JSONL format::
+
+        {"id": "names-with-j", "label": "PERSON", "patterns": ["Jon", "John", "John Snow"]}
+        {"id": "names-with-s", "label": "PERSON", "patterns": ["Stark", "Snow"]}
+        {"id": "names-with-e", "label": "PERSON", "patterns": ["Eddard", "Eddard Stark"]}
+
+    In order to use a CSV file, an additional parameter "delimiter" needs to be
+    set. In this case, the delimiter might be set by using
+    ``.setPatternsResource("patterns.csv", ReadAs.TEXT, {"format": "csv", "delimiter": "|")})``::
+
+        PERSON|Jon
+        PERSON|John
+        PERSON|John Snow
+        LOCATION|Winterfell
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT, TOKEN``    ``CHUNK``
+    ====================== ======================
+
+    Parameters
+    ----------
+    patternsResource
+        Resource in JSON or CSV format to map entities to patterns
+    useStorage
+        Whether to use RocksDB storage to serialize patterns
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from sparknlp.common import *
+    >>> from pyspark.ml import Pipeline
+
+    In this example, the entities file as the form of::
+
+        PERSON|Jon
+        PERSON|John
+        PERSON|John Snow
+        LOCATION|Winterfell
+
+    where each line represents an entity and the associated string delimited by "|".
+
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("token")
+    >>> entityRuler = EntityRulerApproach() \\
+    ...     .setInputCols(["document", "token"]) \\
+    ...     .setOutputCol("entities") \\
+    ...     .setPatternsResource(
+    ...       "patterns.csv",
+    ...       ReadAs.TEXT,
+    ...       {"format": "csv", "delimiter": "\\\\|"}
+    ...     )
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     tokenizer,
+    ...     entityRuler
+    ... ])
+    >>> data = spark.createDataFrame([["Jon Snow wants to be lord of Winterfell."]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.selectExpr("explode(entities)").show(truncate=False)
+    +--------------------------------------------------------------------+
+    |col                                                                 |
+    +--------------------------------------------------------------------+
+    |[chunk, 0, 2, Jon, [entity -> PERSON, sentence -> 0], []]           |
+    |[chunk, 29, 38, Winterfell, [entity -> LOCATION, sentence -> 0], []]|
+    +--------------------------------------------------------------------+
+    """
+    name = "EntityRulerApproach"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+    optionalInputAnnotatorTypes = [AnnotatorType.TOKEN]
+
+    outputAnnotatorType = AnnotatorType.CHUNK
+
+    patternsResource = Param(Params._dummy(),
+                             "patternsResource",
+                             "Resource in JSON or CSV format to map entities to patterns",
+                             typeConverter=TypeConverters.identity)
+
+    useStorage = Param(Params._dummy(),
+                       "useStorage",
+                       "Whether to use RocksDB storage to serialize patterns",
+                       typeConverter=TypeConverters.toBoolean)
+
+    sentenceMatch = Param(Params._dummy(),
+                          "sentenceMatch",
+                          "Whether to find match at sentence level. True: sentence level. False: token level",
+                          typeConverter=TypeConverters.toBoolean)
+
+    alphabet = Param(Params._dummy(),
+                     "alphabet",
+                     "Alphabet resource path to plain text file with all characters in a given alphabet",
+                     typeConverter=TypeConverters.identity)
+
+    @keyword_only
+    def __init__(self):
+        super(EntityRulerApproach, self).__init__(
+            classname="com.johnsnowlabs.nlp.annotators.er.EntityRulerApproach")
+
+    def setPatternsResource(self, path, read_as=ReadAs.TEXT, options={"format": "JSON"}):
+        """Sets Resource in JSON or CSV format to map entities to patterns.
+
+        Parameters
+        ----------
+        path : str
+            Path to the resource
+        read_as : str, optional
+            How to interpret the resource, by default ReadAs.TEXT
+        options : dict, optional
+            Options for parsing the resource, by default {"format": "JSON"}
+        """
+        return self._set(patternsResource=ExternalResource(path, read_as, options))
+
+    def setUseStorage(self, value):
+        """Sets whether to use RocksDB storage to serialize patterns.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to use RocksDB storage to serialize patterns.
+        """
+        return self._set(useStorage=value)
+
+    def setSentenceMatch(self, value):
+        """Sets whether to find match at sentence level.
+
+        Parameters
+        ----------
+        value : bool
+            True: sentence level. False: token level
+        """
+        return self._set(sentenceMatch=value)
+
+    def setAlphabetResource(self, path):
+        """Alphabet Resource (a simple plain text with all language characters)
+
+        Parameters
+        ----------
+        path : str
+            Path to the resource
+        """
+        return self._set(alphabet=ExternalResource(path, read_as=ReadAs.TEXT, options={}))
+
+    def _create_model(self, java_model):
+        return EntityRulerModel(java_model=java_model)
+
+
+class EntityRulerModel(AnnotatorModel, HasStorageModel):
+    """Instantiated model of the EntityRulerApproach.
+    For usage and examples see the documentation of the main class.
+
+    ====================== ======================
+    Input Annotation types Output Annotation type
+    ====================== ======================
+    ``DOCUMENT, TOKEN``    ``CHUNK``
+    ====================== ======================
+    """
+    name = "EntityRulerModel"
+    database = ['ENTITY_PATTERNS']
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT]
+    optionalInputAnnotatorTypes = [AnnotatorType.TOKEN]
+
+    outputAnnotatorType = AnnotatorType.CHUNK
+
+    autoMode = Param(
+        Params._dummy(),
+        "autoMode",
+        "Enable built-in regex presets that combine related entity patterns (e.g., 'communication_entities', 'network_entities', 'media_entities', etc.).",
+        typeConverter=TypeConverters.toString
+    )
+
+    extractEntities = Param(
+        Params._dummy(),
+        "extractEntities",
+        "List of entity types to extract. If not set, all entities in the active autoMode or from regexPatterns are used.",
+        typeConverter=TypeConverters.toListString
+    )
+
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.er.EntityRulerModel", java_model=None):
+        super(EntityRulerModel, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+
+    @staticmethod
+    def pretrained(name, lang="en", remote_loc=None):
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(EntityRulerModel, name, lang, remote_loc)
+
+    @staticmethod
+    def loadStorage(path, spark, storage_ref):
+        HasStorageModel.loadStorages(path, spark, storage_ref, EntityRulerModel.database)
+
+
+    def setAutoMode(self, value):
+        """Sets the auto mode for predefined regex entity groups.
+
+        Parameters
+        ----------
+        value : str
+            Name of the auto mode to activate (e.g., 'communication_entities', 'network_entities', etc.)
+        """
+        return self._set(autoMode=value)
+
+
+    def setExtractEntities(self, value):
+        """Sets specific entities to extract, filtering only those defined in regexPatterns or autoMode.
+
+        Parameters
+        ----------
+        value : list[str]
+            List of entity names to extract, e.g., ['EMAIL_ADDRESS_PATTERN', 'IPV4_PATTERN'].
+        """
+        return self._set(extractEntities=value)

sparknlp/annotator/graph_extraction.py

@@ -0,0 +1,368 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Contains classes for GraphExtraction."""
+from sparknlp.common import *
+
+
+class GraphExtraction(AnnotatorModel):
+    """Extracts a dependency graph between entities.
+
+    The GraphExtraction class takes e.g. extracted entities from a
+    :class:`.NerDLModel` and creates a dependency tree which describes how the
+    entities relate to each other. For that a triple store format is used. Nodes
+    represent the entities and the edges represent the relations between those
+    entities. The graph can then be used to find relevant relationships between
+    words.
+
+    Both the :class:`.DependencyParserModel` and
+    :class:`.TypedDependencyParserModel` need to be
+    present in the pipeline. There are two ways to set them:
+
+    #. Both Annotators are present in the pipeline already. The dependencies are
+       taken implicitly from these two Annotators.
+    #. Setting :meth:`.setMergeEntities` to ``True`` will download the
+       default pretrained models for those two Annotators automatically. The
+       specific models can also be set with :meth:`.setDependencyParserModel`
+       and :meth:`.setTypedDependencyParserModel`:
+
+        >>> graph_extraction = GraphExtraction() \\
+        ...     .setInputCols(["document", "token", "ner"]) \\
+        ...     .setOutputCol("graph") \\
+        ...     .setRelationshipTypes(["prefer-LOC"]) \\
+        ...     .setMergeEntities(True)
+        >>>     #.setDependencyParserModel(["dependency_conllu", "en",  "public/models"])
+        >>>     #.setTypedDependencyParserModel(["dependency_typed_conllu", "en",  "public/models"])
+
+    ================================= ======================
+    Input Annotation types            Output Annotation type
+    ================================= ======================
+    ``DOCUMENT, TOKEN, NAMED_ENTITY`` ``NODE``
+    ================================= ======================
+
+    Parameters
+    ----------
+    relationshipTypes
+        Paths to find between a pair of token and entity
+    entityTypes
+        Paths to find between a pair of entities
+    explodeEntities
+        When set to true find paths between entities
+    rootTokens
+        Tokens to be consider as root to start traversing the paths. Use it
+        along with explodeEntities
+    maxSentenceSize
+        Maximum sentence size that the annotator will process, by default 1000.
+        Above this, the sentence is skipped
+    minSentenceSize
+        Minimum sentence size that the annotator will process, by default 2.
+        Below this, the sentence is skipped.
+    mergeEntities
+        Merge same neighboring entities as a single token
+    includeEdges
+        Whether to include edges when building paths
+    delimiter
+        Delimiter symbol used for path output
+    posModel
+        Coordinates (name, lang, remoteLoc) to a pretrained POS model
+    dependencyParserModel
+        Coordinates (name, lang, remoteLoc) to a pretrained Dependency Parser
+        model
+    typedDependencyParserModel
+        Coordinates (name, lang, remoteLoc) to a pretrained Typed Dependency
+        Parser model
+
+    Examples
+    --------
+    >>> import sparknlp
+    >>> from sparknlp.base import *
+    >>> from sparknlp.annotator import *
+    >>> from pyspark.ml import Pipeline
+    >>> documentAssembler = DocumentAssembler() \\
+    ...     .setInputCol("text") \\
+    ...     .setOutputCol("document")
+    >>> sentence = SentenceDetector() \\
+    ...     .setInputCols(["document"]) \\
+    ...     .setOutputCol("sentence")
+    >>> tokenizer = Tokenizer() \\
+    ...     .setInputCols(["sentence"]) \\
+    ...     .setOutputCol("token")
+    >>> embeddings = WordEmbeddingsModel.pretrained() \\
+    ...     .setInputCols(["sentence", "token"]) \\
+    ...     .setOutputCol("embeddings")
+    >>> nerTagger = NerDLModel.pretrained() \\
+    ...     .setInputCols(["sentence", "token", "embeddings"]) \\
+    ...     .setOutputCol("ner")
+    >>> posTagger = PerceptronModel.pretrained() \\
+    ...     .setInputCols(["sentence", "token"]) \\
+    ...     .setOutputCol("pos")
+    >>> dependencyParser = DependencyParserModel.pretrained() \\
+    ...     .setInputCols(["sentence", "pos", "token"]) \\
+    ...     .setOutputCol("dependency")
+    >>> typedDependencyParser = TypedDependencyParserModel.pretrained() \\
+    ...     .setInputCols(["dependency", "pos", "token"]) \\
+    ...     .setOutputCol("dependency_type")
+    >>> graph_extraction = GraphExtraction() \\
+    ...     .setInputCols(["document", "token", "ner"]) \\
+    ...     .setOutputCol("graph") \\
+    ...     .setRelationshipTypes(["prefer-LOC"])
+    >>> pipeline = Pipeline().setStages([
+    ...     documentAssembler,
+    ...     sentence,
+    ...     tokenizer,
+    ...     embeddings,
+    ...     nerTagger,
+    ...     posTagger,
+    ...     dependencyParser,
+    ...     typedDependencyParser,
+    ...     graph_extraction
+    ... ])
+    >>> data = spark.createDataFrame([["You and John prefer the morning flight through Denver"]]).toDF("text")
+    >>> result = pipeline.fit(data).transform(data)
+    >>> result.select("graph").show(truncate=False)
+    +-----------------------------------------------------------------------------------------------------------------+
+    |graph                                                                                                            |
+    +-----------------------------------------------------------------------------------------------------------------+
+    |[[node, 13, 18, prefer, [relationship -> prefer,LOC, path1 -> prefer,nsubj,morning,flat,flight,flat,Denver], []]]|
+    +-----------------------------------------------------------------------------------------------------------------+
+
+    See Also
+    --------
+    GraphFinisher : to output the paths in a more generic format, like RDF
+    """
+    name = "GraphExtraction"
+
+    inputAnnotatorTypes = [AnnotatorType.DOCUMENT, AnnotatorType.TOKEN, AnnotatorType.NAMED_ENTITY]
+
+    optionalInputAnnotatorTypes = [AnnotatorType.DEPENDENCY, AnnotatorType.LABELED_DEPENDENCY]
+
+    outputAnnotatorType = AnnotatorType.NODE
+
+    relationshipTypes = Param(Params._dummy(),
+                              "relationshipTypes",
+                              "Find paths between a pair of token and entity",
+                              typeConverter=TypeConverters.toListString)
+
+    entityTypes = Param(Params._dummy(),
+                        "entityTypes",
+                        "Find paths between a pair of entities",
+                        typeConverter=TypeConverters.toListString)
+
+    explodeEntities = Param(Params._dummy(),
+                            "explodeEntities",
+                            "When set to true find paths between entities",
+                            typeConverter=TypeConverters.toBoolean)
+
+    rootTokens = Param(Params._dummy(),
+                       "rootTokens",
+                       "Tokens to be consider as root to start traversing the paths. Use it along with explodeEntities",
+                       typeConverter=TypeConverters.toListString)
+
+    maxSentenceSize = Param(Params._dummy(),
+                            "maxSentenceSize",
+                            "Maximum sentence size that the annotator will process. Above this, the sentence is skipped",
+                            typeConverter=TypeConverters.toInt)
+
+    minSentenceSize = Param(Params._dummy(),
+                            "minSentenceSize",
+                            "Minimum sentence size that the annotator will process. Above this, the sentence is skipped",
+                            typeConverter=TypeConverters.toInt)
+
+    mergeEntities = Param(Params._dummy(),
+                          "mergeEntities",
+                          "Merge same neighboring entities as a single token",
+                          typeConverter=TypeConverters.toBoolean)
+
+    mergeEntitiesIOBFormat = Param(Params._dummy(),
+                                   "mergeEntitiesIOBFormat",
+                                   "IOB format to apply when merging entities",
+                                   typeConverter=TypeConverters.toString)
+
+    includeEdges = Param(Params._dummy(),
+                         "includeEdges",
+                         "Whether to include edges when building paths",
+                         typeConverter=TypeConverters.toBoolean)
+
+    delimiter = Param(Params._dummy(),
+                      "delimiter",
+                      "Delimiter symbol used for path output",
+                      typeConverter=TypeConverters.toString)
+
+    posModel = Param(Params._dummy(),
+                     "posModel",
+                     "Coordinates (name, lang, remoteLoc) to a pretrained POS model",
+                     typeConverter=TypeConverters.toListString)
+
+    dependencyParserModel = Param(Params._dummy(),
+                                  "dependencyParserModel",
+                                  "Coordinates (name, lang, remoteLoc) to a pretrained Dependency Parser model",
+                                  typeConverter=TypeConverters.toListString)
+
+    typedDependencyParserModel = Param(Params._dummy(),
+                                       "typedDependencyParserModel",
+                                       "Coordinates (name, lang, remoteLoc) to a pretrained Typed Dependency Parser model",
+                                       typeConverter=TypeConverters.toListString)
+
+    def setRelationshipTypes(self, value):
+        """Sets paths to find between a pair of token and entity.
+
+        Parameters
+        ----------
+        value : List[str]
+            Paths to find between a pair of token and entity
+        """
+        return self._set(relationshipTypes=value)
+
+    def setEntityTypes(self, value):
+        """Sets paths to find between a pair of entities.
+
+        Parameters
+        ----------
+        value : List[str]
+            Paths to find between a pair of entities
+        """
+        return self._set(entityTypes=value)
+
+    def setExplodeEntities(self, value):
+        """Sets whether to find paths between entities.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to find paths between entities.
+        """
+        return self._set(explodeEntities=value)
+
+    def setRootTokens(self, value):
+        """Sets tokens to be considered as the root to start traversing the paths.
+
+        Use it along with explodeEntities.
+
+        Parameters
+        ----------
+        value : List[str]
+            Sets Tokens to be consider as root to start traversing the paths.
+        """
+        return self._set(rootTokens=value)
+
+    def setMaxSentenceSize(self, value):
+        """Sets Maximum sentence size that the annotator will process, by
+        default 1000.
+
+        Above this, the sentence is skipped.
+
+        Parameters
+        ----------
+        value : int
+            Maximum sentence size that the annotator will process
+        """
+        return self._set(maxSentenceSize=value)
+
+    def setMinSentenceSize(self, value):
+        """Sets Minimum sentence size that the annotator will process, by
+        default 2.
+
+        Below this, the sentence is skipped.
+
+        Parameters
+        ----------
+        value : int
+            Minimum sentence size that the annotator will process
+        """
+        return self._set(minSentenceSize=value)
+
+    def setMergeEntities(self, value):
+        """Sets whether to merge same neighboring entities as a single token.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to merge same neighboring entities as a single token.
+        """
+        return self._set(mergeEntities=value)
+
+    def setMergeEntitiesIOBFormat(self, value):
+        """Sets IOB format to apply when merging entities.
+
+        Parameters
+        ----------
+        value : str
+            IOB format to apply when merging entities. Values IOB or IOB2
+        """
+        return self._set(mergeEntitiesIOBFormat=value)
+
+    def setIncludeEdges(self, value):
+        """Sets whether to include edges when building paths.
+
+        Parameters
+        ----------
+        value : bool
+            Whether to include edges when building paths
+        """
+        return self._set(includeEdges=value)
+
+    def setDelimiter(self, value):
+        """Sets delimiter symbol used for path output.
+
+        Parameters
+        ----------
+        value : str
+            Delimiter symbol used for path output
+        """
+        return self._set(delimiter=value)
+
+    def setPosModel(self, value):
+        """Sets Coordinates (name, lang, remoteLoc) to a pretrained POS model.
+
+        Parameters
+        ----------
+        value : List[str]
+            Coordinates (name, lang, remoteLoc) to a pretrained POS model
+        """
+        return self._set(posModel=value)
+
+    def setDependencyParserModel(self, value):
+        """Sets Coordinates (name, lang, remoteLoc) to a pretrained Dependency
+        Parser model.
+
+        Parameters
+        ----------
+        value : List[str]
+            Coordinates (name, lang, remoteLoc) to a pretrained Dependency
+            Parser model
+        """
+        return self._set(dependencyParserModel=value)
+
+    def setTypedDependencyParserModel(self, value):
+        """Sets Coordinates (name, lang, remoteLoc) to a pretrained Typed
+        Dependency Parser model.
+
+        Parameters
+        ----------
+        value : List[str]
+            Coordinates (name, lang, remoteLoc) to a pretrained Typed Dependency
+            Parser model
+        """
+        return self._set(typedDependencyParserModel=value)
+
+    @keyword_only
+    def __init__(self, classname="com.johnsnowlabs.nlp.annotators.GraphExtraction", java_model=None):
+        super(GraphExtraction, self).__init__(
+            classname=classname,
+            java_model=java_model
+        )
+        self._setDefault(
+            maxSentenceSize=1000,
+            minSentenceSize=2
+        )
+

sparknlp/annotator/keyword_extraction/__init__.py

@@ -0,0 +1,16 @@
+#  Copyright 2017-2022 John Snow Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Module of annotators for keyword extraction."""
+from sparknlp.annotator.keyword_extraction.yake_keyword_extraction import *