monapipe 0.1.0__py3-none-any.whl

monapipe/__init__.py

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+import sentry_sdk
+
+from monapipe.silence import (
+    silence_logging,
+    silence_tensorflow,
+    silence_tqdm,
+    silence_transformers,
+    silence_warnings,
+)
+
+silence_logging()
+silence_tensorflow()
+silence_tqdm()
+silence_transformers()
+silence_warnings()

monapipe/annotation.py

@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+from typing import Any, Iterable, List, Optional, Set
+
+
+class Annotation:
+    """A class that stores information from an annotation."""
+
+    def __init__(
+        self, tag, tagset, property_values_dict, tokens, id_value, strings, string_positions
+    ):
+        self.tag = tag
+        self.tagset = tagset
+        self.property_values = property_values_dict
+        self.tokens = tokens
+        self.id = id_value
+        self.strings = strings
+        self.string_positions = string_positions
+
+    def __repr__(self):
+        return str(vars(self))
+
+    def __str__(self):
+        return str(vars(self))
+
+
+class AnnotationList(list):
+    """A class that stores a list of Annotation-objectes."""
+
+    def __init__(self, annotation_list: Optional[List[Annotation]] = None):
+        if annotation_list is None:
+            annotation_list = []
+        super().__init__(annotation_list)
+
+    def get_annotations(
+        self, tags: Optional[Iterable[str]] = None, tagset: Optional[str] = None
+    ) -> Any:
+        """Find Annotation-objects with specified tags and tagset.
+
+        Args:
+            tags: Tags to search for.
+            tagset: Parameter to filter for specific tagset.
+
+        Returns:
+            `AnnotationList`: all annotation-objects with specified tags and tagset.
+                If `tags` is empty, all tags of the given tagset are searched.
+                If `tagset` is None, no filtering for a specific tagset is applied.
+
+        """
+        annotations_to_return = AnnotationList()
+        for annotation in self:
+            if (tags is None or annotation.tag in tags) and (
+                tagset is None or annotation.tagset.startswith(tagset)
+            ):
+                annotations_to_return.append(annotation)
+        return annotations_to_return
+
+    def get_tags(
+        self, tags: Optional[Iterable[str]] = None, tagset: Optional[str] = None
+    ) -> Set[str]:
+        """Find all or specified tags in a AnnotationList.
+
+        Args:
+            tags: Parameter to filter for specific tags.
+            tagset: Parameter to filter for specific tagset.
+
+        Returns:
+            All or specified tags that occour in the AnnotationList.
+
+        """
+        tags_to_return = set()
+        for annotation in self:
+            if (tags is None or annotation.tag in tags) and (
+                tagset is None or annotation.tagset.startswith(tagset)
+            ):
+                tags_to_return.add(annotation.tag)
+        return tags_to_return

monapipe/config.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+import os
+
+from torch import cuda
+
+DATAVERSE = {
+    "api_token": "",
+    "doi_attribution": "doi:10.25625/2D9CAV&version=1.0",
+    "doi_event_classification": "doi:10.25625/0GUOMC&version=1.1",
+    "doi_generalizing_passages_identification_bert": "doi:10.25625/2PHXNC&version=1.1",
+    "doi_heideltime": "doi:10.25625/SIPQEF&version=1.0",
+    "doi_open_multilingual_wordnet": "doi:10.25625/LE57DV&version=1.0",
+    "doi_parsing": "doi:10.25625/S2LPJP&version=1.1",
+    "doi_reflective_passages_identification_bert": "doi:10.25625/0HXWYG&version=1.1",
+}
+
+LOCAL_PATHS = {"germanet": os.path.join(os.path.dirname(__file__), "..", "..", "..", "germanet")}
+
+SETTINGS = {
+    "spacy_max_length": 12000000,
+    "torch_device": ("cuda" if cuda.is_available() else "cpu"),
+}

monapipe/lang/de/auxiliary_verbs.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+AUXILIARY_VERBS = {
+    "AUX1": "haben|sein",  # perfect aspect
+    "AUX2": "sein|werden",  # passive voice
+    "AUX3": "werden",  # future tense
+    "AUX4": "sein",  # copula
+}

monapipe/lang/de/inflection_table.py

@@ -0,0 +1,450 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+from monapipe.lang.de.auxiliary_verbs import AUXILIARY_VERBS
+
+AUX1 = AUXILIARY_VERBS["AUX1"]  # perfect aspect
+AUX2 = AUXILIARY_VERBS["AUX2"]  # passive voice
+AUX3 = AUXILIARY_VERBS["AUX3"]  # future tense
+AUX4 = AUXILIARY_VERBS["AUX4"]  # copula
+
+INFLECTION_TABLE = [
+    (
+        [{"VerbForm": "Inf"}, {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Act",
+        },
+    ),  # [er] wird sehen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Pass",
+        },
+    ),  # [er] wird gesehen werden/sein
+    (
+        [{"VerbForm": "Inf"}, {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] werde sehen
+    (
+        [{"VerbForm": "Inf"}, {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] würde sehen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] werde gesehen werden/sein
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] würde gesehen werden/sein
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Perf",
+            "Mood": "Ind",
+            "Voice": "Act",
+        },
+    ),  # [er] wird gesehen haben
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Perf",
+            "Mood": "Ind",
+            "Voice": "Pass",
+        },
+    ),  # [er] wird gesehen worden/gewesen sein
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Perf",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] werde gesehen haben
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Perf",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] würde gesehen haben
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Perf",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] werde gesehen worden/gewesen sein
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Inf"},
+            {"lemma": AUX3, "VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Fut",
+            "Aspect": "Perf",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] würde gesehen worden/gewesen sein
+    (
+        [{"VerbForm": "Fin", "Tense": "Past", "Mood": "Ind"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Act",
+        },
+    ),  # [er] sah
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Fin", "Tense": "Past", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Pass",
+        },
+    ),  # [er] war gesehen worden/gewesen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Fin", "Tense": "Past", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Pass",
+        },
+    ),  # [er] wurde/war gesehen
+    (
+        [{"VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] sähe
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] wäre gesehen worden/gewesen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] würde/wäre gesehen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Fin", "Tense": "Past", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Perf",
+            "Mood": "Ind",
+            "Voice": "Act",
+        },
+    ),  # [er] hatte gesehen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Fin", "Tense": "Past", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Past",
+            "Aspect": "Perf",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] hätte gesehen
+    (
+        [{"VerbForm": "Fin", "Tense": "Pres", "Mood": "Imp"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Imp",
+            "Voice": "Act",
+        },
+    ),  # sieh!
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Imp"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Imp",
+            "Voice": "Pass",
+        },
+    ),  # werde/sei gesehen!
+    (
+        [{"VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Act",
+        },
+    ),  # [er] sieht
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Pass",
+        },
+    ),  # [er] ist gesehen worden/gewesen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Ind",
+            "Voice": "Pass",
+        },
+    ),  # [er] wird/ist gesehen
+    (
+        [{"VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"}],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] sehe
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] sei gesehen worden/gewesen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Imp",
+            "Mood": "Sub",
+            "Voice": "Pass",
+        },
+    ),  # [er] werde/sei gesehen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Imp"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Perf",
+            "Mood": "Imp",
+            "Voice": "Act",
+        },
+    ),  # habe gesehen!
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Imp"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Perf",
+            "Mood": "Imp",
+            "Voice": "Pass",
+        },
+    ),  # sei gesehen worden/gewesen!
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Perf",
+            "Mood": "Ind",
+            "Voice": "Act",
+        },
+    ),  # [er] hat gesehen
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX1, "VerbForm": "Fin", "Tense": "Pres", "Mood": "Sub"},
+        ],
+        {
+            "VerbForm": "Fin",
+            "Tense": "Pres",
+            "Aspect": "Perf",
+            "Mood": "Sub",
+            "Voice": "Act",
+        },
+    ),  # [er] habe gesehen
+    (
+        [{"VerbForm": "Inf"}],
+        {"VerbForm": "Inf", "Aspect": "Imp", "Voice": "Act"},
+    ),  # (zu) sehen
+    (
+        [{"VerbForm": "Part", "Aspect": "Perf"}, {"lemma": AUX2, "VerbForm": "Inf"}],
+        {"VerbForm": "Inf", "Aspect": "Imp", "Voice": "Pass"},
+    ),  # gesehen (zu) werden/sein
+    (
+        [
+            {"VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX2, "VerbForm": "Part", "Aspect": "Perf"},
+            {"lemma": AUX4, "VerbForm": "Inf"},
+        ],
+        {"VerbForm": "Inf", "Aspect": "Imp", "Voice": "Pass"},
+    ),  # gesehen worden/gewesen (zu) sein
+    (
+        [{"VerbForm": "Part", "Aspect": "Perf"}, {"lemma": AUX1, "VerbForm": "Inf"}],
+        {"VerbForm": "Inf", "Aspect": "Perf", "Voice": "Act"},
+    ),  # gesehen (zu) haben
+    (
+        [{"VerbForm": "Part", "Aspect": "Imp"}],
+        {"VerbForm": "Part", "Tense": "Pres", "Aspect": "Imp"},
+    ),  # sehend
+    (
+        [{"VerbForm": "Part", "Aspect": "Perf"}],
+        {"VerbForm": "Part", "Tense": "Past", "Aspect": "Perf"},
+    ),  # gesehen
+]

monapipe/lang/de/modal_verbs.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+MODAL_VERBS = set(["dürfen", "können", "mögen", "müssen", "sollen", "wollen"])

monapipe/lang/de/quotation_marks.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+QUOTATION_MARKS = [["„", "»", '"'], ["“", "«", '"']]

monapipe/lang/de/semi_modal_verbs.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+SEMI_MODAL_VERBS = set(
+    ["drohen", "lassen", "pflegen", "scheinen", "vermögen", "versprechen", "wissen"]
+)

monapipe/lang/de/speech_verbs.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: 2022 Georg-August-Universität Göttingen
+#
+# SPDX-License-Identifier: CC0-1.0
+
+SPEECH_VERBS = set(
+    [
+        "antworten",
+        "äußern",
+        "blaffen",
+        "brüllen",
+        "denken",
+        "erwidern",
+        "erzählen",
+        "flüstern",
+        "fragen",
+        "glauben",
+        "meinen",
+        "mutmaßen",
+        "reden",
+        "rufen",
+        "sagen",
+        "schreien",
+        "seufzen",
+        "singen",
+        "sprechen",
+        "vermuten",
+        "weinen",
+        "wispern",
+    ]
+)