germania 1.30.0__tar.gz

germania-1.30.0/MANIFEST.in

@@ -0,0 +1,11 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+include germania_data/*.dict
+graft germania_data/ltk_data

germania-1.30.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: germania
+Version: 1.30.0
+Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anaticulae/germania
+Project-URL: Repository, https://github.com/anaticulae/germania
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: utilo<3.0.0,>=2.108.0
+Requires-Dist: konradus<2.0.0,>=1.0.1
+Requires-Dist: iamraw<5.0.0,>=4.91.4
+Requires-Dist: configos<2.0.0,>=1.0.4
+Requires-Dist: nltk<4.0.0,>=3.9.4
+Requires-Dist: ltk_data<2.0.0,>=1.0.2
+Requires-Dist: sdatum<2.0.0,>=1.0.0
+Requires-Dist: analp<2.0.0,>=1.0.0
+Provides-Extra: dev
+Requires-Dist: utilotest<2.0.0,>=1.0.1; extra == "dev"
+Requires-Dist: textbone<2.0.0,>=1.0.0; extra == "dev"
+
+# germania

germania-1.30.0/README

@@ -0,0 +1 @@
+# germania

germania-1.30.0/germania/__init__.py

@@ -0,0 +1,106 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+"""germania
+======
+
+The `germania` package is a wrapper for external tooling which have some
+`magic` inside to determine the type of a word or split a text into
+sentences into words.
+"""
+
+import os
+
+import ltk_data
+import nltk
+
+import germania.sentence
+from germania.abbrev import find_abbrev
+from germania.error.finding import TextError
+from germania.error.finding import TextErrors
+from germania.error.finding import TextErrorType
+from germania.error.machine import TextErrorMachine
+from germania.error.text import TextMachine
+from germania.improve.abbreviation import abbreviation_magic
+from germania.improve.highnote import highnote_magic
+from germania.improve.href import href_magic
+from germania.improve.magic import text_magic
+from germania.language import LanguageResult
+from germania.language import determine as lang
+from germania.language import iseng
+from germania.language import isfre
+from germania.language import isger
+from germania.magic import WordType
+from germania.magic import WordTypes
+from germania.magic import iscity
+from germania.magic import isperson
+from germania.magic import ispress
+from germania.magic import isreference
+from germania.magic import isyear
+from germania.magic import wordtype
+from germania.magic import wordtypes
+from germania.pattern import matched
+from germania.pattern.access import accessed
+from germania.pattern.author import authors
+from germania.pattern.author import authors_decide
+from germania.pattern.book import bibtexts
+from germania.pattern.book import doi
+from germania.pattern.book import isbn
+from germania.pattern.book import issn
+from germania.pattern.book import references
+from germania.pattern.book import volumes
+from germania.pattern.date import dates
+from germania.pattern.date import dates_master
+from germania.pattern.date import dates_month_year
+from germania.pattern.date import years
+from germania.pattern.href import hyperlink
+from germania.pattern.href import links
+from germania.pattern.href import locallink
+from germania.pattern.mail import mails
+from germania.pattern.pagination import page_single
+from germania.pattern.pagination import pagenumbers
+from germania.pattern.pagination import pages_complex
+from germania.quotation import extract_quotes
+from germania.quotation import raw_quotation
+from germania.sentence import Sentences
+from germania.sentence import is_sentence
+from germania.sentence import is_sentence_closed
+from germania.sentence import sentence_select
+from germania.sentence import sentence_tokenize
+from germania.sentence import split_token
+from germania.sequence import init
+from germania.sequence import ngram
+from germania.sequence import search
+from germania.sequence import searches
+from germania.sequence import token_plain
+from germania.tagger import word_tag
+from germania.text import words_fromstr
+from germania.utils import collect_and_replace
+from germania.utils.month import MONTH
+from germania.utils.month import MONTH_REGEX
+from germania.utils.month import month
+from germania.word import Words
+from germania.word import contain_quotation_marks
+from germania.word import word_normalize
+from germania.word import word_tokenize
+
+split_words = word_tokenize  # pylint:disable=C0103
+split_sentences = sentence_tokenize  # pylint:disable=C0103
+
+__version__ = '1.30.0'
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+# REMOVE LATER
+pages = page_single
+
+nltk.download('crubadan', quiet=True)
+nltk.download('punkt_tab', quiet=True)
+
+# TODO: REMovE LATER
+germania.sentence.language_select = lambda x: 'german'

germania-1.30.0/germania/abbrev.py

@@ -0,0 +1,40 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import sdatum
+import utilo
+
+import germania
+
+
+def find_abbrev(abbrev: str, words: list) -> str:
+    """\
+    >>> find_abbrev('MNU', 'Steuergebiete zugunsten multinationaler Unternehmen'.split())
+    'multinationaler Unternehmen'
+    >>> import germania
+    >>> find_abbrev('MNU', germania.words_fromstr('Steuergebiete zugunsten multinationaler Unternehmens(MNU) ende'))
+    'multinationaler Unternehmens'
+    """
+    lookup = sdatum.abbrev(abbrev)
+    if lookup is None:
+        return None
+    lookup = [germania.word_normalize(item) for item in lookup]
+    normalized = [
+        germania.word_normalize(item) if isinstance(item, str) else item
+        for item in words
+    ]
+    detected = germania.searches(
+        patterns=lookup,
+        sentence=normalized,
+        tokens_complex=False,
+    )
+    if not detected:
+        return None
+    result = ' '.join(words[index] for index in utilo.rlist(*detected[0]))
+    return result

germania-1.30.0/germania/error/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

germania-1.30.0/germania/error/finding.py

@@ -0,0 +1,43 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import dataclasses
+import enum
+
+import iamraw
+
+
+class TextErrorType(enum.Enum):
+    MISSING = enum.auto()
+    """Text token is missing"""
+    STYLE = enum.auto()
+    """Violation against good style"""
+    RULE = enum.auto()
+    """Writing is against the writing laws"""
+    DUPLICATED = enum.auto()
+    """Copy paste error"""
+    REPLACEMENT = enum.auto()
+    """Replace this content"""
+    UNDEFINED = enum.auto()
+    """No special state"""
+
+
+@dataclasses.dataclass
+class TextError:
+    title: str = None
+    text: str = None
+    state: TextErrorType = None
+    location: iamraw.Location = None
+    raw: str = None
+    better: str = None
+    debug_method: str = None
+    """Name of method which had determined this error."""
+
+
+TextErrors = list[TextError]

germania-1.30.0/germania/error/machine.py

@@ -0,0 +1,84 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import contextlib
+
+import iamraw
+import utilo
+
+import germania.error.finding
+
+
+class TextErrorMachine:
+    """\
+    >>> empty = TextErrorMachine()
+    >>> empty.determine('')
+    []
+    """
+
+    def determine(
+        self,
+        text: str,
+        page: int = None,
+    ) -> germania.error.finding.TextErrors:
+        result = []
+        todo = utilo.methods(self, starts='check_')
+        for method in todo:
+            detected = method(text)
+            if not detected:
+                continue
+            if not utilo.iterable(detected):
+                result.append(detected)
+                detected.debug_method = method.__name__
+                continue
+            for item in detected:
+                item.debug_method = method.__name__
+            result.extend(detected)
+        if page is not None:
+            for item in result:
+                with contextlib.suppress(AttributeError):
+                    item.location.page = page
+        return result
+
+    def location(self, match) -> iamraw.RangedLocation:  # pylint:disable=R0201
+        if not match:
+            return None
+        location = iamraw.RangedLocation(
+            char=match.span()[0],
+            char_end=match.span()[1],
+        )
+        return location
+
+
+class PhysicMachine(TextErrorMachine):
+    """\
+    >>> machine = PhysicMachine()
+    >>> machine.determine('The weight is 200kg. Thats a lot.', page=10)
+    [TextError(...state=<TextErrorType.RULE...>, location=RangedLocation(page=10, char=13, char_end=19), raw='200kg', better='200 kg', debug_method='check_physical_spaces')]
+    """
+
+    MISSING_SPACE_BEFORE_UNIT = utilo.compiles(r"""
+        \W
+        (
+            (?P<value>\d+((\.|\,)\d+){0,1})
+            (?P<unit>%|‰|kg|km/h|mmHg|mm|ms|mg/km|m|cm|V|W|Hz|mW)
+        )
+    """)
+
+    def check_physical_spaces(self, text: str) -> list:
+        result = []
+        for match in self.MISSING_SPACE_BEFORE_UNIT.finditer(text):
+            error = germania.error.finding.TextError(
+                state=germania.error.finding.TextErrorType.RULE,
+                better=match['value'] + ' ' + match['unit'],
+                location=self.location(match),
+                raw=match[1],
+            )
+            result.append(error)
+        return result

germania-1.30.0/germania/error/text.py

@@ -0,0 +1,107 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2022-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import re
+
+import sdatum
+import utilo
+
+import germania
+
+
+class TextMachine(germania.TextErrorMachine):
+
+    MISSING_PAGENUMBER = utilo.compiles(r"""
+        (
+            \s|
+            [^\w]\.| # u.s.
+            [,;:]
+        )
+        (
+            S\.|
+            p\.
+        )
+        [ ]{0,4}\n?
+        (?![\dixv\s])   # s. VII
+    """)
+
+    def check_pagenumber_complete(self, text: str) -> list:
+        r"""\
+        >>> check = TextMachine().check_pagenumber_complete
+        >>> check('Bonn, Herford 1993, S.\nDer Brief')
+        [TextError(...<TextErrorType.MISSING...raw='S.'...)]
+        >>> check('Hier fehlt wohlx S. die Seitennummer')
+        [TextError(...<TextErrorType.MISSING...raw='S.'...)]
+        >>> check('Vgl. Dixon, S.: Twitter: distribution of global audiences 2021')
+        []
+        >>> check('Schols. Hier')
+        []
+        >>> check('Berlin 19982, s. VII ')
+        []
+        >>> check('S. Meuschel, Legitimation und Parteiherrschaft ')
+        []
+        >>> check('in the U.S. | Pew Research')
+        []
+        >>> check('Meuschel S., Legitimation und Parteiherrschaft ')
+        []
+
+        run large text test
+        >>> import textbone;check(textbone.text_improved(1024*1024))
+        []
+        """
+        result = []
+        for match in self.MISSING_PAGENUMBER.finditer(text):
+            after = text[match.span()[1]:]
+            if follows_name(after):
+                continue
+            start, lookback = match.span()[0], 60
+            before = text[max(0, start - lookback):start]
+            if name_before(before):
+                continue
+            error = germania.TextError(
+                state=germania.TextErrorType.MISSING,
+                location=self.location(match),
+                raw=match[0].strip(),
+            )
+            result.append(error)
+        return result
+
+
+def follows_name(text: str) -> bool:
+    """\
+    >>> follows_name('Helmut wohnt hier')
+    True
+    >>> follows_name('Copyright (c) 2022 by Helmut')
+    False
+    >>> follows_name('Der Brief')
+    False
+    """
+    text = text.strip()
+    if re.match(r'^\w\.', text):
+        return True
+    for name in text.split()[0:4]:
+        name = name.strip(':;, ')
+        if sdatum.isname(name):
+            return True
+    return False
+
+
+def name_before(text: str) -> bool:
+    """\
+    >>> name_before('Legitimation und Parteiherrschaft Meuschel')
+    True
+    """
+    token = text.rstrip(':;, ').rsplit(maxsplit=1)
+    if not token:
+        return False
+    # select the right one
+    name = token[-1].strip(':;, ')
+    if sdatum.isname(name):
+        return True
+    return False

germania-1.30.0/germania/improve/__init__.py

@@ -0,0 +1,8 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================

germania-1.30.0/germania/improve/abbreviation.py

@@ -0,0 +1,31 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import konradus
+import utilo
+
+
+@utilo.cacheme
+def abbreviation_magic(text: str) -> str:
+    """\
+    >>> abbreviation_magic('Helmut hier u. a. und mehr.')
+    'Helmut hier u.a. und mehr.'
+    >>> abbreviation_magic('a. a. o.')
+    'a.a.o.'
+    """
+    for token, replace in TEXT_MAGIC:
+        text = text.replace(token, replace)
+    return text
+
+
+TEXT_MAGIC = [
+    ('. '.join(item.split('.')).strip(), item)
+    for item in list(konradus.ABBREVIATION) + list(konradus.ABBREVIATION_LOWER)
+    if item.count('.') > 1
+]

germania-1.30.0/germania/improve/highnote.py

@@ -0,0 +1,28 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import re
+
+import utilo
+
+
+@utilo.cacheme
+def highnote_magic(text: str) -> str:
+    """\
+    >>> highnote_magic('ohnehin unmöglich.89 So')
+    'ohnehin unmöglich. 89 So'
+    """
+    # highnote at the end of sentence
+    text = re.sub(
+        r'([a-z])([\.\!\?])(\d{1,4})([ ]{1,4})',
+        r'\1\2 \3\4',
+        text,
+        flags=re.I,
+    )
+    return text

germania-1.30.0/germania/improve/href.py

@@ -0,0 +1,69 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import re
+
+import utilo
+
+
+@utilo.cacheme
+def href_magic(text: str) -> str:
+    """\
+    >>> href_magic('4. url: https : / / www . apache . org / licenses/LICENS')
+    '4. url: https://www.apache.org/licenses/LICENS'
+    """
+    result = link_fink(text)
+    return result
+
+
+@utilo.cacheme
+def link_fink(text: str) -> str:
+    """\
+    >>> link_fink('url: http : / / www . bitkom . org / files / documents / BITKOM _ Leitfaden')
+    'url: http://www.bitkom.org/files/documents/BITKOM_Leitfaden'
+    >>> link_fink('singulären  bzw.  typischen')
+    'singulären  bzw.  typischen'
+    """
+    if not IS_HTTP.search(text):
+        return text
+    for (token, replacement) in SPACE_PATTERN:
+        text = re.sub(token, replacement, text)
+    return text
+
+
+IS_HTTP = utilo.compiles(r'\bhttp')
+
+SPACE_PATTERN = (
+    ('. org', '.org'),
+    ('. de', '.de'),
+    ('. com', '.com'),
+    ('. net', '.net'),
+    (' .org', '.org'),
+    (' .de', '.de'),
+    (' .com', '.com'),
+    (' .net', '.net'),
+    (' : /', ':/'),
+    (' / ', '/'),
+    (':/ ', ':/'),
+    (' . ', '.'),
+    (' _ ', '_'),
+    ('_ ', '_'),
+    ('/ ', '/'),
+    ('http :', 'http:'),
+    ('https :', 'https:'),
+    ('http ://', 'http://'),
+    ('https ://', 'https://'),
+    ('http: //', 'http://'),
+    ('https: //', 'https://'),
+)
+SPACE_PATTERN = [(re.escape(left), right) for left, right in SPACE_PATTERN]
+SPACE_PATTERN.append((
+    r'(?P<left>[a-z])\s{0,2}\.\s{0,2}(?P<right>[a-z])',
+    r'\g<left>.\g<right>',
+))

germania-1.30.0/germania/improve/magic.py

@@ -0,0 +1,27 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+
+import germania.improve.abbreviation
+import germania.improve.highnote
+import germania.improve.href
+
+TODO = (
+    germania.improve.abbreviation.abbreviation_magic,
+    germania.improve.highnote.highnote_magic,
+    germania.improve.href.href_magic,
+)
+
+
+@utilo.cacheme
+def text_magic(text: str) -> str:
+    for pattern in TODO:
+        text = pattern(text)
+    return text

germania-1.30.0/germania/language.py

@@ -0,0 +1,84 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2023 by Helmut Konrad Schewe. All rights reserved.
+# This file is property of Helmut Konrad Schewe. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""Language Probability
+====================
+
+Use nltk to determine language where sentence is written in.
+"""
+
+import collections
+import contextlib
+
+import konradus
+import utilo
+
+LanguageResult = collections.namedtuple(
+    'LanguageResult',
+    'language, probability',
+)
+
+
+def determine(text: str) -> LanguageResult:
+    if isinstance(text, list):
+        text = konradus.remove_marks(text)
+    if not isinstance(text, str):
+        text = ' '.join(text)
+    cat = textcat()
+    detected = cat.guess_language(text)
+    language = konradus.Language.UNKNOWN
+    with contextlib.suppress(KeyError):
+        language = MAPPING[detected]
+    return LanguageResult(language=language, probability=1.0)
+
+
+def isfre(tokens: str) -> bool:
+    """\
+    >>> isfre('Ich bin Helmut')
+    False
+    >>> isfre('Bonjour monsieur.')
+    True
+
+    verify that interface support tokens
+    >>> isfre('Toujour suis Luis.'.split())
+    True
+    """
+    return determine(tokens).language == konradus.Language.FRENCH
+
+
+def iseng(tokens: str) -> bool:
+    """\
+    >>> iseng('Ich bin Helmut')
+    False
+    >>> iseng('i like fish')
+    True
+    """
+    return determine(tokens).language == konradus.Language.ENGLISH
+
+
+def isger(tokens: str) -> bool:
+    """\
+    >>> isger('Kartoffelsalat')
+    True
+    """
+    return determine(tokens).language == konradus.Language.GERMAN
+
+
+MAPPING = {
+    'deu': konradus.Language.GERMAN,
+    'eng': konradus.Language.ENGLISH,
+    # 'es': konradus.Language.SPANISH,
+    'fra': konradus.Language.FRENCH,
+}
+
+
+@utilo.cacheme
+def textcat():
+    import nltk.classify.textcat
+    result = nltk.classify.textcat.TextCat()
+    return result