analp 1.0.0__tar.gz

analp-1.0.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: analp
+Version: 1.0.0
+Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anaticulae/iamraw
+Project-URL: Repository, https://github.com/anaticulae/iamraw
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: utilo<3.0.0,>=2.107.4
+Requires-Dist: konradus<3.0.0,>=1.0.1
+Requires-Dist: nltk<4.0.0,>=3.9.4
+Requires-Dist: ltk_data<2.0.0,>=1.0.2
+Provides-Extra: dev
+Requires-Dist: utilotest<2.0.0,>=1.0.1; extra == "dev"
+
+# analp

analp-1.0.0/README

@@ -0,0 +1 @@
+# analp

analp-1.0.0/analp/__init__.py

@@ -0,0 +1,33 @@
+#==============================================================================
+# C O P Y R I G H T
+#------------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+#==============================================================================
+
+import importlib.metadata
+import os
+
+import analp.__lazy__
+from analp.pos import sent_pos
+from analp.sentence import normalize as normalize_sentence
+from analp.sentence import sent_tokenize
+from analp.sentiment import sent_sentiment
+from analp.word import isstopword
+from analp.word import stopwords
+from analp.word import word_tokenize
+
+# import german_data
+# import ltk_data
+
+PACKAGE = 'analp'
+__version__ = importlib.metadata.version(PACKAGE)
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+"""Load STOPWORDS on access time."""
+# from analp.corpus import STOPWORDS, see __lazy__
+__getattr__ = lambda name: getattr(analp.__lazy__, name)
+
+# ltk_data.add_nltk_path(german_data.ROOT)

analp-1.0.0/analp/__lazy__.py

@@ -0,0 +1,50 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import collections
+import functools
+import threading
+
+import utilo
+
+Configure = collections.namedtuple(
+    'Configure',
+    'sent_tokenize word_tokenize pos_tag STOPWORDS STEMMER',
+)
+
+
+@functools.lru_cache
+def lazy() -> Configure:
+    utilo.debug('configure nltk')
+    import nltk
+    import nltk.corpus
+    import nltk.stem
+
+    nltk.download('stopwords', quiet=True)
+    nltk.download('punkt_tab', quiet=True)
+    nltk.download('averaged_perceptron_tagger_eng', quiet=True)
+
+    result = Configure(
+        sent_tokenize=nltk.sent_tokenize,
+        word_tokenize=nltk.word_tokenize,
+        pos_tag=nltk.pos_tag,
+        STOPWORDS=nltk.corpus.stopwords.words('german'),
+        STEMMER=nltk.stem.SnowballStemmer('german'),
+    )
+    return result
+
+
+LOCK = threading.Lock()
+
+
+def __getattr__(name):
+    with LOCK:
+        data = lazy()
+        result = getattr(data, name)
+    return result

analp-1.0.0/analp/corpus.py

@@ -0,0 +1,14 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+# SEE __lazy__
+
+# import nltk.corpus
+
+# STOPWORDS = frozenset(nltk.corpus.stopwords.words('german'))

analp-1.0.0/analp/pos.py

@@ -0,0 +1,35 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+
+import analp
+import analp.__lazy__
+
+
+def sent_pos(text: str, language='german') -> list:
+    """\
+    >>> sent_pos('Helmut is speaking.', language='eng')
+    [('Helmut', 'NNP'), ('is', 'VBZ'), ('speaking', 'VBG'), ('.', '.')]
+    >>> sent_pos('Hier spricht Helmut.') # TODO: not supported yet
+    """
+    language = lang(language)
+    tokens = analp.word_tokenize(text)
+    try:
+        tagged = analp.__lazy__.pos_tag(tokens, lang=language)
+    except NotImplementedError:
+        utilo.error(f'language not supported: {language}')
+        return None
+    return tagged
+
+
+def lang(item: str) -> str:
+    item = item.replace('german', 'ger')
+    item = item.replace('english', 'eng')
+    return item

analp-1.0.0/analp/sentence.py

@@ -0,0 +1,44 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import string
+
+import analp.__lazy__
+
+
+def normalize(
+    raw: str,
+    join: bool = True,
+    remove_punctation: bool = True,
+) -> list:
+    """\
+    >>> normalize('Hier spricht Helmut .', join=False)
+    ['hier', 'spricht', 'helmut']
+    >>> normalize('Heute sprechen wir doch über ein Thema')
+    'heut sprech wir doch uber ein thema'
+    """
+    raw = raw.lower()
+    splitted = raw.split()
+    if remove_punctation:
+        splitted = [item for item in splitted if item not in string.punctuation]
+    stemmed = [analp.__lazy__.STEMMER.stem(item) for item in splitted]
+    if join:
+        result = ' '.join(stemmed)
+    else:
+        result = stemmed
+    return result
+
+
+def sent_tokenize(text: str, language='german') -> list:
+    """\
+    >>> sent_tokenize('Das ist ein Text. Und ich bin Satz 2.')
+    ['Das ist ein Text.', 'Und ich bin Satz 2.']
+    """
+    tokenized = analp.__lazy__.sent_tokenize(text, language=language)
+    return list(tokenized)

analp-1.0.0/analp/sentiment.py

@@ -0,0 +1,21 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import dataclasses
+
+
+@dataclasses.dataclass
+class Sentiment:
+
+    polarity: float = None
+    subjectivity: float = None
+
+
+def sent_sentiment(text: str) -> Sentiment:  # pylint:disable=W0613
+    pass

analp-1.0.0/analp/utils.py

@@ -0,0 +1,59 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import unicodedata
+
+
+def hashed(item: str) -> str:
+    """\
+    >>> hashed('‘SCHEN')
+    'LEFTSINGLEQUOTATIONMARKSCHEN'
+    """
+    selected = [
+        unicodedata.name(char) if ord(char) > 256 else char for char in item
+    ]
+    selected = ''.join(selected).replace(' ', '')
+    return selected
+
+
+ESCAPE = """\
+‘schen
+‘SCHEN
+’schen
+’SCHEN
+'schen
+'SCHEN
+‘sche
+’sche
+’SCHE
+'sche
+'SCHE
+‘s
+‘S
+’s
+’S
+'s
+'S
+"""
+ESCAPE: dict = {
+    f'{line} ': f'{hashed(line)} ' for line in ESCAPE.strip().splitlines()
+}
+
+
+def escape(text: str):
+    for key, value in ESCAPE.items():
+        text = text.replace(key, value)
+    return text
+
+
+def deescape(text: list):
+    # escape mann_sche
+    for key, value in ESCAPE.items():
+        text = [item.replace(value.strip(), key.strip()) for item in text]
+    return text

analp-1.0.0/analp/word.py

@@ -0,0 +1,80 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import functools
+import re
+
+import konradus
+
+import analp
+import analp.utils
+
+
+def word_tokenize(sentence: str, language: str = 'german') -> list:
+    """\
+    >>> word_tokenize('Ein Aufzählung z.B. heute oder morgen 1. Frosch.')
+    ['Ein', 'Aufzählung', 'z.B.', 'heute', 'oder', 'morgen', '1', '.', 'Frosch', '.']
+    >>> word_tokenize('Hier lässt sich auch der Luhmann’sche Personenbegriff angliedern.', language='science')
+    ['Hier', 'lässt', 'sich', 'auch', 'der', 'Luhmann’sche', 'Personenbegriff', 'angliedern', '.']
+    >>> word_tokenize('Clay Shirky‘s Writings About the Internet.', language='science')
+    ['Clay', 'Shirky‘s', 'Writings', 'About', 'the', 'Internet', '.']
+    >>> word_tokenize('Kaplan/Haenlein (2009) schreiben dazu: “In our view […] Social Media”12.', language='science')
+    ['Kaplan/Haenlein', '(', '2009', ')', 'schreiben', 'dazu', ':', '“', 'In', 'our', 'view', '[', '…', ']', 'Social', 'Media', '”', '12', '.']
+    >>> word_tokenize('Phänomen‚Protest‘ angemessen erfassen', language='science')
+    ['Phänomen', '‚', 'Protest', '‘', 'angemessen', 'erfassen']
+    """
+    if language == 'science':
+        # TODO: ENABLE SCIENCE LATER
+        language = 'german'
+    language = konradus.complexlang(language)
+    if language == 'unknown':
+        # unknown language is not defined yet.
+        language = 'science'
+    sentence = hack(sentence)
+    sentence = analp.utils.escape(sentence)
+    tokenized = analp.__lazy__.word_tokenize(sentence, language=language)
+    result = analp.utils.deescape(tokenized)
+    return result
+
+
+def hack(line: str) -> str:
+    """Remove after improving `Science` parser.
+
+    >>> hack('Phänomen‚Protest‘ angemessen')
+    'Phänomen ‚ Protest‘ angemessen'
+    """
+    line = re.sub(r'‚(?=\S)', '‚ ', line)
+    line = re.sub(r'(?=\S)‚', ' ‚', line)
+    return line
+
+
+def isstopword(word: str, lang: str = 'german') -> bool:
+    """\
+    >>> isstopword('der')
+    True
+    >>> isstopword('House')
+    False
+    """
+    lang = konradus.complexlang(lang)
+    word = word.lower()
+    return word in stopwords(lang)
+
+
+@functools.lru_cache(maxsize=None)
+def stopwords(lang: str = 'german') -> set:
+    """\
+    >>> sorted(stopwords('english'))
+    ['a', 'about', 'above',...'your', 'yours', 'yourself', 'yourselves']
+    >>> sorted(stopwords('fre'))
+    ['ai', 'aie', 'aient', 'aies',...'étées', 'étés', 'êtes']
+    """
+    import nltk.corpus
+    lang = konradus.complexlang(lang)
+    result = set(nltk.corpus.stopwords.words(lang))
+    return result

analp-1.0.0/analp.egg-info/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: analp
+Version: 1.0.0
+Author-email: Helmut Konrad Schewe <helmutus@outlook.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/anaticulae/iamraw
+Project-URL: Repository, https://github.com/anaticulae/iamraw
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: utilo<3.0.0,>=2.107.4
+Requires-Dist: konradus<3.0.0,>=1.0.1
+Requires-Dist: nltk<4.0.0,>=3.9.4
+Requires-Dist: ltk_data<2.0.0,>=1.0.2
+Provides-Extra: dev
+Requires-Dist: utilotest<2.0.0,>=1.0.1; extra == "dev"
+
+# analp

analp-1.0.0/analp.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README
+pyproject.toml
+analp/__init__.py
+analp/__lazy__.py
+analp/corpus.py
+analp/pos.py
+analp/sentence.py
+analp/sentiment.py
+analp/utils.py
+analp/word.py
+analp.egg-info/PKG-INFO
+analp.egg-info/SOURCES.txt
+analp.egg-info/dependency_links.txt
+analp.egg-info/requires.txt
+analp.egg-info/top_level.txt
+tests/test_imports.py

analp-1.0.0/analp.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

analp-1.0.0/analp.egg-info/requires.txt

@@ -0,0 +1,7 @@
+utilo<3.0.0,>=2.107.4
+konradus<3.0.0,>=1.0.1
+nltk<4.0.0,>=3.9.4
+ltk_data<2.0.0,>=1.0.2
+
+[dev]
+utilotest<2.0.0,>=1.0.1

analp-1.0.0/analp.egg-info/top_level.txt

@@ -0,0 +1 @@
+analp

analp-1.0.0/pyproject.toml

@@ -0,0 +1,81 @@
+[build-system]
+requires = [
+    "setuptools>=82.0.1",
+    "wheel>=0.47.0",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "analp"
+version = "1.0.0"
+description = ""
+requires-python = ">=3.12"
+authors = [
+    { name = "Helmut Konrad Schewe", email = "helmutus@outlook.com" },
+]
+dependencies = [
+    "utilo>=2.107.4,<3.0.0",
+    "konradus>=1.0.1,<3.0.0",
+    "nltk>=3.9.4,<4.0.0",
+    "ltk_data>=1.0.2,<2.0.0",
+]
+keywords = []
+classifiers = [
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+license = "MIT"
+license-files = [
+    "LICENSE",
+]
+
+[project.readme]
+file = "README"
+content-type = "text/markdown"
+
+[project.optional-dependencies]
+dev = [
+    "utilotest>=1.0.1,<2.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/anaticulae/iamraw"
+Repository = "https://github.com/anaticulae/iamraw"
+
+[tool.semantic_release]
+version_toml = [
+    "pyproject.toml:project.version",
+]
+
+[tool.semantic_release.changelog]
+mode = "init"
+output_format = "md"
+
+[tool.semantic_release.changelog.default_templates]
+changelog_file = "CHANGELOG"
+
+[tool.semantic_release.commit_parser_options]
+patch_tags = [
+    "fix",
+    "perf",
+    "build",
+    "chore",
+    "ci",
+    "docs",
+    "style",
+    "refactor",
+    "test",
+    "deps",
+]
+
+[tool.setuptools.packages.find]
+where = [
+    ".",
+]
+include = [
+    "analp",
+]
+exclude = [
+    "tests*",
+]

analp-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

analp-1.0.0/tests/test_imports.py

@@ -0,0 +1,19 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+
+def test_import_stopwords_lazy():
+    import analp
+    assert len(analp.__lazy__.STOPWORDS) > 100
+    assert len(analp.STOPWORDS) > 100
+
+
+def test_import_stopwords_from_module():
+    import analp
+    assert len(analp.STOPWORDS) > 100