tglstemmer 0.1.0__tar.gz

tglstemmer-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Andrian Lloyd Maagma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tglstemmer-0.1.0/PKG-INFO

@@ -0,0 +1,147 @@
+Metadata-Version: 2.4
+Name: tglstemmer
+Version: 0.1.0
+Summary: A library for Tagalog word stemming.
+Author-email: Andrian Lloyd Maagma <maagmaandrian@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 Andrian Lloyd Maagma
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: homepage, https://github.com/andrianllmm/tglstemmer
+Project-URL: issues, https://github.com/andrianllmm/tglstemmer/issues
+Keywords: stemmer,tagalog,affix removal
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: nltk>=3.8.1
+Requires-Dist: tabulate>=0.9.0
+Dynamic: license-file
+
+# TglStemmer
+
+**A Python library for Tagalog word stemming**
+
+## About
+
+TglStemmer is a library that finds the root form of
+<a href="https://www.ethnologue.com/language/tgl" target="_blank">Tagalog</a>
+words. It works on inflected words, even those with mixed Tagalog-English
+(Taglish) terms or those not found in dictionaries. It removes affixes, reduces
+repeated syllables, and applies transformation rules to find possible root
+forms. These are filtered using a list of valid words and conditions. The best
+root is then chosen based on how much was changed during the process.
+
+## Installation
+
+```sh
+pip install tglstemmer
+```
+
+## Usage
+
+TglStemmer acts as a standalone library that can be imported via
+`from tglstemmer import stemmer`.
+
+Use `get_stem` to get the root of a word. This takes a word and returns its stem
+as a `Stem` object (basically a string with affixes, reduplication,
+transformations, etc. as additional attributes).
+
+```python
+stem = stemmer.get_stem("nagsulat")
+print(stem)
+# Output: 'sulat'
+```
+
+Since `get_stem` returns a `Stem` object, the properties used in the stemming
+process can be accessed as attributes.
+
+```python
+prefix = stem.pre
+print(prefix)
+# Output: 'nag'
+
+suffix = stem.suf
+print(suffix)
+# Output: None
+```
+
+Use `get_stems()` to get the root of each word in a text. This takes a text and
+returns the stem of each word as a list of `Stem` objects.
+
+```python
+stems = stemmer.get_stems("nagsulat, binasa, at punitin")
+print(stems)
+# Output: ['sulat', 'basa', 'at', 'punit']
+```
+
+Use `get_stem_candidates` to get all the stem candidates of a word. This takes a
+word and returns the possible stems as a list of `Stem` objects. This is helpful
+for loose checking considering candidate selection is not perfect.
+
+```python
+candidates = stemmer.get_stem_candidates("pinakamahusay't")
+print(candidates)
+# Output: ['husay', 'mahusay', 'pinakamahusay']
+```
+
+## Accuracy
+
+The accuracy was tested using a list of stems and their corresponding
+inflections. The list is manually derived from the examples from the book
+[Balarila ng Wikang Pambansa (Santos, 1939)](https://tl.wikipedia.org/wiki/Balarila_ng_Wikang_Pambansa),
+particularly in sections "Palabuuan ng mga Salita" (pp. 28-34) and "Mga Sangkap
+ng Pananalita" (pp. 35-37). This is not a "gold" standard dataset but is chosen
+for testing as the book provides varied examples of inflections during its
+discussion about the process of affixation. Each inflection was stemmed by
+TglStemmer and then compared to the original stem. The test included 266
+stem-inflection pairs.
+
+| Metric              | Value  |
+| ------------------- | ------ |
+| Accuracy            | 75.19% |
+| Correct Attempts    | 200    |
+| Incorrect Attempts  | 66     |
+| Understemming Avg   | 0.69   |
+| Overstemming Avg    | 0.12   |
+| Understemming Total | 184    |
+| Overstemming Total  | 33     |
+
+## Development
+
+This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
+
+Clone the repo and sync dependencies (including dev and test groups):
+
+```sh
+git clone https://github.com/andrianllmm/tagalog-stemmer.git
+cd tagalog-stemmer
+uv sync --all-groups
+```
+
+Run the tests:
+
+```sh
+uv run pytest
+```

tglstemmer-0.1.0/README.md

@@ -0,0 +1,106 @@
+# TglStemmer
+
+**A Python library for Tagalog word stemming**
+
+## About
+
+TglStemmer is a library that finds the root form of
+<a href="https://www.ethnologue.com/language/tgl" target="_blank">Tagalog</a>
+words. It works on inflected words, even those with mixed Tagalog-English
+(Taglish) terms or those not found in dictionaries. It removes affixes, reduces
+repeated syllables, and applies transformation rules to find possible root
+forms. These are filtered using a list of valid words and conditions. The best
+root is then chosen based on how much was changed during the process.
+
+## Installation
+
+```sh
+pip install tglstemmer
+```
+
+## Usage
+
+TglStemmer acts as a standalone library that can be imported via
+`from tglstemmer import stemmer`.
+
+Use `get_stem` to get the root of a word. This takes a word and returns its stem
+as a `Stem` object (basically a string with affixes, reduplication,
+transformations, etc. as additional attributes).
+
+```python
+stem = stemmer.get_stem("nagsulat")
+print(stem)
+# Output: 'sulat'
+```
+
+Since `get_stem` returns a `Stem` object, the properties used in the stemming
+process can be accessed as attributes.
+
+```python
+prefix = stem.pre
+print(prefix)
+# Output: 'nag'
+
+suffix = stem.suf
+print(suffix)
+# Output: None
+```
+
+Use `get_stems()` to get the root of each word in a text. This takes a text and
+returns the stem of each word as a list of `Stem` objects.
+
+```python
+stems = stemmer.get_stems("nagsulat, binasa, at punitin")
+print(stems)
+# Output: ['sulat', 'basa', 'at', 'punit']
+```
+
+Use `get_stem_candidates` to get all the stem candidates of a word. This takes a
+word and returns the possible stems as a list of `Stem` objects. This is helpful
+for loose checking considering candidate selection is not perfect.
+
+```python
+candidates = stemmer.get_stem_candidates("pinakamahusay't")
+print(candidates)
+# Output: ['husay', 'mahusay', 'pinakamahusay']
+```
+
+## Accuracy
+
+The accuracy was tested using a list of stems and their corresponding
+inflections. The list is manually derived from the examples from the book
+[Balarila ng Wikang Pambansa (Santos, 1939)](https://tl.wikipedia.org/wiki/Balarila_ng_Wikang_Pambansa),
+particularly in sections "Palabuuan ng mga Salita" (pp. 28-34) and "Mga Sangkap
+ng Pananalita" (pp. 35-37). This is not a "gold" standard dataset but is chosen
+for testing as the book provides varied examples of inflections during its
+discussion about the process of affixation. Each inflection was stemmed by
+TglStemmer and then compared to the original stem. The test included 266
+stem-inflection pairs.
+
+| Metric              | Value  |
+| ------------------- | ------ |
+| Accuracy            | 75.19% |
+| Correct Attempts    | 200    |
+| Incorrect Attempts  | 66     |
+| Understemming Avg   | 0.69   |
+| Overstemming Avg    | 0.12   |
+| Understemming Total | 184    |
+| Overstemming Total  | 33     |
+
+## Development
+
+This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
+
+Clone the repo and sync dependencies (including dev and test groups):
+
+```sh
+git clone https://github.com/andrianllmm/tagalog-stemmer.git
+cd tagalog-stemmer
+uv sync --all-groups
+```
+
+Run the tests:
+
+```sh
+uv run pytest
+```

tglstemmer-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tglstemmer"
+version = "0.1.0"
+description = "A library for Tagalog word stemming."
+readme = "README.md"
+license = {file = "LICENSE"}
+authors = [
+  {name = "Andrian Lloyd Maagma", email = "maagmaandrian@gmail.com"}
+]
+keywords = ["stemmer", "tagalog", "affix removal"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3.10",
+  "Operating System :: OS Independent",
+]
+requires-python = ">=3.10"
+
+dependencies = [
+  "nltk>=3.8.1",
+  "tabulate>=0.9.0",
+]
+
+[dependency-groups]
+dev = [
+    "commitizen>=4.16.5",
+    "git-cliff>=2.13.1",
+    "pre-commit>=4.6.1",
+    "pytest>=9.1.1",
+    "pytest-cov>=7.1.0",
+    "ruff>=0.16.0",
+    "twine>=5.1.1",
+    "ty>=0.0.64",
+]
+
+[project.urls]
+homepage = "https://github.com/andrianllmm/tglstemmer"
+issues = "https://github.com/andrianllmm/tglstemmer/issues"
+
+[tool.setuptools]
+package-data = {"tglstemmer" = ["resources/*.txt", "resources/affixes/*.txt"]}
+
+[[tool.uv.index]]
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+publish-url = "https://test.pypi.org/legacy/"
+explicit = true
+

tglstemmer-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tglstemmer-0.1.0/src/tglstemmer/__init__.py

@@ -0,0 +1 @@
+"""A package for Tagalog word stemming."""

tglstemmer-0.1.0/src/tglstemmer/helpers/affixes.py

@@ -0,0 +1,27 @@
+import os
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+
+def get_affixes(type, folder_path=None):
+    """Get list of affixes from file.
+
+    Args:
+            type (str): Type of affixes to get. Value can only be:
+                    - 'pre': Prefixes
+                    - 'in': Infixes
+                    - 'suf': Suffixes
+            folder_path (str, optional): Path to the folder containing .txt files of Tagalog affixes.
+
+    Returns:
+            list: A list of affixes of chosen type.
+    """
+    if folder_path is None:
+        folder_path = os.path.join(script_dir, "../resources/affixes/")
+    with open(os.path.join(folder_path, f"{type}fixes.txt")) as in_file:
+        return sorted([affix.strip() for affix in in_file], key=len)
+
+
+PREFIXES = get_affixes("pre")
+INFIXES = get_affixes("in")
+SUFFIXES = get_affixes("suf")

tglstemmer-0.1.0/src/tglstemmer/helpers/alphabet.py

@@ -0,0 +1,2 @@
+VOWELS = set("aeiou")
+CONSONANTS = set("bcdfghjklmnpqrstvwxyz")

tglstemmer-0.1.0/src/tglstemmer/helpers/manipulation.py

@@ -0,0 +1,39 @@
+from ..stem import Stem
+
+
+def replace_letter(token, index, letter):
+    """Replaces a letter in a token.
+
+    Args:
+            token (str): Word to be updated.
+            index (int): Index of the letter to be replaced.
+            letter (str): Letter used to replace.
+
+    Returns:
+            str: The updated word.
+    """
+    token_as_list = list(token)
+
+    token_as_list[index] = letter
+
+    return Stem("".join(token_as_list), **token.__dict__)
+
+
+def swap_letters(token, index1, index2):
+    """Swaps two letters in a token.
+
+    Args:
+            token (str): Word to be updated.
+            index1 (int): Index of the first letter to be swapped.
+            index2 (int): Index of the second letter to be swapped.
+
+    Returns:
+            str: The updated word.
+    """
+    token_as_list = list(token)
+
+    index1_letter = token_as_list[index1]
+    token_as_list[index1] = token_as_list[index2]
+    token_as_list[index2] = index1_letter
+
+    return Stem("".join(token_as_list), **token.__dict__)

tglstemmer-0.1.0/src/tglstemmer/helpers/validation.py

@@ -0,0 +1,63 @@
+from .alphabet import CONSONANTS, VOWELS
+
+
+def is_valid(token, valid_words=None):
+    """Checks if token is valid against a list of words.
+
+    Args:
+        token (str): Any word to be tested.
+        valid_words (list): List of valid words. Defaults to None.
+
+    Returns:
+        str/bool: The token if token is valid, False otherwise.
+    """
+    if not valid_words:
+        return token
+
+    if token in valid_words:
+        return token
+
+    return False
+
+
+def is_acceptable(token):
+    """Checks if token is acceptable against certain acceptability conditions.
+
+    Args:
+        token (str): Any word to be tested.
+
+    Returns:
+        str/bool: The token if token is acceptable, False otherwise.
+    """
+    if is_vowel(token[0]):
+        if len(token) == 2 or (len(token) >= 3 and any(is_consonant(c) for c in token)):
+            return token
+    elif is_consonant(token[0]) and (
+        len(token) == 3 or (len(token) >= 4 and any(is_vowel(c) for c in token))
+    ):
+        return token
+    return False
+
+
+def is_vowel(*substring):
+    """Checks if a substring is a consonant.
+
+    Args:
+        *substring: Substrings to be tested.
+
+    Returns:
+        bool: True if substring is consonant, False otherwise.
+    """
+    return all(letter.lower() in VOWELS for letter in "".join(substring))
+
+
+def is_consonant(*substring):
+    """Checks if a substring is a consonant.
+
+    Args:
+        *substring: Substrings to be tested.
+
+    Returns:
+        bool: True if substring is consonant, False otherwise.
+    """
+    return all(letter.lower() in CONSONANTS for letter in "".join(substring))

tglstemmer-0.1.0/src/tglstemmer/helpers/words.py

@@ -0,0 +1,17 @@
+import os
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+
+def get_words(file_path=None):
+    """Get list of words from a Tagalog word list txt file.
+
+    Args:
+            file_path (str, optional): Path to the Tagalog word lsit txt file. Defaults to '<script_dir>/../resources/tgl_wordlist.txt'.
+    Returns:
+            list: A list of words.
+    """
+    if file_path is None:
+        file_path = os.path.join(script_dir, "../resources/tgl_wordlist.txt")
+    with open(file_path) as in_file:
+        return [word.strip().lower() for word in in_file]

tglstemmer-0.1.0/src/tglstemmer/resources/affixes/infixes.txt

@@ -0,0 +1,2 @@
+in
+um

tglstemmer-0.1.0/src/tglstemmer/resources/affixes/prefixes.txt

@@ -0,0 +1,102 @@
+i
+ka
+ika
+pa
+pang
+pan
+pam
+pina
+ipa
+ipapa
+paka
+pakaka
+pana
+panana
+pama
+pamama
+pinaka
+pinakama
+ma
+mang
+man
+mam
+mina
+mai
+maii
+maka
+makaka
+makakapag
+makapa
+makapag
+na
+nang
+nan
+nam
+nai
+naii
+naka
+nakaka
+nakakapag
+nakapa
+nakapag
+napa
+napapa
+napaka
+napag
+sa
+sang
+san
+sam
+ni
+mag
+magka
+magkaka
+magpa
+magpapa
+nag
+nagka
+nagkaka
+nagpa
+nagpapa
+pag
+pagka
+pagkaka
+pagpa
+pagpapa
+pinag
+ipag
+ipinag
+pinagka
+pinagkaka
+ipinagka
+ipinagkaka
+tag
+tig
+may
+maki
+makipag
+makipagka
+makikipag
+makikipagka
+naki
+nakipag
+nakipagka
+nakikipag
+nakikipagka
+paki
+pakiki
+pakipag
+pakipagka
+pakikipag
+pakikipagka
+pala
+mala
+tiga
+tigapag
+taga
+tagapag
+sing
+sin
+sim
+kasing
+di-

tglstemmer-0.1.0/src/tglstemmer/resources/affixes/suffixes.txt

@@ -0,0 +1,19 @@
+an
+in
+han
+hin
+nan
+anan
+nin
+anin
+ang
+ing
+ita
+ito
+dor
+syon
+siyon
+ng
+g
+'t
+'y