amharicNLP 0.0.0__tar.gz

amharicnlp-0.0.0/LICENSE

@@ -0,0 +1 @@
+MIT License

amharicnlp-0.0.0/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: amharicNLP
+Version: 0.0.0
+License-File: LICENSE
+Dynamic: license-file

amharicnlp-0.0.0/README.md

@@ -0,0 +1,22 @@
+# 🇪🇹 Amharic NLP Toolkit
+
+A lightweight and easy-to-use Natural Language Processing (NLP) toolkit for the Amharic language.  
+Includes tokenization, normalization, and sentiment analysis tools.
+
+---
+
+## ✨ Features
+
+- 🔠 Amharic Text Tokenization
+- 🧹 Unicode Normalization & Cleaning
+- 😊 Simple Sentiment Analysis (positive/negative)
+- 🧰 Command Line Interface (CLI) support
+
+---
+
+## 📦 Installation
+
+```bash
+git clone https://github.com/yonasab12/amharic-nlp.git
+cd amharic-nlp
+pip install .

amharicnlp-0.0.0/amharicNLP/__init__.py

@@ -0,0 +1,4 @@
+from .resources import *
+from  .example import *
+from .wordnet import *
+from .utilities import *

amharicnlp-0.0.0/amharicNLP/example/__init__.py

@@ -0,0 +1 @@
+from.test import *

amharicnlp-0.0.0/amharicNLP/example/test.py

@@ -0,0 +1,54 @@
+
+
+from amharicNLP.resources.cleaner import AmharicCleaner
+from amharicNLP.resources.normalizer import AmharicNormalizer
+from amharicNLP.resources.lemmatizer import AmharicLemmatizer
+from amharicNLP.resources.stemmer import AmharicStemmer           
+from amharicNLP.resources.stopwrod import AmharicStopwordProcessor
+from amharicNLP.resources.tokenizer import AmharicWordTokenizer
+
+# Sample Amharic text
+sample_text = "በአገራችን ኢትዮጵያ <h1/> �ላ ያሉ ተማሪዎች በትምህርት ላይ ትኩረት ማድረግ አለባቸው። 123 ቁጥር! በላይ ዘለቀ ጀግና የኢትዮጵያ አርበኛ ነበር።"
+
+
+# Initialize all processors
+cleaner = AmharicCleaner()
+normalizer = AmharicNormalizer()
+lemmatizer = AmharicLemmatizer()
+stemmer = AmharicStemmer()
+stopword_processor = AmharicStopwordProcessor()
+
+# Apply each processing step
+print("Original Text:", sample_text)
+
+# 1. Clean the text
+cleaned_texth = cleaner.remove_html(sample_text)
+
+print("\nAfter HTML Removal:", cleaned_texth)
+cleaned_textn = cleaner.remove_noise(cleaned_texth)
+print("\nAfter Cleaning:", cleaned_textn)
+
+
+
+
+# 2. Normalize the text
+normalized_text = normalizer.normalize_amharic_chars(cleaned_textn)
+normalized_text2=normalizer.normalize_punctuation_spacing(normalized_text)
+                                                
+normalized_text3=normalizer.expand_abbreviations(normalized_text2)
+print("\nAfter Normalization:", normalized_text3)
+
+
+
+# 4. Stopword removal
+without_stopwords = stopword_processor.remove_stopwords(normalized_text3)
+print("\nAfter Stopword Removal:", without_stopwords)
+
+stem= stemmer.stem_amharic(without_stopwords)
+print("\nAfter Stemming:", stem)
+# 5. Lemmatization
+#lemmatized_text = [lemmatizer.lemmatize(word) for word in without_stopwords.split()]
+#print("\nAfter Lemmatization:", " ".join(lemmatized_text))
+
+# 6. Stemming
+##print("\nAfter Stemming:", " ".join(stemmed_text))

amharicnlp-0.0.0/amharicNLP/resources/__init__.py

@@ -0,0 +1,9 @@
+from .cleaner import AmharicCleaner
+from .normalizer import AmharicNormalizer
+from .lemmatizer import AmharicLemmatizer
+from .stemmer import AmharicStemmer
+from.utils import AmharicLanguageDetector
+from .stopwrod import AmharicStopwordProcessor
+
+
+__all__ = ['AmharicCleaner', 'AmharicNormalizer', 'AmharicLemmatizer', 'AmharicStemmer','AmharicLanguageDetector', 'AmharicStopwordProcessor','AmharicLemmatizer']

amharicnlp-0.0.0/amharicNLP/resources/cleaner.py

@@ -0,0 +1,58 @@
+from.utils import AmharicLanguageDetector
+import re
+
+class AmharicCleaner:
+    """import re
+from utils import is_amharic_char
+    A class for cleaning text by removing HTML tags, emojis, and non-Amharic characters.
+    """
+    
+    def __init__(self):
+        # Compile regex patterns once when the class is instantiated
+        self.emoji_pattern = re.compile("["
+            u"\U0001F600-\U0001F64F"  
+            u"\U0001F300-\U0001F5FF"  
+            u"\U0001F680-\U0001F6FF"  
+            u"\U0001F1E0-\U0001F1FF"  
+            u"\U00002500-\U00002BEF"  
+            u"\U00002702-\U000027B0"
+            u"\U00002702-\U000027B0"
+            u"\U000024C2-\U0001F251"
+            u"\U0001f926-\U0001f937"
+            u"\U00010000-\U0010ffff"
+            u"\u2640-\u2642" 
+            u"\u2600-\u2B55"
+            u"\u200d"
+            u"\u23cf"
+            u"\u23e9"
+            u"\u231a"
+            u"\ufe0f"  
+            u"\u3030"
+            "]+", flags=re.UNICODE)
+        
+        self.html_pattern = re.compile(r'<.*?>')
+
+    def remove_html(self, text: str) -> str:
+        """Remove HTML tags from text."""
+        return self.html_pattern.sub('', text)
+
+    def remove_noise(self, text: str, keep_amharic: bool = True) -> str:
+        """
+        Remove non-Amharic characters, emojis, and special symbols while preserving Amharic abbreviations.
+        
+        Args:
+            text: Input text to clean
+            keep_amharic: If True, keeps only Amharic characters, numbers, basic punctuation and abbreviations
+        """
+        text = self.emoji_pattern.sub(r'', text)
+        
+        if keep_amharic:
+            text = ''.join(char for char in text if (
+                AmharicLanguageDetector.is_amharic_char(char) or 
+                char.isdigit() or 
+                char in '.,!?;:- ' or
+                char in '/.'  
+            ))
+        
+        text = ' '.join(text.split())
+        return text

amharicnlp-0.0.0/amharicNLP/resources/lemmatizer.py

@@ -0,0 +1,84 @@
+import os
+import json
+
+class AmharicLemmatizer:
+    def __init__(self, wordnet_path=None):
+        # Handle direct dictionary input
+        if isinstance(wordnet_path, dict):
+            self.wordnet = wordnet_path
+        else:
+            # Handle case where wordnet_path might be a list
+            if isinstance(wordnet_path, list) and wordnet_path:
+                wordnet_path = wordnet_path[0]  # Take first element if it's a non-empty list
+            
+            if wordnet_path is None:
+                current_dir = os.path.dirname(__file__)
+                wordnet_path = os.path.join(current_dir, "..", "wordnet", "amh_wordnet.json")
+            
+            # Validate path type
+            if not isinstance(wordnet_path, (str, bytes, os.PathLike)):
+                # Get the actual type name for better error message
+                actual_type = type(wordnet_path).__name__
+                raise TypeError(f"Invalid path type: {actual_type}. Expected string, path-like object, or dictionary.")
+            
+            if not os.path.exists(wordnet_path):
+                raise FileNotFoundError(f"WordNet file not found at {wordnet_path}")
+            
+            self.wordnet = self.load_wordnet(wordnet_path)
+        
+        self.lemma_map = {}
+        self.fallback_map = {}
+        self.build_lemma_maps()
+    
+    def load_wordnet(self, path):
+        # If we already have dictionary data, return it directly
+        if isinstance(path, dict):
+            return path
+        with open(path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    def build_lemma_maps(self):
+        # Ensure we're working with a list of synsets
+        if isinstance(self.wordnet, dict) and "synsets" in self.wordnet:
+            wordnet_data = self.wordnet["synsets"]
+        elif isinstance(self.wordnet, list):
+            wordnet_data = self.wordnet
+        else:
+            # Handle unexpected structure
+            wordnet_data = []
+        
+        for synset in wordnet_data:
+            # Skip synsets without 'lemmas' key
+            if "lemmas" not in synset or not synset["lemmas"]:
+                continue
+                
+            base_lemma = synset["lemmas"][0]
+            pos_tag = synset.get("pos", None)
+            
+            for lemma in synset["lemmas"]:
+                normalized = lemma.lower()
+                
+                # Add to POS-specific map if POS is available
+                if pos_tag:
+                    pos_key = (normalized, pos_tag)
+                    if pos_key not in self.lemma_map:
+                        self.lemma_map[pos_key] = base_lemma
+                
+                # Add to fallback map
+                if normalized not in self.fallback_map:
+                    self.fallback_map[normalized] = base_lemma
+    
+    def lemmatize(self, word, pos=None):
+        normalized = word.lower()
+        
+        if pos is not None:
+            # Try POS-specific lookup
+            pos_key = (normalized, pos)
+            if pos_key in self.lemma_map:
+                return self.lemma_map[pos_key]
+        
+        # Try fallback (POS=None)
+        if normalized in self.fallback_map:
+            return self.fallback_map[normalized]
+        
+        return word

amharicnlp-0.0.0/amharicNLP/resources/normalizer.py

@@ -0,0 +1,300 @@
+import re
+import unicodedata
+from typing import Dict, Optional, List, Tuple, Set
+
+class AmharicNormalizer:
+   
+    AMHARIC_NORMALIZATION_MAP = {
+        # Normalize variants of ሀ series
+        'ሃ': 'ሀ', 'ኅ': 'ሀ', 'ኃ': 'ሀ', 'ሐ': 'ሀ', 'ሓ': 'ሀ', 'ኻ': 'ሀ',
+        'ሑ': 'ሁ', 'ኁ': 'ሁ', 'ዅ': 'ሁ',
+        'ኂ': 'ሂ', 'ሒ': 'ሂ', 'ኺ': 'ሂ',
+        'ኌ': 'ሄ', 'ሔ': 'ሄ', 'ዄ': 'ሄ',
+        'ሕ': 'ህ', 'ኅ': 'ህ',
+        'ኆ': 'ሆ', 'ሖ': 'ሆ', 'ኾ': 'ሆ',
+        
+        # Normalize ሰ series
+        'ሠ': 'ሰ', 'ሡ': 'ሱ', 'ሢ': 'ሲ', 'ሣ': 'ሳ', 
+        'ሤ': 'ሴ', 'ሥ': 'ስ', 'ሦ': 'ሶ',
+        
+        # Normalize አ series
+        'ዓ': 'አ', 'ኣ': 'አ', 'ዐ': 'አ',
+        'ዑ': 'ኡ', 'ዒ': 'ኢ', 'ዔ': 'ኤ', 'ዕ': 'እ', 'ዖ': 'ኦ',
+        
+        # Normalize ፀ series
+        'ጸ': 'ፀ', 'ጹ': 'ፁ', 'ጺ': 'ፂ', 'ጻ': 'ፃ',
+        'ጼ': 'ፄ', 'ጽ': 'ፅ', 'ጾ': 'ፆ',
+        
+        # Other normalizations
+        'ቊ': 'ቁ', 'ኵ': 'ኩ'
+    }
+
+    # Labialized character components
+    LABIALIZED_COMPONENTS = {
+        'ሉ': 'ሏ', 'ሙ': 'ሟ', 'ቱ': 'ቷ', 'ሩ': 'ሯ', 'ሱ': 'ሷ', 'ሹ': 'ሿ',
+        'ቁ': 'ቋ', 'ቡ': 'ቧ', 'ቹ': 'ቿ', 'ሁ': 'ኋ', 'ኑ': 'ኗ', 'ኙ': 'ኟ',
+        'ኩ': 'ኳ', 'ዙ': 'ዟ', 'ጉ': 'ጓ', 'ደ': 'ዷ', 'ጡ': 'ጧ', 'ጩ': 'ጯ',
+        'ጹ': 'ጿ', 'ፉ': 'ፏ'
+    }
+
+    # Punctuation Handling
+    AMHARIC_PUNCTUATION = {'።', '፣', '፤', '፥', '፦', '፡'}  # Amharic-specific punctuation
+
+    PUNCTUATION_NORMALIZATION = [
+        (re.compile(r'\.'), ' . '), 
+        (re.compile(r'፡'), ' ፡ '),
+        (re.compile(r'[!?]'), r' \g<0> '),  
+        (re.compile(r'።'), ' ። '),
+        (re.compile(r'፣'), ' ፣ '),
+        (re.compile(r'፤'), ' ፤ '),
+        (re.compile(r'[-–—]+'), r' \g<0> '),
+    ]
+
+    # Numerals
+    AMHARIC_NUMERALS = {
+        '፩': '1', '፪': '2', '፫': '3', '፬': '4', '፭': '5',
+        '፮': '6', '፯': '7', '፰': '8', '፱': '9', '፲': '10',
+        '፳': '20', '፴': '30', '፵': '40', '፶': '50',
+        '፷': '60', '፸': '70', '፹': '80', '፺': '90', '፻': '100'
+    }
+
+    # Spelling Corrections
+    COMMON_SPELLING_CORRECTIONS = {
+        'አበ': 'አበራ',
+        'ሠላም': 'ሰላም',
+        # Add more common corrections here
+    }
+
+    # Abbreviations
+    AMHARIC_ABBREVIATIONS = {
+        'ት/ቤት': 'ትምህርት ቤት',
+        'ት/ርት': 'ትምህርት',
+        'ት/ክፍል': 'ትምህርት ክፍል',
+        'ሃ/አለቃ': 'ሀምሳ አለቃ',
+        'ሃ/ስላሴ': 'ሀይለ ስላሴ',
+        'ደ/ዘይት': 'ደብረ ዘይት',
+        'ደ/ታቦር': 'ደብረ ታቦር',
+        'መ/ር': 'መምህር',
+        'መ/ቤት': 'መስሪያ ቤት',
+        'መ/አለቃ': 'መቶ አለቃ',
+        'ክ/ከተማ': 'ክፍለ ከተማ',
+        'ክ/ሀገር': 'ክፍለ ሀገር',
+        'ወ/ር': 'ወታደር',
+        'ወ/ሮ': 'ወይዘሮ',
+        'ወ/ሪት': 'ወይዘሪት',
+        'ወ/ስላሴ': 'ወሌተ ስላሴ',
+        'ፍ/ስላሴ': 'ፍቅረ �ስላሴ',
+        'ፍ/ቤት': 'ፍርድ ቤት',
+        'ጽ/ቤት': 'ጽህፈት ቤት',
+        'ሲ/ር': 'ሲስተር',
+        'ጠ/ሚኒስትር': 'ጠቅላይ ሚኒስትር',
+        'ዶ/ር': 'ዶክተር',
+        'ገ/ጊዮርጊስ': 'ገብረ ጊዮርጊስ',
+        'ቤ/ክርስትያን': 'ቤተ ክርስትያን',
+        'ም/ስራ': 'ምክትል ስራ',
+        'ም/ቤት': 'ምክር ቤት',
+        'ተ/ሃይማኖት': 'ተክለ ሃይማኖት',
+        'ሚ/ር': 'ሚኒስትር',
+        'ኮ/ል': 'ኮለኔል',
+        'ሜ/ጀነራል': 'ሜጀር ጀነራል',
+        'ብ/ጀነራል': 'ብርጋዳር ጀነራል',
+        'ሌ/ኮለኔል': 'ሌተናንት ኮለኔል',
+        'ሊ/መንበር': 'ሊቀ መንበር',
+        'አ/አ': 'አዲስ አበባ',
+        'ር/መምህር': 'ርእሰ መምህር',
+        'ፕ/ት': 'ፕሬዝዳንት',
+        'ዓ.ም': 'አመተ ምህረት',
+        'ዓ.ዓ': 'አዲስ አበባ',
+        'ዶ.ር': 'ዶክተር',
+        'ፕ/ር': 'ፕሮፌሰር'
+    }
+
+    # ====================== CORE FUNCTIONS ======================
+
+    @staticmethod
+    def normalize_unicode(text: str, form: str = 'NFC') -> str:
+        """Normalize Unicode text to specified form (default: NFC)."""
+        return unicodedata.normalize(form, text)
+
+    def normalize_amharic_chars(self, text: str) -> str:
+        """Normalize Amharic characters to their canonical forms with enhanced labialization handling."""
+        normalized = []
+        i = 0
+        n = len(text)
+        
+        while i < n:
+            current_char = text[i]
+            
+            # Check for labialized character patterns (base + ዋ or አ)
+            if i + 1 < n and current_char in self.LABIALIZED_COMPONENTS:
+                next_char = text[i+1]
+                if next_char in ['ዋ', 'አ']:
+                    normalized.append(self.LABIALIZED_COMPONENTS[current_char])
+                    i += 2
+                    continue
+            
+            # Handle standard character mappings
+            normalized.append(self.AMHARIC_NORMALIZATION_MAP.get(current_char, current_char))
+            i += 1
+        
+        return ''.join(normalized)
+
+    def normalize_punctuation_spacing(self, text: str) -> str:
+        """Ensure proper spacing around punctuation marks."""
+        for pattern, replacement in self.PUNCTUATION_NORMALIZATION:
+            text = pattern.sub(replacement, text)
+        return text
+
+    def remove_punctuation(self, text: str, 
+                         keep_basic: bool = True,
+                         keep_amharic: bool = True) -> str:
+        """
+        Enhanced punctuation removal with Amharic support.
+        
+        Args:
+            text: Input text
+            keep_basic: Keep basic punctuation (.!?,;:-)
+            keep_amharic: Keep Amharic-specific punctuation
+        """
+        keep = []
+        if keep_basic:
+            keep.append(r'\.!?,;:-')
+        if keep_amharic:
+            keep.append(''.join(self.AMHARIC_PUNCTUATION))
+        
+        if keep:
+            punctuation = rf'[^\w\s{"".join(keep)}]'
+        else:
+            punctuation = r'[^\w\s]'
+        
+        return re.sub(punctuation, '', text)
+
+    @staticmethod
+    def normalize_whitespace(text: str) -> str:
+        """Normalize all whitespace characters to single space."""
+        text = re.sub(r'[\s\u2000-\u200F\u2028-\u202F\u205F\u3000]+', ' ', text)
+        return text.strip()
+
+    def normalize_numbers(self, text: str, to_western: bool = False) -> str:
+        """
+        Normalize Amharic numbers to either Western or keep Amharic.
+        
+        Args:
+            to_western: If True, converts Amharic numbers to Western (0-9)
+        """
+        if not to_western:
+            return text
+        
+        return ''.join(self.AMHARIC_NUMERALS.get(char, char) for char in text)
+
+    def fix_common_spelling(self, text: str, custom_map: Optional[Dict[str, str]] = None) -> str:
+        """
+        Fix common Amharic spelling variations.
+        
+        Args:
+            custom_map: Additional spelling corrections to apply
+        """
+        corrections = self.COMMON_SPELLING_CORRECTIONS.copy()
+        if custom_map:
+            corrections.update(custom_map)
+        
+        for wrong, correct in corrections.items():
+            text = text.replace(wrong, correct)
+        
+        return text
+
+    def expand_abbreviations(self, text: str, custom_abbr: Optional[Dict[str, str]] = None) -> str:
+        """
+        Expand Amharic abbreviations to their full forms.
+        
+        Args:
+            text: Input text containing abbreviations
+            custom_abbr: Additional abbreviations to expand
+        
+        Returns:
+            Text with abbreviations expanded
+        """
+        abbreviations = self.AMHARIC_ABBREVIATIONS.copy()
+        if custom_abbr:
+            abbreviations.update(custom_abbr)
+        
+        # Sort by length to match longer abbreviations first
+        for abbr in sorted(abbreviations, key=len, reverse=True):
+            text = text.replace(abbr, abbreviations[abbr])
+        
+        return text
+
+    # ====================== MAIN NORMALIZATION FUNCTION ======================
+
+    def normalize_text(
+        self,
+        text: str,
+        *,
+        unicode_form: str = 'NFC',
+        normalize_chars: bool = True,
+        normalize_punct_spacing: bool = True,
+        remove_punct: bool = False,
+        keep_basic_punct: bool = True,
+        keep_amharic_punct: bool = True,
+        normalize_ws: bool = True,
+        normalize_nums: bool = False,
+        fix_spelling: bool = False,
+        expand_abbr: bool = True,
+        custom_spelling_map: Optional[Dict[str, str]] = None,
+        custom_abbr_map: Optional[Dict[str, str]] = None
+    ) -> str:
+        """
+        Comprehensive Amharic text normalization pipeline.
+        
+        Args:
+            text: Input text to normalize
+            unicode_form: Unicode normalization form (NFC, NFD, NFKC, NFKD)
+            normalize_chars: Normalize Amharic character variants
+            normalize_punct_spacing: Add spaces around punctuation
+            remove_punct: Remove punctuation
+            keep_basic_punct: Keep basic punctuation if removing punctuation
+            keep_amharic_punct: Keep Amharic punctuation if removing punctuation
+            normalize_ws: Normalize whitespace
+            normalize_nums: Convert Amharic numbers to Western numerals
+            fix_spelling: Apply common spelling corrections
+            expand_abbr: Expand common Amharic abbreviations
+            custom_spelling_map: Custom spelling corrections dictionary
+            custom_abbr_map: Custom abbreviations dictionary
+        
+        Returns:
+            Normalized text string
+        """
+        # Normalize Unicode first
+        text = self.normalize_unicode(text, unicode_form)
+        
+        # Expand abbreviations
+        if expand_abbr:
+            text = self.expand_abbreviations(text, custom_abbr_map)
+        
+        # Normalize Amharic characters
+        if normalize_chars:
+            text = self.normalize_amharic_chars(text)
+        
+        # Normalize punctuation spacing
+        if normalize_punct_spacing:
+            text = self.normalize_punctuation_spacing(text)
+        
+        # Remove punctuation (if enabled)
+        if remove_punct:
+            text = self.remove_punctuation(text, 
+                                        keep_basic=keep_basic_punct,
+                                        keep_amharic=keep_amharic_punct)
+        
+        # Normalize whitespace
+        if normalize_ws:
+            text = self.normalize_whitespace(text)
+        
+        # Normalize numbers
+        if normalize_nums:
+            text = self.normalize_numbers(text, to_western=True)
+        
+        # Fix common spelling
+        if fix_spelling:
+            text = self.fix_common_spelling(text, custom_spelling_map)
+        
+        return text

amharicnlp-0.0.0/amharicNLP/resources/stemmer.py

@@ -0,0 +1,85 @@
+class AmharicStemmer:
+    # Comprehensive Amharic Stemmer
+    # Based on linguistic research from Addis Ababa University and Amharic NLP Toolkit
+
+    # Curated affix lists from academic sources
+    PREFIXES = [
+        # Subject markers
+        "እን", "ይ", "ት", "እ", "ም", "ተ", "አል", "አት", "አይ", "ን", 
+        "ታ", "ይተ", "አል", "አም", "ተም", "አ",
+        
+        # Prepositions and conjunctions
+        "የ", "በ", "ከ", "ለ", "ስለ", "እስከ", "በስተ", "በኩል", "በውስጥ", 
+        "ከውስጥ", "ለምሳሌ", "እንደ", "ያለ", "ከላይ", "በላይ",
+        
+        # Verb derivation markers
+        "አስ", "ተስ", "አስተ", "ማ", "መ", "ታ", "ት", "ብ", "ን", "ድ", 
+        "ግ", "ል", "ም", "ስ", "ቅ", "ት", "ው", "ዝ", "ሻ", "ካ", "ያ", 
+        "ዋ", "ኃ", "ኅ", "ኻ", "ል", "ት", "ክ"
+    ]
+
+    SUFFIXES = [
+        # Object markers
+        "ኝ", "ህ", "ሽ", "ችሁ", "ኟቸው", "ዋት", "አቸው", "አችኋል", 
+        "ኧቸው", "ኣቸው", "ኣችሁ", "ኣችኋል", "ዎት", "ዎች", "ዎችን",
+        
+        # Tense/aspect markers
+        "አለ", "አል", "ኣል", "ኣለ", "ኧል", "ኧለ", "አችሁ", "አችኋል",
+        "ኧችሁ", "ኧችኋል", "አሉ", "ኧሉ", "ኣሉ", "አችሁ", "ኧሁ", "ኧህ", 
+        "ኧሽ", "ኧን", "ኧ", "ኦ", "ኡ", "ኢ", "ኤ",
+        
+        # Verb suffixes
+        "ነው", "ነበር", "ኧው", "ኧዎ", "ኧና", "ኧም", "ኧኛ", "ኧዎች", "ኧዋ", 
+        "ኧኣት", "ኣት", "ኧለች", "ኣለች", "ኣለሁ", "ኧለሁ",
+        
+        # Nominal suffixes
+        "ች", "ው", "ዎ", "ና", "ት", "ም", "ኛ", "ዎች", "ዋ", "ዋል", "ን", "ቸው"
+    ]
+
+    # Words exempt from stemming
+    PROTECTED_WORDS = {
+           "ውስጥ", "ላይ", "በላይ", "ከታች", "ፊት", "ኋላ", "ትርጉም", 
+        "ስም", "ስራ", "ቤት", "አባት", "እናት", "ልጅ", "ወንድ", "ሴት",
+        "እኔ", "አንተ", "አንቺ", "እሱ", "እሷ", "እኛ", "እናንተ", "እሳቸው",
+        "ኢትዮጵያ", "አዲስ", "አበባ", "ሕግ", "ፍትህ", "ዴሞክራሲ"
+    }
+
+    def __init__(self):
+        # Sort by length for longest-match-first
+        self.PREFIXES.sort(key=len, reverse=True)
+        self.SUFFIXES.sort(key=len, reverse=True)
+
+    def stem_amharic(self, word):
+        """Stem Amharic words using morphological analysis"""
+        # Preserve protected words and short words
+        if word in self.PROTECTED_WORDS or len(word) <= 3:
+            return word
+            
+        original = word
+        stem = word
+        
+        # Remove prefixes (max 2 iterations)
+        for _ in range(2):
+            removed = False
+            for prefix in self.PREFIXES:
+                if stem.startswith(prefix) and len(stem) > len(prefix) + 2:
+                    stem = stem[len(prefix):]
+                    removed = True
+                    break
+            if not removed:
+                break
+        
+        # Remove suffixes (max 2 iterations)
+        for _ in range(2):
+            removed = False
+            for suffix in self.SUFFIXES:
+                if stem.endswith(suffix) and len(stem) > len(suffix) + 2:
+                    stem = stem[:-len(suffix)]
+                    removed = True
+                    break
+            if not removed:
+                break
+        
+        # Fallback to original if stem is invalid
+        return stem if len(stem) >= 3 else original
+

amharicnlp-0.0.0/amharicNLP/resources/stopwrod.py

@@ -0,0 +1,38 @@
+import os
+import re
+
+class AmharicStopwordProcessor:
+    def __init__(self, stopwords_file=None):
+        """
+        Initialize stopword processor with optional custom stopwords file
+        :param stopwords_file: Optional path to custom stopwords file
+        """
+        if stopwords_file is None:
+            # Automatically locate the stopwords file in package resources
+            current_dir = os.path.dirname(os.path.abspath(__file__))
+            stopwords_file = os.path.join(current_dir, "amharic_stopwords.txt")
+        
+        self.stopwords = self.load_stopwords(stopwords_file)
+    
+    @staticmethod
+    def load_stopwords(file_path):
+        """Load stopwords from a text file."""
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"Stopwords file not found: {file_path}")
+        
+        with open(file_path, 'r', encoding='utf-8') as file:
+            return set(line.strip() for line in file)
+    
+    def remove_stopwords(self, text):
+        """
+        Remove Amharic stopwords from text
+        :param text: Input text to process
+        :return: Filtered text with stopwords removed
+        """
+        # Tokenize while preserving Amharic characters
+        words = re.findall(r'[\w\u1200-\u137F]+', text, re.UNICODE)
+        
+        # Filter out stopwords (case-insensitive)
+        filtered_words = [word for word in words if word.lower() not in self.stopwords]
+        
+        return ' '.join(filtered_words)

amharicnlp-0.0.0/amharicNLP/resources/tests/conftest.py

@@ -0,0 +1,14 @@
+import pytest
+from amharicNLP.resources import AmharicCleaner, AmharicNormalizer
+
+@pytest.fixture
+def sample_amharic_text():
+    return "ይህ የሙከራ ጽሑፍ ነው።"
+
+@pytest.fixture
+def cleaner():
+    return AmharicCleaner()
+
+@pytest.fixture
+def normalizer():
+    return AmharicNormalizer()

amharicnlp-0.0.0/amharicNLP/resources/tests/test_cleaner.py

@@ -0,0 +1,14 @@
+import pytest
+from amharicNLP .resources.cleaner import AmharicCleaner
+
+class TestAmharicCleaner:
+    @pytest.fixture
+    def cleaner(self):
+        return AmharicCleaner()
+    
+    def test_remove_html(self, cleaner):
+        assert cleaner.remove_html("<p>Hello</p>") == "Hello"
+    
+    def test_remove_noise(self, cleaner):
+        text = "ሰላም! 😊 Hello 123"
+        assert cleaner.remove_noise(text) == "ሰላም! 123"

amharicnlp-0.0.0/amharicNLP/resources/tests/test_normalizer.py

@@ -0,0 +1,6 @@
+from amharicNLP .resources.normalizer import AmharicNormalizer
+
+class TestAmharicNormalizer:
+    def test_normalize(self):
+        normalizer = AmharicNormalizer()
+        assert normalizer.normalize("ኣብዚ") == "አብዚ"  # Example normalization

amharicnlp-0.0.0/amharicNLP/resources/tests/test_utils.py

@@ -0,0 +1,7 @@
+from amharicNLP.resources.utils import AmharicLanguageDetector
+
+class TestAmharicLanguageDetector:
+    def test_is_amharic_text(self):
+        detector = AmharicLanguageDetector()
+        assert detector.is_amharic_text("ሰላም") is True
+        assert detector.is_amharic_text("Hello") is False

amharicnlp-0.0.0/amharicNLP/resources/tokenizer.py

@@ -0,0 +1,64 @@
+import re
+
+class AmharicWordTokenizer:
+    """
+    Tokenizer for Amharic text.
+    Handles Amharic punctuation, contractions, and parentheses/brackets.
+    """
+    
+    # Amharic starting quotes and punctuation
+    STARTING_QUOTES = [
+        (re.compile(r'^«'), r'`'),
+        (re.compile(r'^»'), r"'"),
+        (re.compile(r'^"'), r''),
+    ]
+
+    # Amharic punctuation marks
+
+ 
+    PUNCTUATION= [ 
+        (re.compile(r'\.'), r' . '), 
+        (re.compile(r'፡'), r' ፡ '),
+        (re.compile(r'[!?]'), r' \g<0> '),  
+        (re.compile(r'።'), r' ። '),
+        (re.compile(r'፣'), r' ፣ '),
+        (re.compile(r'፤'), r' ፤ '),
+        (re.compile(r'[-–—]+'), r' \g<0> '),
+    ]
+
+    # Parentheses and brackets (replace with labels)
+  
+                        
+    PARENS_BRACKETS = [
+       (re.compile(r'([\(\)\[\]\{\}])'), r' \1 ')
+    ]
+
+
+    # Amharic contractions and compound words
+    CONTRACTIONS = [
+        (re.compile(r'(?i)\b(የ|በ|ከ|ለ|ውስጥ|ላይ|በላይ)([^\s]+)\b'), r'\1 \2'),  # Prepositions
+        (re.compile(r'(?i)\b(እኔ|አንቺ|እሱ|እሷ)([^\s]+)\b'), r'\1 \2'),  # Pronouns
+    ]
+
+    def tokenize(self, text):
+        # Apply starting quotes
+        for regexp, substitution in self.STARTING_QUOTES:
+            text = regexp.sub(substitution, text)
+
+        # Handle punctuation
+        for regexp, substitution in self.PUNCTUATION:
+            text = regexp.sub(substitution, text)
+
+        # Replace parentheses/brackets
+        for regexp, substitution in self.PARENS_BRACKETS:
+            text = regexp.sub(substitution, text)
+
+        # Split contractions and compound words
+        for regexp, substitution in self.CONTRACTIONS:
+            text = regexp.sub(substitution, text)
+
+        # Split on whitespace and return tokens
+        tokens = text.strip().split()
+        return tokens
+
+

amharicnlp-0.0.0/amharicNLP/resources/utils.py

@@ -0,0 +1,21 @@
+class AmharicLanguageDetector:
+    def is_amharic_char(char: str) -> bool:
+        """Check if a character is in the Amharic Unicode block."""
+        # Amharic Unicode range: U+1200 to U+137F
+        return '\u1200' <= char <= '\u137F'
+
+    def is_amharic_text(text: str, threshold: float = 0.7) -> bool:
+        """
+        Check if text is primarily Amharic.
+        
+        Args:
+            text: Input text to check
+            threshold: Minimum proportion of Amharic characters to consider as Amharic text
+        """
+        if not text:
+            return False
+        
+        amharic_count = sum(1 for char in text if AmharicLanguageDetector.is_amharic_char(char))
+        total_chars = len(text)
+        
+        return (amharic_count / total_chars) >= threshold

amharicnlp-0.0.0/amharicNLP/utilities/__init__.py

@@ -0,0 +1 @@
+from .data_augmenter import *

amharicnlp-0.0.0/amharicNLP/utilities/data_augmenter.py

@@ -0,0 +1,61 @@
+import random
+
+
+from amharicNLP.resources.tokenizer import AmharicWordTokenizer
+from amharicNLP.resources.cleaner import AmharicCleaner
+from amharicNLP.resources.normalizer import AmharicNormalizer
+from amharicNLP.resources.lemmatizer import AmharicLemmatizer
+from amharicNLP.resources.stemmer import AmharicStemmer        
+# Initialize class-based components
+tokenizer = AmharicWordTokenizer()
+cleaner = AmharicCleaner()
+
+SYNONYMS = {
+    "እንዴት": ["እንደምን", "በምን አይነት"],
+    "ነህ": ["ነሽ", "ናችሁ"],
+    "ደስ": ["ሐሰት", "ደስታ"],
+    "ብሎኛል": ["አደሰኝ", "ደስ አለኝ"]
+}
+
+def random_deletion(words, p=0.2):
+    if len(words) < 2:
+        return words
+    return [w for w in words if random.random() > p] or [random.choice(words)]
+
+def synonym_replacement(words, n=1):
+    new_words = words.copy()
+    indices = [i for i, w in enumerate(new_words) if w in SYNONYMS]
+    random.shuffle(indices)
+    
+    for i in indices[:min(n, len(indices))]:
+        new_words[i] = random.choice(SYNONYMS[new_words[i]])
+    return new_words
+
+def random_swap(words, n=1):
+    new_words = words.copy()
+    for _ in range(min(n, len(words) // 2)):
+        idx1, idx2 = random.sample(range(len(new_words)), 2)
+        new_words[idx1], new_words[idx2] = new_words[idx2], new_words[idx1]
+    return new_words
+
+def augment_sentence(sentence, aug_prob=0.3):
+    words = tokenizer.tokenize(sentence)
+    if random.random() < aug_prob:
+        words = random_deletion(words)
+    if random.random() < aug_prob:
+        words = synonym_replacement(words, n=1)
+    if random.random() < aug_prob:
+        words = random_swap(words)
+    return tokenizer.detokenize(words)
+
+# Preserving your original example usage
+if __name__ == "__main__":
+    sentence = "እንዴት ነህ ደስ ብሎኛል"
+    print("Original:", sentence)
+    
+    words = tokenizer.tokenize(sentence)
+    
+    print("Random Deletion:", tokenizer.detokenize(random_deletion(words, p=0.3)))
+    print("Synonym Replacement:", tokenizer.detokenize(synonym_replacement(words, n=2)))
+    print("Random Swap:", tokenizer.detokenize(random_swap(words, n=2)))
+    print("Remove Diacritics:", cleaner.remove_diacritics(sentence))

amharicnlp-0.0.0/amharicNLP/wordnet/__init__.py

@@ -0,0 +1,14 @@
+import json
+from pathlib import Path
+from ..resources.lemmatizer import AmharicLemmatizer  
+
+# Load WordNet data
+_wordnet_path = Path(__file__).parent / "amh_wordnet.json"
+with open(_wordnet_path, encoding='utf-8') as f:
+    wordnet_data = json.load(f)
+
+# Initialize lemmatizer (without stemmer)
+lemmatizer = AmharicLemmatizer(wordnet_data)
+
+# Make available at package level
+__all__ = ['wordnet_data', 'lemmatizer']

amharicnlp-0.0.0/amharicNLP/wordnet/loader.py

@@ -0,0 +1,10 @@
+import json
+import os
+
+# Automatically find the file relative to this script
+this_dir = os.path.dirname(__file__)
+wordnet_path = os.path.join(this_dir, "am_wordnet.json")
+
+def load_amharic_wordnet():
+    with open(wordnet_path, encoding="utf-8") as f:
+        return json.load(f)

amharicnlp-0.0.0/amharicNLP.egg-info/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: amharicNLP
+Version: 0.0.0
+License-File: LICENSE
+Dynamic: license-file

amharicnlp-0.0.0/amharicNLP.egg-info/SOURCES.txt

@@ -0,0 +1,30 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+setup.py
+amharicNLP/__init__.py
+amharicNLP.egg-info/PKG-INFO
+amharicNLP.egg-info/SOURCES.txt
+amharicNLP.egg-info/dependency_links.txt
+amharicNLP.egg-info/top_level.txt
+amharicNLP/example/__init__.py
+amharicNLP/example/test.py
+amharicNLP/resources/__init__.py
+amharicNLP/resources/cleaner.py
+amharicNLP/resources/lemmatizer.py
+amharicNLP/resources/normalizer.py
+amharicNLP/resources/stemmer.py
+amharicNLP/resources/stopwrod.py
+amharicNLP/resources/tokenizer.py
+amharicNLP/resources/utils.py
+amharicNLP/resources/tests/__init__.py
+amharicNLP/resources/tests/conftest.py
+amharicNLP/resources/tests/test_cleaner.py
+amharicNLP/resources/tests/test_lemmatizer.py
+amharicNLP/resources/tests/test_normalizer.py
+amharicNLP/resources/tests/test_utils.py
+amharicNLP/utilities/__init__.py
+amharicNLP/utilities/data_augmenter.py
+amharicNLP/wordnet/__init__.py
+amharicNLP/wordnet/loader.py

amharicnlp-0.0.0/amharicNLP.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

amharicnlp-0.0.0/amharicNLP.egg-info/top_level.txt

@@ -0,0 +1 @@
+amharicNLP

amharicnlp-0.0.0/pyproject.toml

@@ -0,0 +1 @@
+version = "0.1.2"

amharicnlp-0.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

amharicnlp-0.0.0/setup.py

@@ -0,0 +1,5 @@
+# setup.py
+from setuptools import setup
+
+if __name__ == "__main__":
+    setup()