webscout 7.0__py3-none-any.whl → 7.2__py3-none-any.whl

webstoken/stemmer.py

@@ -1,73 +1,73 @@
-"""
-Word stemming utilities.
-"""
-
-from typing import Set
-
-
-class Stemmer:
-    """Simple rule-based stemmer implementing Porter-like rules."""
-    
-    def __init__(self):
-        self.vowels: Set[str] = {'a', 'e', 'i', 'o', 'u', 'y'}
-        self.doubles: Set[str] = {'bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'}
-        
-    def is_vowel(self, char: str, prev_char: str = None) -> bool:
-        """Check if a character is a vowel, considering 'y' special cases."""
-        return char in self.vowels or (char == 'y' and prev_char and prev_char not in self.vowels)
-    
-    def count_syllables(self, word: str) -> int:
-        """Count syllables in a word based on vowel sequences."""
-        count = 0
-        prev_char = None
-        for i, char in enumerate(word.lower()):
-            if self.is_vowel(char, prev_char) and (i == 0 or not self.is_vowel(prev_char, word[i-2] if i > 1 else None)):
-                count += 1
-            prev_char = char
-        return count or 1
-    
-    def stem(self, word: str) -> str:
-        """Apply stemming rules to reduce word to its root form."""
-        if len(word) <= 3:
-            return word
-            
-        word = word.lower()
-        
-        # Step 1: Handle plurals and past participles
-        if word.endswith('sses'):
-            word = word[:-2]
-        elif word.endswith('ies'):
-            word = word[:-2]
-        elif word.endswith('ss'):
-            pass
-        elif word.endswith('s') and len(word) > 4:
-            word = word[:-1]
-            
-        # Step 2: Handle -ed and -ing
-        if word.endswith('ed') and self.count_syllables(word[:-2]) > 1:
-            word = word[:-2]
-        elif word.endswith('ing') and self.count_syllables(word[:-3]) > 1:
-            word = word[:-3]
-            
-        # Step 3: Handle miscellaneous endings
-        if len(word) > 5:
-            if word.endswith('ement'):
-                word = word[:-5]
-            elif word.endswith('ment'):
-                word = word[:-4]
-            elif word.endswith('ent'):
-                word = word[:-3]
-                
-        # Step 4: Handle -ity endings
-        if word.endswith('ity') and len(word) > 6:
-            word = word[:-3]
-            if word.endswith('abil'):
-                word = word[:-4] + 'able'
-            elif word.endswith('ic'):
-                word = word[:-2]
-                
-        # Final step: Remove double consonants at the end
-        if len(word) > 2 and word[-2:] in self.doubles:
-            word = word[:-1]
-            
-        return word
+"""
+Word stemming utilities.
+"""
+
+from typing import Set
+
+
+class Stemmer:
+    """Simple rule-based stemmer implementing Porter-like rules."""
+    
+    def __init__(self):
+        self.vowels: Set[str] = {'a', 'e', 'i', 'o', 'u', 'y'}
+        self.doubles: Set[str] = {'bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'}
+        
+    def is_vowel(self, char: str, prev_char: str = None) -> bool:
+        """Check if a character is a vowel, considering 'y' special cases."""
+        return char in self.vowels or (char == 'y' and prev_char and prev_char not in self.vowels)
+    
+    def count_syllables(self, word: str) -> int:
+        """Count syllables in a word based on vowel sequences."""
+        count = 0
+        prev_char = None
+        for i, char in enumerate(word.lower()):
+            if self.is_vowel(char, prev_char) and (i == 0 or not self.is_vowel(prev_char, word[i-2] if i > 1 else None)):
+                count += 1
+            prev_char = char
+        return count or 1
+    
+    def stem(self, word: str) -> str:
+        """Apply stemming rules to reduce word to its root form."""
+        if len(word) <= 3:
+            return word
+            
+        word = word.lower()
+        
+        # Step 1: Handle plurals and past participles
+        if word.endswith('sses'):
+            word = word[:-2]
+        elif word.endswith('ies'):
+            word = word[:-2]
+        elif word.endswith('ss'):
+            pass
+        elif word.endswith('s') and len(word) > 4:
+            word = word[:-1]
+            
+        # Step 2: Handle -ed and -ing
+        if word.endswith('ed') and self.count_syllables(word[:-2]) > 1:
+            word = word[:-2]
+        elif word.endswith('ing') and self.count_syllables(word[:-3]) > 1:
+            word = word[:-3]
+            
+        # Step 3: Handle miscellaneous endings
+        if len(word) > 5:
+            if word.endswith('ement'):
+                word = word[:-5]
+            elif word.endswith('ment'):
+                word = word[:-4]
+            elif word.endswith('ent'):
+                word = word[:-3]
+                
+        # Step 4: Handle -ity endings
+        if word.endswith('ity') and len(word) > 6:
+            word = word[:-3]
+            if word.endswith('abil'):
+                word = word[:-4] + 'able'
+            elif word.endswith('ic'):
+                word = word[:-2]
+                
+        # Final step: Remove double consonants at the end
+        if len(word) > 2 and word[-2:] in self.doubles:
+            word = word[:-1]
+            
+        return word

webstoken/tagger.py

@@ -1,60 +1,60 @@
-"""
-Part-of-Speech tagging utilities.
-"""
-
-from typing import List, Set, Tuple
-
-
-class POSTagger:
-    """Simple rule-based Part-of-Speech tagger."""
-    
-    def __init__(self):
-        # Basic rules for POS tagging
-        self.noun_suffixes: Set[str] = {'ness', 'ment', 'ship', 'dom', 'hood', 'er', 'or', 'ist'}
-        self.verb_suffixes: Set[str] = {'ize', 'ate', 'ify', 'ing', 'ed'}
-        self.adj_suffixes: Set[str] = {'able', 'ible', 'al', 'ful', 'ous', 'ive', 'less'}
-        self.adv_suffixes: Set[str] = {'ly'}
-        
-        # Common words by POS
-        self.determiners: Set[str] = {'the', 'a', 'an', 'this', 'that', 'these', 'those'}
-        self.prepositions: Set[str] = {'in', 'on', 'at', 'by', 'with', 'from', 'to', 'for'}
-        self.pronouns: Set[str] = {'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her'}
-        
-    def tag(self, tokens: List[str]) -> List[Tuple[str, str]]:
-        """Assign POS tags to tokens based on rules."""
-        tagged = []
-        prev_tag = None
-        
-        for i, token in enumerate(tokens):
-            word = token.lower()
-            
-            # Check special cases first
-            if word in self.determiners:
-                tag = 'DET'
-            elif word in self.prepositions:
-                tag = 'PREP'
-            elif word in self.pronouns:
-                tag = 'PRON'
-            # Check suffixes
-            elif any(word.endswith(suffix) for suffix in self.noun_suffixes):
-                tag = 'NOUN'
-            elif any(word.endswith(suffix) for suffix in self.verb_suffixes):
-                tag = 'VERB'
-            elif any(word.endswith(suffix) for suffix in self.adj_suffixes):
-                tag = 'ADJ'
-            elif any(word.endswith(suffix) for suffix in self.adv_suffixes):
-                tag = 'ADV'
-            # Default cases
-            elif word[0].isupper() and i > 0:
-                tag = 'PROPN'  # Proper noun
-            elif word.isdigit():
-                tag = 'NUM'
-            elif not word.isalnum():
-                tag = 'PUNCT'
-            else:
-                tag = 'NOUN'  # Default to noun
-            
-            tagged.append((token, tag))
-            prev_tag = tag
-            
-        return tagged
+"""
+Part-of-Speech tagging utilities.
+"""
+
+from typing import List, Set, Tuple
+
+
+class POSTagger:
+    """Simple rule-based Part-of-Speech tagger."""
+    
+    def __init__(self):
+        # Basic rules for POS tagging
+        self.noun_suffixes: Set[str] = {'ness', 'ment', 'ship', 'dom', 'hood', 'er', 'or', 'ist'}
+        self.verb_suffixes: Set[str] = {'ize', 'ate', 'ify', 'ing', 'ed'}
+        self.adj_suffixes: Set[str] = {'able', 'ible', 'al', 'ful', 'ous', 'ive', 'less'}
+        self.adv_suffixes: Set[str] = {'ly'}
+        
+        # Common words by POS
+        self.determiners: Set[str] = {'the', 'a', 'an', 'this', 'that', 'these', 'those'}
+        self.prepositions: Set[str] = {'in', 'on', 'at', 'by', 'with', 'from', 'to', 'for'}
+        self.pronouns: Set[str] = {'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her'}
+        
+    def tag(self, tokens: List[str]) -> List[Tuple[str, str]]:
+        """Assign POS tags to tokens based on rules."""
+        tagged = []
+        prev_tag = None
+        
+        for i, token in enumerate(tokens):
+            word = token.lower()
+            
+            # Check special cases first
+            if word in self.determiners:
+                tag = 'DET'
+            elif word in self.prepositions:
+                tag = 'PREP'
+            elif word in self.pronouns:
+                tag = 'PRON'
+            # Check suffixes
+            elif any(word.endswith(suffix) for suffix in self.noun_suffixes):
+                tag = 'NOUN'
+            elif any(word.endswith(suffix) for suffix in self.verb_suffixes):
+                tag = 'VERB'
+            elif any(word.endswith(suffix) for suffix in self.adj_suffixes):
+                tag = 'ADJ'
+            elif any(word.endswith(suffix) for suffix in self.adv_suffixes):
+                tag = 'ADV'
+            # Default cases
+            elif word[0].isupper() and i > 0:
+                tag = 'PROPN'  # Proper noun
+            elif word.isdigit():
+                tag = 'NUM'
+            elif not word.isalnum():
+                tag = 'PUNCT'
+            else:
+                tag = 'NOUN'  # Default to noun
+            
+            tagged.append((token, tag))
+            prev_tag = tag
+            
+        return tagged

webstoken/tokenizer.py

@@ -1,158 +1,158 @@
-"""
-Tokenization utilities for sentence and word-level tokenization.
-"""
-
-from typing import List, Dict, Set, Pattern
-import re
-
-
-class SentenceTokenizer:
-    """Advanced sentence tokenizer with support for complex cases and proper formatting."""
-    
-    def __init__(self) -> None:
-        # Common abbreviations by category
-        self.TITLES: Set[str] = {
-            'mr', 'mrs', 'ms', 'dr', 'prof', 'rev', 'sr', 'jr', 'esq',
-            'hon', 'pres', 'gov', 'atty', 'supt', 'det', 'rev', 'col','maj', 'gen', 'capt', 'cmdr',
-            'lt', 'sgt', 'cpl', 'pvt'
-        }
-        
-        self.ACADEMIC: Set[str] = {
-            'ph.d', 'phd', 'm.d', 'md', 'b.a', 'ba', 'm.a', 'ma', 'd.d.s', 'dds',
-            'm.b.a', 'mba', 'b.sc', 'bsc', 'm.sc', 'msc', 'llb', 'll.b', 'bl'
-        }
-        
-        self.ORGANIZATIONS: Set[str] = {
-            'inc', 'ltd', 'co', 'corp', 'llc', 'llp', 'assn', 'bros', 'plc', 'cos',
-            'intl', 'dept', 'est', 'dist', 'mfg', 'div'
-        }
-        
-        self.MONTHS: Set[str] = {
-            'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
-        }
-        
-        self.UNITS: Set[str] = {
-            'oz', 'pt', 'qt', 'gal', 'ml', 'cc', 'km', 'cm', 'mm', 'ft', 'in',
-            'kg', 'lb', 'lbs', 'hz', 'khz', 'mhz', 'ghz', 'kb', 'mb', 'gb', 'tb'
-        }
-        
-        self.TECHNOLOGY: Set[str] = {
-            'v', 'ver', 'app', 'sys', 'dir', 'exe', 'lib', 'api', 'sdk', 'url',
-            'cpu', 'gpu', 'ram', 'rom', 'hdd', 'ssd', 'lan', 'wan', 'sql', 'html'
-        }
-        
-        self.MISC: Set[str] = {
-            'vs', 'etc', 'ie', 'eg', 'no', 'al', 'ca', 'cf', 'pp', 'est', 'st',
-            'approx', 'appt', 'apt', 'dept', 'depts', 'min', 'max', 'avg'
-        }
-
-        # Combine all abbreviations
-        self.all_abbreviations: Set[str] = (
-            self.TITLES | self.ACADEMIC | self.ORGANIZATIONS |
-            self.MONTHS | self.UNITS | self.TECHNOLOGY | self.MISC
-        )
-
-        # Special patterns
-        self.ELLIPSIS: str = r'\.{2,}|…'
-        self.URL_PATTERN: str = (
-            r'(?:https?:\/\/|www\.)[\w\-\.]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?'
-        )
-        self.EMAIL_PATTERN: str = r'[\w\.-]+@[\w\.-]+\.\w+'
-        self.NUMBER_PATTERN: str = (
-            r'\d+(?:\.\d+)?(?:%|°|km|cm|mm|m|kg|g|lb|ft|in|mph|kmh|hz|mhz|ghz)?'
-        )
-        
-        # Quote and bracket pairs
-        self.QUOTE_PAIRS: Dict[str, str] = {
-            '"': '"', "'": "'", '"': '"', "「": "」", "『": "』",
-            "«": "»", "‹": "›", "'": "'", "‚": "'"
-        }
-        
-        self.BRACKETS: Dict[str, str] = {
-            '(': ')', '[': ']', '{': '}', '⟨': '⟩', '「': '」',
-            '『': '』', '【': '】', '〖': '〗', '｢': '｣'
-        }
-
-        # Compile regex patterns
-        self._compile_patterns()
-
-    def _compile_patterns(self) -> None:
-        """Compile regex patterns for better performance."""
-        # Pattern for finding potential sentence boundaries
-        self.SENTENCE_END: Pattern = re.compile(
-            r'''
-            # Group for sentence endings
-            (?:
-                # Standard endings with optional quotes/brackets
-                (?<=[.!?])[\"\'\)\]\}»›」』\s]*
-                
-                # Ellipsis
-                |(?:\.{2,}|…)
-                
-                # Asian-style endings
-                |(?<=[。！？」』】\s])
-            )
-            
-            # Must be followed by whitespace and capital letter or number
-            (?=\s+(?:[A-Z0-9]|["'({[\[「『《‹〈][A-Z]))
-            ''',
-            re.VERBOSE
-        )
-
-        # Pattern for abbreviations
-        abbrev_pattern = '|'.join(re.escape(abbr) for abbr in self.all_abbreviations)
-        self.ABBREV_PATTERN: Pattern = re.compile(
-            fr'\b(?:{abbrev_pattern})\.?',
-            re.IGNORECASE
-        )
-
-    def tokenize(self, text: str) -> List[str]:
-        """Split text into sentences while handling complex cases."""
-        if not text or not text.strip():
-            return []
-            
-        # Initial split on potential sentence boundaries
-        sentences = self.SENTENCE_END.split(text)
-        
-        # Clean and validate sentences
-        final_sentences = []
-        for sentence in sentences:
-            sentence = sentence.strip()
-            if sentence:
-                final_sentences.append(sentence)
-                
-        return final_sentences
-
-
-class WordTokenizer:
-    """Simple but effective word tokenizer with support for contractions and special cases."""
-    
-    def __init__(self):
-        self.contractions = {
-            "n't": "not", "'ll": "will", "'re": "are", "'s": "is",
-            "'m": "am", "'ve": "have", "'d": "would"
-        }
-        
-        self.word_pattern = re.compile(r"""
-            (?:[A-Za-z]+(?:[''][A-Za-z]+)*)|    # Words with optional internal apostrophes
-            (?:\d+(?:,\d{3})*(?:\.\d+)?)|       # Numbers with commas and decimals
-            (?:[@#]?\w+)|                        # Hashtags and mentions
-            (?:[^\w\s])                          # Punctuation and symbols
-        """, re.VERBOSE)
-    
-    def tokenize(self, text: str) -> List[str]:
-        """Split text into words while handling contractions and special cases."""
-        tokens = []
-        for match in self.word_pattern.finditer(text):
-            word = match.group()
-            # Handle contractions
-            for contraction, expansion in self.contractions.items():
-                if word.endswith(contraction):
-                    base = word[:-len(contraction)]
-                    if base:
-                        tokens.append(base)
-                    tokens.append(expansion)
-                    break
-            else:
-                tokens.append(word)
-        return tokens
+"""
+Tokenization utilities for sentence and word-level tokenization.
+"""
+
+from typing import List, Dict, Set, Pattern
+import re
+
+
+class SentenceTokenizer:
+    """Advanced sentence tokenizer with support for complex cases and proper formatting."""
+    
+    def __init__(self) -> None:
+        # Common abbreviations by category
+        self.TITLES: Set[str] = {
+            'mr', 'mrs', 'ms', 'dr', 'prof', 'rev', 'sr', 'jr', 'esq',
+            'hon', 'pres', 'gov', 'atty', 'supt', 'det', 'rev', 'col','maj', 'gen', 'capt', 'cmdr',
+            'lt', 'sgt', 'cpl', 'pvt'
+        }
+        
+        self.ACADEMIC: Set[str] = {
+            'ph.d', 'phd', 'm.d', 'md', 'b.a', 'ba', 'm.a', 'ma', 'd.d.s', 'dds',
+            'm.b.a', 'mba', 'b.sc', 'bsc', 'm.sc', 'msc', 'llb', 'll.b', 'bl'
+        }
+        
+        self.ORGANIZATIONS: Set[str] = {
+            'inc', 'ltd', 'co', 'corp', 'llc', 'llp', 'assn', 'bros', 'plc', 'cos',
+            'intl', 'dept', 'est', 'dist', 'mfg', 'div'
+        }
+        
+        self.MONTHS: Set[str] = {
+            'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
+        }
+        
+        self.UNITS: Set[str] = {
+            'oz', 'pt', 'qt', 'gal', 'ml', 'cc', 'km', 'cm', 'mm', 'ft', 'in',
+            'kg', 'lb', 'lbs', 'hz', 'khz', 'mhz', 'ghz', 'kb', 'mb', 'gb', 'tb'
+        }
+        
+        self.TECHNOLOGY: Set[str] = {
+            'v', 'ver', 'app', 'sys', 'dir', 'exe', 'lib', 'api', 'sdk', 'url',
+            'cpu', 'gpu', 'ram', 'rom', 'hdd', 'ssd', 'lan', 'wan', 'sql', 'html'
+        }
+        
+        self.MISC: Set[str] = {
+            'vs', 'etc', 'ie', 'eg', 'no', 'al', 'ca', 'cf', 'pp', 'est', 'st',
+            'approx', 'appt', 'apt', 'dept', 'depts', 'min', 'max', 'avg'
+        }
+
+        # Combine all abbreviations
+        self.all_abbreviations: Set[str] = (
+            self.TITLES | self.ACADEMIC | self.ORGANIZATIONS |
+            self.MONTHS | self.UNITS | self.TECHNOLOGY | self.MISC
+        )
+
+        # Special patterns
+        self.ELLIPSIS: str = r'\.{2,}|…'
+        self.URL_PATTERN: str = (
+            r'(?:https?:\/\/|www\.)[\w\-\.]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?'
+        )
+        self.EMAIL_PATTERN: str = r'[\w\.-]+@[\w\.-]+\.\w+'
+        self.NUMBER_PATTERN: str = (
+            r'\d+(?:\.\d+)?(?:%|°|km|cm|mm|m|kg|g|lb|ft|in|mph|kmh|hz|mhz|ghz)?'
+        )
+        
+        # Quote and bracket pairs
+        self.QUOTE_PAIRS: Dict[str, str] = {
+            '"': '"', "'": "'", '"': '"', "「": "」", "『": "』",
+            "«": "»", "‹": "›", "'": "'", "‚": "'"
+        }
+        
+        self.BRACKETS: Dict[str, str] = {
+            '(': ')', '[': ']', '{': '}', '⟨': '⟩', '「': '」',
+            '『': '』', '【': '】', '〖': '〗', '｢': '｣'
+        }
+
+        # Compile regex patterns
+        self._compile_patterns()
+
+    def _compile_patterns(self) -> None:
+        """Compile regex patterns for better performance."""
+        # Pattern for finding potential sentence boundaries
+        self.SENTENCE_END: Pattern = re.compile(
+            r'''
+            # Group for sentence endings
+            (?:
+                # Standard endings with optional quotes/brackets
+                (?<=[.!?])[\"\'\)\]\}»›」』\s]*
+                
+                # Ellipsis
+                |(?:\.{2,}|…)
+                
+                # Asian-style endings
+                |(?<=[。！？」』】\s])
+            )
+            
+            # Must be followed by whitespace and capital letter or number
+            (?=\s+(?:[A-Z0-9]|["'({[\[「『《‹〈][A-Z]))
+            ''',
+            re.VERBOSE
+        )
+
+        # Pattern for abbreviations
+        abbrev_pattern = '|'.join(re.escape(abbr) for abbr in self.all_abbreviations)
+        self.ABBREV_PATTERN: Pattern = re.compile(
+            fr'\b(?:{abbrev_pattern})\.?',
+            re.IGNORECASE
+        )
+
+    def tokenize(self, text: str) -> List[str]:
+        """Split text into sentences while handling complex cases."""
+        if not text or not text.strip():
+            return []
+            
+        # Initial split on potential sentence boundaries
+        sentences = self.SENTENCE_END.split(text)
+        
+        # Clean and validate sentences
+        final_sentences = []
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if sentence:
+                final_sentences.append(sentence)
+                
+        return final_sentences
+
+
+class WordTokenizer:
+    """Simple but effective word tokenizer with support for contractions and special cases."""
+    
+    def __init__(self):
+        self.contractions = {
+            "n't": "not", "'ll": "will", "'re": "are", "'s": "is",
+            "'m": "am", "'ve": "have", "'d": "would"
+        }
+        
+        self.word_pattern = re.compile(r"""
+            (?:[A-Za-z]+(?:[''][A-Za-z]+)*)|    # Words with optional internal apostrophes
+            (?:\d+(?:,\d{3})*(?:\.\d+)?)|       # Numbers with commas and decimals
+            (?:[@#]?\w+)|                        # Hashtags and mentions
+            (?:[^\w\s])                          # Punctuation and symbols
+        """, re.VERBOSE)
+    
+    def tokenize(self, text: str) -> List[str]:
+        """Split text into words while handling contractions and special cases."""
+        tokens = []
+        for match in self.word_pattern.finditer(text):
+            word = match.group()
+            # Handle contractions
+            for contraction, expansion in self.contractions.items():
+                if word.endswith(contraction):
+                    base = word[:-len(contraction)]
+                    if base:
+                        tokens.append(base)
+                    tokens.append(expansion)
+                    break
+            else:
+                tokens.append(word)
+        return tokens