webscout 6.5__py3-none-any.whl → 6.6__py3-none-any.whl

webstoken/stemmer.py

@@ -0,0 +1,73 @@
+"""
+Word stemming utilities.
+"""
+
+from typing import Set
+
+
+class Stemmer:
+    """Simple rule-based stemmer implementing Porter-like rules."""
+    
+    def __init__(self):
+        self.vowels: Set[str] = {'a', 'e', 'i', 'o', 'u', 'y'}
+        self.doubles: Set[str] = {'bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'}
+        
+    def is_vowel(self, char: str, prev_char: str = None) -> bool:
+        """Check if a character is a vowel, considering 'y' special cases."""
+        return char in self.vowels or (char == 'y' and prev_char and prev_char not in self.vowels)
+    
+    def count_syllables(self, word: str) -> int:
+        """Count syllables in a word based on vowel sequences."""
+        count = 0
+        prev_char = None
+        for i, char in enumerate(word.lower()):
+            if self.is_vowel(char, prev_char) and (i == 0 or not self.is_vowel(prev_char, word[i-2] if i > 1 else None)):
+                count += 1
+            prev_char = char
+        return count or 1
+    
+    def stem(self, word: str) -> str:
+        """Apply stemming rules to reduce word to its root form."""
+        if len(word) <= 3:
+            return word
+            
+        word = word.lower()
+        
+        # Step 1: Handle plurals and past participles
+        if word.endswith('sses'):
+            word = word[:-2]
+        elif word.endswith('ies'):
+            word = word[:-2]
+        elif word.endswith('ss'):
+            pass
+        elif word.endswith('s') and len(word) > 4:
+            word = word[:-1]
+            
+        # Step 2: Handle -ed and -ing
+        if word.endswith('ed') and self.count_syllables(word[:-2]) > 1:
+            word = word[:-2]
+        elif word.endswith('ing') and self.count_syllables(word[:-3]) > 1:
+            word = word[:-3]
+            
+        # Step 3: Handle miscellaneous endings
+        if len(word) > 5:
+            if word.endswith('ement'):
+                word = word[:-5]
+            elif word.endswith('ment'):
+                word = word[:-4]
+            elif word.endswith('ent'):
+                word = word[:-3]
+                
+        # Step 4: Handle -ity endings
+        if word.endswith('ity') and len(word) > 6:
+            word = word[:-3]
+            if word.endswith('abil'):
+                word = word[:-4] + 'able'
+            elif word.endswith('ic'):
+                word = word[:-2]
+                
+        # Final step: Remove double consonants at the end
+        if len(word) > 2 and word[-2:] in self.doubles:
+            word = word[:-1]
+            
+        return word

webstoken/t.py

@@ -0,0 +1,75 @@
+from webstoken import (
+    process_text, NamedEntityRecognizer, TextClassifier,
+    TopicClassifier, LanguageDetector, SentimentAnalyzer,
+    KeywordExtractor
+)
+from rich import print
+# Example text
+text = """
+Dr. John Smith from Microsoft Corporation visited New York City on January 15th, 2024.
+He presented an excellent paper about artificial intelligence and machine learning at
+the International Technology Conference. The research was incredibly well-received,
+and many attendees were excited about its potential applications in healthcare.
+"""
+
+print("1. Basic Text Processing")
+print("-" * 50)
+result = process_text(text)
+for sentence_data in result['sentences']:
+    print("Original:", sentence_data['original'])
+    print("Tokens:", sentence_data['tokens'])
+    print("POS Tags:", sentence_data['pos_tags'])
+    print("Stems:", sentence_data['stems'])
+    print()
+
+print("\n2. Named Entity Recognition")
+print("-" * 50)
+ner = NamedEntityRecognizer()
+entities = ner.extract_entities(text)
+for entity_type, entity_list in entities.items():
+    if entity_list:
+        print(f"{entity_type}:", entity_list)
+
+print("\n3. Topic Classification")
+print("-" * 50)
+topic_classifier = TopicClassifier()
+topics = topic_classifier.classify(text)
+print("Topics (with confidence):")
+for topic, confidence in topics[:3]:  # Top 3 topics
+    print(f"{topic}: {confidence:.2f}")
+
+print("\n4. Language Detection")
+print("-" * 50)
+lang_detector = LanguageDetector()
+languages = lang_detector.detect(text)
+print("Detected Languages (with confidence):")
+for lang, confidence in languages:
+    print(f"{lang}: {confidence:.2f}")
+
+print("\n5. Sentiment Analysis")
+print("-" * 50)
+sentiment_analyzer = SentimentAnalyzer()
+sentiment = sentiment_analyzer.analyze_sentiment(text)
+print("Sentiment Scores:")
+print(f"Polarity: {sentiment['polarity']:.2f}")
+print(f"Subjectivity: {sentiment['subjectivity']:.2f}")
+print(f"Confidence: {sentiment['confidence']:.2f}")
+
+print("\nEmotions:")
+emotions = sentiment_analyzer.analyze_emotions(text)
+for emotion, score in emotions:
+    if score > 0.1:  # Only show significant emotions
+        print(f"{emotion}: {score:.2f}")
+
+print("\n6. Keyword Extraction")
+print("-" * 50)
+keyword_extractor = KeywordExtractor()
+print("Keywords:")
+keywords = keyword_extractor.extract_keywords(text, num_keywords=5)
+for keyword, score in keywords:
+    print(f"{keyword}: {score:.2f}")
+
+print("\nKey Phrases:")
+keyphrases = keyword_extractor.extract_keyphrases(text, num_phrases=3)
+for phrase, score in keyphrases:
+    print(f"{phrase}: {score:.2f}")

webstoken/tagger.py

@@ -0,0 +1,60 @@
+"""
+Part-of-Speech tagging utilities.
+"""
+
+from typing import List, Set, Tuple
+
+
+class POSTagger:
+    """Simple rule-based Part-of-Speech tagger."""
+    
+    def __init__(self):
+        # Basic rules for POS tagging
+        self.noun_suffixes: Set[str] = {'ness', 'ment', 'ship', 'dom', 'hood', 'er', 'or', 'ist'}
+        self.verb_suffixes: Set[str] = {'ize', 'ate', 'ify', 'ing', 'ed'}
+        self.adj_suffixes: Set[str] = {'able', 'ible', 'al', 'ful', 'ous', 'ive', 'less'}
+        self.adv_suffixes: Set[str] = {'ly'}
+        
+        # Common words by POS
+        self.determiners: Set[str] = {'the', 'a', 'an', 'this', 'that', 'these', 'those'}
+        self.prepositions: Set[str] = {'in', 'on', 'at', 'by', 'with', 'from', 'to', 'for'}
+        self.pronouns: Set[str] = {'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her'}
+        
+    def tag(self, tokens: List[str]) -> List[Tuple[str, str]]:
+        """Assign POS tags to tokens based on rules."""
+        tagged = []
+        prev_tag = None
+        
+        for i, token in enumerate(tokens):
+            word = token.lower()
+            
+            # Check special cases first
+            if word in self.determiners:
+                tag = 'DET'
+            elif word in self.prepositions:
+                tag = 'PREP'
+            elif word in self.pronouns:
+                tag = 'PRON'
+            # Check suffixes
+            elif any(word.endswith(suffix) for suffix in self.noun_suffixes):
+                tag = 'NOUN'
+            elif any(word.endswith(suffix) for suffix in self.verb_suffixes):
+                tag = 'VERB'
+            elif any(word.endswith(suffix) for suffix in self.adj_suffixes):
+                tag = 'ADJ'
+            elif any(word.endswith(suffix) for suffix in self.adv_suffixes):
+                tag = 'ADV'
+            # Default cases
+            elif word[0].isupper() and i > 0:
+                tag = 'PROPN'  # Proper noun
+            elif word.isdigit():
+                tag = 'NUM'
+            elif not word.isalnum():
+                tag = 'PUNCT'
+            else:
+                tag = 'NOUN'  # Default to noun
+            
+            tagged.append((token, tag))
+            prev_tag = tag
+            
+        return tagged

webstoken/tokenizer.py

@@ -0,0 +1,158 @@
+"""
+Tokenization utilities for sentence and word-level tokenization.
+"""
+
+from typing import List, Dict, Set, Pattern
+import re
+
+
+class SentenceTokenizer:
+    """Advanced sentence tokenizer with support for complex cases and proper formatting."""
+    
+    def __init__(self) -> None:
+        # Common abbreviations by category
+        self.TITLES: Set[str] = {
+            'mr', 'mrs', 'ms', 'dr', 'prof', 'rev', 'sr', 'jr', 'esq',
+            'hon', 'pres', 'gov', 'atty', 'supt', 'det', 'rev', 'col','maj', 'gen', 'capt', 'cmdr',
+            'lt', 'sgt', 'cpl', 'pvt'
+        }
+        
+        self.ACADEMIC: Set[str] = {
+            'ph.d', 'phd', 'm.d', 'md', 'b.a', 'ba', 'm.a', 'ma', 'd.d.s', 'dds',
+            'm.b.a', 'mba', 'b.sc', 'bsc', 'm.sc', 'msc', 'llb', 'll.b', 'bl'
+        }
+        
+        self.ORGANIZATIONS: Set[str] = {
+            'inc', 'ltd', 'co', 'corp', 'llc', 'llp', 'assn', 'bros', 'plc', 'cos',
+            'intl', 'dept', 'est', 'dist', 'mfg', 'div'
+        }
+        
+        self.MONTHS: Set[str] = {
+            'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
+        }
+        
+        self.UNITS: Set[str] = {
+            'oz', 'pt', 'qt', 'gal', 'ml', 'cc', 'km', 'cm', 'mm', 'ft', 'in',
+            'kg', 'lb', 'lbs', 'hz', 'khz', 'mhz', 'ghz', 'kb', 'mb', 'gb', 'tb'
+        }
+        
+        self.TECHNOLOGY: Set[str] = {
+            'v', 'ver', 'app', 'sys', 'dir', 'exe', 'lib', 'api', 'sdk', 'url',
+            'cpu', 'gpu', 'ram', 'rom', 'hdd', 'ssd', 'lan', 'wan', 'sql', 'html'
+        }
+        
+        self.MISC: Set[str] = {
+            'vs', 'etc', 'ie', 'eg', 'no', 'al', 'ca', 'cf', 'pp', 'est', 'st',
+            'approx', 'appt', 'apt', 'dept', 'depts', 'min', 'max', 'avg'
+        }
+
+        # Combine all abbreviations
+        self.all_abbreviations: Set[str] = (
+            self.TITLES | self.ACADEMIC | self.ORGANIZATIONS |
+            self.MONTHS | self.UNITS | self.TECHNOLOGY | self.MISC
+        )
+
+        # Special patterns
+        self.ELLIPSIS: str = r'\.{2,}|…'
+        self.URL_PATTERN: str = (
+            r'(?:https?:\/\/|www\.)[\w\-\.]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?'
+        )
+        self.EMAIL_PATTERN: str = r'[\w\.-]+@[\w\.-]+\.\w+'
+        self.NUMBER_PATTERN: str = (
+            r'\d+(?:\.\d+)?(?:%|°|km|cm|mm|m|kg|g|lb|ft|in|mph|kmh|hz|mhz|ghz)?'
+        )
+        
+        # Quote and bracket pairs
+        self.QUOTE_PAIRS: Dict[str, str] = {
+            '"': '"', "'": "'", '"': '"', "「": "」", "『": "』",
+            "«": "»", "‹": "›", "'": "'", "‚": "'"
+        }
+        
+        self.BRACKETS: Dict[str, str] = {
+            '(': ')', '[': ']', '{': '}', '⟨': '⟩', '「': '」',
+            '『': '』', '【': '】', '〖': '〗', '｢': '｣'
+        }
+
+        # Compile regex patterns
+        self._compile_patterns()
+
+    def _compile_patterns(self) -> None:
+        """Compile regex patterns for better performance."""
+        # Pattern for finding potential sentence boundaries
+        self.SENTENCE_END: Pattern = re.compile(
+            r'''
+            # Group for sentence endings
+            (?:
+                # Standard endings with optional quotes/brackets
+                (?<=[.!?])[\"\'\)\]\}»›」』\s]*
+                
+                # Ellipsis
+                |(?:\.{2,}|…)
+                
+                # Asian-style endings
+                |(?<=[。！？」』】\s])
+            )
+            
+            # Must be followed by whitespace and capital letter or number
+            (?=\s+(?:[A-Z0-9]|["'({[\[「『《‹〈][A-Z]))
+            ''',
+            re.VERBOSE
+        )
+
+        # Pattern for abbreviations
+        abbrev_pattern = '|'.join(re.escape(abbr) for abbr in self.all_abbreviations)
+        self.ABBREV_PATTERN: Pattern = re.compile(
+            fr'\b(?:{abbrev_pattern})\.?',
+            re.IGNORECASE
+        )
+
+    def tokenize(self, text: str) -> List[str]:
+        """Split text into sentences while handling complex cases."""
+        if not text or not text.strip():
+            return []
+            
+        # Initial split on potential sentence boundaries
+        sentences = self.SENTENCE_END.split(text)
+        
+        # Clean and validate sentences
+        final_sentences = []
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if sentence:
+                final_sentences.append(sentence)
+                
+        return final_sentences
+
+
+class WordTokenizer:
+    """Simple but effective word tokenizer with support for contractions and special cases."""
+    
+    def __init__(self):
+        self.contractions = {
+            "n't": "not", "'ll": "will", "'re": "are", "'s": "is",
+            "'m": "am", "'ve": "have", "'d": "would"
+        }
+        
+        self.word_pattern = re.compile(r"""
+            (?:[A-Za-z]+(?:[''][A-Za-z]+)*)|    # Words with optional internal apostrophes
+            (?:\d+(?:,\d{3})*(?:\.\d+)?)|       # Numbers with commas and decimals
+            (?:[@#]?\w+)|                        # Hashtags and mentions
+            (?:[^\w\s])                          # Punctuation and symbols
+        """, re.VERBOSE)
+    
+    def tokenize(self, text: str) -> List[str]:
+        """Split text into words while handling contractions and special cases."""
+        tokens = []
+        for match in self.word_pattern.finditer(text):
+            word = match.group()
+            # Handle contractions
+            for contraction, expansion in self.contractions.items():
+                if word.endswith(contraction):
+                    base = word[:-len(contraction)]
+                    if base:
+                        tokens.append(base)
+                    tokens.append(expansion)
+                    break
+            else:
+                tokens.append(word)
+        return tokens