webscout 8.2.6__py3-none-any.whl → 8.2.7__py3-none-any.whl

webstoken/sentiment.py

@@ -1,206 +0,0 @@
-"""
-Sentiment analysis module for determining text sentiment and emotion.
-"""
-
-from typing import Dict, List, Set, Tuple
-import re
-
-from .tokenizer import WordTokenizer
-from .normalizer import TextNormalizer
-
-
-class SentimentAnalyzer:
-    """Rule-based sentiment analysis using lexicon approach."""
-    
-    def __init__(self):
-        self.word_tokenizer = WordTokenizer()
-        self.normalizer = TextNormalizer()
-        
-        # Sentiment lexicons
-        self.positive_words: Set[str] = {
-            'good', 'great', 'awesome', 'excellent', 'happy', 'wonderful',
-            'fantastic', 'amazing', 'love', 'beautiful', 'best', 'perfect',
-            'brilliant', 'outstanding', 'superb', 'nice', 'pleasant', 'delightful',
-            'positive', 'remarkable', 'terrific', 'incredible', 'enjoyable',
-            'favorable', 'marvelous', 'splendid', 'superior', 'worthy', 'right'
-        }
-        
-        self.negative_words: Set[str] = {
-            'bad', 'terrible', 'awful', 'horrible', 'sad', 'poor', 'wrong',
-            'worse', 'worst', 'hate', 'dislike', 'disappointing', 'negative',
-            'inferior', 'useless', 'worthless', 'mediocre', 'inadequate',
-            'unpleasant', 'unfavorable', 'disagreeable', 'offensive', 'annoying',
-            'frustrating', 'irritating', 'disgusting', 'dreadful', 'pathetic'
-        }
-        
-        # Emotion lexicons
-        self.emotion_words = {
-            'JOY': {
-                'happy', 'joyful', 'delighted', 'excited', 'pleased', 'glad',
-                'cheerful', 'content', 'satisfied', 'elated', 'jubilant',
-                'thrilled', 'ecstatic', 'merry', 'peaceful', 'upbeat'
-            },
-            'SADNESS': {
-                'sad', 'unhappy', 'depressed', 'gloomy', 'miserable', 'down',
-                'heartbroken', 'disappointed', 'upset', 'distressed', 'grief',
-                'sorrow', 'melancholy', 'despair', 'hopeless', 'blue'
-            },
-            'ANGER': {
-                'angry', 'mad', 'furious', 'outraged', 'irritated', 'annoyed',
-                'frustrated', 'enraged', 'hostile', 'bitter', 'hateful', 'rage',
-                'resentful', 'violent', 'aggressive', 'irate'
-            },
-            'FEAR': {
-                'afraid', 'scared', 'frightened', 'terrified', 'anxious', 'worried',
-                'nervous', 'fearful', 'panicked', 'alarmed', 'horrified', 'dread',
-                'uneasy', 'stressed', 'concerned', 'apprehensive'
-            },
-            'SURPRISE': {
-                'surprised', 'amazed', 'astonished', 'shocked', 'stunned',
-                'startled', 'unexpected', 'incredible', 'unbelievable', 'wonder',
-                'awe', 'remarkable', 'mysterious', 'sudden', 'strange'
-            }
-        }
-        
-        # Intensity modifiers
-        self.intensifiers = {
-            'very': 1.5,
-            'really': 1.5,
-            'extremely': 2.0,
-            'incredibly': 2.0,
-            'absolutely': 2.0,
-            'totally': 1.5,
-            'completely': 1.5,
-            'utterly': 2.0,
-            'highly': 1.5,
-            'especially': 1.5
-        }
-        
-        self.diminishers = {
-            'somewhat': 0.5,
-            'slightly': 0.5,
-            'barely': 0.3,
-            'hardly': 0.3,
-            'sort of': 0.5,
-            'kind of': 0.5,
-            'a bit': 0.5,
-            'a little': 0.5,
-            'not very': 0.3,
-            'less': 0.5
-        }
-        
-        # Negation words
-        self.negation_words = {
-            'not', 'no', 'never', 'none', 'nobody', 'nothing', 'neither',
-            'nowhere', 'hardly', 'scarcely', 'barely', "don't", "doesn't",
-            "didn't", "won't", "wouldn't", "shouldn't", "couldn't", "can't"
-        }
-        
-        # Compile patterns
-        self.word_pattern = re.compile(r'\b\w+\b')
-    
-    def _get_window_around_word(self, words: List[str], index: int, window_size: int = 3) -> List[str]:
-        """Get a window of words around a given index."""
-        start = max(0, index - window_size)
-        end = min(len(words), index + window_size + 1)
-        return words[start:end]
-    
-    def _is_negated(self, words: List[str], index: int) -> bool:
-        """Check if a word is negated by looking at surrounding context."""
-        window = self._get_window_around_word(words, index)
-        return any(word in self.negation_words for word in window[:index-window[0]])
-    
-    def _get_intensity_multiplier(self, words: List[str], index: int) -> float:
-        """Get intensity multiplier based on modifiers."""
-        window = self._get_window_around_word(words, index)
-        multiplier = 1.0
-        
-        for word in window[:index-window[0]]:
-            if word in self.intensifiers:
-                multiplier *= self.intensifiers[word]
-            elif word in self.diminishers:
-                multiplier *= self.diminishers[word]
-        
-        return multiplier
-    
-    def analyze_sentiment(self, text: str) -> Dict[str, float]:
-        """
-        Analyze sentiment of text.
-        
-        Returns:
-            Dict with sentiment scores:
-            {
-                'polarity': float (-1 to 1),
-                'subjectivity': float (0 to 1),
-                'confidence': float (0 to 1)
-            }
-        """
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        words = self.word_tokenizer.tokenize(text)
-        
-        positive_score = 0
-        negative_score = 0
-        word_count = len(words)
-        
-        for i, word in enumerate(words):
-            word = word.lower()
-            multiplier = self._get_intensity_multiplier(words, i)
-            is_negated = self._is_negated(words, i)
-            
-            if word in self.positive_words:
-                score = 1.0 * multiplier
-                positive_score += -score if is_negated else score
-            elif word in self.negative_words:
-                score = 1.0 * multiplier
-                negative_score += -score if is_negated else score
-        
-        # Calculate metrics
-        total_score = positive_score + negative_score
-        total_magnitude = abs(positive_score) + abs(negative_score)
-        
-        if word_count == 0:
-            return {'polarity': 0.0, 'subjectivity': 0.0, 'confidence': 0.0}
-        
-        polarity = total_score / (word_count or 1)  # Normalize to [-1, 1]
-        subjectivity = total_magnitude / (word_count or 1)  # Normalize to [0, 1]
-        confidence = min(1.0, total_magnitude / (word_count / 2))  # Confidence based on magnitude
-        
-        return {
-            'polarity': max(-1.0, min(1.0, polarity)),
-            'subjectivity': min(1.0, subjectivity),
-            'confidence': confidence
-        }
-    
-    def analyze_emotions(self, text: str) -> List[Tuple[str, float]]:
-        """
-        Analyze emotions in text.
-        
-        Returns:
-            List of (emotion, score) tuples, sorted by score
-        """
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        words = self.word_tokenizer.tokenize(text)
-        
-        emotion_scores = {emotion: 0.0 for emotion in self.emotion_words}
-        
-        for i, word in enumerate(words):
-            word = word.lower()
-            multiplier = self._get_intensity_multiplier(words, i)
-            is_negated = self._is_negated(words, i)
-            
-            for emotion, emotion_set in self.emotion_words.items():
-                if word in emotion_set:
-                    score = 1.0 * multiplier
-                    emotion_scores[emotion] += -score if is_negated else score
-        
-        # Normalize scores
-        max_score = max(abs(score) for score in emotion_scores.values()) or 1
-        normalized_scores = [
-            (emotion, score/max_score)
-            for emotion, score in emotion_scores.items()
-        ]
-        
-        # Sort by score
-        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)

webstoken/stemmer.py

@@ -1,73 +0,0 @@
-"""
-Word stemming utilities.
-"""
-
-from typing import Set
-
-
-class Stemmer:
-    """Simple rule-based stemmer implementing Porter-like rules."""
-    
-    def __init__(self):
-        self.vowels: Set[str] = {'a', 'e', 'i', 'o', 'u', 'y'}
-        self.doubles: Set[str] = {'bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'}
-        
-    def is_vowel(self, char: str, prev_char: str = None) -> bool:
-        """Check if a character is a vowel, considering 'y' special cases."""
-        return char in self.vowels or (char == 'y' and prev_char and prev_char not in self.vowels)
-    
-    def count_syllables(self, word: str) -> int:
-        """Count syllables in a word based on vowel sequences."""
-        count = 0
-        prev_char = None
-        for i, char in enumerate(word.lower()):
-            if self.is_vowel(char, prev_char) and (i == 0 or not self.is_vowel(prev_char, word[i-2] if i > 1 else None)):
-                count += 1
-            prev_char = char
-        return count or 1
-    
-    def stem(self, word: str) -> str:
-        """Apply stemming rules to reduce word to its root form."""
-        if len(word) <= 3:
-            return word
-            
-        word = word.lower()
-        
-        # Step 1: Handle plurals and past participles
-        if word.endswith('sses'):
-            word = word[:-2]
-        elif word.endswith('ies'):
-            word = word[:-2]
-        elif word.endswith('ss'):
-            pass
-        elif word.endswith('s') and len(word) > 4:
-            word = word[:-1]
-            
-        # Step 2: Handle -ed and -ing
-        if word.endswith('ed') and self.count_syllables(word[:-2]) > 1:
-            word = word[:-2]
-        elif word.endswith('ing') and self.count_syllables(word[:-3]) > 1:
-            word = word[:-3]
-            
-        # Step 3: Handle miscellaneous endings
-        if len(word) > 5:
-            if word.endswith('ement'):
-                word = word[:-5]
-            elif word.endswith('ment'):
-                word = word[:-4]
-            elif word.endswith('ent'):
-                word = word[:-3]
-                
-        # Step 4: Handle -ity endings
-        if word.endswith('ity') and len(word) > 6:
-            word = word[:-3]
-            if word.endswith('abil'):
-                word = word[:-4] + 'able'
-            elif word.endswith('ic'):
-                word = word[:-2]
-                
-        # Final step: Remove double consonants at the end
-        if len(word) > 2 and word[-2:] in self.doubles:
-            word = word[:-1]
-            
-        return word

webstoken/tagger.py

@@ -1,60 +0,0 @@
-"""
-Part-of-Speech tagging utilities.
-"""
-
-from typing import List, Set, Tuple
-
-
-class POSTagger:
-    """Simple rule-based Part-of-Speech tagger."""
-    
-    def __init__(self):
-        # Basic rules for POS tagging
-        self.noun_suffixes: Set[str] = {'ness', 'ment', 'ship', 'dom', 'hood', 'er', 'or', 'ist'}
-        self.verb_suffixes: Set[str] = {'ize', 'ate', 'ify', 'ing', 'ed'}
-        self.adj_suffixes: Set[str] = {'able', 'ible', 'al', 'ful', 'ous', 'ive', 'less'}
-        self.adv_suffixes: Set[str] = {'ly'}
-        
-        # Common words by POS
-        self.determiners: Set[str] = {'the', 'a', 'an', 'this', 'that', 'these', 'those'}
-        self.prepositions: Set[str] = {'in', 'on', 'at', 'by', 'with', 'from', 'to', 'for'}
-        self.pronouns: Set[str] = {'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her'}
-        
-    def tag(self, tokens: List[str]) -> List[Tuple[str, str]]:
-        """Assign POS tags to tokens based on rules."""
-        tagged = []
-        prev_tag = None
-        
-        for i, token in enumerate(tokens):
-            word = token.lower()
-            
-            # Check special cases first
-            if word in self.determiners:
-                tag = 'DET'
-            elif word in self.prepositions:
-                tag = 'PREP'
-            elif word in self.pronouns:
-                tag = 'PRON'
-            # Check suffixes
-            elif any(word.endswith(suffix) for suffix in self.noun_suffixes):
-                tag = 'NOUN'
-            elif any(word.endswith(suffix) for suffix in self.verb_suffixes):
-                tag = 'VERB'
-            elif any(word.endswith(suffix) for suffix in self.adj_suffixes):
-                tag = 'ADJ'
-            elif any(word.endswith(suffix) for suffix in self.adv_suffixes):
-                tag = 'ADV'
-            # Default cases
-            elif word[0].isupper() and i > 0:
-                tag = 'PROPN'  # Proper noun
-            elif word.isdigit():
-                tag = 'NUM'
-            elif not word.isalnum():
-                tag = 'PUNCT'
-            else:
-                tag = 'NOUN'  # Default to noun
-            
-            tagged.append((token, tag))
-            prev_tag = tag
-            
-        return tagged

webstoken/tokenizer.py

@@ -1,158 +0,0 @@
-"""
-Tokenization utilities for sentence and word-level tokenization.
-"""
-
-from typing import List, Dict, Set, Pattern
-import re
-
-
-class SentenceTokenizer:
-    """Advanced sentence tokenizer with support for complex cases and proper formatting."""
-    
-    def __init__(self) -> None:
-        # Common abbreviations by category
-        self.TITLES: Set[str] = {
-            'mr', 'mrs', 'ms', 'dr', 'prof', 'rev', 'sr', 'jr', 'esq',
-            'hon', 'pres', 'gov', 'atty', 'supt', 'det', 'rev', 'col','maj', 'gen', 'capt', 'cmdr',
-            'lt', 'sgt', 'cpl', 'pvt'
-        }
-        
-        self.ACADEMIC: Set[str] = {
-            'ph.d', 'phd', 'm.d', 'md', 'b.a', 'ba', 'm.a', 'ma', 'd.d.s', 'dds',
-            'm.b.a', 'mba', 'b.sc', 'bsc', 'm.sc', 'msc', 'llb', 'll.b', 'bl'
-        }
-        
-        self.ORGANIZATIONS: Set[str] = {
-            'inc', 'ltd', 'co', 'corp', 'llc', 'llp', 'assn', 'bros', 'plc', 'cos',
-            'intl', 'dept', 'est', 'dist', 'mfg', 'div'
-        }
-        
-        self.MONTHS: Set[str] = {
-            'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
-        }
-        
-        self.UNITS: Set[str] = {
-            'oz', 'pt', 'qt', 'gal', 'ml', 'cc', 'km', 'cm', 'mm', 'ft', 'in',
-            'kg', 'lb', 'lbs', 'hz', 'khz', 'mhz', 'ghz', 'kb', 'mb', 'gb', 'tb'
-        }
-        
-        self.TECHNOLOGY: Set[str] = {
-            'v', 'ver', 'app', 'sys', 'dir', 'exe', 'lib', 'api', 'sdk', 'url',
-            'cpu', 'gpu', 'ram', 'rom', 'hdd', 'ssd', 'lan', 'wan', 'sql', 'html'
-        }
-        
-        self.MISC: Set[str] = {
-            'vs', 'etc', 'ie', 'eg', 'no', 'al', 'ca', 'cf', 'pp', 'est', 'st',
-            'approx', 'appt', 'apt', 'dept', 'depts', 'min', 'max', 'avg'
-        }
-
-        # Combine all abbreviations
-        self.all_abbreviations: Set[str] = (
-            self.TITLES | self.ACADEMIC | self.ORGANIZATIONS |
-            self.MONTHS | self.UNITS | self.TECHNOLOGY | self.MISC
-        )
-
-        # Special patterns
-        self.ELLIPSIS: str = r'\.{2,}|…'
-        self.URL_PATTERN: str = (
-            r'(?:https?:\/\/|www\.)[\w\-\.]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?'
-        )
-        self.EMAIL_PATTERN: str = r'[\w\.-]+@[\w\.-]+\.\w+'
-        self.NUMBER_PATTERN: str = (
-            r'\d+(?:\.\d+)?(?:%|°|km|cm|mm|m|kg|g|lb|ft|in|mph|kmh|hz|mhz|ghz)?'
-        )
-        
-        # Quote and bracket pairs
-        self.QUOTE_PAIRS: Dict[str, str] = {
-            '"': '"', "'": "'", '"': '"', "「": "」", "『": "』",
-            "«": "»", "‹": "›", "'": "'", "‚": "'"
-        }
-        
-        self.BRACKETS: Dict[str, str] = {
-            '(': ')', '[': ']', '{': '}', '⟨': '⟩', '「': '」',
-            '『': '』', '【': '】', '〖': '〗', '｢': '｣'
-        }
-
-        # Compile regex patterns
-        self._compile_patterns()
-
-    def _compile_patterns(self) -> None:
-        """Compile regex patterns for better performance."""
-        # Pattern for finding potential sentence boundaries
-        self.SENTENCE_END: Pattern = re.compile(
-            r'''
-            # Group for sentence endings
-            (?:
-                # Standard endings with optional quotes/brackets
-                (?<=[.!?])[\"\'\)\]\}»›」』\s]*
-                
-                # Ellipsis
-                |(?:\.{2,}|…)
-                
-                # Asian-style endings
-                |(?<=[。！？」』】\s])
-            )
-            
-            # Must be followed by whitespace and capital letter or number
-            (?=\s+(?:[A-Z0-9]|["'({[\[「『《‹〈][A-Z]))
-            ''',
-            re.VERBOSE
-        )
-
-        # Pattern for abbreviations
-        abbrev_pattern = '|'.join(re.escape(abbr) for abbr in self.all_abbreviations)
-        self.ABBREV_PATTERN: Pattern = re.compile(
-            fr'\b(?:{abbrev_pattern})\.?',
-            re.IGNORECASE
-        )
-
-    def tokenize(self, text: str) -> List[str]:
-        """Split text into sentences while handling complex cases."""
-        if not text or not text.strip():
-            return []
-            
-        # Initial split on potential sentence boundaries
-        sentences = self.SENTENCE_END.split(text)
-        
-        # Clean and validate sentences
-        final_sentences = []
-        for sentence in sentences:
-            sentence = sentence.strip()
-            if sentence:
-                final_sentences.append(sentence)
-                
-        return final_sentences
-
-
-class WordTokenizer:
-    """Simple but effective word tokenizer with support for contractions and special cases."""
-    
-    def __init__(self):
-        self.contractions = {
-            "n't": "not", "'ll": "will", "'re": "are", "'s": "is",
-            "'m": "am", "'ve": "have", "'d": "would"
-        }
-        
-        self.word_pattern = re.compile(r"""
-            (?:[A-Za-z]+(?:[''][A-Za-z]+)*)|    # Words with optional internal apostrophes
-            (?:\d+(?:,\d{3})*(?:\.\d+)?)|       # Numbers with commas and decimals
-            (?:[@#]?\w+)|                        # Hashtags and mentions
-            (?:[^\w\s])                          # Punctuation and symbols
-        """, re.VERBOSE)
-    
-    def tokenize(self, text: str) -> List[str]:
-        """Split text into words while handling contractions and special cases."""
-        tokens = []
-        for match in self.word_pattern.finditer(text):
-            word = match.group()
-            # Handle contractions
-            for contraction, expansion in self.contractions.items():
-                if word.endswith(contraction):
-                    base = word[:-len(contraction)]
-                    if base:
-                        tokens.append(base)
-                    tokens.append(expansion)
-                    break
-            else:
-                tokens.append(word)
-        return tokens