webscout 8.2.6__py3-none-any.whl → 8.2.8__py3-none-any.whl

webstoken/ner.py

@@ -1,164 +0,0 @@
-"""
-Named Entity Recognition (NER) module for identifying and classifying named entities.
-"""
-
-from typing import List, Tuple, Dict, Set
-import re
-
-
-class NamedEntityRecognizer:
-    """Rule-based Named Entity Recognition."""
-    
-    def __init__(self):
-        # Common entity patterns
-        self.PERSON_TITLES = {
-            'mr', 'mrs', 'ms', 'miss', 'dr', 'prof', 'sir', 'madam',
-            'lord', 'lady', 'president', 'ceo', 'director'
-        }
-        
-        self.ORGANIZATION_SUFFIXES = {
-            'inc', 'corp', 'ltd', 'llc', 'company', 'corporation',
-            'associates', 'partners', 'foundation', 'institute'
-        }
-        
-        self.LOCATION_INDICATORS = {
-            'street', 'road', 'avenue', 'boulevard', 'lane', 'drive',
-            'circle', 'square', 'park', 'bridge', 'river', 'lake',
-            'mountain', 'forest', 'city', 'town', 'village', 'country'
-        }
-        
-        self.DATE_MONTHS = {
-            'january', 'february', 'march', 'april', 'may', 'june',
-            'july', 'august', 'september', 'october', 'november', 'december'
-        }
-        
-        # Compile regex patterns
-        self.patterns = {
-            'EMAIL': re.compile(r'\b[\w\.-]+@[\w\.-]+\.\w+\b'),
-            'URL': re.compile(r'https?://(?:[\w-]|(?:%[\da-fA-F]{2}))+'),
-            'PHONE': re.compile(r'\+?\d{1,3}[-.\s]?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}'),
-            'DATE': re.compile(r'\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b'),
-            'TIME': re.compile(r'\b\d{1,2}:\d{2}(?::\d{2})?(?:\s*[AaPp][Mm])?\b'),
-            'MONEY': re.compile(r'\$\d+(?:,\d{3})*(?:\.\d{2})?|\d+(?:,\d{3})*(?:\.\d{2})?\s*(?:dollars|USD|EUR|GBP)'),
-            'PERCENTAGE': re.compile(r'\b\d+(?:\.\d+)?%\b')
-        }
-    
-    def is_capitalized(self, word: str) -> bool:
-        """Check if a word is capitalized."""
-        return word and word[0].isupper()
-    
-    def extract_entities(self, text: str) -> Dict[str, List[Tuple[str, str]]]:
-        """
-        Extract named entities from text.
-        
-        Returns:
-            Dict mapping entity types to list of (text, label) tuples
-        """
-        entities = {
-            'PERSON': [],
-            'ORGANIZATION': [],
-            'LOCATION': [],
-            'DATE': [],
-            'TIME': [],
-            'MONEY': [],
-            'EMAIL': [],
-            'URL': [],
-            'PHONE': [],
-            'PERCENTAGE': []
-        }
-        
-        # First find regex pattern matches
-        for label, pattern in self.patterns.items():
-            for match in pattern.finditer(text):
-                entities[label].append((match.group(), label))
-        
-        # Process text word by word for other entities
-        words = text.split()
-        i = 0
-        while i < len(words):
-            word = words[i]
-            next_word = words[i + 1] if i + 1 < len(words) else None
-            
-            # Check for person names
-            if word.lower() in self.PERSON_TITLES and next_word and self.is_capitalized(next_word):
-                name_parts = []
-                j = i + 1
-                while j < len(words) and self.is_capitalized(words[j]):
-                    name_parts.append(words[j])
-                    j += 1
-                if name_parts:
-                    entities['PERSON'].append((' '.join(name_parts), 'PERSON'))
-                i = j
-                continue
-            
-            # Check for organizations
-            if self.is_capitalized(word):
-                org_parts = [word]
-                j = i + 1
-                while j < len(words) and (
-                    self.is_capitalized(words[j]) or 
-                    words[j].lower() in self.ORGANIZATION_SUFFIXES
-                ):
-                    org_parts.append(words[j])
-                    j += 1
-                if len(org_parts) > 1 or (
-                    len(org_parts) == 1 and 
-                    any(suff in word.lower() for suff in self.ORGANIZATION_SUFFIXES)
-                ):
-                    entities['ORGANIZATION'].append((' '.join(org_parts), 'ORGANIZATION'))
-                i = j
-                continue
-            
-            # Check for locations
-            if word.lower() in self.LOCATION_INDICATORS and i > 0:
-                if self.is_capitalized(words[i - 1]):
-                    entities['LOCATION'].append((words[i - 1] + ' ' + word, 'LOCATION'))
-            
-            i += 1
-        
-        return entities
-        
-    def tag_text(self, text: str) -> List[Tuple[str, str]]:
-        """
-        Tag each word in text with its entity type.
-        
-        Returns:
-            List of (word, entity_type) tuples
-        """
-        entities = self.extract_entities(text)
-        tagged = []
-        
-        # Create a map of word positions to entity labels
-        position_labels = {}
-        text_lower = text.lower()
-        
-        for entity_type, entity_list in entities.items():
-            for entity_text, _ in entity_list:
-                start = text_lower.find(entity_text.lower())
-                if start != -1:
-                    end = start + len(entity_text)
-                    for pos in range(start, end):
-                        position_labels[pos] = entity_type
-        
-        # Tag each character position
-        current_pos = 0
-        current_word = []
-        current_label = 'O'  # Outside any entity
-        
-        for char in text:
-            if char.isspace():
-                if current_word:
-                    tagged.append((''.join(current_word), current_label))
-                    current_word = []
-                    current_label = 'O'
-            else:
-                current_word.append(char)
-                if current_pos in position_labels:
-                    current_label = position_labels[current_pos]
-            current_pos += 1
-        
-        # Add last word if exists
-        if current_word:
-            tagged.append((''.join(current_word), current_label))
-        
-        return tagged

webstoken/normalizer.py

@@ -1,35 +0,0 @@
-"""
-Text normalization utilities.
-"""
-
-import re
-from typing import List, Set
-
-
-class TextNormalizer:
-    """Text normalization utilities."""
-    
-    def __init__(self):
-        self.stop_words: Set[str] = {
-            'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
-            'has', 'he', 'in', 'is', 'it', 'its', 'of', 'on', 'that', 'the',
-            'to', 'was', 'were', 'will', 'with'
-        }
-        
-    def remove_stop_words(self, tokens: List[str]) -> List[str]:
-        """Remove common stop words from token list."""
-        return [token for token in tokens if token.lower() not in self.stop_words]
-    
-    def normalize(self, text: str) -> str:
-        """Apply various normalization steps to text."""
-        # Convert to lowercase
-        text = text.lower()
-        
-        # Replace multiple spaces with single space
-        text = re.sub(r'\s+', ' ', text)
-        
-        # Remove special characters except apostrophes within words
-        text = re.sub(r'[^a-z0-9\s\']', '', text)
-        text = re.sub(r'\s\'|\'\s', ' ', text)
-        
-        return text.strip()

webstoken/processor.py

@@ -1,77 +0,0 @@
-"""
-Main text processing utilities combining all NLP components.
-"""
-
-from typing import Dict, Any, List, Tuple
-
-from .tokenizer import SentenceTokenizer, WordTokenizer
-from .tagger import POSTagger
-from .stemmer import Stemmer
-from .normalizer import TextNormalizer
-
-
-def process_text(text: str, normalize: bool = True, remove_stops: bool = True) -> Dict[str, Any]:
-    """
-    Process text using all available NLP tools.
-    
-    Args:
-        text (str): Input text to process
-        normalize (bool): Whether to normalize text
-        remove_stops (bool): Whether to remove stop words
-        
-    Returns:
-        Dict containing processed results with the following structure:
-        {
-            'sentences': [
-                {
-                    'original': str,     # Original sentence
-                    'tokens': List[str],  # Word tokens
-                    'pos_tags': List[Tuple[str, str]],  # (word, tag) pairs
-                    'stems': List[Tuple[str, str]]      # (word, stem) pairs
-                },
-                ...
-            ],
-            'num_sentences': int,  # Total number of sentences
-            'num_tokens': int      # Total number of tokens
-        }
-    """
-    # Initialize tools
-    sentence_tokenizer = SentenceTokenizer()
-    word_tokenizer = WordTokenizer()
-    pos_tagger = POSTagger()
-    stemmer = Stemmer()
-    normalizer = TextNormalizer()
-    
-    # Process text
-    if normalize:
-        text = normalizer.normalize(text)
-    
-    # Get sentences
-    sentences = sentence_tokenizer.tokenize(text)
-    
-    # Process each sentence
-    processed_sentences = []
-    for sentence in sentences:
-        # Tokenize words
-        tokens = word_tokenizer.tokenize(sentence)
-        
-        # Remove stop words if requested
-        if remove_stops:
-            tokens = normalizer.remove_stop_words(tokens)
-        
-        # Get POS tags and stems
-        tagged = pos_tagger.tag(tokens)
-        stems = [(token, stemmer.stem(token)) for token, _ in tagged]
-        
-        processed_sentences.append({
-            'original': sentence,
-            'tokens': tokens,
-            'pos_tags': tagged,
-            'stems': stems
-        })
-    
-    return {
-        'sentences': processed_sentences,
-        'num_sentences': len(sentences),
-        'num_tokens': sum(len(s['tokens']) for s in processed_sentences)
-    }

webstoken/sentiment.py

@@ -1,206 +0,0 @@
-"""
-Sentiment analysis module for determining text sentiment and emotion.
-"""
-
-from typing import Dict, List, Set, Tuple
-import re
-
-from .tokenizer import WordTokenizer
-from .normalizer import TextNormalizer
-
-
-class SentimentAnalyzer:
-    """Rule-based sentiment analysis using lexicon approach."""
-    
-    def __init__(self):
-        self.word_tokenizer = WordTokenizer()
-        self.normalizer = TextNormalizer()
-        
-        # Sentiment lexicons
-        self.positive_words: Set[str] = {
-            'good', 'great', 'awesome', 'excellent', 'happy', 'wonderful',
-            'fantastic', 'amazing', 'love', 'beautiful', 'best', 'perfect',
-            'brilliant', 'outstanding', 'superb', 'nice', 'pleasant', 'delightful',
-            'positive', 'remarkable', 'terrific', 'incredible', 'enjoyable',
-            'favorable', 'marvelous', 'splendid', 'superior', 'worthy', 'right'
-        }
-        
-        self.negative_words: Set[str] = {
-            'bad', 'terrible', 'awful', 'horrible', 'sad', 'poor', 'wrong',
-            'worse', 'worst', 'hate', 'dislike', 'disappointing', 'negative',
-            'inferior', 'useless', 'worthless', 'mediocre', 'inadequate',
-            'unpleasant', 'unfavorable', 'disagreeable', 'offensive', 'annoying',
-            'frustrating', 'irritating', 'disgusting', 'dreadful', 'pathetic'
-        }
-        
-        # Emotion lexicons
-        self.emotion_words = {
-            'JOY': {
-                'happy', 'joyful', 'delighted', 'excited', 'pleased', 'glad',
-                'cheerful', 'content', 'satisfied', 'elated', 'jubilant',
-                'thrilled', 'ecstatic', 'merry', 'peaceful', 'upbeat'
-            },
-            'SADNESS': {
-                'sad', 'unhappy', 'depressed', 'gloomy', 'miserable', 'down',
-                'heartbroken', 'disappointed', 'upset', 'distressed', 'grief',
-                'sorrow', 'melancholy', 'despair', 'hopeless', 'blue'
-            },
-            'ANGER': {
-                'angry', 'mad', 'furious', 'outraged', 'irritated', 'annoyed',
-                'frustrated', 'enraged', 'hostile', 'bitter', 'hateful', 'rage',
-                'resentful', 'violent', 'aggressive', 'irate'
-            },
-            'FEAR': {
-                'afraid', 'scared', 'frightened', 'terrified', 'anxious', 'worried',
-                'nervous', 'fearful', 'panicked', 'alarmed', 'horrified', 'dread',
-                'uneasy', 'stressed', 'concerned', 'apprehensive'
-            },
-            'SURPRISE': {
-                'surprised', 'amazed', 'astonished', 'shocked', 'stunned',
-                'startled', 'unexpected', 'incredible', 'unbelievable', 'wonder',
-                'awe', 'remarkable', 'mysterious', 'sudden', 'strange'
-            }
-        }
-        
-        # Intensity modifiers
-        self.intensifiers = {
-            'very': 1.5,
-            'really': 1.5,
-            'extremely': 2.0,
-            'incredibly': 2.0,
-            'absolutely': 2.0,
-            'totally': 1.5,
-            'completely': 1.5,
-            'utterly': 2.0,
-            'highly': 1.5,
-            'especially': 1.5
-        }
-        
-        self.diminishers = {
-            'somewhat': 0.5,
-            'slightly': 0.5,
-            'barely': 0.3,
-            'hardly': 0.3,
-            'sort of': 0.5,
-            'kind of': 0.5,
-            'a bit': 0.5,
-            'a little': 0.5,
-            'not very': 0.3,
-            'less': 0.5
-        }
-        
-        # Negation words
-        self.negation_words = {
-            'not', 'no', 'never', 'none', 'nobody', 'nothing', 'neither',
-            'nowhere', 'hardly', 'scarcely', 'barely', "don't", "doesn't",
-            "didn't", "won't", "wouldn't", "shouldn't", "couldn't", "can't"
-        }
-        
-        # Compile patterns
-        self.word_pattern = re.compile(r'\b\w+\b')
-    
-    def _get_window_around_word(self, words: List[str], index: int, window_size: int = 3) -> List[str]:
-        """Get a window of words around a given index."""
-        start = max(0, index - window_size)
-        end = min(len(words), index + window_size + 1)
-        return words[start:end]
-    
-    def _is_negated(self, words: List[str], index: int) -> bool:
-        """Check if a word is negated by looking at surrounding context."""
-        window = self._get_window_around_word(words, index)
-        return any(word in self.negation_words for word in window[:index-window[0]])
-    
-    def _get_intensity_multiplier(self, words: List[str], index: int) -> float:
-        """Get intensity multiplier based on modifiers."""
-        window = self._get_window_around_word(words, index)
-        multiplier = 1.0
-        
-        for word in window[:index-window[0]]:
-            if word in self.intensifiers:
-                multiplier *= self.intensifiers[word]
-            elif word in self.diminishers:
-                multiplier *= self.diminishers[word]
-        
-        return multiplier
-    
-    def analyze_sentiment(self, text: str) -> Dict[str, float]:
-        """
-        Analyze sentiment of text.
-        
-        Returns:
-            Dict with sentiment scores:
-            {
-                'polarity': float (-1 to 1),
-                'subjectivity': float (0 to 1),
-                'confidence': float (0 to 1)
-            }
-        """
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        words = self.word_tokenizer.tokenize(text)
-        
-        positive_score = 0
-        negative_score = 0
-        word_count = len(words)
-        
-        for i, word in enumerate(words):
-            word = word.lower()
-            multiplier = self._get_intensity_multiplier(words, i)
-            is_negated = self._is_negated(words, i)
-            
-            if word in self.positive_words:
-                score = 1.0 * multiplier
-                positive_score += -score if is_negated else score
-            elif word in self.negative_words:
-                score = 1.0 * multiplier
-                negative_score += -score if is_negated else score
-        
-        # Calculate metrics
-        total_score = positive_score + negative_score
-        total_magnitude = abs(positive_score) + abs(negative_score)
-        
-        if word_count == 0:
-            return {'polarity': 0.0, 'subjectivity': 0.0, 'confidence': 0.0}
-        
-        polarity = total_score / (word_count or 1)  # Normalize to [-1, 1]
-        subjectivity = total_magnitude / (word_count or 1)  # Normalize to [0, 1]
-        confidence = min(1.0, total_magnitude / (word_count / 2))  # Confidence based on magnitude
-        
-        return {
-            'polarity': max(-1.0, min(1.0, polarity)),
-            'subjectivity': min(1.0, subjectivity),
-            'confidence': confidence
-        }
-    
-    def analyze_emotions(self, text: str) -> List[Tuple[str, float]]:
-        """
-        Analyze emotions in text.
-        
-        Returns:
-            List of (emotion, score) tuples, sorted by score
-        """
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        words = self.word_tokenizer.tokenize(text)
-        
-        emotion_scores = {emotion: 0.0 for emotion in self.emotion_words}
-        
-        for i, word in enumerate(words):
-            word = word.lower()
-            multiplier = self._get_intensity_multiplier(words, i)
-            is_negated = self._is_negated(words, i)
-            
-            for emotion, emotion_set in self.emotion_words.items():
-                if word in emotion_set:
-                    score = 1.0 * multiplier
-                    emotion_scores[emotion] += -score if is_negated else score
-        
-        # Normalize scores
-        max_score = max(abs(score) for score in emotion_scores.values()) or 1
-        normalized_scores = [
-            (emotion, score/max_score)
-            for emotion, score in emotion_scores.items()
-        ]
-        
-        # Sort by score
-        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)

webstoken/stemmer.py

@@ -1,73 +0,0 @@
-"""
-Word stemming utilities.
-"""
-
-from typing import Set
-
-
-class Stemmer:
-    """Simple rule-based stemmer implementing Porter-like rules."""
-    
-    def __init__(self):
-        self.vowels: Set[str] = {'a', 'e', 'i', 'o', 'u', 'y'}
-        self.doubles: Set[str] = {'bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'}
-        
-    def is_vowel(self, char: str, prev_char: str = None) -> bool:
-        """Check if a character is a vowel, considering 'y' special cases."""
-        return char in self.vowels or (char == 'y' and prev_char and prev_char not in self.vowels)
-    
-    def count_syllables(self, word: str) -> int:
-        """Count syllables in a word based on vowel sequences."""
-        count = 0
-        prev_char = None
-        for i, char in enumerate(word.lower()):
-            if self.is_vowel(char, prev_char) and (i == 0 or not self.is_vowel(prev_char, word[i-2] if i > 1 else None)):
-                count += 1
-            prev_char = char
-        return count or 1
-    
-    def stem(self, word: str) -> str:
-        """Apply stemming rules to reduce word to its root form."""
-        if len(word) <= 3:
-            return word
-            
-        word = word.lower()
-        
-        # Step 1: Handle plurals and past participles
-        if word.endswith('sses'):
-            word = word[:-2]
-        elif word.endswith('ies'):
-            word = word[:-2]
-        elif word.endswith('ss'):
-            pass
-        elif word.endswith('s') and len(word) > 4:
-            word = word[:-1]
-            
-        # Step 2: Handle -ed and -ing
-        if word.endswith('ed') and self.count_syllables(word[:-2]) > 1:
-            word = word[:-2]
-        elif word.endswith('ing') and self.count_syllables(word[:-3]) > 1:
-            word = word[:-3]
-            
-        # Step 3: Handle miscellaneous endings
-        if len(word) > 5:
-            if word.endswith('ement'):
-                word = word[:-5]
-            elif word.endswith('ment'):
-                word = word[:-4]
-            elif word.endswith('ent'):
-                word = word[:-3]
-                
-        # Step 4: Handle -ity endings
-        if word.endswith('ity') and len(word) > 6:
-            word = word[:-3]
-            if word.endswith('abil'):
-                word = word[:-4] + 'able'
-            elif word.endswith('ic'):
-                word = word[:-2]
-                
-        # Final step: Remove double consonants at the end
-        if len(word) > 2 and word[-2:] in self.doubles:
-            word = word[:-1]
-            
-        return word

webstoken/tagger.py

@@ -1,60 +0,0 @@
-"""
-Part-of-Speech tagging utilities.
-"""
-
-from typing import List, Set, Tuple
-
-
-class POSTagger:
-    """Simple rule-based Part-of-Speech tagger."""
-    
-    def __init__(self):
-        # Basic rules for POS tagging
-        self.noun_suffixes: Set[str] = {'ness', 'ment', 'ship', 'dom', 'hood', 'er', 'or', 'ist'}
-        self.verb_suffixes: Set[str] = {'ize', 'ate', 'ify', 'ing', 'ed'}
-        self.adj_suffixes: Set[str] = {'able', 'ible', 'al', 'ful', 'ous', 'ive', 'less'}
-        self.adv_suffixes: Set[str] = {'ly'}
-        
-        # Common words by POS
-        self.determiners: Set[str] = {'the', 'a', 'an', 'this', 'that', 'these', 'those'}
-        self.prepositions: Set[str] = {'in', 'on', 'at', 'by', 'with', 'from', 'to', 'for'}
-        self.pronouns: Set[str] = {'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her'}
-        
-    def tag(self, tokens: List[str]) -> List[Tuple[str, str]]:
-        """Assign POS tags to tokens based on rules."""
-        tagged = []
-        prev_tag = None
-        
-        for i, token in enumerate(tokens):
-            word = token.lower()
-            
-            # Check special cases first
-            if word in self.determiners:
-                tag = 'DET'
-            elif word in self.prepositions:
-                tag = 'PREP'
-            elif word in self.pronouns:
-                tag = 'PRON'
-            # Check suffixes
-            elif any(word.endswith(suffix) for suffix in self.noun_suffixes):
-                tag = 'NOUN'
-            elif any(word.endswith(suffix) for suffix in self.verb_suffixes):
-                tag = 'VERB'
-            elif any(word.endswith(suffix) for suffix in self.adj_suffixes):
-                tag = 'ADJ'
-            elif any(word.endswith(suffix) for suffix in self.adv_suffixes):
-                tag = 'ADV'
-            # Default cases
-            elif word[0].isupper() and i > 0:
-                tag = 'PROPN'  # Proper noun
-            elif word.isdigit():
-                tag = 'NUM'
-            elif not word.isalnum():
-                tag = 'PUNCT'
-            else:
-                tag = 'NOUN'  # Default to noun
-            
-            tagged.append((token, tag))
-            prev_tag = tag
-            
-        return tagged