webscout 6.5__py3-none-any.whl → 6.7__py3-none-any.whl

webstoken/language.py

@@ -0,0 +1,128 @@
+"""
+Language detection module using character and word frequency analysis.
+"""
+
+from typing import Dict, List, Set, Tuple
+from collections import Counter
+import re
+
+
+class LanguageDetector:
+    """Language detection using character n-gram frequencies."""
+    
+    def __init__(self):
+        # Language profiles based on common character sequences
+        self.language_profiles = {
+            'ENGLISH': {
+                'chars': 'etaoinshrdlcumwfgypbvkjxqz',
+                'ngrams': {'th', 'he', 'in', 'er', 'an', 're', 'on', 'at', 'en', 'nd',
+                          'ti', 'es', 'or', 'te', 'of', 'ed', 'is', 'it', 'al', 'ar',
+                          'st', 'to', 'nt', 'ng', 'se', 'ha', 'as', 'ou', 'io', 'le'},
+                'words': {'the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have',
+                         'i', 'it', 'for', 'not', 'on', 'with', 'he', 'as', 'you',
+                         'do', 'at'}
+            },
+            'SPANISH': {
+                'chars': 'eaosrnidlctumpbgvyqhfzjñxwk',
+                'ngrams': {'de', 'en', 'el', 'la', 'os', 'es', 'as', 'ar', 'er', 'ra',
+                          'al', 'an', 'do', 'or', 'ta', 'ue', 'io', 'on', 'ro', 'ad',
+                          'te', 'co', 'st', 'ci', 'nt', 'to', 'lo', 'no', 'po', 'ac'},
+                'words': {'de', 'la', 'que', 'el', 'en', 'y', 'a', 'los', 'se', 'del',
+                         'las', 'un', 'por', 'con', 'no', 'una', 'su', 'para', 'es',
+                         'al'}
+            },
+            'FRENCH': {
+                'chars': 'esaitnrulodcpmévqfbghàjxèêyçwzùâîôûëïüœ',
+                'ngrams': {'es', 'le', 'en', 'de', 'nt', 'on', 're', 'er', 'ai', 'te',
+                          'la', 'an', 'ou', 'it', 'ur', 'et', 'el', 'se', 'qu', 'me',
+                          'is', 'ar', 'ce', 'ns', 'us', 'ue', 'ss', 'ie', 'em', 'tr'},
+                'words': {'le', 'de', 'un', 'être', 'et', 'à', 'il', 'avoir', 'ne',
+                         'je', 'son', 'que', 'se', 'qui', 'ce', 'dans', 'en', 'du',
+                         'elle', 'au'}
+            },
+            'GERMAN': {
+                'chars': 'enisratdhulcgmobwfkzvüpäößjyqxéèêëàáâãåāăąćčĉċďđ',
+                'ngrams': {'en', 'er', 'ch', 'de', 'ei', 'in', 'te', 'nd', 'ie', 'ge',
+                          'st', 'ne', 'be', 'es', 'un', 'zu', 'an', 'ng', 'au', 'it',
+                          'is', 'he', 'ht', 'se', 'ck', 'ic', 're', 'ns', 'sc', 'tz'},
+                'words': {'der', 'die', 'und', 'in', 'den', 'von', 'zu', 'das', 'mit',
+                         'sich', 'des', 'auf', 'für', 'ist', 'im', 'dem', 'nicht',
+                         'ein', 'eine', 'als'}
+            }
+        }
+        
+        # Compile word patterns
+        self.word_pattern = re.compile(r'\b\w+\b')
+    
+    def _extract_ngrams(self, text: str, n: int = 2) -> List[str]:
+        """Extract character n-grams from text."""
+        text = text.lower()
+        return [text[i:i+n] for i in range(len(text)-n+1)]
+    
+    def _calculate_char_frequencies(self, text: str) -> Dict[str, float]:
+        """Calculate character frequencies in text."""
+        text = text.lower()
+        char_count = Counter(c for c in text if c.isalpha())
+        total = sum(char_count.values()) or 1
+        return {char: count/total for char, count in char_count.items()}
+    
+    def _calculate_ngram_frequencies(self, text: str) -> Dict[str, float]:
+        """Calculate n-gram frequencies in text."""
+        ngrams = self._extract_ngrams(text)
+        ngram_count = Counter(ngrams)
+        total = sum(ngram_count.values()) or 1
+        return {ngram: count/total for ngram, count in ngram_count.items()}
+    
+    def _calculate_word_frequencies(self, text: str) -> Dict[str, float]:
+        """Calculate word frequencies in text."""
+        words = self.word_pattern.findall(text.lower())
+        word_count = Counter(words)
+        total = sum(word_count.values()) or 1
+        return {word: count/total for word, count in word_count.items()}
+    
+    def _calculate_similarity(self, freq1: Dict[str, float], freq2: Dict[str, float]) -> float:
+        """Calculate similarity between two frequency distributions."""
+        common_keys = set(freq1.keys()) & set(freq2.keys())
+        if not common_keys:
+            return 0.0
+        
+        similarity = sum(min(freq1.get(k, 0), freq2.get(k, 0)) for k in common_keys)
+        return similarity
+    
+    def detect(self, text: str) -> List[Tuple[str, float]]:
+        """
+        Detect the language of text with confidence scores.
+        
+        Returns:
+            List of (language, confidence) tuples, sorted by confidence
+        """
+        if not text:
+            return []
+        
+        # Calculate frequencies for input text
+        char_freqs = self._calculate_char_frequencies(text)
+        ngram_freqs = self._calculate_ngram_frequencies(text)
+        word_freqs = self._calculate_word_frequencies(text)
+        
+        # Calculate similarity scores for each language
+        scores = []
+        for lang, profile in self.language_profiles.items():
+            # Character similarity
+            char_sim = sum(char_freqs.get(c, 0) for c in profile['chars'])
+            
+            # N-gram similarity
+            ngram_sim = sum(ngram_freqs.get(ng, 0) for ng in profile['ngrams'])
+            
+            # Word similarity
+            word_sim = sum(word_freqs.get(w, 0) for w in profile['words'])
+            
+            # Combined score (weighted average)
+            total_score = (0.3 * char_sim + 0.4 * ngram_sim + 0.3 * word_sim)
+            scores.append((lang, total_score))
+        
+        # Normalize scores
+        total = sum(score for _, score in scores) or 1
+        normalized_scores = [(lang, score/total) for lang, score in scores]
+        
+        # Sort by confidence
+        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)

webstoken/ner.py

@@ -0,0 +1,164 @@
+"""
+Named Entity Recognition (NER) module for identifying and classifying named entities.
+"""
+
+from typing import List, Tuple, Dict, Set
+import re
+
+
+class NamedEntityRecognizer:
+    """Rule-based Named Entity Recognition."""
+    
+    def __init__(self):
+        # Common entity patterns
+        self.PERSON_TITLES = {
+            'mr', 'mrs', 'ms', 'miss', 'dr', 'prof', 'sir', 'madam',
+            'lord', 'lady', 'president', 'ceo', 'director'
+        }
+        
+        self.ORGANIZATION_SUFFIXES = {
+            'inc', 'corp', 'ltd', 'llc', 'company', 'corporation',
+            'associates', 'partners', 'foundation', 'institute'
+        }
+        
+        self.LOCATION_INDICATORS = {
+            'street', 'road', 'avenue', 'boulevard', 'lane', 'drive',
+            'circle', 'square', 'park', 'bridge', 'river', 'lake',
+            'mountain', 'forest', 'city', 'town', 'village', 'country'
+        }
+        
+        self.DATE_MONTHS = {
+            'january', 'february', 'march', 'april', 'may', 'june',
+            'july', 'august', 'september', 'october', 'november', 'december'
+        }
+        
+        # Compile regex patterns
+        self.patterns = {
+            'EMAIL': re.compile(r'\b[\w\.-]+@[\w\.-]+\.\w+\b'),
+            'URL': re.compile(r'https?://(?:[\w-]|(?:%[\da-fA-F]{2}))+'),
+            'PHONE': re.compile(r'\+?\d{1,3}[-.\s]?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}'),
+            'DATE': re.compile(r'\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b'),
+            'TIME': re.compile(r'\b\d{1,2}:\d{2}(?::\d{2})?(?:\s*[AaPp][Mm])?\b'),
+            'MONEY': re.compile(r'\$\d+(?:,\d{3})*(?:\.\d{2})?|\d+(?:,\d{3})*(?:\.\d{2})?\s*(?:dollars|USD|EUR|GBP)'),
+            'PERCENTAGE': re.compile(r'\b\d+(?:\.\d+)?%\b')
+        }
+    
+    def is_capitalized(self, word: str) -> bool:
+        """Check if a word is capitalized."""
+        return word and word[0].isupper()
+    
+    def extract_entities(self, text: str) -> Dict[str, List[Tuple[str, str]]]:
+        """
+        Extract named entities from text.
+        
+        Returns:
+            Dict mapping entity types to list of (text, label) tuples
+        """
+        entities = {
+            'PERSON': [],
+            'ORGANIZATION': [],
+            'LOCATION': [],
+            'DATE': [],
+            'TIME': [],
+            'MONEY': [],
+            'EMAIL': [],
+            'URL': [],
+            'PHONE': [],
+            'PERCENTAGE': []
+        }
+        
+        # First find regex pattern matches
+        for label, pattern in self.patterns.items():
+            for match in pattern.finditer(text):
+                entities[label].append((match.group(), label))
+        
+        # Process text word by word for other entities
+        words = text.split()
+        i = 0
+        while i < len(words):
+            word = words[i]
+            next_word = words[i + 1] if i + 1 < len(words) else None
+            
+            # Check for person names
+            if word.lower() in self.PERSON_TITLES and next_word and self.is_capitalized(next_word):
+                name_parts = []
+                j = i + 1
+                while j < len(words) and self.is_capitalized(words[j]):
+                    name_parts.append(words[j])
+                    j += 1
+                if name_parts:
+                    entities['PERSON'].append((' '.join(name_parts), 'PERSON'))
+                i = j
+                continue
+            
+            # Check for organizations
+            if self.is_capitalized(word):
+                org_parts = [word]
+                j = i + 1
+                while j < len(words) and (
+                    self.is_capitalized(words[j]) or 
+                    words[j].lower() in self.ORGANIZATION_SUFFIXES
+                ):
+                    org_parts.append(words[j])
+                    j += 1
+                if len(org_parts) > 1 or (
+                    len(org_parts) == 1 and 
+                    any(suff in word.lower() for suff in self.ORGANIZATION_SUFFIXES)
+                ):
+                    entities['ORGANIZATION'].append((' '.join(org_parts), 'ORGANIZATION'))
+                i = j
+                continue
+            
+            # Check for locations
+            if word.lower() in self.LOCATION_INDICATORS and i > 0:
+                if self.is_capitalized(words[i - 1]):
+                    entities['LOCATION'].append((words[i - 1] + ' ' + word, 'LOCATION'))
+            
+            i += 1
+        
+        return entities
+        
+    def tag_text(self, text: str) -> List[Tuple[str, str]]:
+        """
+        Tag each word in text with its entity type.
+        
+        Returns:
+            List of (word, entity_type) tuples
+        """
+        entities = self.extract_entities(text)
+        tagged = []
+        
+        # Create a map of word positions to entity labels
+        position_labels = {}
+        text_lower = text.lower()
+        
+        for entity_type, entity_list in entities.items():
+            for entity_text, _ in entity_list:
+                start = text_lower.find(entity_text.lower())
+                if start != -1:
+                    end = start + len(entity_text)
+                    for pos in range(start, end):
+                        position_labels[pos] = entity_type
+        
+        # Tag each character position
+        current_pos = 0
+        current_word = []
+        current_label = 'O'  # Outside any entity
+        
+        for char in text:
+            if char.isspace():
+                if current_word:
+                    tagged.append((''.join(current_word), current_label))
+                    current_word = []
+                    current_label = 'O'
+            else:
+                current_word.append(char)
+                if current_pos in position_labels:
+                    current_label = position_labels[current_pos]
+            current_pos += 1
+        
+        # Add last word if exists
+        if current_word:
+            tagged.append((''.join(current_word), current_label))
+        
+        return tagged

webstoken/normalizer.py

@@ -0,0 +1,35 @@
+"""
+Text normalization utilities.
+"""
+
+import re
+from typing import List, Set
+
+
+class TextNormalizer:
+    """Text normalization utilities."""
+    
+    def __init__(self):
+        self.stop_words: Set[str] = {
+            'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
+            'has', 'he', 'in', 'is', 'it', 'its', 'of', 'on', 'that', 'the',
+            'to', 'was', 'were', 'will', 'with'
+        }
+        
+    def remove_stop_words(self, tokens: List[str]) -> List[str]:
+        """Remove common stop words from token list."""
+        return [token for token in tokens if token.lower() not in self.stop_words]
+    
+    def normalize(self, text: str) -> str:
+        """Apply various normalization steps to text."""
+        # Convert to lowercase
+        text = text.lower()
+        
+        # Replace multiple spaces with single space
+        text = re.sub(r'\s+', ' ', text)
+        
+        # Remove special characters except apostrophes within words
+        text = re.sub(r'[^a-z0-9\s\']', '', text)
+        text = re.sub(r'\s\'|\'\s', ' ', text)
+        
+        return text.strip()

webstoken/processor.py

@@ -0,0 +1,77 @@
+"""
+Main text processing utilities combining all NLP components.
+"""
+
+from typing import Dict, Any, List, Tuple
+
+from .tokenizer import SentenceTokenizer, WordTokenizer
+from .tagger import POSTagger
+from .stemmer import Stemmer
+from .normalizer import TextNormalizer
+
+
+def process_text(text: str, normalize: bool = True, remove_stops: bool = True) -> Dict[str, Any]:
+    """
+    Process text using all available NLP tools.
+    
+    Args:
+        text (str): Input text to process
+        normalize (bool): Whether to normalize text
+        remove_stops (bool): Whether to remove stop words
+        
+    Returns:
+        Dict containing processed results with the following structure:
+        {
+            'sentences': [
+                {
+                    'original': str,     # Original sentence
+                    'tokens': List[str],  # Word tokens
+                    'pos_tags': List[Tuple[str, str]],  # (word, tag) pairs
+                    'stems': List[Tuple[str, str]]      # (word, stem) pairs
+                },
+                ...
+            ],
+            'num_sentences': int,  # Total number of sentences
+            'num_tokens': int      # Total number of tokens
+        }
+    """
+    # Initialize tools
+    sentence_tokenizer = SentenceTokenizer()
+    word_tokenizer = WordTokenizer()
+    pos_tagger = POSTagger()
+    stemmer = Stemmer()
+    normalizer = TextNormalizer()
+    
+    # Process text
+    if normalize:
+        text = normalizer.normalize(text)
+    
+    # Get sentences
+    sentences = sentence_tokenizer.tokenize(text)
+    
+    # Process each sentence
+    processed_sentences = []
+    for sentence in sentences:
+        # Tokenize words
+        tokens = word_tokenizer.tokenize(sentence)
+        
+        # Remove stop words if requested
+        if remove_stops:
+            tokens = normalizer.remove_stop_words(tokens)
+        
+        # Get POS tags and stems
+        tagged = pos_tagger.tag(tokens)
+        stems = [(token, stemmer.stem(token)) for token, _ in tagged]
+        
+        processed_sentences.append({
+            'original': sentence,
+            'tokens': tokens,
+            'pos_tags': tagged,
+            'stems': stems
+        })
+    
+    return {
+        'sentences': processed_sentences,
+        'num_sentences': len(sentences),
+        'num_tokens': sum(len(s['tokens']) for s in processed_sentences)
+    }

webstoken/sentiment.py

@@ -0,0 +1,206 @@
+"""
+Sentiment analysis module for determining text sentiment and emotion.
+"""
+
+from typing import Dict, List, Set, Tuple
+import re
+
+from .tokenizer import WordTokenizer
+from .normalizer import TextNormalizer
+
+
+class SentimentAnalyzer:
+    """Rule-based sentiment analysis using lexicon approach."""
+    
+    def __init__(self):
+        self.word_tokenizer = WordTokenizer()
+        self.normalizer = TextNormalizer()
+        
+        # Sentiment lexicons
+        self.positive_words: Set[str] = {
+            'good', 'great', 'awesome', 'excellent', 'happy', 'wonderful',
+            'fantastic', 'amazing', 'love', 'beautiful', 'best', 'perfect',
+            'brilliant', 'outstanding', 'superb', 'nice', 'pleasant', 'delightful',
+            'positive', 'remarkable', 'terrific', 'incredible', 'enjoyable',
+            'favorable', 'marvelous', 'splendid', 'superior', 'worthy', 'right'
+        }
+        
+        self.negative_words: Set[str] = {
+            'bad', 'terrible', 'awful', 'horrible', 'sad', 'poor', 'wrong',
+            'worse', 'worst', 'hate', 'dislike', 'disappointing', 'negative',
+            'inferior', 'useless', 'worthless', 'mediocre', 'inadequate',
+            'unpleasant', 'unfavorable', 'disagreeable', 'offensive', 'annoying',
+            'frustrating', 'irritating', 'disgusting', 'dreadful', 'pathetic'
+        }
+        
+        # Emotion lexicons
+        self.emotion_words = {
+            'JOY': {
+                'happy', 'joyful', 'delighted', 'excited', 'pleased', 'glad',
+                'cheerful', 'content', 'satisfied', 'elated', 'jubilant',
+                'thrilled', 'ecstatic', 'merry', 'peaceful', 'upbeat'
+            },
+            'SADNESS': {
+                'sad', 'unhappy', 'depressed', 'gloomy', 'miserable', 'down',
+                'heartbroken', 'disappointed', 'upset', 'distressed', 'grief',
+                'sorrow', 'melancholy', 'despair', 'hopeless', 'blue'
+            },
+            'ANGER': {
+                'angry', 'mad', 'furious', 'outraged', 'irritated', 'annoyed',
+                'frustrated', 'enraged', 'hostile', 'bitter', 'hateful', 'rage',
+                'resentful', 'violent', 'aggressive', 'irate'
+            },
+            'FEAR': {
+                'afraid', 'scared', 'frightened', 'terrified', 'anxious', 'worried',
+                'nervous', 'fearful', 'panicked', 'alarmed', 'horrified', 'dread',
+                'uneasy', 'stressed', 'concerned', 'apprehensive'
+            },
+            'SURPRISE': {
+                'surprised', 'amazed', 'astonished', 'shocked', 'stunned',
+                'startled', 'unexpected', 'incredible', 'unbelievable', 'wonder',
+                'awe', 'remarkable', 'mysterious', 'sudden', 'strange'
+            }
+        }
+        
+        # Intensity modifiers
+        self.intensifiers = {
+            'very': 1.5,
+            'really': 1.5,
+            'extremely': 2.0,
+            'incredibly': 2.0,
+            'absolutely': 2.0,
+            'totally': 1.5,
+            'completely': 1.5,
+            'utterly': 2.0,
+            'highly': 1.5,
+            'especially': 1.5
+        }
+        
+        self.diminishers = {
+            'somewhat': 0.5,
+            'slightly': 0.5,
+            'barely': 0.3,
+            'hardly': 0.3,
+            'sort of': 0.5,
+            'kind of': 0.5,
+            'a bit': 0.5,
+            'a little': 0.5,
+            'not very': 0.3,
+            'less': 0.5
+        }
+        
+        # Negation words
+        self.negation_words = {
+            'not', 'no', 'never', 'none', 'nobody', 'nothing', 'neither',
+            'nowhere', 'hardly', 'scarcely', 'barely', "don't", "doesn't",
+            "didn't", "won't", "wouldn't", "shouldn't", "couldn't", "can't"
+        }
+        
+        # Compile patterns
+        self.word_pattern = re.compile(r'\b\w+\b')
+    
+    def _get_window_around_word(self, words: List[str], index: int, window_size: int = 3) -> List[str]:
+        """Get a window of words around a given index."""
+        start = max(0, index - window_size)
+        end = min(len(words), index + window_size + 1)
+        return words[start:end]
+    
+    def _is_negated(self, words: List[str], index: int) -> bool:
+        """Check if a word is negated by looking at surrounding context."""
+        window = self._get_window_around_word(words, index)
+        return any(word in self.negation_words for word in window[:index-window[0]])
+    
+    def _get_intensity_multiplier(self, words: List[str], index: int) -> float:
+        """Get intensity multiplier based on modifiers."""
+        window = self._get_window_around_word(words, index)
+        multiplier = 1.0
+        
+        for word in window[:index-window[0]]:
+            if word in self.intensifiers:
+                multiplier *= self.intensifiers[word]
+            elif word in self.diminishers:
+                multiplier *= self.diminishers[word]
+        
+        return multiplier
+    
+    def analyze_sentiment(self, text: str) -> Dict[str, float]:
+        """
+        Analyze sentiment of text.
+        
+        Returns:
+            Dict with sentiment scores:
+            {
+                'polarity': float (-1 to 1),
+                'subjectivity': float (0 to 1),
+                'confidence': float (0 to 1)
+            }
+        """
+        # Normalize and tokenize text
+        text = self.normalizer.normalize(text)
+        words = self.word_tokenizer.tokenize(text)
+        
+        positive_score = 0
+        negative_score = 0
+        word_count = len(words)
+        
+        for i, word in enumerate(words):
+            word = word.lower()
+            multiplier = self._get_intensity_multiplier(words, i)
+            is_negated = self._is_negated(words, i)
+            
+            if word in self.positive_words:
+                score = 1.0 * multiplier
+                positive_score += -score if is_negated else score
+            elif word in self.negative_words:
+                score = 1.0 * multiplier
+                negative_score += -score if is_negated else score
+        
+        # Calculate metrics
+        total_score = positive_score + negative_score
+        total_magnitude = abs(positive_score) + abs(negative_score)
+        
+        if word_count == 0:
+            return {'polarity': 0.0, 'subjectivity': 0.0, 'confidence': 0.0}
+        
+        polarity = total_score / (word_count or 1)  # Normalize to [-1, 1]
+        subjectivity = total_magnitude / (word_count or 1)  # Normalize to [0, 1]
+        confidence = min(1.0, total_magnitude / (word_count / 2))  # Confidence based on magnitude
+        
+        return {
+            'polarity': max(-1.0, min(1.0, polarity)),
+            'subjectivity': min(1.0, subjectivity),
+            'confidence': confidence
+        }
+    
+    def analyze_emotions(self, text: str) -> List[Tuple[str, float]]:
+        """
+        Analyze emotions in text.
+        
+        Returns:
+            List of (emotion, score) tuples, sorted by score
+        """
+        # Normalize and tokenize text
+        text = self.normalizer.normalize(text)
+        words = self.word_tokenizer.tokenize(text)
+        
+        emotion_scores = {emotion: 0.0 for emotion in self.emotion_words}
+        
+        for i, word in enumerate(words):
+            word = word.lower()
+            multiplier = self._get_intensity_multiplier(words, i)
+            is_negated = self._is_negated(words, i)
+            
+            for emotion, emotion_set in self.emotion_words.items():
+                if word in emotion_set:
+                    score = 1.0 * multiplier
+                    emotion_scores[emotion] += -score if is_negated else score
+        
+        # Normalize scores
+        max_score = max(abs(score) for score in emotion_scores.values()) or 1
+        normalized_scores = [
+            (emotion, score/max_score)
+            for emotion, score in emotion_scores.items()
+        ]
+        
+        # Sort by score
+        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)