webscout 7.0__py3-none-any.whl → 7.2__py3-none-any.whl

webstoken/language.py

@@ -1,128 +1,128 @@
-"""
-Language detection module using character and word frequency analysis.
-"""
-
-from typing import Dict, List, Set, Tuple
-from collections import Counter
-import re
-
-
-class LanguageDetector:
-    """Language detection using character n-gram frequencies."""
-    
-    def __init__(self):
-        # Language profiles based on common character sequences
-        self.language_profiles = {
-            'ENGLISH': {
-                'chars': 'etaoinshrdlcumwfgypbvkjxqz',
-                'ngrams': {'th', 'he', 'in', 'er', 'an', 're', 'on', 'at', 'en', 'nd',
-                          'ti', 'es', 'or', 'te', 'of', 'ed', 'is', 'it', 'al', 'ar',
-                          'st', 'to', 'nt', 'ng', 'se', 'ha', 'as', 'ou', 'io', 'le'},
-                'words': {'the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have',
-                         'i', 'it', 'for', 'not', 'on', 'with', 'he', 'as', 'you',
-                         'do', 'at'}
-            },
-            'SPANISH': {
-                'chars': 'eaosrnidlctumpbgvyqhfzjñxwk',
-                'ngrams': {'de', 'en', 'el', 'la', 'os', 'es', 'as', 'ar', 'er', 'ra',
-                          'al', 'an', 'do', 'or', 'ta', 'ue', 'io', 'on', 'ro', 'ad',
-                          'te', 'co', 'st', 'ci', 'nt', 'to', 'lo', 'no', 'po', 'ac'},
-                'words': {'de', 'la', 'que', 'el', 'en', 'y', 'a', 'los', 'se', 'del',
-                         'las', 'un', 'por', 'con', 'no', 'una', 'su', 'para', 'es',
-                         'al'}
-            },
-            'FRENCH': {
-                'chars': 'esaitnrulodcpmévqfbghàjxèêyçwzùâîôûëïüœ',
-                'ngrams': {'es', 'le', 'en', 'de', 'nt', 'on', 're', 'er', 'ai', 'te',
-                          'la', 'an', 'ou', 'it', 'ur', 'et', 'el', 'se', 'qu', 'me',
-                          'is', 'ar', 'ce', 'ns', 'us', 'ue', 'ss', 'ie', 'em', 'tr'},
-                'words': {'le', 'de', 'un', 'être', 'et', 'à', 'il', 'avoir', 'ne',
-                         'je', 'son', 'que', 'se', 'qui', 'ce', 'dans', 'en', 'du',
-                         'elle', 'au'}
-            },
-            'GERMAN': {
-                'chars': 'enisratdhulcgmobwfkzvüpäößjyqxéèêëàáâãåāăąćčĉċďđ',
-                'ngrams': {'en', 'er', 'ch', 'de', 'ei', 'in', 'te', 'nd', 'ie', 'ge',
-                          'st', 'ne', 'be', 'es', 'un', 'zu', 'an', 'ng', 'au', 'it',
-                          'is', 'he', 'ht', 'se', 'ck', 'ic', 're', 'ns', 'sc', 'tz'},
-                'words': {'der', 'die', 'und', 'in', 'den', 'von', 'zu', 'das', 'mit',
-                         'sich', 'des', 'auf', 'für', 'ist', 'im', 'dem', 'nicht',
-                         'ein', 'eine', 'als'}
-            }
-        }
-        
-        # Compile word patterns
-        self.word_pattern = re.compile(r'\b\w+\b')
-    
-    def _extract_ngrams(self, text: str, n: int = 2) -> List[str]:
-        """Extract character n-grams from text."""
-        text = text.lower()
-        return [text[i:i+n] for i in range(len(text)-n+1)]
-    
-    def _calculate_char_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate character frequencies in text."""
-        text = text.lower()
-        char_count = Counter(c for c in text if c.isalpha())
-        total = sum(char_count.values()) or 1
-        return {char: count/total for char, count in char_count.items()}
-    
-    def _calculate_ngram_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate n-gram frequencies in text."""
-        ngrams = self._extract_ngrams(text)
-        ngram_count = Counter(ngrams)
-        total = sum(ngram_count.values()) or 1
-        return {ngram: count/total for ngram, count in ngram_count.items()}
-    
-    def _calculate_word_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate word frequencies in text."""
-        words = self.word_pattern.findall(text.lower())
-        word_count = Counter(words)
-        total = sum(word_count.values()) or 1
-        return {word: count/total for word, count in word_count.items()}
-    
-    def _calculate_similarity(self, freq1: Dict[str, float], freq2: Dict[str, float]) -> float:
-        """Calculate similarity between two frequency distributions."""
-        common_keys = set(freq1.keys()) & set(freq2.keys())
-        if not common_keys:
-            return 0.0
-        
-        similarity = sum(min(freq1.get(k, 0), freq2.get(k, 0)) for k in common_keys)
-        return similarity
-    
-    def detect(self, text: str) -> List[Tuple[str, float]]:
-        """
-        Detect the language of text with confidence scores.
-        
-        Returns:
-            List of (language, confidence) tuples, sorted by confidence
-        """
-        if not text:
-            return []
-        
-        # Calculate frequencies for input text
-        char_freqs = self._calculate_char_frequencies(text)
-        ngram_freqs = self._calculate_ngram_frequencies(text)
-        word_freqs = self._calculate_word_frequencies(text)
-        
-        # Calculate similarity scores for each language
-        scores = []
-        for lang, profile in self.language_profiles.items():
-            # Character similarity
-            char_sim = sum(char_freqs.get(c, 0) for c in profile['chars'])
-            
-            # N-gram similarity
-            ngram_sim = sum(ngram_freqs.get(ng, 0) for ng in profile['ngrams'])
-            
-            # Word similarity
-            word_sim = sum(word_freqs.get(w, 0) for w in profile['words'])
-            
-            # Combined score (weighted average)
-            total_score = (0.3 * char_sim + 0.4 * ngram_sim + 0.3 * word_sim)
-            scores.append((lang, total_score))
-        
-        # Normalize scores
-        total = sum(score for _, score in scores) or 1
-        normalized_scores = [(lang, score/total) for lang, score in scores]
-        
-        # Sort by confidence
-        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)
+"""
+Language detection module using character and word frequency analysis.
+"""
+
+from typing import Dict, List, Set, Tuple
+from collections import Counter
+import re
+
+
+class LanguageDetector:
+    """Language detection using character n-gram frequencies."""
+    
+    def __init__(self):
+        # Language profiles based on common character sequences
+        self.language_profiles = {
+            'ENGLISH': {
+                'chars': 'etaoinshrdlcumwfgypbvkjxqz',
+                'ngrams': {'th', 'he', 'in', 'er', 'an', 're', 'on', 'at', 'en', 'nd',
+                          'ti', 'es', 'or', 'te', 'of', 'ed', 'is', 'it', 'al', 'ar',
+                          'st', 'to', 'nt', 'ng', 'se', 'ha', 'as', 'ou', 'io', 'le'},
+                'words': {'the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have',
+                         'i', 'it', 'for', 'not', 'on', 'with', 'he', 'as', 'you',
+                         'do', 'at'}
+            },
+            'SPANISH': {
+                'chars': 'eaosrnidlctumpbgvyqhfzjñxwk',
+                'ngrams': {'de', 'en', 'el', 'la', 'os', 'es', 'as', 'ar', 'er', 'ra',
+                          'al', 'an', 'do', 'or', 'ta', 'ue', 'io', 'on', 'ro', 'ad',
+                          'te', 'co', 'st', 'ci', 'nt', 'to', 'lo', 'no', 'po', 'ac'},
+                'words': {'de', 'la', 'que', 'el', 'en', 'y', 'a', 'los', 'se', 'del',
+                         'las', 'un', 'por', 'con', 'no', 'una', 'su', 'para', 'es',
+                         'al'}
+            },
+            'FRENCH': {
+                'chars': 'esaitnrulodcpmévqfbghàjxèêyçwzùâîôûëïüœ',
+                'ngrams': {'es', 'le', 'en', 'de', 'nt', 'on', 're', 'er', 'ai', 'te',
+                          'la', 'an', 'ou', 'it', 'ur', 'et', 'el', 'se', 'qu', 'me',
+                          'is', 'ar', 'ce', 'ns', 'us', 'ue', 'ss', 'ie', 'em', 'tr'},
+                'words': {'le', 'de', 'un', 'être', 'et', 'à', 'il', 'avoir', 'ne',
+                         'je', 'son', 'que', 'se', 'qui', 'ce', 'dans', 'en', 'du',
+                         'elle', 'au'}
+            },
+            'GERMAN': {
+                'chars': 'enisratdhulcgmobwfkzvüpäößjyqxéèêëàáâãåāăąćčĉċďđ',
+                'ngrams': {'en', 'er', 'ch', 'de', 'ei', 'in', 'te', 'nd', 'ie', 'ge',
+                          'st', 'ne', 'be', 'es', 'un', 'zu', 'an', 'ng', 'au', 'it',
+                          'is', 'he', 'ht', 'se', 'ck', 'ic', 're', 'ns', 'sc', 'tz'},
+                'words': {'der', 'die', 'und', 'in', 'den', 'von', 'zu', 'das', 'mit',
+                         'sich', 'des', 'auf', 'für', 'ist', 'im', 'dem', 'nicht',
+                         'ein', 'eine', 'als'}
+            }
+        }
+        
+        # Compile word patterns
+        self.word_pattern = re.compile(r'\b\w+\b')
+    
+    def _extract_ngrams(self, text: str, n: int = 2) -> List[str]:
+        """Extract character n-grams from text."""
+        text = text.lower()
+        return [text[i:i+n] for i in range(len(text)-n+1)]
+    
+    def _calculate_char_frequencies(self, text: str) -> Dict[str, float]:
+        """Calculate character frequencies in text."""
+        text = text.lower()
+        char_count = Counter(c for c in text if c.isalpha())
+        total = sum(char_count.values()) or 1
+        return {char: count/total for char, count in char_count.items()}
+    
+    def _calculate_ngram_frequencies(self, text: str) -> Dict[str, float]:
+        """Calculate n-gram frequencies in text."""
+        ngrams = self._extract_ngrams(text)
+        ngram_count = Counter(ngrams)
+        total = sum(ngram_count.values()) or 1
+        return {ngram: count/total for ngram, count in ngram_count.items()}
+    
+    def _calculate_word_frequencies(self, text: str) -> Dict[str, float]:
+        """Calculate word frequencies in text."""
+        words = self.word_pattern.findall(text.lower())
+        word_count = Counter(words)
+        total = sum(word_count.values()) or 1
+        return {word: count/total for word, count in word_count.items()}
+    
+    def _calculate_similarity(self, freq1: Dict[str, float], freq2: Dict[str, float]) -> float:
+        """Calculate similarity between two frequency distributions."""
+        common_keys = set(freq1.keys()) & set(freq2.keys())
+        if not common_keys:
+            return 0.0
+        
+        similarity = sum(min(freq1.get(k, 0), freq2.get(k, 0)) for k in common_keys)
+        return similarity
+    
+    def detect(self, text: str) -> List[Tuple[str, float]]:
+        """
+        Detect the language of text with confidence scores.
+        
+        Returns:
+            List of (language, confidence) tuples, sorted by confidence
+        """
+        if not text:
+            return []
+        
+        # Calculate frequencies for input text
+        char_freqs = self._calculate_char_frequencies(text)
+        ngram_freqs = self._calculate_ngram_frequencies(text)
+        word_freqs = self._calculate_word_frequencies(text)
+        
+        # Calculate similarity scores for each language
+        scores = []
+        for lang, profile in self.language_profiles.items():
+            # Character similarity
+            char_sim = sum(char_freqs.get(c, 0) for c in profile['chars'])
+            
+            # N-gram similarity
+            ngram_sim = sum(ngram_freqs.get(ng, 0) for ng in profile['ngrams'])
+            
+            # Word similarity
+            word_sim = sum(word_freqs.get(w, 0) for w in profile['words'])
+            
+            # Combined score (weighted average)
+            total_score = (0.3 * char_sim + 0.4 * ngram_sim + 0.3 * word_sim)
+            scores.append((lang, total_score))
+        
+        # Normalize scores
+        total = sum(score for _, score in scores) or 1
+        normalized_scores = [(lang, score/total) for lang, score in scores]
+        
+        # Sort by confidence
+        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)

webstoken/ner.py

@@ -1,164 +1,164 @@
-"""
-Named Entity Recognition (NER) module for identifying and classifying named entities.
-"""
-
-from typing import List, Tuple, Dict, Set
-import re
-
-
-class NamedEntityRecognizer:
-    """Rule-based Named Entity Recognition."""
-    
-    def __init__(self):
-        # Common entity patterns
-        self.PERSON_TITLES = {
-            'mr', 'mrs', 'ms', 'miss', 'dr', 'prof', 'sir', 'madam',
-            'lord', 'lady', 'president', 'ceo', 'director'
-        }
-        
-        self.ORGANIZATION_SUFFIXES = {
-            'inc', 'corp', 'ltd', 'llc', 'company', 'corporation',
-            'associates', 'partners', 'foundation', 'institute'
-        }
-        
-        self.LOCATION_INDICATORS = {
-            'street', 'road', 'avenue', 'boulevard', 'lane', 'drive',
-            'circle', 'square', 'park', 'bridge', 'river', 'lake',
-            'mountain', 'forest', 'city', 'town', 'village', 'country'
-        }
-        
-        self.DATE_MONTHS = {
-            'january', 'february', 'march', 'april', 'may', 'june',
-            'july', 'august', 'september', 'october', 'november', 'december'
-        }
-        
-        # Compile regex patterns
-        self.patterns = {
-            'EMAIL': re.compile(r'\b[\w\.-]+@[\w\.-]+\.\w+\b'),
-            'URL': re.compile(r'https?://(?:[\w-]|(?:%[\da-fA-F]{2}))+'),
-            'PHONE': re.compile(r'\+?\d{1,3}[-.\s]?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}'),
-            'DATE': re.compile(r'\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b'),
-            'TIME': re.compile(r'\b\d{1,2}:\d{2}(?::\d{2})?(?:\s*[AaPp][Mm])?\b'),
-            'MONEY': re.compile(r'\$\d+(?:,\d{3})*(?:\.\d{2})?|\d+(?:,\d{3})*(?:\.\d{2})?\s*(?:dollars|USD|EUR|GBP)'),
-            'PERCENTAGE': re.compile(r'\b\d+(?:\.\d+)?%\b')
-        }
-    
-    def is_capitalized(self, word: str) -> bool:
-        """Check if a word is capitalized."""
-        return word and word[0].isupper()
-    
-    def extract_entities(self, text: str) -> Dict[str, List[Tuple[str, str]]]:
-        """
-        Extract named entities from text.
-        
-        Returns:
-            Dict mapping entity types to list of (text, label) tuples
-        """
-        entities = {
-            'PERSON': [],
-            'ORGANIZATION': [],
-            'LOCATION': [],
-            'DATE': [],
-            'TIME': [],
-            'MONEY': [],
-            'EMAIL': [],
-            'URL': [],
-            'PHONE': [],
-            'PERCENTAGE': []
-        }
-        
-        # First find regex pattern matches
-        for label, pattern in self.patterns.items():
-            for match in pattern.finditer(text):
-                entities[label].append((match.group(), label))
-        
-        # Process text word by word for other entities
-        words = text.split()
-        i = 0
-        while i < len(words):
-            word = words[i]
-            next_word = words[i + 1] if i + 1 < len(words) else None
-            
-            # Check for person names
-            if word.lower() in self.PERSON_TITLES and next_word and self.is_capitalized(next_word):
-                name_parts = []
-                j = i + 1
-                while j < len(words) and self.is_capitalized(words[j]):
-                    name_parts.append(words[j])
-                    j += 1
-                if name_parts:
-                    entities['PERSON'].append((' '.join(name_parts), 'PERSON'))
-                i = j
-                continue
-            
-            # Check for organizations
-            if self.is_capitalized(word):
-                org_parts = [word]
-                j = i + 1
-                while j < len(words) and (
-                    self.is_capitalized(words[j]) or 
-                    words[j].lower() in self.ORGANIZATION_SUFFIXES
-                ):
-                    org_parts.append(words[j])
-                    j += 1
-                if len(org_parts) > 1 or (
-                    len(org_parts) == 1 and 
-                    any(suff in word.lower() for suff in self.ORGANIZATION_SUFFIXES)
-                ):
-                    entities['ORGANIZATION'].append((' '.join(org_parts), 'ORGANIZATION'))
-                i = j
-                continue
-            
-            # Check for locations
-            if word.lower() in self.LOCATION_INDICATORS and i > 0:
-                if self.is_capitalized(words[i - 1]):
-                    entities['LOCATION'].append((words[i - 1] + ' ' + word, 'LOCATION'))
-            
-            i += 1
-        
-        return entities
-        
-    def tag_text(self, text: str) -> List[Tuple[str, str]]:
-        """
-        Tag each word in text with its entity type.
-        
-        Returns:
-            List of (word, entity_type) tuples
-        """
-        entities = self.extract_entities(text)
-        tagged = []
-        
-        # Create a map of word positions to entity labels
-        position_labels = {}
-        text_lower = text.lower()
-        
-        for entity_type, entity_list in entities.items():
-            for entity_text, _ in entity_list:
-                start = text_lower.find(entity_text.lower())
-                if start != -1:
-                    end = start + len(entity_text)
-                    for pos in range(start, end):
-                        position_labels[pos] = entity_type
-        
-        # Tag each character position
-        current_pos = 0
-        current_word = []
-        current_label = 'O'  # Outside any entity
-        
-        for char in text:
-            if char.isspace():
-                if current_word:
-                    tagged.append((''.join(current_word), current_label))
-                    current_word = []
-                    current_label = 'O'
-            else:
-                current_word.append(char)
-                if current_pos in position_labels:
-                    current_label = position_labels[current_pos]
-            current_pos += 1
-        
-        # Add last word if exists
-        if current_word:
-            tagged.append((''.join(current_word), current_label))
-        
-        return tagged
+"""
+Named Entity Recognition (NER) module for identifying and classifying named entities.
+"""
+
+from typing import List, Tuple, Dict, Set
+import re
+
+
+class NamedEntityRecognizer:
+    """Rule-based Named Entity Recognition."""
+    
+    def __init__(self):
+        # Common entity patterns
+        self.PERSON_TITLES = {
+            'mr', 'mrs', 'ms', 'miss', 'dr', 'prof', 'sir', 'madam',
+            'lord', 'lady', 'president', 'ceo', 'director'
+        }
+        
+        self.ORGANIZATION_SUFFIXES = {
+            'inc', 'corp', 'ltd', 'llc', 'company', 'corporation',
+            'associates', 'partners', 'foundation', 'institute'
+        }
+        
+        self.LOCATION_INDICATORS = {
+            'street', 'road', 'avenue', 'boulevard', 'lane', 'drive',
+            'circle', 'square', 'park', 'bridge', 'river', 'lake',
+            'mountain', 'forest', 'city', 'town', 'village', 'country'
+        }
+        
+        self.DATE_MONTHS = {
+            'january', 'february', 'march', 'april', 'may', 'june',
+            'july', 'august', 'september', 'october', 'november', 'december'
+        }
+        
+        # Compile regex patterns
+        self.patterns = {
+            'EMAIL': re.compile(r'\b[\w\.-]+@[\w\.-]+\.\w+\b'),
+            'URL': re.compile(r'https?://(?:[\w-]|(?:%[\da-fA-F]{2}))+'),
+            'PHONE': re.compile(r'\+?\d{1,3}[-.\s]?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}'),
+            'DATE': re.compile(r'\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b'),
+            'TIME': re.compile(r'\b\d{1,2}:\d{2}(?::\d{2})?(?:\s*[AaPp][Mm])?\b'),
+            'MONEY': re.compile(r'\$\d+(?:,\d{3})*(?:\.\d{2})?|\d+(?:,\d{3})*(?:\.\d{2})?\s*(?:dollars|USD|EUR|GBP)'),
+            'PERCENTAGE': re.compile(r'\b\d+(?:\.\d+)?%\b')
+        }
+    
+    def is_capitalized(self, word: str) -> bool:
+        """Check if a word is capitalized."""
+        return word and word[0].isupper()
+    
+    def extract_entities(self, text: str) -> Dict[str, List[Tuple[str, str]]]:
+        """
+        Extract named entities from text.
+        
+        Returns:
+            Dict mapping entity types to list of (text, label) tuples
+        """
+        entities = {
+            'PERSON': [],
+            'ORGANIZATION': [],
+            'LOCATION': [],
+            'DATE': [],
+            'TIME': [],
+            'MONEY': [],
+            'EMAIL': [],
+            'URL': [],
+            'PHONE': [],
+            'PERCENTAGE': []
+        }
+        
+        # First find regex pattern matches
+        for label, pattern in self.patterns.items():
+            for match in pattern.finditer(text):
+                entities[label].append((match.group(), label))
+        
+        # Process text word by word for other entities
+        words = text.split()
+        i = 0
+        while i < len(words):
+            word = words[i]
+            next_word = words[i + 1] if i + 1 < len(words) else None
+            
+            # Check for person names
+            if word.lower() in self.PERSON_TITLES and next_word and self.is_capitalized(next_word):
+                name_parts = []
+                j = i + 1
+                while j < len(words) and self.is_capitalized(words[j]):
+                    name_parts.append(words[j])
+                    j += 1
+                if name_parts:
+                    entities['PERSON'].append((' '.join(name_parts), 'PERSON'))
+                i = j
+                continue
+            
+            # Check for organizations
+            if self.is_capitalized(word):
+                org_parts = [word]
+                j = i + 1
+                while j < len(words) and (
+                    self.is_capitalized(words[j]) or 
+                    words[j].lower() in self.ORGANIZATION_SUFFIXES
+                ):
+                    org_parts.append(words[j])
+                    j += 1
+                if len(org_parts) > 1 or (
+                    len(org_parts) == 1 and 
+                    any(suff in word.lower() for suff in self.ORGANIZATION_SUFFIXES)
+                ):
+                    entities['ORGANIZATION'].append((' '.join(org_parts), 'ORGANIZATION'))
+                i = j
+                continue
+            
+            # Check for locations
+            if word.lower() in self.LOCATION_INDICATORS and i > 0:
+                if self.is_capitalized(words[i - 1]):
+                    entities['LOCATION'].append((words[i - 1] + ' ' + word, 'LOCATION'))
+            
+            i += 1
+        
+        return entities
+        
+    def tag_text(self, text: str) -> List[Tuple[str, str]]:
+        """
+        Tag each word in text with its entity type.
+        
+        Returns:
+            List of (word, entity_type) tuples
+        """
+        entities = self.extract_entities(text)
+        tagged = []
+        
+        # Create a map of word positions to entity labels
+        position_labels = {}
+        text_lower = text.lower()
+        
+        for entity_type, entity_list in entities.items():
+            for entity_text, _ in entity_list:
+                start = text_lower.find(entity_text.lower())
+                if start != -1:
+                    end = start + len(entity_text)
+                    for pos in range(start, end):
+                        position_labels[pos] = entity_type
+        
+        # Tag each character position
+        current_pos = 0
+        current_word = []
+        current_label = 'O'  # Outside any entity
+        
+        for char in text:
+            if char.isspace():
+                if current_word:
+                    tagged.append((''.join(current_word), current_label))
+                    current_word = []
+                    current_label = 'O'
+            else:
+                current_word.append(char)
+                if current_pos in position_labels:
+                    current_label = position_labels[current_pos]
+            current_pos += 1
+        
+        # Add last word if exists
+        if current_word:
+            tagged.append((''.join(current_word), current_label))
+        
+        return tagged