webscout 8.2.6__py3-none-any.whl → 8.2.7__py3-none-any.whl

webstoken/keywords.py

@@ -1,216 +0,0 @@
-"""
-Keyword extraction module using statistical and graph-based approaches.
-"""
-
-from typing import Dict, List, Set, Tuple
-from collections import Counter, defaultdict
-import math
-import re
-
-from .tokenizer import WordTokenizer
-from .normalizer import TextNormalizer
-
-
-class KeywordExtractor:
-    """Keyword extraction using TF-IDF and TextRank-inspired algorithms."""
-    
-    def __init__(self):
-        self.word_tokenizer = WordTokenizer()
-        self.normalizer = TextNormalizer()
-        
-        # Common words to filter out beyond basic stop words
-        self.filter_words: Set[str] = {
-            'would', 'could', 'should', 'said', 'also', 'may', 'might',
-            'must', 'need', 'shall', 'want', 'way', 'time', 'just',
-            'now', 'like', 'make', 'made', 'well', 'back', 'even',
-            'still', 'way', 'take', 'took', 'get', 'got', 'go', 'went'
-        }
-    
-    def _split_into_sentences(self, text: str) -> List[str]:
-        """Split text into sentences using simple rules."""
-        text = re.sub(r'\s+', ' ', text)
-        sentences = re.split(r'[.!?]+', text)
-        return [s.strip() for s in sentences if s.strip()]
-    
-    def _calculate_word_scores(self, text: str) -> Dict[str, float]:
-        """Calculate word importance scores using frequency and position."""
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        sentences = self._split_into_sentences(text)
-        
-        word_scores: Dict[str, float] = defaultdict(float)
-        word_positions: Dict[str, List[int]] = defaultdict(list)
-        
-        # Calculate word frequencies and positions
-        for i, sentence in enumerate(sentences):
-            words = self.word_tokenizer.tokenize(sentence)
-            for j, word in enumerate(words):
-                word = word.lower()
-                if (word.isalnum() and 
-                    len(word) > 2 and
-                    word not in self.filter_words and
-                    word not in self.normalizer.stop_words):
-                    word_scores[word] += 1
-                    word_positions[word].append(i)
-        
-        # Adjust scores based on position
-        num_sentences = len(sentences)
-        for word, positions in word_positions.items():
-            # Words appearing in first or last sentences get bonus
-            if 0 in positions:
-                word_scores[word] *= 1.2
-            if num_sentences - 1 in positions:
-                word_scores[word] *= 1.1
-            
-            # Words appearing throughout text get bonus
-            coverage = len(set(positions)) / num_sentences
-            word_scores[word] *= (1 + coverage)
-        
-        return word_scores
-    
-    def _calculate_word_cooccurrence(self, text: str, window_size: int = 3) -> Dict[str, Dict[str, int]]:
-        """Calculate word co-occurrence matrix."""
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        words = self.word_tokenizer.tokenize(text)
-        
-        # Filter words
-        filtered_words = [
-            word.lower() for word in words
-            if (word.isalnum() and
-                len(word) > 2 and
-                word.lower() not in self.filter_words and
-                word.lower() not in self.normalizer.stop_words)
-        ]
-        
-        # Build co-occurrence matrix
-        cooccurrence: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
-        
-        for i, word in enumerate(filtered_words):
-            for j in range(max(0, i - window_size), min(len(filtered_words), i + window_size + 1)):
-                if i != j:
-                    cooccurrence[word][filtered_words[j]] += 1
-                    cooccurrence[filtered_words[j]][word] += 1
-        
-        return cooccurrence
-    
-    def _textrank_scores(self, cooccurrence: Dict[str, Dict[str, int]], damping: float = 0.85,
-                        iterations: int = 30) -> Dict[str, float]:
-        """Calculate TextRank scores from co-occurrence matrix."""
-        scores = {word: 1.0 for word in cooccurrence}
-        
-        for _ in range(iterations):
-            new_scores = {}
-            for word in scores:
-                if not cooccurrence[word]:
-                    continue
-                    
-                incoming_score = sum(
-                    scores[other] * cooccurrence[word][other] / sum(cooccurrence[other].values())
-                    for other in cooccurrence[word]
-                )
-                new_scores[word] = (1 - damping) + damping * incoming_score
-            
-            # Check convergence
-            score_diff = sum(abs(new_scores[w] - scores[w]) for w in scores)
-            scores = new_scores
-            if score_diff < 0.0001:
-                break
-        
-        return scores
-    
-    def extract_keywords(self, text: str, num_keywords: int = 10,
-                        use_textrank: bool = True) -> List[Tuple[str, float]]:
-        """
-        Extract keywords from text using combined frequency and graph-based approach.
-        
-        Args:
-            text: Input text
-            num_keywords: Number of keywords to return
-            use_textrank: Whether to use TextRank algorithm
-            
-        Returns:
-            List of (keyword, score) tuples, sorted by score
-        """
-        if not text:
-            return []
-        
-        # Get frequency-based scores
-        freq_scores = self._calculate_word_scores(text)
-        
-        if use_textrank:
-            # Get TextRank scores
-            cooccurrence = self._calculate_word_cooccurrence(text)
-            textrank_scores = self._textrank_scores(cooccurrence)
-            
-            # Combine scores
-            combined_scores = {
-                word: freq_scores[word] * textrank_scores.get(word, 0)
-                for word in freq_scores
-            }
-        else:
-            combined_scores = freq_scores
-        
-        # Sort and return top keywords
-        sorted_words = sorted(
-            combined_scores.items(),
-            key=lambda x: x[1],
-            reverse=True
-        )
-        
-        return sorted_words[:num_keywords]
-    
-    def extract_keyphrases(self, text: str, num_phrases: int = 5,
-                          min_words: int = 2, max_words: int = 4) -> List[Tuple[str, float]]:
-        """
-        Extract key phrases from text.
-        
-        Args:
-            text: Input text
-            num_phrases: Number of phrases to return
-            min_words: Minimum words in phrase
-            max_words: Maximum words in phrase
-            
-        Returns:
-            List of (phrase, score) tuples, sorted by score
-        """
-        # Normalize and split into sentences
-        text = self.normalizer.normalize(text)
-        sentences = self._split_into_sentences(text)
-        
-        # Get word importance scores
-        word_scores = self._calculate_word_scores(text)
-        
-        # Extract candidate phrases
-        phrases: Dict[str, float] = {}
-        
-        for sentence in sentences:
-            words = self.word_tokenizer.tokenize(sentence)
-            
-            # Generate phrases of different lengths
-            for i in range(len(words)):
-                for length in range(min_words, min(max_words + 1, len(words) - i + 1)):
-                    phrase_words = words[i:i+length]
-                    
-                    # Filter phrases
-                    if all(
-                        word.isalnum() and
-                        len(word) > 2 and
-                        word.lower() not in self.filter_words and
-                        word.lower() not in self.normalizer.stop_words
-                        for word in phrase_words
-                    ):
-                        phrase = ' '.join(phrase_words)
-                        # Score is average of word scores
-                        score = sum(word_scores.get(word.lower(), 0) for word in phrase_words)
-                        score /= len(phrase_words)
-                        phrases[phrase] = score
-        
-        # Sort and return top phrases
-        sorted_phrases = sorted(
-            phrases.items(),
-            key=lambda x: x[1],
-            reverse=True
-        )
-        
-        return sorted_phrases[:num_phrases]

webstoken/language.py

@@ -1,128 +0,0 @@
-"""
-Language detection module using character and word frequency analysis.
-"""
-
-from typing import Dict, List, Set, Tuple
-from collections import Counter
-import re
-
-
-class LanguageDetector:
-    """Language detection using character n-gram frequencies."""
-    
-    def __init__(self):
-        # Language profiles based on common character sequences
-        self.language_profiles = {
-            'ENGLISH': {
-                'chars': 'etaoinshrdlcumwfgypbvkjxqz',
-                'ngrams': {'th', 'he', 'in', 'er', 'an', 're', 'on', 'at', 'en', 'nd',
-                          'ti', 'es', 'or', 'te', 'of', 'ed', 'is', 'it', 'al', 'ar',
-                          'st', 'to', 'nt', 'ng', 'se', 'ha', 'as', 'ou', 'io', 'le'},
-                'words': {'the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have',
-                         'i', 'it', 'for', 'not', 'on', 'with', 'he', 'as', 'you',
-                         'do', 'at'}
-            },
-            'SPANISH': {
-                'chars': 'eaosrnidlctumpbgvyqhfzjñxwk',
-                'ngrams': {'de', 'en', 'el', 'la', 'os', 'es', 'as', 'ar', 'er', 'ra',
-                          'al', 'an', 'do', 'or', 'ta', 'ue', 'io', 'on', 'ro', 'ad',
-                          'te', 'co', 'st', 'ci', 'nt', 'to', 'lo', 'no', 'po', 'ac'},
-                'words': {'de', 'la', 'que', 'el', 'en', 'y', 'a', 'los', 'se', 'del',
-                         'las', 'un', 'por', 'con', 'no', 'una', 'su', 'para', 'es',
-                         'al'}
-            },
-            'FRENCH': {
-                'chars': 'esaitnrulodcpmévqfbghàjxèêyçwzùâîôûëïüœ',
-                'ngrams': {'es', 'le', 'en', 'de', 'nt', 'on', 're', 'er', 'ai', 'te',
-                          'la', 'an', 'ou', 'it', 'ur', 'et', 'el', 'se', 'qu', 'me',
-                          'is', 'ar', 'ce', 'ns', 'us', 'ue', 'ss', 'ie', 'em', 'tr'},
-                'words': {'le', 'de', 'un', 'être', 'et', 'à', 'il', 'avoir', 'ne',
-                         'je', 'son', 'que', 'se', 'qui', 'ce', 'dans', 'en', 'du',
-                         'elle', 'au'}
-            },
-            'GERMAN': {
-                'chars': 'enisratdhulcgmobwfkzvüpäößjyqxéèêëàáâãåāăąćčĉċďđ',
-                'ngrams': {'en', 'er', 'ch', 'de', 'ei', 'in', 'te', 'nd', 'ie', 'ge',
-                          'st', 'ne', 'be', 'es', 'un', 'zu', 'an', 'ng', 'au', 'it',
-                          'is', 'he', 'ht', 'se', 'ck', 'ic', 're', 'ns', 'sc', 'tz'},
-                'words': {'der', 'die', 'und', 'in', 'den', 'von', 'zu', 'das', 'mit',
-                         'sich', 'des', 'auf', 'für', 'ist', 'im', 'dem', 'nicht',
-                         'ein', 'eine', 'als'}
-            }
-        }
-        
-        # Compile word patterns
-        self.word_pattern = re.compile(r'\b\w+\b')
-    
-    def _extract_ngrams(self, text: str, n: int = 2) -> List[str]:
-        """Extract character n-grams from text."""
-        text = text.lower()
-        return [text[i:i+n] for i in range(len(text)-n+1)]
-    
-    def _calculate_char_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate character frequencies in text."""
-        text = text.lower()
-        char_count = Counter(c for c in text if c.isalpha())
-        total = sum(char_count.values()) or 1
-        return {char: count/total for char, count in char_count.items()}
-    
-    def _calculate_ngram_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate n-gram frequencies in text."""
-        ngrams = self._extract_ngrams(text)
-        ngram_count = Counter(ngrams)
-        total = sum(ngram_count.values()) or 1
-        return {ngram: count/total for ngram, count in ngram_count.items()}
-    
-    def _calculate_word_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate word frequencies in text."""
-        words = self.word_pattern.findall(text.lower())
-        word_count = Counter(words)
-        total = sum(word_count.values()) or 1
-        return {word: count/total for word, count in word_count.items()}
-    
-    def _calculate_similarity(self, freq1: Dict[str, float], freq2: Dict[str, float]) -> float:
-        """Calculate similarity between two frequency distributions."""
-        common_keys = set(freq1.keys()) & set(freq2.keys())
-        if not common_keys:
-            return 0.0
-        
-        similarity = sum(min(freq1.get(k, 0), freq2.get(k, 0)) for k in common_keys)
-        return similarity
-    
-    def detect(self, text: str) -> List[Tuple[str, float]]:
-        """
-        Detect the language of text with confidence scores.
-        
-        Returns:
-            List of (language, confidence) tuples, sorted by confidence
-        """
-        if not text:
-            return []
-        
-        # Calculate frequencies for input text
-        char_freqs = self._calculate_char_frequencies(text)
-        ngram_freqs = self._calculate_ngram_frequencies(text)
-        word_freqs = self._calculate_word_frequencies(text)
-        
-        # Calculate similarity scores for each language
-        scores = []
-        for lang, profile in self.language_profiles.items():
-            # Character similarity
-            char_sim = sum(char_freqs.get(c, 0) for c in profile['chars'])
-            
-            # N-gram similarity
-            ngram_sim = sum(ngram_freqs.get(ng, 0) for ng in profile['ngrams'])
-            
-            # Word similarity
-            word_sim = sum(word_freqs.get(w, 0) for w in profile['words'])
-            
-            # Combined score (weighted average)
-            total_score = (0.3 * char_sim + 0.4 * ngram_sim + 0.3 * word_sim)
-            scores.append((lang, total_score))
-        
-        # Normalize scores
-        total = sum(score for _, score in scores) or 1
-        normalized_scores = [(lang, score/total) for lang, score in scores]
-        
-        # Sort by confidence
-        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)

webstoken/ner.py

@@ -1,164 +0,0 @@
-"""
-Named Entity Recognition (NER) module for identifying and classifying named entities.
-"""
-
-from typing import List, Tuple, Dict, Set
-import re
-
-
-class NamedEntityRecognizer:
-    """Rule-based Named Entity Recognition."""
-    
-    def __init__(self):
-        # Common entity patterns
-        self.PERSON_TITLES = {
-            'mr', 'mrs', 'ms', 'miss', 'dr', 'prof', 'sir', 'madam',
-            'lord', 'lady', 'president', 'ceo', 'director'
-        }
-        
-        self.ORGANIZATION_SUFFIXES = {
-            'inc', 'corp', 'ltd', 'llc', 'company', 'corporation',
-            'associates', 'partners', 'foundation', 'institute'
-        }
-        
-        self.LOCATION_INDICATORS = {
-            'street', 'road', 'avenue', 'boulevard', 'lane', 'drive',
-            'circle', 'square', 'park', 'bridge', 'river', 'lake',
-            'mountain', 'forest', 'city', 'town', 'village', 'country'
-        }
-        
-        self.DATE_MONTHS = {
-            'january', 'february', 'march', 'april', 'may', 'june',
-            'july', 'august', 'september', 'october', 'november', 'december'
-        }
-        
-        # Compile regex patterns
-        self.patterns = {
-            'EMAIL': re.compile(r'\b[\w\.-]+@[\w\.-]+\.\w+\b'),
-            'URL': re.compile(r'https?://(?:[\w-]|(?:%[\da-fA-F]{2}))+'),
-            'PHONE': re.compile(r'\+?\d{1,3}[-.\s]?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}'),
-            'DATE': re.compile(r'\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b'),
-            'TIME': re.compile(r'\b\d{1,2}:\d{2}(?::\d{2})?(?:\s*[AaPp][Mm])?\b'),
-            'MONEY': re.compile(r'\$\d+(?:,\d{3})*(?:\.\d{2})?|\d+(?:,\d{3})*(?:\.\d{2})?\s*(?:dollars|USD|EUR|GBP)'),
-            'PERCENTAGE': re.compile(r'\b\d+(?:\.\d+)?%\b')
-        }
-    
-    def is_capitalized(self, word: str) -> bool:
-        """Check if a word is capitalized."""
-        return word and word[0].isupper()
-    
-    def extract_entities(self, text: str) -> Dict[str, List[Tuple[str, str]]]:
-        """
-        Extract named entities from text.
-        
-        Returns:
-            Dict mapping entity types to list of (text, label) tuples
-        """
-        entities = {
-            'PERSON': [],
-            'ORGANIZATION': [],
-            'LOCATION': [],
-            'DATE': [],
-            'TIME': [],
-            'MONEY': [],
-            'EMAIL': [],
-            'URL': [],
-            'PHONE': [],
-            'PERCENTAGE': []
-        }
-        
-        # First find regex pattern matches
-        for label, pattern in self.patterns.items():
-            for match in pattern.finditer(text):
-                entities[label].append((match.group(), label))
-        
-        # Process text word by word for other entities
-        words = text.split()
-        i = 0
-        while i < len(words):
-            word = words[i]
-            next_word = words[i + 1] if i + 1 < len(words) else None
-            
-            # Check for person names
-            if word.lower() in self.PERSON_TITLES and next_word and self.is_capitalized(next_word):
-                name_parts = []
-                j = i + 1
-                while j < len(words) and self.is_capitalized(words[j]):
-                    name_parts.append(words[j])
-                    j += 1
-                if name_parts:
-                    entities['PERSON'].append((' '.join(name_parts), 'PERSON'))
-                i = j
-                continue
-            
-            # Check for organizations
-            if self.is_capitalized(word):
-                org_parts = [word]
-                j = i + 1
-                while j < len(words) and (
-                    self.is_capitalized(words[j]) or 
-                    words[j].lower() in self.ORGANIZATION_SUFFIXES
-                ):
-                    org_parts.append(words[j])
-                    j += 1
-                if len(org_parts) > 1 or (
-                    len(org_parts) == 1 and 
-                    any(suff in word.lower() for suff in self.ORGANIZATION_SUFFIXES)
-                ):
-                    entities['ORGANIZATION'].append((' '.join(org_parts), 'ORGANIZATION'))
-                i = j
-                continue
-            
-            # Check for locations
-            if word.lower() in self.LOCATION_INDICATORS and i > 0:
-                if self.is_capitalized(words[i - 1]):
-                    entities['LOCATION'].append((words[i - 1] + ' ' + word, 'LOCATION'))
-            
-            i += 1
-        
-        return entities
-        
-    def tag_text(self, text: str) -> List[Tuple[str, str]]:
-        """
-        Tag each word in text with its entity type.
-        
-        Returns:
-            List of (word, entity_type) tuples
-        """
-        entities = self.extract_entities(text)
-        tagged = []
-        
-        # Create a map of word positions to entity labels
-        position_labels = {}
-        text_lower = text.lower()
-        
-        for entity_type, entity_list in entities.items():
-            for entity_text, _ in entity_list:
-                start = text_lower.find(entity_text.lower())
-                if start != -1:
-                    end = start + len(entity_text)
-                    for pos in range(start, end):
-                        position_labels[pos] = entity_type
-        
-        # Tag each character position
-        current_pos = 0
-        current_word = []
-        current_label = 'O'  # Outside any entity
-        
-        for char in text:
-            if char.isspace():
-                if current_word:
-                    tagged.append((''.join(current_word), current_label))
-                    current_word = []
-                    current_label = 'O'
-            else:
-                current_word.append(char)
-                if current_pos in position_labels:
-                    current_label = position_labels[current_pos]
-            current_pos += 1
-        
-        # Add last word if exists
-        if current_word:
-            tagged.append((''.join(current_word), current_label))
-        
-        return tagged

webstoken/normalizer.py

@@ -1,35 +0,0 @@
-"""
-Text normalization utilities.
-"""
-
-import re
-from typing import List, Set
-
-
-class TextNormalizer:
-    """Text normalization utilities."""
-    
-    def __init__(self):
-        self.stop_words: Set[str] = {
-            'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
-            'has', 'he', 'in', 'is', 'it', 'its', 'of', 'on', 'that', 'the',
-            'to', 'was', 'were', 'will', 'with'
-        }
-        
-    def remove_stop_words(self, tokens: List[str]) -> List[str]:
-        """Remove common stop words from token list."""
-        return [token for token in tokens if token.lower() not in self.stop_words]
-    
-    def normalize(self, text: str) -> str:
-        """Apply various normalization steps to text."""
-        # Convert to lowercase
-        text = text.lower()
-        
-        # Replace multiple spaces with single space
-        text = re.sub(r'\s+', ' ', text)
-        
-        # Remove special characters except apostrophes within words
-        text = re.sub(r'[^a-z0-9\s\']', '', text)
-        text = re.sub(r'\s\'|\'\s', ' ', text)
-        
-        return text.strip()

webstoken/processor.py

@@ -1,77 +0,0 @@
-"""
-Main text processing utilities combining all NLP components.
-"""
-
-from typing import Dict, Any, List, Tuple
-
-from .tokenizer import SentenceTokenizer, WordTokenizer
-from .tagger import POSTagger
-from .stemmer import Stemmer
-from .normalizer import TextNormalizer
-
-
-def process_text(text: str, normalize: bool = True, remove_stops: bool = True) -> Dict[str, Any]:
-    """
-    Process text using all available NLP tools.
-    
-    Args:
-        text (str): Input text to process
-        normalize (bool): Whether to normalize text
-        remove_stops (bool): Whether to remove stop words
-        
-    Returns:
-        Dict containing processed results with the following structure:
-        {
-            'sentences': [
-                {
-                    'original': str,     # Original sentence
-                    'tokens': List[str],  # Word tokens
-                    'pos_tags': List[Tuple[str, str]],  # (word, tag) pairs
-                    'stems': List[Tuple[str, str]]      # (word, stem) pairs
-                },
-                ...
-            ],
-            'num_sentences': int,  # Total number of sentences
-            'num_tokens': int      # Total number of tokens
-        }
-    """
-    # Initialize tools
-    sentence_tokenizer = SentenceTokenizer()
-    word_tokenizer = WordTokenizer()
-    pos_tagger = POSTagger()
-    stemmer = Stemmer()
-    normalizer = TextNormalizer()
-    
-    # Process text
-    if normalize:
-        text = normalizer.normalize(text)
-    
-    # Get sentences
-    sentences = sentence_tokenizer.tokenize(text)
-    
-    # Process each sentence
-    processed_sentences = []
-    for sentence in sentences:
-        # Tokenize words
-        tokens = word_tokenizer.tokenize(sentence)
-        
-        # Remove stop words if requested
-        if remove_stops:
-            tokens = normalizer.remove_stop_words(tokens)
-        
-        # Get POS tags and stems
-        tagged = pos_tagger.tag(tokens)
-        stems = [(token, stemmer.stem(token)) for token, _ in tagged]
-        
-        processed_sentences.append({
-            'original': sentence,
-            'tokens': tokens,
-            'pos_tags': tagged,
-            'stems': stems
-        })
-    
-    return {
-        'sentences': processed_sentences,
-        'num_sentences': len(sentences),
-        'num_tokens': sum(len(s['tokens']) for s in processed_sentences)
-    }