webscout 8.2.6__py3-none-any.whl → 8.2.8__py3-none-any.whl

webscout-8.2.6.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-WEBS = webscout.cli:cli
-webscout = webscout.cli:cli

webscout-8.2.6.dist-info/top_level.txt

@@ -1,2 +0,0 @@
-webscout
-webstoken

webstoken/__init__.py

@@ -1,30 +0,0 @@
-"""
-Webstoken - A pure Python NLP toolkit for text processing
-"""
-
-from .tokenizer import SentenceTokenizer, WordTokenizer
-from .tagger import POSTagger
-from .stemmer import Stemmer
-from .normalizer import TextNormalizer
-from .processor import process_text
-from .ner import NamedEntityRecognizer
-from .classifier import TextClassifier, TopicClassifier
-from .language import LanguageDetector
-from .sentiment import SentimentAnalyzer
-from .keywords import KeywordExtractor
-
-__version__ = '0.1.0'
-__all__ = [
-    'SentenceTokenizer',
-    'WordTokenizer',
-    'POSTagger',
-    'Stemmer',
-    'TextNormalizer',
-    'process_text',
-    'NamedEntityRecognizer',
-    'TextClassifier',
-    'TopicClassifier',
-    'LanguageDetector',
-    'SentimentAnalyzer',
-    'KeywordExtractor'
-]

webstoken/classifier.py

@@ -1,189 +0,0 @@
-"""
-Text classification module using rule-based and statistical approaches.
-"""
-
-from typing import Dict, List, Set, Tuple
-from collections import Counter
-import math
-import re
-
-from .normalizer import TextNormalizer
-from .tokenizer import WordTokenizer
-
-
-class TextClassifier:
-    """Simple text classifier using TF-IDF and cosine similarity."""
-    
-    def __init__(self):
-        self.word_tokenizer = WordTokenizer()
-        self.normalizer = TextNormalizer()
-        self.documents: Dict[str, List[str]] = {}  # category -> list of documents
-        self.vocabulary: Set[str] = set()
-        self.idf_scores: Dict[str, float] = {}
-        self.category_vectors: Dict[str, Dict[str, float]] = {}
-    
-    def train(self, documents: Dict[str, List[str]]) -> None:
-        """
-        Train the classifier on labeled documents.
-        
-        Args:
-            documents: Dict mapping categories to lists of documents
-        """
-        self.documents = documents
-        
-        # Build vocabulary and document frequencies
-        doc_frequencies: Dict[str, int] = Counter()
-        total_docs = sum(len(docs) for docs in documents.values())
-        
-        for category, docs in documents.items():
-            for doc in docs:
-                # Normalize and tokenize
-                doc = self.normalizer.normalize(doc)
-                tokens = self.word_tokenizer.tokenize(doc)
-                
-                # Update vocabulary and document frequencies
-                unique_tokens = set(tokens)
-                self.vocabulary.update(unique_tokens)
-                doc_frequencies.update(unique_tokens)
-        
-        # Calculate IDF scores
-        self.idf_scores = {
-            word: math.log(total_docs / (freq + 1))
-            for word, freq in doc_frequencies.items()
-        }
-        
-        # Calculate TF-IDF vectors for each category
-        for category, docs in documents.items():
-            category_vector: Dict[str, float] = {word: 0.0 for word in self.vocabulary}
-            
-            for doc in docs:
-                # Get term frequencies
-                doc = self.normalizer.normalize(doc)
-                tokens = self.word_tokenizer.tokenize(doc)
-                term_freqs = Counter(tokens)
-                
-                # Update category vector with TF-IDF scores
-                for word, tf in term_freqs.items():
-                    if word in self.idf_scores:
-                        category_vector[word] += tf * self.idf_scores[word]
-            
-            # Average the scores
-            for word in category_vector:
-                category_vector[word] /= len(docs)
-            
-            self.category_vectors[category] = category_vector
-    
-    def _calculate_vector(self, text: str) -> Dict[str, float]:
-        """Calculate TF-IDF vector for input text."""
-        # Normalize and tokenize
-        text = self.normalizer.normalize(text)
-        tokens = self.word_tokenizer.tokenize(text)
-        term_freqs = Counter(tokens)
-        
-        # Calculate TF-IDF scores
-        vector = {word: 0.0 for word in self.vocabulary}
-        for word, tf in term_freqs.items():
-            if word in self.idf_scores:
-                vector[word] = tf * self.idf_scores[word]
-        
-        return vector
-    
-    def _cosine_similarity(self, vec1: Dict[str, float], vec2: Dict[str, float]) -> float:
-        """Calculate cosine similarity between two vectors."""
-        dot_product = sum(vec1[word] * vec2[word] for word in vec1)
-        norm1 = math.sqrt(sum(score * score for score in vec1.values()))
-        norm2 = math.sqrt(sum(score * score for score in vec2.values()))
-        
-        if norm1 == 0 or norm2 == 0:
-            return 0.0
-        return dot_product / (norm1 * norm2)
-    
-    def classify(self, text: str) -> List[Tuple[str, float]]:
-        """
-        Classify text into categories with confidence scores.
-        
-        Returns:
-            List of (category, confidence) tuples, sorted by confidence
-        """
-        if not self.category_vectors:
-            raise ValueError("Classifier must be trained before classification")
-        
-        # Calculate vector for input text
-        text_vector = self._calculate_vector(text)
-        
-        # Calculate similarity with each category
-        similarities = [
-            (category, self._cosine_similarity(text_vector, category_vec))
-            for category, category_vec in self.category_vectors.items()
-        ]
-        
-        # Sort by similarity score
-        return sorted(similarities, key=lambda x: x[1], reverse=True)
-
-
-class TopicClassifier:
-    """Rule-based topic classifier using keyword matching."""
-    
-    def __init__(self):
-        # Define topic keywords
-        self.topic_keywords = {
-            'TECHNOLOGY': {
-                'computer', 'software', 'hardware', 'internet', 'programming',
-                'digital', 'data', 'algorithm', 'code', 'web', 'app', 'mobile',
-                'cyber', 'robot', 'ai', 'artificial intelligence', 'machine learning'
-            },
-            'SCIENCE': {
-                'research', 'experiment', 'laboratory', 'scientific', 'physics',
-                'chemistry', 'biology', 'mathematics', 'theory', 'hypothesis',
-                'study', 'discovery', 'innovation', 'analysis', 'observation'
-            },
-            'BUSINESS': {
-                'company', 'market', 'finance', 'investment', 'stock', 'trade',
-                'economy', 'business', 'corporate', 'startup', 'entrepreneur',
-                'profit', 'revenue', 'management', 'strategy', 'commercial'
-            },
-            'POLITICS': {
-                'government', 'policy', 'election', 'political', 'democracy',
-                'parliament', 'congress', 'law', 'legislation', 'party',
-                'vote', 'campaign', 'president', 'minister', 'diplomatic'
-            },
-            'SPORTS': {
-                'game', 'team', 'player', 'competition', 'tournament',
-                'championship', 'score', 'match', 'athlete', 'sport',
-                'win', 'lose', 'victory', 'defeat', 'coach', 'training'
-            },
-            'ENTERTAINMENT': {
-                'movie', 'film', 'music', 'song', 'concert', 'actor',
-                'actress', 'celebrity', 'show', 'performance', 'art',
-                'entertainment', 'theater', 'dance', 'festival', 'media'
-            }
-        }
-        
-        # Compile regex patterns for each topic
-        self.topic_patterns = {
-            topic: re.compile(r'\b(' + '|'.join(re.escape(kw) for kw in keywords) + r')\b', re.IGNORECASE)
-            for topic, keywords in self.topic_keywords.items()
-        }
-    
-    def classify(self, text: str) -> List[Tuple[str, float]]:
-        """
-        Classify text into topics with confidence scores.
-        
-        Returns:
-            List of (topic, confidence) tuples, sorted by confidence
-        """
-        # Count keyword matches for each topic
-        topic_matches = {
-            topic: len(pattern.findall(text))
-            for topic, pattern in self.topic_patterns.items()
-        }
-        
-        # Calculate confidence scores
-        total_matches = sum(topic_matches.values()) or 1  # Avoid division by zero
-        topic_scores = [
-            (topic, count / total_matches)
-            for topic, count in topic_matches.items()
-        ]
-        
-        # Sort by score
-        return sorted(topic_scores, key=lambda x: x[1], reverse=True)

webstoken/keywords.py

@@ -1,216 +0,0 @@
-"""
-Keyword extraction module using statistical and graph-based approaches.
-"""
-
-from typing import Dict, List, Set, Tuple
-from collections import Counter, defaultdict
-import math
-import re
-
-from .tokenizer import WordTokenizer
-from .normalizer import TextNormalizer
-
-
-class KeywordExtractor:
-    """Keyword extraction using TF-IDF and TextRank-inspired algorithms."""
-    
-    def __init__(self):
-        self.word_tokenizer = WordTokenizer()
-        self.normalizer = TextNormalizer()
-        
-        # Common words to filter out beyond basic stop words
-        self.filter_words: Set[str] = {
-            'would', 'could', 'should', 'said', 'also', 'may', 'might',
-            'must', 'need', 'shall', 'want', 'way', 'time', 'just',
-            'now', 'like', 'make', 'made', 'well', 'back', 'even',
-            'still', 'way', 'take', 'took', 'get', 'got', 'go', 'went'
-        }
-    
-    def _split_into_sentences(self, text: str) -> List[str]:
-        """Split text into sentences using simple rules."""
-        text = re.sub(r'\s+', ' ', text)
-        sentences = re.split(r'[.!?]+', text)
-        return [s.strip() for s in sentences if s.strip()]
-    
-    def _calculate_word_scores(self, text: str) -> Dict[str, float]:
-        """Calculate word importance scores using frequency and position."""
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        sentences = self._split_into_sentences(text)
-        
-        word_scores: Dict[str, float] = defaultdict(float)
-        word_positions: Dict[str, List[int]] = defaultdict(list)
-        
-        # Calculate word frequencies and positions
-        for i, sentence in enumerate(sentences):
-            words = self.word_tokenizer.tokenize(sentence)
-            for j, word in enumerate(words):
-                word = word.lower()
-                if (word.isalnum() and 
-                    len(word) > 2 and
-                    word not in self.filter_words and
-                    word not in self.normalizer.stop_words):
-                    word_scores[word] += 1
-                    word_positions[word].append(i)
-        
-        # Adjust scores based on position
-        num_sentences = len(sentences)
-        for word, positions in word_positions.items():
-            # Words appearing in first or last sentences get bonus
-            if 0 in positions:
-                word_scores[word] *= 1.2
-            if num_sentences - 1 in positions:
-                word_scores[word] *= 1.1
-            
-            # Words appearing throughout text get bonus
-            coverage = len(set(positions)) / num_sentences
-            word_scores[word] *= (1 + coverage)
-        
-        return word_scores
-    
-    def _calculate_word_cooccurrence(self, text: str, window_size: int = 3) -> Dict[str, Dict[str, int]]:
-        """Calculate word co-occurrence matrix."""
-        # Normalize and tokenize text
-        text = self.normalizer.normalize(text)
-        words = self.word_tokenizer.tokenize(text)
-        
-        # Filter words
-        filtered_words = [
-            word.lower() for word in words
-            if (word.isalnum() and
-                len(word) > 2 and
-                word.lower() not in self.filter_words and
-                word.lower() not in self.normalizer.stop_words)
-        ]
-        
-        # Build co-occurrence matrix
-        cooccurrence: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
-        
-        for i, word in enumerate(filtered_words):
-            for j in range(max(0, i - window_size), min(len(filtered_words), i + window_size + 1)):
-                if i != j:
-                    cooccurrence[word][filtered_words[j]] += 1
-                    cooccurrence[filtered_words[j]][word] += 1
-        
-        return cooccurrence
-    
-    def _textrank_scores(self, cooccurrence: Dict[str, Dict[str, int]], damping: float = 0.85,
-                        iterations: int = 30) -> Dict[str, float]:
-        """Calculate TextRank scores from co-occurrence matrix."""
-        scores = {word: 1.0 for word in cooccurrence}
-        
-        for _ in range(iterations):
-            new_scores = {}
-            for word in scores:
-                if not cooccurrence[word]:
-                    continue
-                    
-                incoming_score = sum(
-                    scores[other] * cooccurrence[word][other] / sum(cooccurrence[other].values())
-                    for other in cooccurrence[word]
-                )
-                new_scores[word] = (1 - damping) + damping * incoming_score
-            
-            # Check convergence
-            score_diff = sum(abs(new_scores[w] - scores[w]) for w in scores)
-            scores = new_scores
-            if score_diff < 0.0001:
-                break
-        
-        return scores
-    
-    def extract_keywords(self, text: str, num_keywords: int = 10,
-                        use_textrank: bool = True) -> List[Tuple[str, float]]:
-        """
-        Extract keywords from text using combined frequency and graph-based approach.
-        
-        Args:
-            text: Input text
-            num_keywords: Number of keywords to return
-            use_textrank: Whether to use TextRank algorithm
-            
-        Returns:
-            List of (keyword, score) tuples, sorted by score
-        """
-        if not text:
-            return []
-        
-        # Get frequency-based scores
-        freq_scores = self._calculate_word_scores(text)
-        
-        if use_textrank:
-            # Get TextRank scores
-            cooccurrence = self._calculate_word_cooccurrence(text)
-            textrank_scores = self._textrank_scores(cooccurrence)
-            
-            # Combine scores
-            combined_scores = {
-                word: freq_scores[word] * textrank_scores.get(word, 0)
-                for word in freq_scores
-            }
-        else:
-            combined_scores = freq_scores
-        
-        # Sort and return top keywords
-        sorted_words = sorted(
-            combined_scores.items(),
-            key=lambda x: x[1],
-            reverse=True
-        )
-        
-        return sorted_words[:num_keywords]
-    
-    def extract_keyphrases(self, text: str, num_phrases: int = 5,
-                          min_words: int = 2, max_words: int = 4) -> List[Tuple[str, float]]:
-        """
-        Extract key phrases from text.
-        
-        Args:
-            text: Input text
-            num_phrases: Number of phrases to return
-            min_words: Minimum words in phrase
-            max_words: Maximum words in phrase
-            
-        Returns:
-            List of (phrase, score) tuples, sorted by score
-        """
-        # Normalize and split into sentences
-        text = self.normalizer.normalize(text)
-        sentences = self._split_into_sentences(text)
-        
-        # Get word importance scores
-        word_scores = self._calculate_word_scores(text)
-        
-        # Extract candidate phrases
-        phrases: Dict[str, float] = {}
-        
-        for sentence in sentences:
-            words = self.word_tokenizer.tokenize(sentence)
-            
-            # Generate phrases of different lengths
-            for i in range(len(words)):
-                for length in range(min_words, min(max_words + 1, len(words) - i + 1)):
-                    phrase_words = words[i:i+length]
-                    
-                    # Filter phrases
-                    if all(
-                        word.isalnum() and
-                        len(word) > 2 and
-                        word.lower() not in self.filter_words and
-                        word.lower() not in self.normalizer.stop_words
-                        for word in phrase_words
-                    ):
-                        phrase = ' '.join(phrase_words)
-                        # Score is average of word scores
-                        score = sum(word_scores.get(word.lower(), 0) for word in phrase_words)
-                        score /= len(phrase_words)
-                        phrases[phrase] = score
-        
-        # Sort and return top phrases
-        sorted_phrases = sorted(
-            phrases.items(),
-            key=lambda x: x[1],
-            reverse=True
-        )
-        
-        return sorted_phrases[:num_phrases]

webstoken/language.py

@@ -1,128 +0,0 @@
-"""
-Language detection module using character and word frequency analysis.
-"""
-
-from typing import Dict, List, Set, Tuple
-from collections import Counter
-import re
-
-
-class LanguageDetector:
-    """Language detection using character n-gram frequencies."""
-    
-    def __init__(self):
-        # Language profiles based on common character sequences
-        self.language_profiles = {
-            'ENGLISH': {
-                'chars': 'etaoinshrdlcumwfgypbvkjxqz',
-                'ngrams': {'th', 'he', 'in', 'er', 'an', 're', 'on', 'at', 'en', 'nd',
-                          'ti', 'es', 'or', 'te', 'of', 'ed', 'is', 'it', 'al', 'ar',
-                          'st', 'to', 'nt', 'ng', 'se', 'ha', 'as', 'ou', 'io', 'le'},
-                'words': {'the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have',
-                         'i', 'it', 'for', 'not', 'on', 'with', 'he', 'as', 'you',
-                         'do', 'at'}
-            },
-            'SPANISH': {
-                'chars': 'eaosrnidlctumpbgvyqhfzjñxwk',
-                'ngrams': {'de', 'en', 'el', 'la', 'os', 'es', 'as', 'ar', 'er', 'ra',
-                          'al', 'an', 'do', 'or', 'ta', 'ue', 'io', 'on', 'ro', 'ad',
-                          'te', 'co', 'st', 'ci', 'nt', 'to', 'lo', 'no', 'po', 'ac'},
-                'words': {'de', 'la', 'que', 'el', 'en', 'y', 'a', 'los', 'se', 'del',
-                         'las', 'un', 'por', 'con', 'no', 'una', 'su', 'para', 'es',
-                         'al'}
-            },
-            'FRENCH': {
-                'chars': 'esaitnrulodcpmévqfbghàjxèêyçwzùâîôûëïüœ',
-                'ngrams': {'es', 'le', 'en', 'de', 'nt', 'on', 're', 'er', 'ai', 'te',
-                          'la', 'an', 'ou', 'it', 'ur', 'et', 'el', 'se', 'qu', 'me',
-                          'is', 'ar', 'ce', 'ns', 'us', 'ue', 'ss', 'ie', 'em', 'tr'},
-                'words': {'le', 'de', 'un', 'être', 'et', 'à', 'il', 'avoir', 'ne',
-                         'je', 'son', 'que', 'se', 'qui', 'ce', 'dans', 'en', 'du',
-                         'elle', 'au'}
-            },
-            'GERMAN': {
-                'chars': 'enisratdhulcgmobwfkzvüpäößjyqxéèêëàáâãåāăąćčĉċďđ',
-                'ngrams': {'en', 'er', 'ch', 'de', 'ei', 'in', 'te', 'nd', 'ie', 'ge',
-                          'st', 'ne', 'be', 'es', 'un', 'zu', 'an', 'ng', 'au', 'it',
-                          'is', 'he', 'ht', 'se', 'ck', 'ic', 're', 'ns', 'sc', 'tz'},
-                'words': {'der', 'die', 'und', 'in', 'den', 'von', 'zu', 'das', 'mit',
-                         'sich', 'des', 'auf', 'für', 'ist', 'im', 'dem', 'nicht',
-                         'ein', 'eine', 'als'}
-            }
-        }
-        
-        # Compile word patterns
-        self.word_pattern = re.compile(r'\b\w+\b')
-    
-    def _extract_ngrams(self, text: str, n: int = 2) -> List[str]:
-        """Extract character n-grams from text."""
-        text = text.lower()
-        return [text[i:i+n] for i in range(len(text)-n+1)]
-    
-    def _calculate_char_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate character frequencies in text."""
-        text = text.lower()
-        char_count = Counter(c for c in text if c.isalpha())
-        total = sum(char_count.values()) or 1
-        return {char: count/total for char, count in char_count.items()}
-    
-    def _calculate_ngram_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate n-gram frequencies in text."""
-        ngrams = self._extract_ngrams(text)
-        ngram_count = Counter(ngrams)
-        total = sum(ngram_count.values()) or 1
-        return {ngram: count/total for ngram, count in ngram_count.items()}
-    
-    def _calculate_word_frequencies(self, text: str) -> Dict[str, float]:
-        """Calculate word frequencies in text."""
-        words = self.word_pattern.findall(text.lower())
-        word_count = Counter(words)
-        total = sum(word_count.values()) or 1
-        return {word: count/total for word, count in word_count.items()}
-    
-    def _calculate_similarity(self, freq1: Dict[str, float], freq2: Dict[str, float]) -> float:
-        """Calculate similarity between two frequency distributions."""
-        common_keys = set(freq1.keys()) & set(freq2.keys())
-        if not common_keys:
-            return 0.0
-        
-        similarity = sum(min(freq1.get(k, 0), freq2.get(k, 0)) for k in common_keys)
-        return similarity
-    
-    def detect(self, text: str) -> List[Tuple[str, float]]:
-        """
-        Detect the language of text with confidence scores.
-        
-        Returns:
-            List of (language, confidence) tuples, sorted by confidence
-        """
-        if not text:
-            return []
-        
-        # Calculate frequencies for input text
-        char_freqs = self._calculate_char_frequencies(text)
-        ngram_freqs = self._calculate_ngram_frequencies(text)
-        word_freqs = self._calculate_word_frequencies(text)
-        
-        # Calculate similarity scores for each language
-        scores = []
-        for lang, profile in self.language_profiles.items():
-            # Character similarity
-            char_sim = sum(char_freqs.get(c, 0) for c in profile['chars'])
-            
-            # N-gram similarity
-            ngram_sim = sum(ngram_freqs.get(ng, 0) for ng in profile['ngrams'])
-            
-            # Word similarity
-            word_sim = sum(word_freqs.get(w, 0) for w in profile['words'])
-            
-            # Combined score (weighted average)
-            total_score = (0.3 * char_sim + 0.4 * ngram_sim + 0.3 * word_sim)
-            scores.append((lang, total_score))
-        
-        # Normalize scores
-        total = sum(score for _, score in scores) or 1
-        normalized_scores = [(lang, score/total) for lang, score in scores]
-        
-        # Sort by confidence
-        return sorted(normalized_scores, key=lambda x: x[1], reverse=True)