webscout 6.4__py3-none-any.whl → 6.6__py3-none-any.whl

webstoken/stemmer.py

@@ -0,0 +1,73 @@
+"""
+Word stemming utilities.
+"""
+
+from typing import Set
+
+
+class Stemmer:
+    """Simple rule-based stemmer implementing Porter-like rules."""
+    
+    def __init__(self):
+        self.vowels: Set[str] = {'a', 'e', 'i', 'o', 'u', 'y'}
+        self.doubles: Set[str] = {'bb', 'dd', 'ff', 'gg', 'mm', 'nn', 'pp', 'rr', 'tt'}
+        
+    def is_vowel(self, char: str, prev_char: str = None) -> bool:
+        """Check if a character is a vowel, considering 'y' special cases."""
+        return char in self.vowels or (char == 'y' and prev_char and prev_char not in self.vowels)
+    
+    def count_syllables(self, word: str) -> int:
+        """Count syllables in a word based on vowel sequences."""
+        count = 0
+        prev_char = None
+        for i, char in enumerate(word.lower()):
+            if self.is_vowel(char, prev_char) and (i == 0 or not self.is_vowel(prev_char, word[i-2] if i > 1 else None)):
+                count += 1
+            prev_char = char
+        return count or 1
+    
+    def stem(self, word: str) -> str:
+        """Apply stemming rules to reduce word to its root form."""
+        if len(word) <= 3:
+            return word
+            
+        word = word.lower()
+        
+        # Step 1: Handle plurals and past participles
+        if word.endswith('sses'):
+            word = word[:-2]
+        elif word.endswith('ies'):
+            word = word[:-2]
+        elif word.endswith('ss'):
+            pass
+        elif word.endswith('s') and len(word) > 4:
+            word = word[:-1]
+            
+        # Step 2: Handle -ed and -ing
+        if word.endswith('ed') and self.count_syllables(word[:-2]) > 1:
+            word = word[:-2]
+        elif word.endswith('ing') and self.count_syllables(word[:-3]) > 1:
+            word = word[:-3]
+            
+        # Step 3: Handle miscellaneous endings
+        if len(word) > 5:
+            if word.endswith('ement'):
+                word = word[:-5]
+            elif word.endswith('ment'):
+                word = word[:-4]
+            elif word.endswith('ent'):
+                word = word[:-3]
+                
+        # Step 4: Handle -ity endings
+        if word.endswith('ity') and len(word) > 6:
+            word = word[:-3]
+            if word.endswith('abil'):
+                word = word[:-4] + 'able'
+            elif word.endswith('ic'):
+                word = word[:-2]
+                
+        # Final step: Remove double consonants at the end
+        if len(word) > 2 and word[-2:] in self.doubles:
+            word = word[:-1]
+            
+        return word

webstoken/t.py

@@ -0,0 +1,75 @@
+from webstoken import (
+    process_text, NamedEntityRecognizer, TextClassifier,
+    TopicClassifier, LanguageDetector, SentimentAnalyzer,
+    KeywordExtractor
+)
+from rich import print
+# Example text
+text = """
+Dr. John Smith from Microsoft Corporation visited New York City on January 15th, 2024.
+He presented an excellent paper about artificial intelligence and machine learning at
+the International Technology Conference. The research was incredibly well-received,
+and many attendees were excited about its potential applications in healthcare.
+"""
+
+print("1. Basic Text Processing")
+print("-" * 50)
+result = process_text(text)
+for sentence_data in result['sentences']:
+    print("Original:", sentence_data['original'])
+    print("Tokens:", sentence_data['tokens'])
+    print("POS Tags:", sentence_data['pos_tags'])
+    print("Stems:", sentence_data['stems'])
+    print()
+
+print("\n2. Named Entity Recognition")
+print("-" * 50)
+ner = NamedEntityRecognizer()
+entities = ner.extract_entities(text)
+for entity_type, entity_list in entities.items():
+    if entity_list:
+        print(f"{entity_type}:", entity_list)
+
+print("\n3. Topic Classification")
+print("-" * 50)
+topic_classifier = TopicClassifier()
+topics = topic_classifier.classify(text)
+print("Topics (with confidence):")
+for topic, confidence in topics[:3]:  # Top 3 topics
+    print(f"{topic}: {confidence:.2f}")
+
+print("\n4. Language Detection")
+print("-" * 50)
+lang_detector = LanguageDetector()
+languages = lang_detector.detect(text)
+print("Detected Languages (with confidence):")
+for lang, confidence in languages:
+    print(f"{lang}: {confidence:.2f}")
+
+print("\n5. Sentiment Analysis")
+print("-" * 50)
+sentiment_analyzer = SentimentAnalyzer()
+sentiment = sentiment_analyzer.analyze_sentiment(text)
+print("Sentiment Scores:")
+print(f"Polarity: {sentiment['polarity']:.2f}")
+print(f"Subjectivity: {sentiment['subjectivity']:.2f}")
+print(f"Confidence: {sentiment['confidence']:.2f}")
+
+print("\nEmotions:")
+emotions = sentiment_analyzer.analyze_emotions(text)
+for emotion, score in emotions:
+    if score > 0.1:  # Only show significant emotions
+        print(f"{emotion}: {score:.2f}")
+
+print("\n6. Keyword Extraction")
+print("-" * 50)
+keyword_extractor = KeywordExtractor()
+print("Keywords:")
+keywords = keyword_extractor.extract_keywords(text, num_keywords=5)
+for keyword, score in keywords:
+    print(f"{keyword}: {score:.2f}")
+
+print("\nKey Phrases:")
+keyphrases = keyword_extractor.extract_keyphrases(text, num_phrases=3)
+for phrase, score in keyphrases:
+    print(f"{phrase}: {score:.2f}")

webstoken/tagger.py

@@ -0,0 +1,60 @@
+"""
+Part-of-Speech tagging utilities.
+"""
+
+from typing import List, Set, Tuple
+
+
+class POSTagger:
+    """Simple rule-based Part-of-Speech tagger."""
+    
+    def __init__(self):
+        # Basic rules for POS tagging
+        self.noun_suffixes: Set[str] = {'ness', 'ment', 'ship', 'dom', 'hood', 'er', 'or', 'ist'}
+        self.verb_suffixes: Set[str] = {'ize', 'ate', 'ify', 'ing', 'ed'}
+        self.adj_suffixes: Set[str] = {'able', 'ible', 'al', 'ful', 'ous', 'ive', 'less'}
+        self.adv_suffixes: Set[str] = {'ly'}
+        
+        # Common words by POS
+        self.determiners: Set[str] = {'the', 'a', 'an', 'this', 'that', 'these', 'those'}
+        self.prepositions: Set[str] = {'in', 'on', 'at', 'by', 'with', 'from', 'to', 'for'}
+        self.pronouns: Set[str] = {'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her'}
+        
+    def tag(self, tokens: List[str]) -> List[Tuple[str, str]]:
+        """Assign POS tags to tokens based on rules."""
+        tagged = []
+        prev_tag = None
+        
+        for i, token in enumerate(tokens):
+            word = token.lower()
+            
+            # Check special cases first
+            if word in self.determiners:
+                tag = 'DET'
+            elif word in self.prepositions:
+                tag = 'PREP'
+            elif word in self.pronouns:
+                tag = 'PRON'
+            # Check suffixes
+            elif any(word.endswith(suffix) for suffix in self.noun_suffixes):
+                tag = 'NOUN'
+            elif any(word.endswith(suffix) for suffix in self.verb_suffixes):
+                tag = 'VERB'
+            elif any(word.endswith(suffix) for suffix in self.adj_suffixes):
+                tag = 'ADJ'
+            elif any(word.endswith(suffix) for suffix in self.adv_suffixes):
+                tag = 'ADV'
+            # Default cases
+            elif word[0].isupper() and i > 0:
+                tag = 'PROPN'  # Proper noun
+            elif word.isdigit():
+                tag = 'NUM'
+            elif not word.isalnum():
+                tag = 'PUNCT'
+            else:
+                tag = 'NOUN'  # Default to noun
+            
+            tagged.append((token, tag))
+            prev_tag = tag
+            
+        return tagged

webstoken/tokenizer.py

@@ -0,0 +1,158 @@
+"""
+Tokenization utilities for sentence and word-level tokenization.
+"""
+
+from typing import List, Dict, Set, Pattern
+import re
+
+
+class SentenceTokenizer:
+    """Advanced sentence tokenizer with support for complex cases and proper formatting."""
+    
+    def __init__(self) -> None:
+        # Common abbreviations by category
+        self.TITLES: Set[str] = {
+            'mr', 'mrs', 'ms', 'dr', 'prof', 'rev', 'sr', 'jr', 'esq',
+            'hon', 'pres', 'gov', 'atty', 'supt', 'det', 'rev', 'col','maj', 'gen', 'capt', 'cmdr',
+            'lt', 'sgt', 'cpl', 'pvt'
+        }
+        
+        self.ACADEMIC: Set[str] = {
+            'ph.d', 'phd', 'm.d', 'md', 'b.a', 'ba', 'm.a', 'ma', 'd.d.s', 'dds',
+            'm.b.a', 'mba', 'b.sc', 'bsc', 'm.sc', 'msc', 'llb', 'll.b', 'bl'
+        }
+        
+        self.ORGANIZATIONS: Set[str] = {
+            'inc', 'ltd', 'co', 'corp', 'llc', 'llp', 'assn', 'bros', 'plc', 'cos',
+            'intl', 'dept', 'est', 'dist', 'mfg', 'div'
+        }
+        
+        self.MONTHS: Set[str] = {
+            'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'
+        }
+        
+        self.UNITS: Set[str] = {
+            'oz', 'pt', 'qt', 'gal', 'ml', 'cc', 'km', 'cm', 'mm', 'ft', 'in',
+            'kg', 'lb', 'lbs', 'hz', 'khz', 'mhz', 'ghz', 'kb', 'mb', 'gb', 'tb'
+        }
+        
+        self.TECHNOLOGY: Set[str] = {
+            'v', 'ver', 'app', 'sys', 'dir', 'exe', 'lib', 'api', 'sdk', 'url',
+            'cpu', 'gpu', 'ram', 'rom', 'hdd', 'ssd', 'lan', 'wan', 'sql', 'html'
+        }
+        
+        self.MISC: Set[str] = {
+            'vs', 'etc', 'ie', 'eg', 'no', 'al', 'ca', 'cf', 'pp', 'est', 'st',
+            'approx', 'appt', 'apt', 'dept', 'depts', 'min', 'max', 'avg'
+        }
+
+        # Combine all abbreviations
+        self.all_abbreviations: Set[str] = (
+            self.TITLES | self.ACADEMIC | self.ORGANIZATIONS |
+            self.MONTHS | self.UNITS | self.TECHNOLOGY | self.MISC
+        )
+
+        # Special patterns
+        self.ELLIPSIS: str = r'\.{2,}|…'
+        self.URL_PATTERN: str = (
+            r'(?:https?:\/\/|www\.)[\w\-\.]+\.[a-zA-Z]{2,}(?:\/[^\s]*)?'
+        )
+        self.EMAIL_PATTERN: str = r'[\w\.-]+@[\w\.-]+\.\w+'
+        self.NUMBER_PATTERN: str = (
+            r'\d+(?:\.\d+)?(?:%|°|km|cm|mm|m|kg|g|lb|ft|in|mph|kmh|hz|mhz|ghz)?'
+        )
+        
+        # Quote and bracket pairs
+        self.QUOTE_PAIRS: Dict[str, str] = {
+            '"': '"', "'": "'", '"': '"', "「": "」", "『": "』",
+            "«": "»", "‹": "›", "'": "'", "‚": "'"
+        }
+        
+        self.BRACKETS: Dict[str, str] = {
+            '(': ')', '[': ']', '{': '}', '⟨': '⟩', '「': '」',
+            '『': '』', '【': '】', '〖': '〗', '｢': '｣'
+        }
+
+        # Compile regex patterns
+        self._compile_patterns()
+
+    def _compile_patterns(self) -> None:
+        """Compile regex patterns for better performance."""
+        # Pattern for finding potential sentence boundaries
+        self.SENTENCE_END: Pattern = re.compile(
+            r'''
+            # Group for sentence endings
+            (?:
+                # Standard endings with optional quotes/brackets
+                (?<=[.!?])[\"\'\)\]\}»›」』\s]*
+                
+                # Ellipsis
+                |(?:\.{2,}|…)
+                
+                # Asian-style endings
+                |(?<=[。！？」』】\s])
+            )
+            
+            # Must be followed by whitespace and capital letter or number
+            (?=\s+(?:[A-Z0-9]|["'({[\[「『《‹〈][A-Z]))
+            ''',
+            re.VERBOSE
+        )
+
+        # Pattern for abbreviations
+        abbrev_pattern = '|'.join(re.escape(abbr) for abbr in self.all_abbreviations)
+        self.ABBREV_PATTERN: Pattern = re.compile(
+            fr'\b(?:{abbrev_pattern})\.?',
+            re.IGNORECASE
+        )
+
+    def tokenize(self, text: str) -> List[str]:
+        """Split text into sentences while handling complex cases."""
+        if not text or not text.strip():
+            return []
+            
+        # Initial split on potential sentence boundaries
+        sentences = self.SENTENCE_END.split(text)
+        
+        # Clean and validate sentences
+        final_sentences = []
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if sentence:
+                final_sentences.append(sentence)
+                
+        return final_sentences
+
+
+class WordTokenizer:
+    """Simple but effective word tokenizer with support for contractions and special cases."""
+    
+    def __init__(self):
+        self.contractions = {
+            "n't": "not", "'ll": "will", "'re": "are", "'s": "is",
+            "'m": "am", "'ve": "have", "'d": "would"
+        }
+        
+        self.word_pattern = re.compile(r"""
+            (?:[A-Za-z]+(?:[''][A-Za-z]+)*)|    # Words with optional internal apostrophes
+            (?:\d+(?:,\d{3})*(?:\.\d+)?)|       # Numbers with commas and decimals
+            (?:[@#]?\w+)|                        # Hashtags and mentions
+            (?:[^\w\s])                          # Punctuation and symbols
+        """, re.VERBOSE)
+    
+    def tokenize(self, text: str) -> List[str]:
+        """Split text into words while handling contractions and special cases."""
+        tokens = []
+        for match in self.word_pattern.finditer(text):
+            word = match.group()
+            # Handle contractions
+            for contraction, expansion in self.contractions.items():
+                if word.endswith(contraction):
+                    base = word[:-len(contraction)]
+                    if base:
+                        tokens.append(base)
+                    tokens.append(expansion)
+                    break
+            else:
+                tokens.append(word)
+        return tokens

webscout/Agents/Onlinesearcher.py

@@ -1,182 +0,0 @@
-import json
-import httpx
-from bs4 import BeautifulSoup
-from typing import List, Dict
-from webscout import GoogleS, GEMINIAPI
-import re
-from concurrent.futures import ThreadPoolExecutor, as_completed
-
-
-class WebSearchAgent:
-    def __init__(self):
-        self.webs = GoogleS()
-        self.ai = GEMINIAPI(is_conversation=False, api_key='AIzaSyAYlT5-V0MXZwaLYpXCF1Z-Yvy_tx1jylA')
-
-    def generate_search_queries(self, information: str, num_queries: int = 10) -> List[str]:
-        prompt = f""" Task: Generate exactly {num_queries} optimal search queries based on the given information.  
-Instructions: 
-1. Analyze the provided information thoroughly. 
-2. Identify key concepts, entities, and relationships. 
-3. Formulate {num_queries} concise and specific search queries that will yield relevant and diverse results. 
-4. Each query should focus on a different aspect or angle of the information. 
-5. The queries should be in natural language, not in the form of keywords. 
-6. Avoid unnecessary words or phrases that might limit the search results. 
-7. **Important**: Return the response **ONLY** in JSON format without any additional text or code blocks.  
-Your response must be in the following JSON format: {{
-    "search_queries": [
-        "Your first search query here",
-        "Your second search query here",
-        "...",
-        "Your last search query here"
-    ]
-}}  
-Ensure that: 
-- You provide exactly {num_queries} search queries. 
-- Each query is unique and focuses on a different aspect of the information. 
-- The queries are in plain text, suitable for a web search engine.  
-
-Information to base the search queries on: 
-{information}  
-
-Now, generate the optimal search queries: """
-
-        response = ""
-        for chunk in self.ai.chat(prompt):
-            response += chunk
-
-        json_match = re.search(r'\{.*\}', response, re.DOTALL)
-        if json_match:
-            json_str = json_match.group(0)
-            try:
-                json_response = json.loads(json_str)
-                print(json_response['search_queries'])
-                return json_response["search_queries"]
-            except json.JSONDecodeError:
-                pass
-
-        queries = re.findall(r'"([^"]+)"', response)
-        if len(queries) >= num_queries:
-            return queries[:num_queries]
-        elif queries:
-            return queries
-        else:
-            return [information]
-
-    def search(self, information: str, region: str = 'wt-wt', safe: str = 'off',
-               max_results: int = 10) -> List[Dict]:
-        search_queries = self.generate_search_queries(information, num_queries=10)
-        all_results = []
-
-        for query in search_queries:
-            results = []
-            with self.webs as webs:
-                for result in webs.search(query, region=region, safe=safe,
-                                       max_results=max_results):
-                    results.append(result)
-            all_results.extend(results)
-
-        return all_results
-
-    def extract_urls(self, results: List[Dict]) -> List[str]:
-        urls = [result.get('href') for result in results if result.get('href')]
-        unique_urls = list(set(urls))
-        return unique_urls
-
-    def fetch_webpage(self, url: str) -> Dict[str, str]:
-        try:
-            with httpx.Client(timeout=120) as client:
-                response = client.get(url)
-                if response.status_code == 200:
-                    html = response.text
-                    soup = BeautifulSoup(html, 'html.parser')
-                    paragraphs = soup.find_all('p')
-                    text = ' '.join([p.get_text() for p in paragraphs])
-                    words = text.split()
-                    if len(words) > 600:
-                        text = ' '.join(words[:600]) + '...'
-                    return {"url": url, "content": text}
-                else:
-                    return {"url": url, "content": f"Failed to fetch {url}: HTTP {response.status_code}"}
-        except Exception as e:
-            return {"url": url, "content": f"Error fetching {url}: {str(e)}"}
-
-    def fetch_all_webpages(self, urls: List[str], max_workers: int = 10) -> List[Dict[str, str]]:
-        contents = []
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            future_to_url = {executor.submit(self.fetch_webpage, url): url for url in urls}
-            for future in as_completed(future_to_url):
-                result = future.result()
-                contents.append(result)
-        return contents
-
-
-class OnlineSearcher:
-    def __init__(self):
-        self.agent = WebSearchAgent()
-        self.ai = GEMINIAPI(is_conversation=False, api_key='GOOGLE GEMINI API')
-
-    def answer_question(self, question: str) -> None:
-        search_results = self.agent.search(question, max_results=10)
-        urls = self.agent.extract_urls(search_results)
-        webpage_contents = self.agent.fetch_all_webpages(urls)
-
-        context = "Web search results and extracted content:\n\n"
-        for i, result in enumerate(search_results, 1):
-            title = result.get('title', 'No Title')
-            href = result.get('href', 'No URL')
-            snippet = result.get('body', 'No Snippet')
-            context += f"{i}. **Title:** {title}\n   **URL:** {href}\n   **Snippet:** {snippet}\n\n"
-
-        context += "Extracted webpage contents:\n"
-        for i, webpage in enumerate(webpage_contents, 1):
-            content = webpage['content']
-            content_preview = content[:600] + '...' if len(content) > 600 else content
-            context += f"{i}. **URL:** {webpage['url']}\n   **Content:** {content_preview}\n\n"
-
-        prompt = f""" Task: Provide a comprehensive, insightful, and well-structured answer to the given question based on the provided web search results and your general knowledge.  
-Question: {question}  
-Context: {context}  
-Instructions:  
-1. Carefully analyze the provided web search results and extracted content.  
-2. Synthesize the information to form a coherent and comprehensive answer.  
-3. If the search results contain relevant information, incorporate it into your answer seamlessly.  
-4. Avoid providing irrelevant information, and do not reference "according to web page".  
-5. If the search results don't contain sufficient information, clearly state this and provide the best answer based on your general knowledge.  
-6. Ensure your answer is well-structured, factual, and directly addresses the question.  
-7. Use clear headings, bullet points, or other formatting tools to enhance readability where appropriate.  
-8. Strive for a tone and style similar to that of professional, authoritative sources like Perplexity, ensuring clarity and depth in your response.  
-Your response should be informative, accurate, and properly sourced when possible. Begin your answer now: """
-
-        for chunk in self.ai.chat(prompt, stream=True):
-            print(chunk, end='', flush=True)
-
-
-
-# Usage example
-if __name__ == "__main__":
-    assistant = OnlineSearcher()
-    while True:
-        try:
-            question = input(">>> ")
-            if question.lower() == 'quit':
-                break
-            print("=" * 50)
-            assistant.answer_question(question) 
-            print("=" * 50)
-        except KeyboardInterrupt:
-            print("\nExiting.")
-            break
-        except Exception as e:
-            print(f"An error occurred: {e}")
-
-"""
-def format_prompt(messages: Messages, add_special_tokens=False) -> str:
-
-    if not add_special_tokens and len(messages) <= 1:
-        return messages[0]["content"]
-    formatted = "\n".join([
-        f'{message["role"].capitalize()}: {message["content"]}'
-        for message in messages
-    ])
-    return f"{formatted}\nAssistant:
-"""

webscout/Agents/__init__.py

@@ -1,2 +0,0 @@
-from .Onlinesearcher import *
-from .functioncall import *