janito 0.12.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

janito/tools/fetch_webpage/extractors.py

@@ -1,276 +0,0 @@
-"""
-Content extraction methods for web pages.
-"""
-
-from typing import List, Dict, Union, Optional
-from bs4 import BeautifulSoup
-import trafilatura
-from newspaper import Article
-import re
-
-from janito.tools.rich_console import print_info, print_success, print_warning
-from janito.tools.fetch_webpage.utils import clean_text, similar_text
-
-
-def extract_clean_text(html_content: str, method: str = 'trafilatura', 
-                       url: Optional[str] = None, target_strings: List[str] = None) -> str:
-    """
-    Extract clean, relevant text from HTML content using various methods.
-    
-    Args:
-        html_content: The HTML content to extract text from
-        method: The extraction method to use ('trafilatura', 'newspaper', 'beautifulsoup', 'all')
-        url: Optional URL for methods that require it (like newspaper)
-        target_strings: Optional list of strings to target specific content sections
-        
-    Returns:
-        Extracted text content
-    """
-    print_info(f"Extracting content using method: {method}", "Content Extraction")
-    
-    extracted_text = ""
-    
-    if method == 'trafilatura' or method == 'all':
-        try:
-            traf_text = trafilatura.extract(html_content, include_links=False, 
-                                           include_tables=False, include_images=False,
-                                           favor_precision=True)
-            if traf_text and len(traf_text) > 100:
-                if method == 'trafilatura':
-                    print_success("Successfully extracted content with Trafilatura", "Content Extraction")
-                    return clean_text(traf_text)
-                extracted_text = traf_text
-                print_success("Successfully extracted content with Trafilatura", "Content Extraction")
-        except Exception as e:
-            print_warning(f"Content Extraction: Trafilatura extraction error: {str(e)}")
-    
-    if method == 'newspaper' or method == 'all':
-        if not url:
-            print_warning("Content Extraction: URL required for newspaper extraction but not provided")
-        else:
-            try:
-                article = Article(url)
-                article.download(html_content)
-                article.parse()
-                np_text = article.text
-                if np_text and len(np_text) > 100:
-                    if method == 'newspaper':
-                        print_success("Successfully extracted content with Newspaper3k", "Content Extraction")
-                        return clean_text(np_text)
-                    if not extracted_text or len(np_text) > len(extracted_text):
-                        extracted_text = np_text
-                        print_success("Successfully extracted content with Newspaper3k", "Content Extraction")
-            except Exception as e:
-                print_warning(f"Content Extraction: Newspaper extraction error: {str(e)}")
-    
-    if method == 'beautifulsoup' or method == 'all':
-        try:
-            soup = BeautifulSoup(html_content, 'html.parser')
-            
-            # Remove script, style, and other non-content elements
-            for element in soup(['script', 'style', 'header', 'footer', 'nav', 'aside']):
-                element.decompose()
-            
-            # Extract text from paragraph and heading tags
-            paragraphs = []
-            for tag in soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'article']):
-                text = tag.get_text(strip=True)
-                if text and len(text) > 20:  # Skip very short pieces that might be UI elements
-                    paragraphs.append(text)
-            
-            bs_text = "\n\n".join(paragraphs)
-            if bs_text and len(bs_text) > 100:
-                if method == 'beautifulsoup':
-                    print_success("Successfully extracted content with BeautifulSoup", "Content Extraction")
-                    return clean_text(bs_text)
-                if not extracted_text or len(bs_text) > len(extracted_text):
-                    extracted_text = bs_text
-                    print_success("Successfully extracted content with BeautifulSoup", "Content Extraction")
-        except Exception as e:
-            print_warning(f"Content Extraction: BeautifulSoup extraction error: {str(e)}")
-    
-    if not extracted_text:
-        print_warning("Content Extraction: Could not extract meaningful content with any method")
-        # Fall back to the raw text with HTML tags removed
-        extracted_text = BeautifulSoup(html_content, 'html.parser').get_text(separator='\n')
-    
-    return clean_text(extracted_text)
-
-
-def extract_targeted_content(html_content: str, target_strings: List[str], 
-                          context_size: int = 500) -> str:
-    """
-    Extract content sections that contain specific target strings.
-    
-    Args:
-        html_content: The HTML content to search within
-        target_strings: List of strings to search for in the content
-        context_size: Number of characters to include before and after each match
-        
-    Returns:
-        Extracted content focusing on sections containing target strings
-    """
-    if not target_strings:
-        return ""
-    
-    print_info(f"Extracting content targeted around {len(target_strings)} search strings", "Targeted Extraction")
-    
-    # First clean the HTML to make text extraction easier
-    soup = BeautifulSoup(html_content, 'html.parser')
-    
-    # Remove script, style, and other non-content elements
-    for element in soup(['script', 'style', 'header', 'footer', 'nav']):
-        element.decompose()
-    
-    # Get the full text content
-    full_text = soup.get_text(' ', strip=True)
-    full_text = re.sub(r'\s+', ' ', full_text)  # Normalize whitespace
-    
-    matched_sections = []
-    for target in target_strings:
-        if not target or len(target) < 3:
-            continue
-            
-        # Try exact match first
-        if target in full_text:
-            indices = [m.start() for m in re.finditer(re.escape(target), full_text)]
-            for idx in indices:
-                start = max(0, idx - context_size)
-                end = min(len(full_text), idx + len(target) + context_size)
-                section = full_text[start:end]
-                # Add ellipsis if we're showing a fragment
-                if start > 0:
-                    section = "..." + section
-                if end < len(full_text):
-                    section = section + "..."
-                matched_sections.append(section)
-        else:
-            # Try fuzzy search if no exact match (looking for words in the target string)
-            words = [w for w in target.lower().split() if len(w) > 3]
-            if words:
-                for word in words:
-                    pattern = r'\b' + re.escape(word) + r'\b'
-                    matches = list(re.finditer(pattern, full_text.lower()))
-                    for match in matches[:3]:  # Limit to first 3 matches per word
-                        idx = match.start()
-                        start = max(0, idx - context_size)
-                        end = min(len(full_text), idx + len(word) + context_size)
-                        section = full_text[start:end]
-                        if start > 0:
-                            section = "..." + section
-                        if end < len(full_text):
-                            section = section + "..."
-                        matched_sections.append(section)
-    
-    # Deduplicate similar sections
-    unique_sections = []
-    for section in matched_sections:
-        if not any(similar_text(section, existing, threshold=0.7) for existing in unique_sections):
-            unique_sections.append(section)
-    
-    if not unique_sections:
-        print_warning("Targeted Extraction: No content sections found matching the target strings")
-        return ""
-    
-    # Join the sections with paragraph breaks
-    result = "\n\n".join(unique_sections)
-    print_success(f"Found {len(unique_sections)} relevant content sections", "Targeted Extraction")
-    
-    return clean_text(result)
-
-
-def extract_structured_content(html_content: str, url: str = None, 
-                              target_strings: List[str] = None) -> Dict[str, Union[str, List[str]]]:
-    """
-    Extract structured content from a webpage, including title, main text, and key points.
-    
-    Args:
-        html_content: The HTML content to extract from
-        url: Optional URL for methods that require it
-        target_strings: Optional list of strings to target specific content sections
-        
-    Returns:
-        Dictionary with structured content elements
-    """
-    soup = BeautifulSoup(html_content, 'html.parser')
-    
-    # Extract title
-    title = ""
-    if soup.title:
-        title = soup.title.text.strip()
-    
-    # Try to get more specific title from h1 if title looks generic
-    if not title or len(title) < 10:
-        h1_tags = soup.find_all('h1')
-        if h1_tags and len(h1_tags[0].text.strip()) > 10:
-            title = h1_tags[0].text.strip()
-    
-    # Extract main content using trafilatura (our primary extractor)
-    main_text = extract_clean_text(html_content, method='trafilatura', url=url)
-    
-    # If target strings are provided, prioritize content around those strings
-    targeted_text = ""
-    if target_strings:
-        targeted_text = extract_targeted_content(html_content, target_strings)
-        if targeted_text:
-            main_text = targeted_text
-    
-    # Extract key points (using headers)
-    key_points = []
-    for header in soup.find_all(['h1', 'h2', 'h3']):
-        text = header.text.strip()
-        if text and len(text) > 5 and text not in key_points:
-            key_points.append(text)
-    
-    # For news aggregators like Google News, look for news article titles specifically
-    if url and ('news.google.com' in url or 'news.yahoo.com' in url or 'msn.com/news' in url):
-        print_info("Detected news aggregator site, searching for article titles", "Content Extraction")
-        
-        # Look for common news article title patterns
-        article_titles = []
-        
-        # Google News specific article elements
-        articles = soup.find_all('article')
-        for article in articles[:20]:  # Limit to first 20 articles
-            # Try to find the headline
-            headline = article.find(['h3', 'h4'])
-            if headline:
-                title = headline.text.strip()
-                if title and len(title) > 15 and title not in article_titles:  # Skip short titles
-                    article_titles.append(title)
-                    
-        # Add these to our key points
-        if article_titles:
-            key_points = article_titles + key_points
-            
-    # Limit key points to most important ones
-    key_points = key_points[:15]
-    
-    # Extract potential highlights (often in <strong>, <b>, <em> tags)
-    highlights = []
-    for tag in soup.find_all(['strong', 'b', 'em']):
-        text = tag.text.strip()
-        if text and len(text) > 15 and text not in highlights:
-            highlights.append(text)
-    
-    # Limit highlights to most important ones
-    highlights = highlights[:5]
-    
-    # Create a summary of the extracted content
-    summary = ""
-    if len(main_text) > 200:
-        # Extract first paragraph or two for summary
-        paragraphs = main_text.split('\n\n')
-        summary = '\n\n'.join(paragraphs[:2])
-        if len(summary) > 500:
-            summary = summary[:500] + "..."
-    
-    return {
-        "title": title,
-        "main_text": main_text,
-        "key_points": key_points,
-        "highlights": highlights,
-        "summary": summary,
-        "word_count": len(main_text.split()),
-        "targeted_extraction": bool(target_strings and targeted_text)
-    }

janito/tools/fetch_webpage/news.py

@@ -1,137 +0,0 @@
-"""
-Specialized functionality for handling news aggregator sites.
-"""
-
-from typing import Tuple, List
-from urllib.parse import urlparse
-from bs4 import BeautifulSoup
-import re
-
-from janito.tools.rich_console import print_info, print_success, print_warning
-from janito.tools.usage_tracker import track_usage
-from janito.tools.fetch_webpage.utils import SITE_SPECIFIC_STRATEGIES
-# Import moved to function to avoid circular imports
-from janito.tools.fetch_webpage.extractors import extract_clean_text
-from janito.tools.fetch_webpage.chunking import chunk_large_content
-
-
-@track_usage('web_content')
-def fetch_and_extract_news_aggregator(url: str, max_stories: int = 15) -> Tuple[str, bool]:
-    """
-    Specialized extraction for news aggregator sites like Google News.
-    
-    Args:
-        url: The URL of the news aggregator site
-        max_stories: Maximum number of stories to extract
-        
-    Returns:
-        A tuple containing (extracted_content, is_error)
-    """
-    domain = urlparse(url).netloc
-    
-    # Check if we have a specific strategy for this domain
-    strategy = None
-    for site_domain, site_strategy in SITE_SPECIFIC_STRATEGIES.items():
-        if site_domain in domain:
-            strategy = site_strategy
-            break
-    
-    if not strategy:
-        print_warning(f"News Extraction: No specific strategy found for {domain}. Using general extraction.")
-        from janito.tools.fetch_webpage.core import fetch_and_extract
-        return fetch_and_extract(url)
-    
-    print_info(f"Using specialized extraction for {domain}", "News Extraction")
-    
-    # Import here to avoid circular imports
-    from janito.tools.fetch_webpage.core import fetch_webpage
-    
-    # Fetch the page
-    html_content, is_error = fetch_webpage(url, max_size=2000000)  # Limit to 2MB for news sites
-    
-    if is_error:
-        return html_content, True
-    
-    # Extract content using the site-specific strategy
-    extracted_text = extract_clean_text(
-        html_content, 
-        method=strategy.get("method", "beautifulsoup"),
-        url=url,
-        target_strings=strategy.get("target_strings", [])
-    )
-    
-    if not extracted_text or len(extracted_text) < 100:
-        return f"Could not extract meaningful content from {url}", True
-    
-    # Get article titles and snippets using BeautifulSoup
-    soup = BeautifulSoup(html_content, 'html.parser')
-    
-    article_titles = []
-    article_snippets = []
-    
-    # Use site-specific selectors
-    selectors = strategy.get("article_selectors", ["article", "h3", "h4"])
-    
-    # Find article elements
-    for selector in selectors:
-        elements = soup.select(selector)
-        for element in elements[:max_stories*2]:  # Get more than we need, then filter
-            text = element.get_text(strip=True)
-            if text and len(text) > 15:
-                if len(text) < 200:  # Likely a title
-                    if text not in article_titles:
-                        article_titles.append(text)
-                else:  # Likely a snippet
-                    if text not in article_snippets:
-                        article_snippets.append(text)
-    
-    # Limit to requested number of stories
-    article_titles = article_titles[:max_stories]
-    article_snippets = article_snippets[:max_stories]
-    
-    # Format the result
-    result = ["# Top Stories\n"]
-    
-    # Add titles and snippets
-    for i, title in enumerate(article_titles):
-        result.append(f"## {title}")
-        # Try to find a matching snippet
-        snippet_added = False
-        for snippet in article_snippets:
-            # Check if any significant words from title appear in snippet
-            title_words = set(re.findall(r'\b\w{5,}\b', title.lower()))
-            if any(word in snippet.lower() for word in title_words if len(word) > 4):
-                result.append(f"{snippet[:300]}...")
-                snippet_added = True
-                break
-        
-        if not snippet_added and i < len(article_snippets):
-            result.append(f"{article_snippets[i][:300]}...")
-        
-        result.append("")  # Add spacing between articles
-    
-    # If we didn't get enough specific articles, add some generic extracted content
-    if len(article_titles) < 3:
-        # Chunk the generic extracted content
-        chunks = chunk_large_content(extracted_text, chunk_size=2000, overlap=200)
-        relevant_chunks = []
-        
-        # Find chunks that look like news
-        for chunk in chunks[:10]:
-            if any(marker in chunk for marker in [":", " - ", "reports", "according to", "says"]):
-                relevant_chunks.append(chunk)
-        
-        if relevant_chunks:
-            result.append("# Additional News Content\n")
-            result.append("\n".join(relevant_chunks[:3]))
-    
-    max_length = strategy.get("max_length", 15000)
-    final_text = "\n".join(result)
-    
-    # Truncate if needed
-    if len(final_text) > max_length:
-        print_info(f"Truncating content from {len(final_text)} to {max_length} characters", "News Extraction")
-        final_text = final_text[:max_length] + "..."
-    
-    print_success(f"Successfully extracted {len(article_titles)} news stories", "News Extraction")
-    return final_text, False

janito/tools/fetch_webpage/utils.py

@@ -1,108 +0,0 @@
-"""
-Utility functions and constants for the fetch_webpage package.
-"""
-
-import re
-import html
-import unicodedata
-
-# Dictionary of known content types and extraction strategies
-SITE_SPECIFIC_STRATEGIES = {
-    "news.google.com": {
-        "method": "beautifulsoup",
-        "target_strings": [
-            "Top stories", "Headlines", "For you",
-            "U.S.", "World", "Business", "Technology",
-            "Entertainment", "Sports", "Science", "Health"
-        ],
-        "max_length": 20000,
-        "article_selectors": ["article", "h3", "h4", ".ipQwMb", ".BOz6fb", ".MgUUmf"]
-    },
-    "news.yahoo.com": {
-        "method": "beautifulsoup",
-        "target_strings": ["Top Stories", "Trending News"],
-        "max_length": 20000,
-        "article_selectors": [".js-stream-content", ".js-content", "h3", "h2"]
-    },
-    "msn.com": {
-        "method": "newspaper",
-        "max_length": 20000,
-        "target_strings": ["Top stories", "Headlines"]
-    },
-    "reddit.com": {
-        "method": "trafilatura",
-        "target_strings": ["comments", "Posted by", "communities"],
-        "max_length": 15000,
-        "article_selectors": [".Post", "h1", "h2", ".title"]
-    },
-    "twitter.com": {
-        "method": "beautifulsoup",
-        "target_strings": ["Trending", "Following", "For you"],
-        "max_length": 15000,
-        "article_selectors": [".tweet", ".content", "[data-testid='tweet']"]
-    }
-}
-
-
-def clean_text(text: str) -> str:
-    """
-    Clean extracted text by removing extra whitespace, normalizing Unicode, etc.
-    
-    Args:
-        text: The text to clean
-        
-    Returns:
-        Cleaned text
-    """
-    # Decode HTML entities
-    text = html.unescape(text)
-    
-    # Normalize Unicode characters
-    text = unicodedata.normalize('NFKC', text)
-    
-    # Remove excess whitespace
-    text = re.sub(r'\s+', ' ', text)
-    
-    # Remove duplicate newlines (but preserve paragraph breaks)
-    text = re.sub(r'\n{3,}', '\n\n', text)
-    
-    # Remove very short lines that are often menu items or UI elements
-    lines = [line for line in text.split('\n') if len(line.strip()) > 20]
-    text = '\n'.join(lines)
-    
-    return text.strip()
-
-
-def similar_text(text1: str, text2: str, threshold: float = 0.7) -> bool:
-    """
-    Check if two text strings are similar using a simple similarity metric.
-    
-    Args:
-        text1: First text string
-        text2: Second text string
-        threshold: Similarity threshold (0-1)
-        
-    Returns:
-        True if texts are similar, False otherwise
-    """
-    # Simple character-based similarity
-    if len(text1) == 0 or len(text2) == 0:
-        return False
-    
-    # If one string is much shorter than the other, they're not similar
-    if len(text1) < len(text2) * 0.5 or len(text2) < len(text1) * 0.5:
-        return False
-    
-    # Check for substring relationship
-    if text1 in text2 or text2 in text1:
-        return True
-    
-    # Simple character-based similarity for short strings
-    if len(text1) < 200 and len(text2) < 200:
-        shorter = text1 if len(text1) <= len(text2) else text2
-        longer = text2 if len(text1) <= len(text2) else text1
-        
-        matches = sum(c1 == c2 for c1, c2 in zip(shorter, longer))
-        return matches / len(shorter) >= threshold
-    
-    return False