videonut 1.0.1 → 1.1.0

package/tools/downloaders/article_screenshotter.py

@@ -0,0 +1,388 @@
+#!/usr/bin/env python3
+"""
+Smart Article Screenshotter for VideoNut
+Takes screenshots of news articles with specific quotes highlighted.
+Can scroll to specific text, highlight it, and capture focused screenshots.
+
+USAGE:
+  # Find and highlight a specific quote
+  python article_screenshotter.py --url "https://example.com/article" --quote "exact words to find" --output "quote.png"
+  
+  # Screenshot without highlighting
+  python article_screenshotter.py --url "https://example.com" --quote "text" --no-highlight --output "quote.png"
+"""
+
+import sys
+import os
+import argparse
+import time
+from random import uniform
+
+try:
+    from playwright.sync_api import sync_playwright
+except ImportError:
+    print("Error: Playwright not installed. Install with: pip install playwright && playwright install chromium")
+    sys.exit(1)
+
+
+def normalize_text(text):
+    """Normalize text for comparison - remove extra spaces, newlines."""
+    import re
+    return re.sub(r'\s+', ' ', text.strip().lower())
+
+
+def find_quote_in_page(page, quote):
+    """
+    Find the specific element containing the quote using multiple strategies.
+    Returns the element locator or None.
+    """
+    quote_normalized = normalize_text(quote)
+    
+    # Strategy 1: Try exact text match with Playwright
+    print(f"  Strategy 1: Exact text match...")
+    locator = page.get_by_text(quote, exact=False)
+    if locator.count() > 0:
+        print(f"  ✅ Found with Strategy 1")
+        return locator.first
+    
+    # Strategy 2: Try first few words (in case quote is long)
+    words = quote.split()
+    if len(words) > 5:
+        short_quote = ' '.join(words[:5])
+        print(f"  Strategy 2: First 5 words: '{short_quote}'...")
+        locator = page.get_by_text(short_quote, exact=False)
+        if locator.count() > 0:
+            print(f"  ✅ Found with Strategy 2")
+            return locator.first
+    
+    # Strategy 3: JavaScript search across all text nodes
+    print(f"  Strategy 3: JavaScript deep search...")
+    element_handle = page.evaluate_handle('''(searchText) => {
+        const normalizeText = (t) => t.replace(/\\s+/g, ' ').trim().toLowerCase();
+        const searchNorm = normalizeText(searchText);
+        
+        // Search in common content elements
+        const selectors = ['p', 'span', 'div', 'blockquote', 'h1', 'h2', 'h3', 'h4', 'li', 'td', 'article'];
+        
+        for (const selector of selectors) {
+            const elements = document.querySelectorAll(selector);
+            for (const el of elements) {
+                const text = normalizeText(el.innerText || '');
+                if (text.includes(searchNorm)) {
+                    return el;
+                }
+            }
+        }
+        
+        // Fallback: search entire body
+        const allElements = document.querySelectorAll('*');
+        for (const el of allElements) {
+            if (el.innerText) {
+                const text = normalizeText(el.innerText);
+                if (text.includes(searchNorm) && el.innerText.length < 2000) {
+                    return el;
+                }
+            }
+        }
+        
+        return null;
+    }''', quote)
+    
+    if element_handle:
+        element = element_handle.as_element()
+        if element:
+            print(f"  ✅ Found with Strategy 3 (JavaScript)")
+            return element
+    
+    print(f"  ❌ Quote not found with any strategy")
+    return None
+
+
+def take_quote_screenshot(url, output_path, quote=None, highlight=True, width=1280, height=900):
+    """
+    Take a screenshot of a webpage, focusing on a specific quote.
+    
+    Args:
+        url: URL of the article
+        output_path: Where to save the screenshot
+        quote: Text to find and focus on (REQUIRED for meaningful screenshot)
+        highlight: Whether to highlight the found text
+        width: Viewport width
+        height: Viewport height
+    
+    Returns:
+        Dict with success status and details
+    """
+    # Rate limiting
+    delay = uniform(1, 2)
+    print(f"⏳ Rate limiting: Waiting {delay:.2f} seconds...")
+    time.sleep(delay)
+    
+    result = {
+        'success': False,
+        'quote_found': False,
+        'message': '',
+        'output_path': output_path
+    }
+    
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(
+            viewport={'width': width, 'height': height},
+            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        )
+        page = context.new_page()
+        
+        # Set headers to appear like real browser
+        page.set_extra_http_headers({
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Accept-Encoding": "gzip, deflate",
+            "Connection": "keep-alive",
+        })
+        
+        try:
+            print(f"🌐 Navigating to {url}...")
+            # Use networkidle for better dynamic content handling
+            try:
+                page.goto(url, timeout=45000, wait_until='networkidle')
+            except:
+                # Fallback to domcontentloaded if networkidle times out
+                print("  ⚠️ networkidle timeout, using domcontentloaded...")
+                page.goto(url, timeout=30000, wait_until='domcontentloaded')
+            
+            # Wait for dynamic content (increased from 3s to 5s for JS-heavy sites)
+            print("  ⏳ Waiting for dynamic content to load...")
+            page.wait_for_timeout(5000)
+            
+            # Try to close cookie popups, modals, and ads
+            print("🧹 Closing popups...")
+            for selector in [
+                'button:has-text("Accept")', 
+                'button:has-text("I Agree")', 
+                'button:has-text("Got it")',
+                'button:has-text("Continue")',
+                '.close-button', 
+                '[aria-label="Close"]', 
+                '.modal-close',
+                '.popup-close',
+                '#close-btn'
+            ]:
+                try:
+                    if page.locator(selector).count() > 0:
+                        page.locator(selector).first.click(timeout=1000)
+                        page.wait_for_timeout(500)
+                except:
+                    pass
+            
+            if quote:
+                print(f"🔍 Searching for quote: '{quote[:60]}{'...' if len(quote) > 60 else ''}'")
+                
+                # Find the quote element
+                element = find_quote_in_page(page, quote)
+                
+                if element:
+                    result['quote_found'] = True
+                    
+                    # Step 1: Try multiple scroll methods (some sites block certain approaches)
+                    print("📍 Scrolling quote to center of viewport...")
+                    
+                    # Method 1: scrollIntoView with block center (most reliable)
+                    try:
+                        page.evaluate('''(el) => {
+                            el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' });
+                        }''', element)
+                        page.wait_for_timeout(500)
+                        print("  ✅ Scroll method 1 (scrollIntoView) succeeded")
+                    except Exception as scroll_err:
+                        print(f"  ⚠️ Scroll method 1 failed: {scroll_err}")
+                        
+                        # Method 2: Manual scrollTo calculation as fallback
+                        try:
+                            page.evaluate('''(el) => {
+                                const rect = el.getBoundingClientRect();
+                                const scrollTop = window.pageYOffset + rect.top - (window.innerHeight / 2) + (rect.height / 2);
+                                window.scrollTo({ top: Math.max(0, scrollTop), behavior: 'instant' });
+                            }''', element)
+                            page.wait_for_timeout(500)
+                            print("  ✅ Scroll method 2 (scrollTo) succeeded")
+                        except Exception as scroll_err2:
+                            print(f"  ⚠️ Scroll method 2 also failed: {scroll_err2}")
+                    
+                    # Step 1.5: Verify element is now visible in viewport
+                    is_visible = page.evaluate('''(el) => {
+                        const rect = el.getBoundingClientRect();
+                        return rect.top >= 0 && rect.bottom <= window.innerHeight;
+                    }''', element)
+                    
+                    if not is_visible:
+                        print("  ⚠️ Element not fully visible, trying Playwright scroll...")
+                        try:
+                            element.scroll_into_view_if_needed()
+                            page.wait_for_timeout(500)
+                        except:
+                            pass
+                    
+                    # Step 2: Highlight the element
+                    if highlight:
+                        print("🎨 Highlighting quote...")
+                        page.evaluate('''(el) => {
+                            // Save original styles
+                            el.setAttribute('data-original-style', el.getAttribute('style') || '');
+                            
+                            // Apply highlight styles with !important to override site CSS
+                            el.style.setProperty('background-color', '#ffff00', 'important');
+                            el.style.setProperty('color', '#000000', 'important');
+                            el.style.setProperty('padding', '10px', 'important');
+                            el.style.setProperty('border-radius', '4px', 'important');
+                            el.style.setProperty('border', '4px solid #ff6600', 'important');
+                            el.style.setProperty('box-shadow', '0 0 30px rgba(255, 102, 0, 0.8)', 'important');
+                            el.style.setProperty('position', 'relative', 'important');
+                            el.style.setProperty('z-index', '99999', 'important');
+                            el.style.setProperty('display', 'block', 'important');
+                        }''', element)
+                        print("  ✅ Quote highlighted with yellow background + orange border")
+                    
+                    # Step 3: Wait for CSS to apply and re-render
+                    page.wait_for_timeout(800)
+                    
+                    # Step 4: Take the screenshot
+                    print("📸 Taking screenshot...")
+                    page.screenshot(path=output_path)
+                    result['success'] = True
+                    result['message'] = f"Quote found, centered, and captured: '{quote[:40]}...'"
+                    
+                else:
+                    # Quote NOT found - try fuzzy fallback
+                    print("⚠️ Exact quote not found. Trying fuzzy search...")
+                    
+                    # Try with just the first 3 words
+                    words = quote.split()
+                    if len(words) >= 3:
+                        fuzzy_quote = ' '.join(words[:3])
+                        fuzzy_element = find_quote_in_page(page, fuzzy_quote)
+                        
+                        if fuzzy_element:
+                            print(f"  ✅ Found partial match with: '{fuzzy_quote}'")
+                            
+                            # Scroll and highlight
+                            page.evaluate('''(el) => {
+                                const rect = el.getBoundingClientRect();
+                                const scrollTop = window.pageYOffset + rect.top - (window.innerHeight / 2);
+                                window.scrollTo({ top: scrollTop, behavior: 'instant' });
+                            }''', fuzzy_element)
+                            
+                            if highlight:
+                                page.evaluate('''(el) => {
+                                    el.style.backgroundColor = '#ffff00';
+                                    el.style.border = '3px solid #ff6600';
+                                    el.style.padding = '8px';
+                                }''', fuzzy_element)
+                            
+                            page.wait_for_timeout(300)
+                            page.screenshot(path=output_path)
+                            result['success'] = True
+                            result['quote_found'] = True
+                            result['message'] = f"Partial match found: '{fuzzy_quote}'"
+                        else:
+                            # Complete failure
+                            result['success'] = False
+                            result['quote_found'] = False
+                            result['message'] = f"ERROR: Quote not found on page: '{quote[:50]}...'"
+                            print(f"  ❌ {result['message']}")
+                            # Don't take useless screenshot
+                    else:
+                        result['success'] = False
+                        result['message'] = f"ERROR: Quote too short and not found: '{quote}'"
+            
+            else:
+                # No quote provided - just screenshot the article content
+                print("📸 No quote specified. Taking article screenshot...")
+                
+                # Try to find and scroll to main article content
+                for selector in ['article', '.article-content', '.story-content', 
+                                '.post-content', 'main', '#content', '.entry-content']:
+                    if page.locator(selector).count() > 0:
+                        page.locator(selector).first.scroll_into_view_if_needed()
+                        break
+                
+                page.screenshot(path=output_path)
+                result['success'] = True
+                result['message'] = "Article screenshot captured (no specific quote)"
+            
+            # Validate file was created
+            if result['success'] and os.path.exists(output_path):
+                size = os.path.getsize(output_path)
+                if size > 0:
+                    print(f"✅ Screenshot saved: {output_path} ({size:,} bytes)")
+                else:
+                    result['success'] = False
+                    result['message'] = "Screenshot file is empty"
+                    
+        except Exception as e:
+            result['message'] = f"Error: {str(e)}"
+            print(f"❌ Error: {e}")
+            
+        finally:
+            browser.close()
+    
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Take screenshots of news articles with quote highlighting.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Find and highlight a specific quote (RECOMMENDED)
+  python article_screenshotter.py --url "https://timesofindia.com/article" --quote "PM Modi said this is important" --output "quote.png"
+  
+  # Screenshot without highlighting
+  python article_screenshotter.py --url "https://example.com" --quote "important text" --no-highlight --output "quote.png"
+  
+  # Just capture article (no specific quote)
+  python article_screenshotter.py --url "https://example.com" --output "article.png"
+
+NOTE: Always provide --quote for meaningful screenshots. Without it, you just get the page header.
+        """
+    )
+    
+    parser.add_argument("--url", "-u", required=True, help="URL of the article")
+    parser.add_argument("--output", "-o", required=True, help="Output file path for screenshot")
+    parser.add_argument("--quote", "-q", help="Specific quote/text to find, center, and highlight (REQUIRED for useful screenshots)")
+    parser.add_argument("--no-highlight", action="store_true", help="Don't highlight the found text")
+    parser.add_argument("--width", "-w", type=int, default=1280, help="Viewport width (default: 1280)")
+    parser.add_argument("--height", "-H", type=int, default=900, help="Viewport height (default: 900)")
+    
+    args = parser.parse_args()
+    
+    # Warn if no quote provided
+    if not args.quote:
+        print("⚠️ WARNING: No --quote provided. Screenshot will just be the page header.")
+        print("   For useful screenshots, always provide the specific text you want to capture.")
+    
+    # Ensure output directory exists
+    output_dir = os.path.dirname(args.output)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+    
+    result = take_quote_screenshot(
+        url=args.url,
+        output_path=args.output,
+        quote=args.quote,
+        highlight=not args.no_highlight,
+        width=args.width,
+        height=args.height
+    )
+    
+    if result['success']:
+        print(f"\n✅ SUCCESS: {result['message']}")
+        sys.exit(0)
+    else:
+        print(f"\n❌ FAILED: {result['message']}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/tools/downloaders/caption_reader.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""
+YouTube Caption/Transcript Reader for VideoNut
+Extracts captions from YouTube videos with optional timestamp display.
+"""
+
+import sys
+import argparse
+import json
+from youtube_transcript_api import YouTubeTranscriptApi
+from youtube_transcript_api.formatters import TextFormatter, JSONFormatter
+import re
+
+
+def extract_video_id(url):
+    """
+    Extract YouTube video ID from various URL formats
+    """
+    # Patterns for different YouTube URL formats
+    patterns = [
+        r'(?:https?:\/\/)?(?:www\.)?youtube\.com\/watch\?v=([a-zA-Z0-9_-]{11,12})',
+        r'(?:https?:\/\/)?(?:www\.)?youtu\.be\/([a-zA-Z0-9_-]{11,12})',
+        r'(?:https?:\/\/)?(?:www\.)?youtube\.com\/embed\/([a-zA-Z0-9_-]{11,12})',
+        r'(?:https?:\/\/)?(?:www\.)?youtube\.com\/v\/([a-zA-Z0-9_-]{11,12})',
+    ]
+    
+    for pattern in patterns:
+        match = re.search(pattern, url)
+        if match:
+            return match.group(1)
+    
+    return None
+
+
+def format_timestamp(seconds):
+    """Convert seconds to HH:MM:SS format"""
+    hours = int(seconds // 3600)
+    minutes = int((seconds % 3600) // 60)
+    secs = int(seconds % 60)
+    
+    if hours > 0:
+        return f"{hours:02d}:{minutes:02d}:{secs:02d}"
+    else:
+        return f"{minutes:02d}:{secs:02d}"
+
+
+def get_youtube_captions(url, languages=None, with_timestamps=False, search_term=None):
+    """
+    Get YouTube video captions/transcript
+    
+    Args:
+        url: YouTube video URL
+        languages: List of preferred language codes
+        with_timestamps: If True, include timestamps with each line
+        search_term: If provided, only return lines containing this term (with timestamps)
+    
+    Returns:
+        Formatted transcript string
+    """
+    if languages is None:
+        # Default to English and other common languages
+        languages = ['en', 'en-US', 'en-GB', 'hi', 'te', 'ta', 'mr', 'es', 'fr', 'de']
+    
+    video_id = extract_video_id(url)
+    
+    if not video_id:
+        print(f"Error: Could not extract video ID from URL: {url}", file=sys.stderr)
+        sys.exit(1)
+    
+    try:
+        # Instantiate the API class
+        api = YouTubeTranscriptApi()
+        
+        # Fetch the transcript data directly using the instance method
+        transcript_data = api.fetch(video_id, languages=languages)
+        
+        # If searching for a term, filter and return with timestamps
+        if search_term:
+            search_lower = search_term.lower()
+            matches = []
+            for entry in transcript_data:
+                if search_lower in entry.text.lower():
+                    timestamp = format_timestamp(entry.start)
+                    duration = getattr(entry, 'duration', 0)
+                    end_timestamp = format_timestamp(entry.start + duration)
+                    matches.append({
+                        'timestamp': timestamp,
+                        'end_timestamp': end_timestamp,
+                        'start_seconds': entry.start,
+                        'text': entry.text
+                    })
+            
+            if not matches:
+                return f"No matches found for '{search_term}' in transcript."
+            
+            output = [f"\n🔍 Found {len(matches)} matches for '{search_term}':\n"]
+            for match in matches:
+                output.append(f"[{match['timestamp']}] {match['text']}")
+            output.append(f"\n📋 Suggested clip range: {matches[0]['timestamp']} - {matches[-1]['end_timestamp']}")
+            return '\n'.join(output)
+        
+        # If with_timestamps, format each line with timestamp
+        if with_timestamps:
+            output = []
+            output.append(f"\n📝 Transcript with Timestamps:\n")
+            output.append("=" * 60)
+            for entry in transcript_data:
+                timestamp = format_timestamp(entry.start)
+                output.append(f"[{timestamp}] {entry.text}")
+            return '\n'.join(output)
+        
+        # Default: plain text format - join all text entries
+        plain_text = ' '.join([entry.text for entry in transcript_data])
+        
+        return plain_text
+        
+    except Exception as e:
+        print(f"Error retrieving captions: {str(e)}", file=sys.stderr)
+        sys.exit(1)
+
+
+def find_timestamp_for_quote(url, quote, context_seconds=30):
+    """
+    Find the timestamp where a specific quote appears in the video.
+    Returns the start and end timestamps for a clip containing that quote.
+    
+    Args:
+        url: YouTube video URL
+        quote: The quote to search for
+        context_seconds: How many seconds of context to include before/after
+    
+    Returns:
+        Dict with start_time, end_time, and surrounding text
+    """
+    video_id = extract_video_id(url)
+    if not video_id:
+        return None
+    
+    try:
+        api = YouTubeTranscriptApi()
+        # Use fetch to get the default transcript or specify languages
+        transcript_data = api.fetch(video_id)
+        
+        quote_lower = quote.lower()
+        
+        for i, entry in enumerate(transcript_data):
+            if quote_lower in entry.text.lower():
+                # Found the quote
+                start_time = max(0, entry.start - context_seconds)
+                end_time = entry.start + getattr(entry, 'duration', 5) + context_seconds
+                
+                # Get surrounding context
+                context_entries = []
+                for j in range(max(0, i-3), min(len(transcript_data), i+4)):
+                    context_entries.append({
+                        'timestamp': format_timestamp(transcript_data[j].start),
+                        'text': transcript_data[j].text
+                    })
+                
+                return {
+                    'found': True,
+                    'quote': entry.text,
+                    'timestamp': format_timestamp(entry.start),
+                    'clip_start': format_timestamp(start_time),
+                    'clip_end': format_timestamp(end_time),
+                    'context': context_entries
+                }
+        
+        return {'found': False, 'message': f"Quote not found: {quote}"}
+        
+    except Exception as e:
+        return {'found': False, 'message': str(e)}
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract captions from YouTube videos with optional timestamps.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Get plain transcript
+  python caption_reader.py --url "https://youtube.com/watch?v=xxx"
+  
+  # Get transcript with timestamps
+  python caption_reader.py --url "https://youtube.com/watch?v=xxx" --timestamps
+  
+  # Search for specific term and get timestamps
+  python caption_reader.py --url "https://youtube.com/watch?v=xxx" --search "electoral bonds"
+  
+  # Find timestamp for a specific quote
+  python caption_reader.py --url "https://youtube.com/watch?v=xxx" --find-quote "corruption" --json
+        """
+    )
+    
+    parser.add_argument("--url", required=True, help="YouTube video URL")
+    parser.add_argument("--languages", nargs="*", default=None, 
+                       help="Preferred language codes (e.g., en hi te). Default: en and Indian languages")
+    parser.add_argument("--timestamps", "-t", action="store_true",
+                       help="Include timestamps with each line")
+    parser.add_argument("--search", "-s", help="Search for specific term and show timestamps")
+    parser.add_argument("--find-quote", "-f", help="Find exact timestamp for a quote")
+    parser.add_argument("--context", "-c", type=int, default=30,
+                       help="Seconds of context around found quote (default: 30)")
+    parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")
+    
+    args = parser.parse_args()
+    
+    if args.find_quote:
+        # Find timestamp for specific quote
+        result = find_timestamp_for_quote(args.url, args.find_quote, args.context)
+        if args.json:
+            print(json.dumps(result, indent=2, ensure_ascii=False))
+        else:
+            if result.get('found'):
+                print(f"\n✅ Quote Found!")
+                print(f"   Timestamp: {result['timestamp']}")
+                print(f"   Text: {result['quote']}")
+                print(f"\n🎬 Suggested Clip:")
+                print(f"   Start: {result['clip_start']}")
+                print(f"   End: {result['clip_end']}")
+                print(f"\n📄 Context:")
+                for entry in result['context']:
+                    print(f"   [{entry['timestamp']}] {entry['text']}")
+            else:
+                print(f"❌ {result.get('message', 'Quote not found')}")
+    else:
+        # Get transcript
+        captions = get_youtube_captions(
+            args.url, 
+            args.languages, 
+            with_timestamps=args.timestamps,
+            search_term=args.search
+        )
+        print(captions)
+
+
+if __name__ == "__main__":
+    main()