videonut 1.0.1 → 1.1.0

package/tools/downloaders/clip_grabber.py

@@ -0,0 +1,83 @@
+import os
+import sys
+import subprocess
+import argparse
+
+def download_clip(url, start_time, end_time, output_path, ffmpeg_path):
+    """
+    Downloads a specific clip from a YouTube video using yt-dlp.
+    """
+    # Ensure output directory exists if it's not the current directory
+    dir_name = os.path.dirname(output_path)
+    if dir_name:
+        os.makedirs(dir_name, exist_ok=True)
+
+    # Construct the yt-dlp command
+    # --download-sections "*start-end" downloads only that range
+    # --force-keyframes-at-cuts ensures precise cutting (requires ffmpeg)
+    cmd = [
+        "yt-dlp",
+        "--verbose",
+        "--download-sections", f"*{start_time}-{end_time}",
+        "--force-keyframes-at-cuts",
+        "--ffmpeg-location", ffmpeg_path,
+        "-o", output_path,
+        url
+    ]
+
+    print(f"Executing: ", ' '.join(cmd))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        print("Download successful.")
+        print(result.stdout)
+
+        # Validate that the file was created and has content
+        if os.path.exists(output_path):
+            file_size = os.path.getsize(output_path)
+            if file_size == 0:
+                print(f"Error: Downloaded file is empty: {output_path}")
+                sys.exit(1)
+            else:
+                print(f"File validation: {output_path} created with size {file_size} bytes")
+        else:
+            print(f"Error: Downloaded file does not exist: {output_path}")
+            sys.exit(1)
+
+    except subprocess.CalledProcessError as e:
+        print("Error during download:")
+        print(e.stderr)
+        sys.exit(1)
+    except FileNotFoundError:
+        print("Error: yt-dlp not found. Please install it (pip install yt-dlp) and ensure it's in your PATH.")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download a video clip.")
+    parser.add_argument("--url", required=True, help="Video URL")
+    parser.add_argument("--start", required=True, help="Start time (e.g., 00:00:10 or 10)")
+    parser.add_argument("--end", required=True, help="End time (e.g., 00:00:20 or 20)")
+    parser.add_argument("--output", required=True, help="Output file path")
+
+    # Try to find ffmpeg in system PATH first
+    import shutil
+    import platform
+    default_ffmpeg = shutil.which("ffmpeg")
+    if not default_ffmpeg:
+        # Fallback to local bin folder relative to this script
+        # Assumes structure: tools/downloaders/clip_grabber.py -> tools/bin/ffmpeg.exe (Windows) or tools/bin/ffmpeg (Unix)
+        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        # Determine the appropriate executable name based on the platform
+        ffmpeg_exe = "ffmpeg.exe" if platform.system().lower() == "windows" else "ffmpeg"
+        default_ffmpeg = os.path.join(base_dir, "bin", ffmpeg_exe)
+
+        # If the fallback path doesn't exist, warn the user
+        if not os.path.exists(default_ffmpeg):
+            print(f"Warning: ffmpeg not found in PATH or at expected location: {default_ffmpeg}")
+            print("Please install ffmpeg or place it in the tools/bin/ directory.")
+
+    parser.add_argument("--ffmpeg", default=default_ffmpeg, help="Path to ffmpeg executable")
+
+    args = parser.parse_args()
+
+    download_clip(args.url, args.start, args.end, args.output, args.ffmpeg)

package/tools/downloaders/image_grabber.py

@@ -0,0 +1,106 @@
+import os
+import sys
+import requests
+import argparse
+import mimetypes
+from urllib.parse import urlparse
+
+def is_safe_image_type(content_type, url):
+    """
+    Check if the content type is a safe image type.
+    """
+    safe_types = {
+        'image/jpeg', 'image/jpg', 'image/png', 'image/gif',
+        'image/webp', 'image/bmp', 'image/svg+xml', 'image/tiff'
+    }
+
+    # Check content-type header
+    if content_type and content_type.lower() in safe_types:
+        return True
+
+    # Fallback: check file extension from URL
+    parsed_url = urlparse(url)
+    file_ext = os.path.splitext(parsed_url.path)[1].lower()
+    mime_type, _ = mimetypes.guess_type(f"dummy{file_ext}")
+
+    if mime_type and mime_type in safe_types:
+        return True
+
+    return False
+
+def get_file_size(response):
+    """
+    Get the file size from the response headers.
+    """
+    content_length = response.headers.get('content-length')
+    if content_length:
+        return int(content_length)
+    return 0
+
+def download_image(url, output_path):
+    """
+    Downloads an image from a URL with security validation.
+    """
+    # Ensure output directory exists if it's not the current directory
+    dir_name = os.path.dirname(output_path)
+    if dir_name:
+        os.makedirs(dir_name, exist_ok=True)
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    }
+
+    try:
+        print(f"Downloading image from: {url}")
+
+        # First, make a HEAD request to check content type and size
+        head_response = requests.head(url, headers=headers, timeout=10)
+        content_type = head_response.headers.get('content-type', '').lower()
+
+        # Validate content type
+        if not is_safe_image_type(content_type, url):
+            print(f"Security Error: Content type '{content_type}' is not a safe image type.")
+            sys.exit(1)
+
+        # Check file size (limit to 50MB)
+        file_size = get_file_size(head_response)
+        if file_size > 50 * 1024 * 1024:  # 50MB
+            print(f"Security Error: File size {file_size} bytes exceeds 50MB limit.")
+            sys.exit(1)
+
+        # Actually download the file
+        response = requests.get(url, headers=headers, stream=True, timeout=10)
+        response.raise_for_status()
+
+        # Double-check content type after download
+        downloaded_content_type = response.headers.get('content-type', '').lower()
+        if not is_safe_image_type(downloaded_content_type, url):
+            print(f"Security Error: Downloaded content type '{downloaded_content_type}' is not a safe image type.")
+            sys.exit(1)
+
+        # Write file in chunks with size validation
+        total_size = 0
+        with open(output_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:  # Filter out keep-alive chunks
+                    total_size += len(chunk)
+                    if total_size > 50 * 1024 * 1024:  # 50MB limit
+                        print(f"Security Error: Downloaded file exceeds 50MB limit.")
+                        os.remove(output_path)  # Clean up partial file
+                        sys.exit(1)
+                    f.write(chunk)
+
+        print(f"Successfully saved to {output_path}")
+
+    except Exception as e:
+        print(f"Failed to download image: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download an image.")
+    parser.add_argument("--url", required=True, help="Image URL")
+    parser.add_argument("--output", required=True, help="Output file path")
+
+    args = parser.parse_args()
+
+    download_image(args.url, args.output)

package/tools/downloaders/pdf_reader.py

@@ -0,0 +1,163 @@
+import sys
+import requests
+import io
+import time
+from random import uniform
+from pypdf import PdfReader
+import argparse
+import re
+
+def read_pdf(url, search_term=None, page_number=None):
+    """
+    Read a PDF from URL with optional search and page selection.
+    
+    Args:
+        url: URL of the PDF
+        search_term: Optional term to search for in the PDF
+        page_number: Optional specific page to read (1-indexed)
+    """
+    # Add random delay to implement rate limiting
+    delay = uniform(1, 3)
+    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
+    time.sleep(delay)
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    }
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+
+        f = io.BytesIO(response.content)
+        reader = PdfReader(f)
+        total_pages = len(reader.pages)
+        
+        print(f"📄 PDF loaded: {total_pages} pages")
+        
+        # If specific page requested
+        if page_number:
+            if 1 <= page_number <= total_pages:
+                text = reader.pages[page_number - 1].extract_text()
+                print(f"\n--- Page {page_number} of {total_pages} ---")
+                print(text)
+                return
+            else:
+                print(f"Error: Page {page_number} not found. PDF has {total_pages} pages.")
+                sys.exit(1)
+        
+        # If search term provided, find all occurrences
+        if search_term:
+            print(f"🔍 Searching for: '{search_term}'")
+            matches = []
+            search_lower = search_term.lower()
+            
+            for i, page in enumerate(reader.pages):
+                page_text = page.extract_text()
+                if page_text and search_lower in page_text.lower():
+                    # Find the context around the match
+                    lines = page_text.split('\n')
+                    for j, line in enumerate(lines):
+                        if search_lower in line.lower():
+                            # Get surrounding context (2 lines before/after)
+                            context_start = max(0, j - 2)
+                            context_end = min(len(lines), j + 3)
+                            context = '\n'.join(lines[context_start:context_end])
+                            matches.append({
+                                'page': i + 1,
+                                'line': line.strip(),
+                                'context': context
+                            })
+            
+            if matches:
+                print(f"\n✅ Found {len(matches)} matches for '{search_term}':\n")
+                for idx, match in enumerate(matches[:10]):  # Limit to first 10 matches
+                    print(f"{'='*60}")
+                    print(f"📍 Match {idx+1} - Page {match['page']}")
+                    print(f"{'='*60}")
+                    print(f"Line: {match['line']}")
+                    print(f"\nContext:")
+                    print(match['context'])
+                    print()
+                
+                if len(matches) > 10:
+                    print(f"... and {len(matches) - 10} more matches")
+                
+                # Suggest best page for screenshot
+                best_page = matches[0]['page']
+                print(f"\n📸 Suggested page for screenshot: Page {best_page}")
+                print(f"   Use: python pdf_reader.py --url \"{url}\" --page {best_page}")
+            else:
+                print(f"❌ No matches found for '{search_term}'")
+            return
+        
+        # Default: Smart extraction with priority for first and last pages
+        MAX_PAGES = 15
+        MAX_CHARS = 20000
+        
+        text = ""
+        pages_to_read = []
+        
+        if total_pages <= MAX_PAGES:
+            pages_to_read = list(range(total_pages))
+        else:
+            # Smart selection: first 7 + last 4
+            first_pages = list(range(min(7, total_pages)))
+            last_pages = list(range(max(0, total_pages - 4), total_pages))
+            pages_to_read = sorted(set(first_pages + last_pages))
+            
+            print(f"📄 Document has {total_pages} pages. Reading pages: {[p+1 for p in pages_to_read]} (first + last priority)")
+        
+        for i in pages_to_read:
+            page_text = reader.pages[i].extract_text()
+            if page_text:
+                text += f"\n--- Page {i+1} ---\n{page_text}"
+        
+        # Smart truncation with intro/conclusion preservation
+        if len(text) > MAX_CHARS:
+            intro = text[:8000]
+            outro = text[-8000:]
+            truncated = len(text) - MAX_CHARS
+            print(f"--- PDF CONTENT START ---")
+            print(intro)
+            print(f"\n\n[... {truncated:,} characters truncated from middle ...]\n\n")
+            print(f"--- PDF CONTENT END ---")
+            print(outro)
+        else:
+            print(text)
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error downloading PDF: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading PDF: {e}")
+        sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Read and search PDF documents from URLs.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Read PDF with smart extraction (first + last pages)
+  python pdf_reader.py --url "https://example.com/report.pdf"
+  
+  # Search for specific term in PDF
+  python pdf_reader.py --url "https://example.com/report.pdf" --search "Prime Minister"
+  
+  # Read specific page
+  python pdf_reader.py --url "https://example.com/report.pdf" --page 5
+        """
+    )
+    
+    parser.add_argument("--url", required=True, help="URL of the PDF document")
+    parser.add_argument("--search", "-s", help="Search for specific term and show context")
+    parser.add_argument("--page", "-p", type=int, help="Read specific page number (1-indexed)")
+    
+    args = parser.parse_args()
+    read_pdf(args.url, search_term=args.search, page_number=args.page)
+
+
+if __name__ == "__main__":
+    main()

package/tools/downloaders/pdf_screenshotter.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""
+PDF Page Screenshotter for VideoNut
+Takes screenshots of specific PDF pages, optionally highlighting search terms.
+Uses Playwright to render PDFs in browser for high-quality screenshots.
+
+USAGE:
+  # Screenshot specific page from PDF
+  python pdf_screenshotter.py --url "https://example.com/report.pdf" --page 3 --output "page3.png"
+  
+  # Search for term and screenshot the page where it's found
+  python pdf_screenshotter.py --url "https://example.com/report.pdf" --search "Prime Minister" --output "pm_quote.png"
+"""
+
+import sys
+import os
+import argparse
+import requests
+import io
+import time
+from random import uniform
+from pypdf import PdfReader
+
+try:
+    from playwright.sync_api import sync_playwright
+except ImportError:
+    print("Error: Playwright not installed. Install with: pip install playwright && playwright install chromium")
+    sys.exit(1)
+
+
+def download_pdf_to_temp(url, temp_path):
+    """Download PDF to a temporary file."""
+    delay = uniform(1, 2)
+    print(f"⏳ Rate limiting: Waiting {delay:.2f} seconds...")
+    time.sleep(delay)
+    
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    }
+    
+    response = requests.get(url, headers=headers, timeout=30)
+    response.raise_for_status()
+    
+    with open(temp_path, 'wb') as f:
+        f.write(response.content)
+    
+    return temp_path
+
+
+def find_page_with_term(pdf_path, search_term):
+    """Find the first page containing the search term."""
+    with open(pdf_path, 'rb') as f:
+        reader = PdfReader(f)
+        search_lower = search_term.lower()
+        
+        for i, page in enumerate(reader.pages):
+            page_text = page.extract_text()
+            if page_text and search_lower in page_text.lower():
+                return i + 1  # Return 1-indexed page number
+    
+    return None
+
+
+def screenshot_pdf_page(pdf_path, page_number, output_path, search_term=None, width=1280, height=1600):
+    """
+    Take a screenshot of a specific PDF page using browser rendering.
+    
+    Args:
+        pdf_path: Local path to PDF file
+        page_number: Page to screenshot (1-indexed)
+        output_path: Where to save the screenshot
+        search_term: Optional term to highlight on the page
+        width: Viewport width
+        height: Viewport height
+    """
+    result = {
+        'success': False,
+        'page': page_number,
+        'search_found': False,
+        'message': '',
+        'output_path': output_path
+    }
+    
+    # Verify PDF exists and page is valid
+    with open(pdf_path, 'rb') as f:
+        reader = PdfReader(f)
+        total_pages = len(reader.pages)
+        
+        if page_number < 1 or page_number > total_pages:
+            result['message'] = f"Page {page_number} not found. PDF has {total_pages} pages."
+            return result
+    
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(
+            viewport={'width': width, 'height': height}
+        )
+        page = context.new_page()
+        
+        try:
+            # Open PDF in browser using file:// protocol
+            abs_path = os.path.abspath(pdf_path)
+            pdf_url = f"file:///{abs_path.replace(os.sep, '/')}"
+            
+            print(f"🌐 Opening PDF in browser...")
+            page.goto(pdf_url, timeout=30000, wait_until='networkidle')
+            
+            # Wait for PDF to render
+            page.wait_for_timeout(3000)
+            
+            # Navigate to specific page
+            # Most PDF viewers use #page=N in URL
+            if page_number > 1:
+                page.goto(f"{pdf_url}#page={page_number}", timeout=30000)
+                page.wait_for_timeout(2000)
+            
+            # If search term provided, try to find and highlight
+            if search_term:
+                print(f"🔍 Searching for '{search_term}'...")
+                # Use browser's find function (Ctrl+F simulation)
+                # This is a workaround since direct PDF text selection is complex
+                page.keyboard.press("Control+f")
+                page.wait_for_timeout(500)
+                page.keyboard.type(search_term)
+                page.wait_for_timeout(1000)
+                page.keyboard.press("Escape")
+                result['search_found'] = True
+            
+            # Take screenshot
+            print(f"📸 Taking screenshot of page {page_number}...")
+            page.screenshot(path=output_path, full_page=False)
+            
+            if os.path.exists(output_path) and os.path.getsize(output_path) > 0:
+                result['success'] = True
+                result['message'] = f"Screenshot saved: {output_path}"
+                print(f"✅ {result['message']}")
+            else:
+                result['message'] = "Screenshot file is empty or not created"
+                
+        except Exception as e:
+            result['message'] = f"Error: {str(e)}"
+            print(f"❌ {result['message']}")
+            
+        finally:
+            browser.close()
+    
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Screenshot specific pages from PDF documents.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Screenshot specific page
+  python pdf_screenshotter.py --url "https://example.com/report.pdf" --page 3 --output "page3.png"
+  
+  # Auto-find page with search term and screenshot
+  python pdf_screenshotter.py --url "https://example.com/report.pdf" --search "Prime Minister" --output "pm_quote.png"
+  
+  # Screenshot local PDF file
+  python pdf_screenshotter.py --file "report.pdf" --page 5 --output "page5.png"
+        """
+    )
+    
+    parser.add_argument("--url", "-u", help="URL of the PDF document")
+    parser.add_argument("--file", "-f", help="Local path to PDF file")
+    parser.add_argument("--page", "-p", type=int, help="Page number to screenshot (1-indexed)")
+    parser.add_argument("--search", "-s", help="Search for term and screenshot that page")
+    parser.add_argument("--output", "-o", required=True, help="Output file path for screenshot")
+    parser.add_argument("--width", type=int, default=1280, help="Viewport width (default: 1280)")
+    parser.add_argument("--height", type=int, default=1600, help="Viewport height (default: 1600)")
+    
+    args = parser.parse_args()
+    
+    # Validate inputs
+    if not args.url and not args.file:
+        print("Error: Either --url or --file must be provided")
+        sys.exit(1)
+    
+    if not args.page and not args.search:
+        print("Error: Either --page or --search must be provided")
+        sys.exit(1)
+    
+    # Ensure output directory exists
+    output_dir = os.path.dirname(args.output)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+    
+    # Get PDF file (download if URL provided)
+    if args.url:
+        temp_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.temp')
+        os.makedirs(temp_dir, exist_ok=True)
+        pdf_path = os.path.join(temp_dir, 'temp_pdf.pdf')
+        
+        print(f"📥 Downloading PDF from {args.url}...")
+        try:
+            download_pdf_to_temp(args.url, pdf_path)
+            print(f"✅ PDF downloaded")
+        except Exception as e:
+            print(f"❌ Failed to download PDF: {e}")
+            sys.exit(1)
+    else:
+        pdf_path = args.file
+        if not os.path.exists(pdf_path):
+            print(f"Error: File not found: {pdf_path}")
+            sys.exit(1)
+    
+    # Determine page number
+    page_number = args.page
+    if args.search:
+        print(f"🔍 Searching for '{args.search}' in PDF...")
+        found_page = find_page_with_term(pdf_path, args.search)
+        if found_page:
+            page_number = found_page
+            print(f"✅ Found '{args.search}' on page {page_number}")
+        else:
+            print(f"❌ '{args.search}' not found in PDF")
+            sys.exit(1)
+    
+    # Take screenshot
+    result = screenshot_pdf_page(
+        pdf_path, 
+        page_number, 
+        args.output,
+        search_term=args.search,
+        width=args.width,
+        height=args.height
+    )
+    
+    if result['success']:
+        sys.exit(0)
+    else:
+        print(f"❌ Failed: {result['message']}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/tools/downloaders/screenshotter.py

@@ -0,0 +1,58 @@
+import sys
+import os
+import argparse
+import time
+from random import uniform
+from playwright.sync_api import sync_playwright
+
+def take_screenshot(url, output_path):
+    # Add random delay to implement rate limiting
+    delay = uniform(1, 3)  # Random delay between 1-3 seconds
+    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
+    time.sleep(delay)
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch()
+        page = browser.new_page()
+
+        # Set additional headers to appear more like a real user
+        page.set_extra_http_headers({
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Accept-Encoding": "gzip, deflate",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        })
+
+        try:
+            print(f"Navigating to {url}...")
+            page.goto(url, timeout=30000)
+            # Wait a bit for dynamic content (e.g., Twitter embeds)
+            page.wait_for_timeout(2000)
+
+            page.screenshot(path=output_path, full_page=False)
+
+            # Validate that the file was created and has content
+            if os.path.exists(output_path):
+                file_size = os.path.getsize(output_path)
+                if file_size == 0:
+                    print(f"Error: Screenshot file is empty: {output_path}")
+                    sys.exit(1)
+                else:
+                    print(f"File validation: {output_path} created with size {file_size} bytes")
+            else:
+                print(f"Error: Screenshot file does not exist: {output_path}")
+                sys.exit(1)
+
+        except Exception as e:
+            print(f"Error taking screenshot: {e}")
+            sys.exit(1)
+        finally:
+            browser.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+    take_screenshot(args.url, args.output)