videonut 1.2.8 → 1.3.0

package/tools/downloaders/image_grabber.py

@@ -1,106 +1,106 @@
-import os
-import sys
-import requests
-import argparse
-import mimetypes
-from urllib.parse import urlparse
-
-def is_safe_image_type(content_type, url):
-    """
-    Check if the content type is a safe image type.
-    """
-    safe_types = {
-        'image/jpeg', 'image/jpg', 'image/png', 'image/gif',
-        'image/webp', 'image/bmp', 'image/svg+xml', 'image/tiff'
-    }
-
-    # Check content-type header
-    if content_type and content_type.lower() in safe_types:
-        return True
-
-    # Fallback: check file extension from URL
-    parsed_url = urlparse(url)
-    file_ext = os.path.splitext(parsed_url.path)[1].lower()
-    mime_type, _ = mimetypes.guess_type(f"dummy{file_ext}")
-
-    if mime_type and mime_type in safe_types:
-        return True
-
-    return False
-
-def get_file_size(response):
-    """
-    Get the file size from the response headers.
-    """
-    content_length = response.headers.get('content-length')
-    if content_length:
-        return int(content_length)
-    return 0
-
-def download_image(url, output_path):
-    """
-    Downloads an image from a URL with security validation.
-    """
-    # Ensure output directory exists if it's not the current directory
-    dir_name = os.path.dirname(output_path)
-    if dir_name:
-        os.makedirs(dir_name, exist_ok=True)
-
-    headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-    }
-
-    try:
-        print(f"Downloading image from: {url}")
-
-        # First, make a HEAD request to check content type and size
-        head_response = requests.head(url, headers=headers, timeout=10)
-        content_type = head_response.headers.get('content-type', '').lower()
-
-        # Validate content type
-        if not is_safe_image_type(content_type, url):
-            print(f"Security Error: Content type '{content_type}' is not a safe image type.")
-            sys.exit(1)
-
-        # Check file size (limit to 50MB)
-        file_size = get_file_size(head_response)
-        if file_size > 50 * 1024 * 1024:  # 50MB
-            print(f"Security Error: File size {file_size} bytes exceeds 50MB limit.")
-            sys.exit(1)
-
-        # Actually download the file
-        response = requests.get(url, headers=headers, stream=True, timeout=10)
-        response.raise_for_status()
-
-        # Double-check content type after download
-        downloaded_content_type = response.headers.get('content-type', '').lower()
-        if not is_safe_image_type(downloaded_content_type, url):
-            print(f"Security Error: Downloaded content type '{downloaded_content_type}' is not a safe image type.")
-            sys.exit(1)
-
-        # Write file in chunks with size validation
-        total_size = 0
-        with open(output_path, 'wb') as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                if chunk:  # Filter out keep-alive chunks
-                    total_size += len(chunk)
-                    if total_size > 50 * 1024 * 1024:  # 50MB limit
-                        print(f"Security Error: Downloaded file exceeds 50MB limit.")
-                        os.remove(output_path)  # Clean up partial file
-                        sys.exit(1)
-                    f.write(chunk)
-
-        print(f"Successfully saved to {output_path}")
-
-    except Exception as e:
-        print(f"Failed to download image: {e}")
-        sys.exit(1)
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Download an image.")
-    parser.add_argument("--url", required=True, help="Image URL")
-    parser.add_argument("--output", required=True, help="Output file path")
-
-    args = parser.parse_args()
-
+import os
+import sys
+import requests
+import argparse
+import mimetypes
+from urllib.parse import urlparse
+
+def is_safe_image_type(content_type, url):
+    """
+    Check if the content type is a safe image type.
+    """
+    safe_types = {
+        'image/jpeg', 'image/jpg', 'image/png', 'image/gif',
+        'image/webp', 'image/bmp', 'image/svg+xml', 'image/tiff'
+    }
+
+    # Check content-type header
+    if content_type and content_type.lower() in safe_types:
+        return True
+
+    # Fallback: check file extension from URL
+    parsed_url = urlparse(url)
+    file_ext = os.path.splitext(parsed_url.path)[1].lower()
+    mime_type, _ = mimetypes.guess_type(f"dummy{file_ext}")
+
+    if mime_type and mime_type in safe_types:
+        return True
+
+    return False
+
+def get_file_size(response):
+    """
+    Get the file size from the response headers.
+    """
+    content_length = response.headers.get('content-length')
+    if content_length:
+        return int(content_length)
+    return 0
+
+def download_image(url, output_path):
+    """
+    Downloads an image from a URL with security validation.
+    """
+    # Ensure output directory exists if it's not the current directory
+    dir_name = os.path.dirname(output_path)
+    if dir_name:
+        os.makedirs(dir_name, exist_ok=True)
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    }
+
+    try:
+        print(f"Downloading image from: {url}")
+
+        # First, make a HEAD request to check content type and size
+        head_response = requests.head(url, headers=headers, timeout=10)
+        content_type = head_response.headers.get('content-type', '').lower()
+
+        # Validate content type
+        if not is_safe_image_type(content_type, url):
+            print(f"Security Error: Content type '{content_type}' is not a safe image type.")
+            sys.exit(1)
+
+        # Check file size (limit to 50MB)
+        file_size = get_file_size(head_response)
+        if file_size > 50 * 1024 * 1024:  # 50MB
+            print(f"Security Error: File size {file_size} bytes exceeds 50MB limit.")
+            sys.exit(1)
+
+        # Actually download the file
+        response = requests.get(url, headers=headers, stream=True, timeout=10)
+        response.raise_for_status()
+
+        # Double-check content type after download
+        downloaded_content_type = response.headers.get('content-type', '').lower()
+        if not is_safe_image_type(downloaded_content_type, url):
+            print(f"Security Error: Downloaded content type '{downloaded_content_type}' is not a safe image type.")
+            sys.exit(1)
+
+        # Write file in chunks with size validation
+        total_size = 0
+        with open(output_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:  # Filter out keep-alive chunks
+                    total_size += len(chunk)
+                    if total_size > 50 * 1024 * 1024:  # 50MB limit
+                        print(f"Security Error: Downloaded file exceeds 50MB limit.")
+                        os.remove(output_path)  # Clean up partial file
+                        sys.exit(1)
+                    f.write(chunk)
+
+        print(f"Successfully saved to {output_path}")
+
+    except Exception as e:
+        print(f"Failed to download image: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download an image.")
+    parser.add_argument("--url", required=True, help="Image URL")
+    parser.add_argument("--output", required=True, help="Output file path")
+
+    args = parser.parse_args()
+
     download_image(args.url, args.output)

package/tools/downloaders/pdf_reader.py

@@ -1,163 +1,163 @@
-import sys
-import requests
-import io
-import time
-from random import uniform
-from pypdf import PdfReader
-import argparse
-import re
-
-def read_pdf(url, search_term=None, page_number=None):
-    """
-    Read a PDF from URL with optional search and page selection.
-    
-    Args:
-        url: URL of the PDF
-        search_term: Optional term to search for in the PDF
-        page_number: Optional specific page to read (1-indexed)
-    """
-    # Add random delay to implement rate limiting
-    delay = uniform(1, 3)
-    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
-    time.sleep(delay)
-
-    headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-    }
-
-    try:
-        response = requests.get(url, headers=headers, timeout=30)
-        response.raise_for_status()
-
-        f = io.BytesIO(response.content)
-        reader = PdfReader(f)
-        total_pages = len(reader.pages)
-        
-        print(f"📄 PDF loaded: {total_pages} pages")
-        
-        # If specific page requested
-        if page_number:
-            if 1 <= page_number <= total_pages:
-                text = reader.pages[page_number - 1].extract_text()
-                print(f"\n--- Page {page_number} of {total_pages} ---")
-                print(text)
-                return
-            else:
-                print(f"Error: Page {page_number} not found. PDF has {total_pages} pages.")
-                sys.exit(1)
-        
-        # If search term provided, find all occurrences
-        if search_term:
-            print(f"🔍 Searching for: '{search_term}'")
-            matches = []
-            search_lower = search_term.lower()
-            
-            for i, page in enumerate(reader.pages):
-                page_text = page.extract_text()
-                if page_text and search_lower in page_text.lower():
-                    # Find the context around the match
-                    lines = page_text.split('\n')
-                    for j, line in enumerate(lines):
-                        if search_lower in line.lower():
-                            # Get surrounding context (2 lines before/after)
-                            context_start = max(0, j - 2)
-                            context_end = min(len(lines), j + 3)
-                            context = '\n'.join(lines[context_start:context_end])
-                            matches.append({
-                                'page': i + 1,
-                                'line': line.strip(),
-                                'context': context
-                            })
-            
-            if matches:
-                print(f"\n✅ Found {len(matches)} matches for '{search_term}':\n")
-                for idx, match in enumerate(matches[:10]):  # Limit to first 10 matches
-                    print(f"{'='*60}")
-                    print(f"📍 Match {idx+1} - Page {match['page']}")
-                    print(f"{'='*60}")
-                    print(f"Line: {match['line']}")
-                    print(f"\nContext:")
-                    print(match['context'])
-                    print()
-                
-                if len(matches) > 10:
-                    print(f"... and {len(matches) - 10} more matches")
-                
-                # Suggest best page for screenshot
-                best_page = matches[0]['page']
-                print(f"\n📸 Suggested page for screenshot: Page {best_page}")
-                print(f"   Use: python pdf_reader.py --url \"{url}\" --page {best_page}")
-            else:
-                print(f"❌ No matches found for '{search_term}'")
-            return
-        
-        # Default: Smart extraction with priority for first and last pages
-        MAX_PAGES = 15
-        MAX_CHARS = 20000
-        
-        text = ""
-        pages_to_read = []
-        
-        if total_pages <= MAX_PAGES:
-            pages_to_read = list(range(total_pages))
-        else:
-            # Smart selection: first 7 + last 4
-            first_pages = list(range(min(7, total_pages)))
-            last_pages = list(range(max(0, total_pages - 4), total_pages))
-            pages_to_read = sorted(set(first_pages + last_pages))
-            
-            print(f"📄 Document has {total_pages} pages. Reading pages: {[p+1 for p in pages_to_read]} (first + last priority)")
-        
-        for i in pages_to_read:
-            page_text = reader.pages[i].extract_text()
-            if page_text:
-                text += f"\n--- Page {i+1} ---\n{page_text}"
-        
-        # Smart truncation with intro/conclusion preservation
-        if len(text) > MAX_CHARS:
-            intro = text[:8000]
-            outro = text[-8000:]
-            truncated = len(text) - MAX_CHARS
-            print(f"--- PDF CONTENT START ---")
-            print(intro)
-            print(f"\n\n[... {truncated:,} characters truncated from middle ...]\n\n")
-            print(f"--- PDF CONTENT END ---")
-            print(outro)
-        else:
-            print(text)
-
-    except requests.exceptions.RequestException as e:
-        print(f"Error downloading PDF: {e}")
-        sys.exit(1)
-    except Exception as e:
-        print(f"Error reading PDF: {e}")
-        sys.exit(1)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Read and search PDF documents from URLs.",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Read PDF with smart extraction (first + last pages)
-  python pdf_reader.py --url "https://example.com/report.pdf"
-  
-  # Search for specific term in PDF
-  python pdf_reader.py --url "https://example.com/report.pdf" --search "Prime Minister"
-  
-  # Read specific page
-  python pdf_reader.py --url "https://example.com/report.pdf" --page 5
-        """
-    )
-    
-    parser.add_argument("--url", required=True, help="URL of the PDF document")
-    parser.add_argument("--search", "-s", help="Search for specific term and show context")
-    parser.add_argument("--page", "-p", type=int, help="Read specific page number (1-indexed)")
-    
-    args = parser.parse_args()
-    read_pdf(args.url, search_term=args.search, page_number=args.page)
-
-
-if __name__ == "__main__":
-    main()
+import sys
+import requests
+import io
+import time
+from random import uniform
+from pypdf import PdfReader
+import argparse
+import re
+
+def read_pdf(url, search_term=None, page_number=None):
+    """
+    Read a PDF from URL with optional search and page selection.
+    
+    Args:
+        url: URL of the PDF
+        search_term: Optional term to search for in the PDF
+        page_number: Optional specific page to read (1-indexed)
+    """
+    # Add random delay to implement rate limiting
+    delay = uniform(1, 3)
+    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
+    time.sleep(delay)
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    }
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+
+        f = io.BytesIO(response.content)
+        reader = PdfReader(f)
+        total_pages = len(reader.pages)
+        
+        print(f"📄 PDF loaded: {total_pages} pages")
+        
+        # If specific page requested
+        if page_number:
+            if 1 <= page_number <= total_pages:
+                text = reader.pages[page_number - 1].extract_text()
+                print(f"\n--- Page {page_number} of {total_pages} ---")
+                print(text)
+                return
+            else:
+                print(f"Error: Page {page_number} not found. PDF has {total_pages} pages.")
+                sys.exit(1)
+        
+        # If search term provided, find all occurrences
+        if search_term:
+            print(f"🔍 Searching for: '{search_term}'")
+            matches = []
+            search_lower = search_term.lower()
+            
+            for i, page in enumerate(reader.pages):
+                page_text = page.extract_text()
+                if page_text and search_lower in page_text.lower():
+                    # Find the context around the match
+                    lines = page_text.split('\n')
+                    for j, line in enumerate(lines):
+                        if search_lower in line.lower():
+                            # Get surrounding context (2 lines before/after)
+                            context_start = max(0, j - 2)
+                            context_end = min(len(lines), j + 3)
+                            context = '\n'.join(lines[context_start:context_end])
+                            matches.append({
+                                'page': i + 1,
+                                'line': line.strip(),
+                                'context': context
+                            })
+            
+            if matches:
+                print(f"\n✅ Found {len(matches)} matches for '{search_term}':\n")
+                for idx, match in enumerate(matches[:10]):  # Limit to first 10 matches
+                    print(f"{'='*60}")
+                    print(f"📍 Match {idx+1} - Page {match['page']}")
+                    print(f"{'='*60}")
+                    print(f"Line: {match['line']}")
+                    print(f"\nContext:")
+                    print(match['context'])
+                    print()
+                
+                if len(matches) > 10:
+                    print(f"... and {len(matches) - 10} more matches")
+                
+                # Suggest best page for screenshot
+                best_page = matches[0]['page']
+                print(f"\n📸 Suggested page for screenshot: Page {best_page}")
+                print(f"   Use: python pdf_reader.py --url \"{url}\" --page {best_page}")
+            else:
+                print(f"❌ No matches found for '{search_term}'")
+            return
+        
+        # Default: Smart extraction with priority for first and last pages
+        MAX_PAGES = 15
+        MAX_CHARS = 20000
+        
+        text = ""
+        pages_to_read = []
+        
+        if total_pages <= MAX_PAGES:
+            pages_to_read = list(range(total_pages))
+        else:
+            # Smart selection: first 7 + last 4
+            first_pages = list(range(min(7, total_pages)))
+            last_pages = list(range(max(0, total_pages - 4), total_pages))
+            pages_to_read = sorted(set(first_pages + last_pages))
+            
+            print(f"📄 Document has {total_pages} pages. Reading pages: {[p+1 for p in pages_to_read]} (first + last priority)")
+        
+        for i in pages_to_read:
+            page_text = reader.pages[i].extract_text()
+            if page_text:
+                text += f"\n--- Page {i+1} ---\n{page_text}"
+        
+        # Smart truncation with intro/conclusion preservation
+        if len(text) > MAX_CHARS:
+            intro = text[:8000]
+            outro = text[-8000:]
+            truncated = len(text) - MAX_CHARS
+            print(f"--- PDF CONTENT START ---")
+            print(intro)
+            print(f"\n\n[... {truncated:,} characters truncated from middle ...]\n\n")
+            print(f"--- PDF CONTENT END ---")
+            print(outro)
+        else:
+            print(text)
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error downloading PDF: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading PDF: {e}")
+        sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Read and search PDF documents from URLs.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Read PDF with smart extraction (first + last pages)
+  python pdf_reader.py --url "https://example.com/report.pdf"
+  
+  # Search for specific term in PDF
+  python pdf_reader.py --url "https://example.com/report.pdf" --search "Prime Minister"
+  
+  # Read specific page
+  python pdf_reader.py --url "https://example.com/report.pdf" --page 5
+        """
+    )
+    
+    parser.add_argument("--url", required=True, help="URL of the PDF document")
+    parser.add_argument("--search", "-s", help="Search for specific term and show context")
+    parser.add_argument("--page", "-p", type=int, help="Read specific page number (1-indexed)")
+    
+    args = parser.parse_args()
+    read_pdf(args.url, search_term=args.search, page_number=args.page)
+
+
+if __name__ == "__main__":
+    main()

package/tools/downloaders/screenshotter.py

@@ -1,58 +1,58 @@
-import sys
-import os
-import argparse
-import time
-from random import uniform
-from playwright.sync_api import sync_playwright
-
-def take_screenshot(url, output_path):
-    # Add random delay to implement rate limiting
-    delay = uniform(1, 3)  # Random delay between 1-3 seconds
-    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
-    time.sleep(delay)
-
-    with sync_playwright() as p:
-        browser = p.chromium.launch()
-        page = browser.new_page()
-
-        # Set additional headers to appear more like a real user
-        page.set_extra_http_headers({
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.5",
-            "Accept-Encoding": "gzip, deflate",
-            "Connection": "keep-alive",
-            "Upgrade-Insecure-Requests": "1",
-        })
-
-        try:
-            print(f"Navigating to {url}...")
-            page.goto(url, timeout=30000)
-            # Wait a bit for dynamic content (e.g., Twitter embeds)
-            page.wait_for_timeout(2000)
-
-            page.screenshot(path=output_path, full_page=False)
-
-            # Validate that the file was created and has content
-            if os.path.exists(output_path):
-                file_size = os.path.getsize(output_path)
-                if file_size == 0:
-                    print(f"Error: Screenshot file is empty: {output_path}")
-                    sys.exit(1)
-                else:
-                    print(f"File validation: {output_path} created with size {file_size} bytes")
-            else:
-                print(f"Error: Screenshot file does not exist: {output_path}")
-                sys.exit(1)
-
-        except Exception as e:
-            print(f"Error taking screenshot: {e}")
-            sys.exit(1)
-        finally:
-            browser.close()
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--url", required=True)
-    parser.add_argument("--output", required=True)
-    args = parser.parse_args()
-    take_screenshot(args.url, args.output)
+import sys
+import os
+import argparse
+import time
+from random import uniform
+from playwright.sync_api import sync_playwright
+
+def take_screenshot(url, output_path):
+    # Add random delay to implement rate limiting
+    delay = uniform(1, 3)  # Random delay between 1-3 seconds
+    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
+    time.sleep(delay)
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch()
+        page = browser.new_page()
+
+        # Set additional headers to appear more like a real user
+        page.set_extra_http_headers({
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Accept-Encoding": "gzip, deflate",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        })
+
+        try:
+            print(f"Navigating to {url}...")
+            page.goto(url, timeout=30000)
+            # Wait a bit for dynamic content (e.g., Twitter embeds)
+            page.wait_for_timeout(2000)
+
+            page.screenshot(path=output_path, full_page=False)
+
+            # Validate that the file was created and has content
+            if os.path.exists(output_path):
+                file_size = os.path.getsize(output_path)
+                if file_size == 0:
+                    print(f"Error: Screenshot file is empty: {output_path}")
+                    sys.exit(1)
+                else:
+                    print(f"File validation: {output_path} created with size {file_size} bytes")
+            else:
+                print(f"Error: Screenshot file does not exist: {output_path}")
+                sys.exit(1)
+
+        except Exception as e:
+            print(f"Error taking screenshot: {e}")
+            sys.exit(1)
+        finally:
+            browser.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+    take_screenshot(args.url, args.output)