videonut 1.2.7 → 1.3.0

package/tools/downloaders/pdf_reader.py

@@ -1,163 +1,163 @@
-import sys
-import requests
-import io
-import time
-from random import uniform
-from pypdf import PdfReader
-import argparse
-import re
-
-def read_pdf(url, search_term=None, page_number=None):
-    """
-    Read a PDF from URL with optional search and page selection.
-    
-    Args:
-        url: URL of the PDF
-        search_term: Optional term to search for in the PDF
-        page_number: Optional specific page to read (1-indexed)
-    """
-    # Add random delay to implement rate limiting
-    delay = uniform(1, 3)
-    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
-    time.sleep(delay)
-
-    headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
-    }
-
-    try:
-        response = requests.get(url, headers=headers, timeout=30)
-        response.raise_for_status()
-
-        f = io.BytesIO(response.content)
-        reader = PdfReader(f)
-        total_pages = len(reader.pages)
-        
-        print(f"📄 PDF loaded: {total_pages} pages")
-        
-        # If specific page requested
-        if page_number:
-            if 1 <= page_number <= total_pages:
-                text = reader.pages[page_number - 1].extract_text()
-                print(f"\n--- Page {page_number} of {total_pages} ---")
-                print(text)
-                return
-            else:
-                print(f"Error: Page {page_number} not found. PDF has {total_pages} pages.")
-                sys.exit(1)
-        
-        # If search term provided, find all occurrences
-        if search_term:
-            print(f"🔍 Searching for: '{search_term}'")
-            matches = []
-            search_lower = search_term.lower()
-            
-            for i, page in enumerate(reader.pages):
-                page_text = page.extract_text()
-                if page_text and search_lower in page_text.lower():
-                    # Find the context around the match
-                    lines = page_text.split('\n')
-                    for j, line in enumerate(lines):
-                        if search_lower in line.lower():
-                            # Get surrounding context (2 lines before/after)
-                            context_start = max(0, j - 2)
-                            context_end = min(len(lines), j + 3)
-                            context = '\n'.join(lines[context_start:context_end])
-                            matches.append({
-                                'page': i + 1,
-                                'line': line.strip(),
-                                'context': context
-                            })
-            
-            if matches:
-                print(f"\n✅ Found {len(matches)} matches for '{search_term}':\n")
-                for idx, match in enumerate(matches[:10]):  # Limit to first 10 matches
-                    print(f"{'='*60}")
-                    print(f"📍 Match {idx+1} - Page {match['page']}")
-                    print(f"{'='*60}")
-                    print(f"Line: {match['line']}")
-                    print(f"\nContext:")
-                    print(match['context'])
-                    print()
-                
-                if len(matches) > 10:
-                    print(f"... and {len(matches) - 10} more matches")
-                
-                # Suggest best page for screenshot
-                best_page = matches[0]['page']
-                print(f"\n📸 Suggested page for screenshot: Page {best_page}")
-                print(f"   Use: python pdf_reader.py --url \"{url}\" --page {best_page}")
-            else:
-                print(f"❌ No matches found for '{search_term}'")
-            return
-        
-        # Default: Smart extraction with priority for first and last pages
-        MAX_PAGES = 15
-        MAX_CHARS = 20000
-        
-        text = ""
-        pages_to_read = []
-        
-        if total_pages <= MAX_PAGES:
-            pages_to_read = list(range(total_pages))
-        else:
-            # Smart selection: first 7 + last 4
-            first_pages = list(range(min(7, total_pages)))
-            last_pages = list(range(max(0, total_pages - 4), total_pages))
-            pages_to_read = sorted(set(first_pages + last_pages))
-            
-            print(f"📄 Document has {total_pages} pages. Reading pages: {[p+1 for p in pages_to_read]} (first + last priority)")
-        
-        for i in pages_to_read:
-            page_text = reader.pages[i].extract_text()
-            if page_text:
-                text += f"\n--- Page {i+1} ---\n{page_text}"
-        
-        # Smart truncation with intro/conclusion preservation
-        if len(text) > MAX_CHARS:
-            intro = text[:8000]
-            outro = text[-8000:]
-            truncated = len(text) - MAX_CHARS
-            print(f"--- PDF CONTENT START ---")
-            print(intro)
-            print(f"\n\n[... {truncated:,} characters truncated from middle ...]\n\n")
-            print(f"--- PDF CONTENT END ---")
-            print(outro)
-        else:
-            print(text)
-
-    except requests.exceptions.RequestException as e:
-        print(f"Error downloading PDF: {e}")
-        sys.exit(1)
-    except Exception as e:
-        print(f"Error reading PDF: {e}")
-        sys.exit(1)
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Read and search PDF documents from URLs.",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Read PDF with smart extraction (first + last pages)
-  python pdf_reader.py --url "https://example.com/report.pdf"
-  
-  # Search for specific term in PDF
-  python pdf_reader.py --url "https://example.com/report.pdf" --search "Prime Minister"
-  
-  # Read specific page
-  python pdf_reader.py --url "https://example.com/report.pdf" --page 5
-        """
-    )
-    
-    parser.add_argument("--url", required=True, help="URL of the PDF document")
-    parser.add_argument("--search", "-s", help="Search for specific term and show context")
-    parser.add_argument("--page", "-p", type=int, help="Read specific page number (1-indexed)")
-    
-    args = parser.parse_args()
-    read_pdf(args.url, search_term=args.search, page_number=args.page)
-
-
-if __name__ == "__main__":
-    main()
+import sys
+import requests
+import io
+import time
+from random import uniform
+from pypdf import PdfReader
+import argparse
+import re
+
+def read_pdf(url, search_term=None, page_number=None):
+    """
+    Read a PDF from URL with optional search and page selection.
+    
+    Args:
+        url: URL of the PDF
+        search_term: Optional term to search for in the PDF
+        page_number: Optional specific page to read (1-indexed)
+    """
+    # Add random delay to implement rate limiting
+    delay = uniform(1, 3)
+    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
+    time.sleep(delay)
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    }
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+
+        f = io.BytesIO(response.content)
+        reader = PdfReader(f)
+        total_pages = len(reader.pages)
+        
+        print(f"📄 PDF loaded: {total_pages} pages")
+        
+        # If specific page requested
+        if page_number:
+            if 1 <= page_number <= total_pages:
+                text = reader.pages[page_number - 1].extract_text()
+                print(f"\n--- Page {page_number} of {total_pages} ---")
+                print(text)
+                return
+            else:
+                print(f"Error: Page {page_number} not found. PDF has {total_pages} pages.")
+                sys.exit(1)
+        
+        # If search term provided, find all occurrences
+        if search_term:
+            print(f"🔍 Searching for: '{search_term}'")
+            matches = []
+            search_lower = search_term.lower()
+            
+            for i, page in enumerate(reader.pages):
+                page_text = page.extract_text()
+                if page_text and search_lower in page_text.lower():
+                    # Find the context around the match
+                    lines = page_text.split('\n')
+                    for j, line in enumerate(lines):
+                        if search_lower in line.lower():
+                            # Get surrounding context (2 lines before/after)
+                            context_start = max(0, j - 2)
+                            context_end = min(len(lines), j + 3)
+                            context = '\n'.join(lines[context_start:context_end])
+                            matches.append({
+                                'page': i + 1,
+                                'line': line.strip(),
+                                'context': context
+                            })
+            
+            if matches:
+                print(f"\n✅ Found {len(matches)} matches for '{search_term}':\n")
+                for idx, match in enumerate(matches[:10]):  # Limit to first 10 matches
+                    print(f"{'='*60}")
+                    print(f"📍 Match {idx+1} - Page {match['page']}")
+                    print(f"{'='*60}")
+                    print(f"Line: {match['line']}")
+                    print(f"\nContext:")
+                    print(match['context'])
+                    print()
+                
+                if len(matches) > 10:
+                    print(f"... and {len(matches) - 10} more matches")
+                
+                # Suggest best page for screenshot
+                best_page = matches[0]['page']
+                print(f"\n📸 Suggested page for screenshot: Page {best_page}")
+                print(f"   Use: python pdf_reader.py --url \"{url}\" --page {best_page}")
+            else:
+                print(f"❌ No matches found for '{search_term}'")
+            return
+        
+        # Default: Smart extraction with priority for first and last pages
+        MAX_PAGES = 15
+        MAX_CHARS = 20000
+        
+        text = ""
+        pages_to_read = []
+        
+        if total_pages <= MAX_PAGES:
+            pages_to_read = list(range(total_pages))
+        else:
+            # Smart selection: first 7 + last 4
+            first_pages = list(range(min(7, total_pages)))
+            last_pages = list(range(max(0, total_pages - 4), total_pages))
+            pages_to_read = sorted(set(first_pages + last_pages))
+            
+            print(f"📄 Document has {total_pages} pages. Reading pages: {[p+1 for p in pages_to_read]} (first + last priority)")
+        
+        for i in pages_to_read:
+            page_text = reader.pages[i].extract_text()
+            if page_text:
+                text += f"\n--- Page {i+1} ---\n{page_text}"
+        
+        # Smart truncation with intro/conclusion preservation
+        if len(text) > MAX_CHARS:
+            intro = text[:8000]
+            outro = text[-8000:]
+            truncated = len(text) - MAX_CHARS
+            print(f"--- PDF CONTENT START ---")
+            print(intro)
+            print(f"\n\n[... {truncated:,} characters truncated from middle ...]\n\n")
+            print(f"--- PDF CONTENT END ---")
+            print(outro)
+        else:
+            print(text)
+
+    except requests.exceptions.RequestException as e:
+        print(f"Error downloading PDF: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading PDF: {e}")
+        sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Read and search PDF documents from URLs.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Read PDF with smart extraction (first + last pages)
+  python pdf_reader.py --url "https://example.com/report.pdf"
+  
+  # Search for specific term in PDF
+  python pdf_reader.py --url "https://example.com/report.pdf" --search "Prime Minister"
+  
+  # Read specific page
+  python pdf_reader.py --url "https://example.com/report.pdf" --page 5
+        """
+    )
+    
+    parser.add_argument("--url", required=True, help="URL of the PDF document")
+    parser.add_argument("--search", "-s", help="Search for specific term and show context")
+    parser.add_argument("--page", "-p", type=int, help="Read specific page number (1-indexed)")
+    
+    args = parser.parse_args()
+    read_pdf(args.url, search_term=args.search, page_number=args.page)
+
+
+if __name__ == "__main__":
+    main()

package/tools/downloaders/screenshotter.py

@@ -1,58 +1,58 @@
-import sys
-import os
-import argparse
-import time
-from random import uniform
-from playwright.sync_api import sync_playwright
-
-def take_screenshot(url, output_path):
-    # Add random delay to implement rate limiting
-    delay = uniform(1, 3)  # Random delay between 1-3 seconds
-    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
-    time.sleep(delay)
-
-    with sync_playwright() as p:
-        browser = p.chromium.launch()
-        page = browser.new_page()
-
-        # Set additional headers to appear more like a real user
-        page.set_extra_http_headers({
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.5",
-            "Accept-Encoding": "gzip, deflate",
-            "Connection": "keep-alive",
-            "Upgrade-Insecure-Requests": "1",
-        })
-
-        try:
-            print(f"Navigating to {url}...")
-            page.goto(url, timeout=30000)
-            # Wait a bit for dynamic content (e.g., Twitter embeds)
-            page.wait_for_timeout(2000)
-
-            page.screenshot(path=output_path, full_page=False)
-
-            # Validate that the file was created and has content
-            if os.path.exists(output_path):
-                file_size = os.path.getsize(output_path)
-                if file_size == 0:
-                    print(f"Error: Screenshot file is empty: {output_path}")
-                    sys.exit(1)
-                else:
-                    print(f"File validation: {output_path} created with size {file_size} bytes")
-            else:
-                print(f"Error: Screenshot file does not exist: {output_path}")
-                sys.exit(1)
-
-        except Exception as e:
-            print(f"Error taking screenshot: {e}")
-            sys.exit(1)
-        finally:
-            browser.close()
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--url", required=True)
-    parser.add_argument("--output", required=True)
-    args = parser.parse_args()
-    take_screenshot(args.url, args.output)
+import sys
+import os
+import argparse
+import time
+from random import uniform
+from playwright.sync_api import sync_playwright
+
+def take_screenshot(url, output_path):
+    # Add random delay to implement rate limiting
+    delay = uniform(1, 3)  # Random delay between 1-3 seconds
+    print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
+    time.sleep(delay)
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch()
+        page = browser.new_page()
+
+        # Set additional headers to appear more like a real user
+        page.set_extra_http_headers({
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Accept-Encoding": "gzip, deflate",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+        })
+
+        try:
+            print(f"Navigating to {url}...")
+            page.goto(url, timeout=30000)
+            # Wait a bit for dynamic content (e.g., Twitter embeds)
+            page.wait_for_timeout(2000)
+
+            page.screenshot(path=output_path, full_page=False)
+
+            # Validate that the file was created and has content
+            if os.path.exists(output_path):
+                file_size = os.path.getsize(output_path)
+                if file_size == 0:
+                    print(f"Error: Screenshot file is empty: {output_path}")
+                    sys.exit(1)
+                else:
+                    print(f"File validation: {output_path} created with size {file_size} bytes")
+            else:
+                print(f"Error: Screenshot file does not exist: {output_path}")
+                sys.exit(1)
+
+        except Exception as e:
+            print(f"Error taking screenshot: {e}")
+            sys.exit(1)
+        finally:
+            browser.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+    take_screenshot(args.url, args.output)

package/tools/downloaders/web_reader.py

@@ -1,69 +1,69 @@
-import sys
-import argparse
-import time
-from random import uniform
-from playwright.sync_api import sync_playwright
-
-def read_webpage(url):
-    try:
-        # Add random delay to implement rate limiting
-        delay = uniform(1, 3)  # Random delay between 1-3 seconds
-        print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
-        time.sleep(delay)
-
-        with sync_playwright() as p:
-            # Launch browser (headless by default)
-            browser = p.chromium.launch()
-            page = browser.new_page()
-
-            # Set additional headers to appear more like a real user
-            page.set_extra_http_headers({
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-                "Accept-Language": "en-US,en;q=0.5",
-                "Accept-Encoding": "gzip, deflate",
-                "Connection": "keep-alive",
-                "Upgrade-Insecure-Requests": "1",
-            })
-
-            # Navigate with a reasonable timeout
-            page.goto(url, timeout=30000)
-
-            # Wait for content to load (basic heuristic)
-            page.wait_for_load_state("domcontentloaded")
-
-            # Get the text content
-            # We use evaluate to get innerText which mimics what a user sees (hidden text is ignored)
-            text = page.evaluate("document.body.innerText")
-
-            # Basic cleanup: Remove excessive newlines
-            clean_text = '\n'.join([line.strip() for line in text.splitlines() if line.strip()])
-
-            # Smart truncation: Preserve intro AND conclusion (critical for research)
-            MAX_TOTAL = 40000  # Increased from 25000
-            INTRO_SIZE = 8000  # First portion (hook/summary)
-            OUTRO_SIZE = 8000  # Last portion (conclusion/recommendations)
-            
-            if len(clean_text) > MAX_TOTAL:
-                intro = clean_text[:INTRO_SIZE]
-                outro = clean_text[-OUTRO_SIZE:]
-                truncated_chars = len(clean_text) - MAX_TOTAL
-                
-                print(f"--- CONTENT START (First {INTRO_SIZE} chars) ---")
-                print(intro)
-                print(f"\n\n[... {truncated_chars:,} CHARACTERS TRUNCATED - Middle section omitted to preserve intro and conclusion ...]\n\n")
-                print(f"--- CONTENT END (Last {OUTRO_SIZE} chars) ---")
-                print(outro)
-            else:
-                print(clean_text)
-
-            browser.close()
-
-    except Exception as e:
-        print(f"Error reading webpage: {e}")
-        sys.exit(1)
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--url", required=True)
-    args = parser.parse_args()
-    read_webpage(args.url)
+import sys
+import argparse
+import time
+from random import uniform
+from playwright.sync_api import sync_playwright
+
+def read_webpage(url):
+    try:
+        # Add random delay to implement rate limiting
+        delay = uniform(1, 3)  # Random delay between 1-3 seconds
+        print(f"Rate limiting: Waiting {delay:.2f} seconds before accessing {url}")
+        time.sleep(delay)
+
+        with sync_playwright() as p:
+            # Launch browser (headless by default)
+            browser = p.chromium.launch()
+            page = browser.new_page()
+
+            # Set additional headers to appear more like a real user
+            page.set_extra_http_headers({
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.5",
+                "Accept-Encoding": "gzip, deflate",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+            })
+
+            # Navigate with a reasonable timeout
+            page.goto(url, timeout=30000)
+
+            # Wait for content to load (basic heuristic)
+            page.wait_for_load_state("domcontentloaded")
+
+            # Get the text content
+            # We use evaluate to get innerText which mimics what a user sees (hidden text is ignored)
+            text = page.evaluate("document.body.innerText")
+
+            # Basic cleanup: Remove excessive newlines
+            clean_text = '\n'.join([line.strip() for line in text.splitlines() if line.strip()])
+
+            # Smart truncation: Preserve intro AND conclusion (critical for research)
+            MAX_TOTAL = 40000  # Increased from 25000
+            INTRO_SIZE = 8000  # First portion (hook/summary)
+            OUTRO_SIZE = 8000  # Last portion (conclusion/recommendations)
+            
+            if len(clean_text) > MAX_TOTAL:
+                intro = clean_text[:INTRO_SIZE]
+                outro = clean_text[-OUTRO_SIZE:]
+                truncated_chars = len(clean_text) - MAX_TOTAL
+                
+                print(f"--- CONTENT START (First {INTRO_SIZE} chars) ---")
+                print(intro)
+                print(f"\n\n[... {truncated_chars:,} CHARACTERS TRUNCATED - Middle section omitted to preserve intro and conclusion ...]\n\n")
+                print(f"--- CONTENT END (Last {OUTRO_SIZE} chars) ---")
+                print(outro)
+            else:
+                print(clean_text)
+
+            browser.close()
+
+    except Exception as e:
+        print(f"Error reading webpage: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", required=True)
+    args = parser.parse_args()
+    read_webpage(args.url)