videonut 1.0.1 → 1.1.0

package/tools/validators/archive_url.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""
+Archive URL Tool for VideoNut
+
+Archives web pages to Archive.is (archive.today) to preserve them before they expire.
+News articles especially tend to disappear or go behind paywalls.
+
+Usage:
+    python archive_url.py --url "https://example.com/article"
+    python archive_url.py --url "https://example.com" --check-only
+    python archive_url.py --batch "urls.txt" --output "archived_urls.txt"
+"""
+
+import sys
+import argparse
+import time
+import json
+from random import uniform
+
+try:
+    import requests
+except ImportError:
+    print("Error: requests not installed. Install with: pip install requests")
+    sys.exit(1)
+
+
+def check_existing_archive(url):
+    """
+    Check if a URL is already archived on Archive.is.
+    Returns the archived URL if found, None otherwise.
+    """
+    check_url = f"https://archive.is/{url}"
+    
+    try:
+        response = requests.get(check_url, timeout=10, allow_redirects=True)
+        
+        # If we get redirected to an archived page, it exists
+        if response.status_code == 200 and "archive.is" in response.url:
+            return response.url
+        return None
+    except Exception as e:
+        print(f"Error checking archive: {e}")
+        return None
+
+
+def archive_url(url, wait_for_complete=True):
+    """
+    Submit a URL to Archive.is for archiving.
+    
+    Args:
+        url: The URL to archive
+        wait_for_complete: Whether to wait for archiving to complete
+    
+    Returns:
+        Dict with status and archived URL
+    """
+    result = {
+        'success': False,
+        'original_url': url,
+        'archived_url': None,
+        'message': ''
+    }
+    
+    # Rate limiting
+    delay = uniform(2, 4)
+    print(f"Rate limiting: Waiting {delay:.2f} seconds...")
+    time.sleep(delay)
+    
+    # First, check if already archived
+    print(f"Checking if {url} is already archived...")
+    existing = check_existing_archive(url)
+    if existing:
+        result['success'] = True
+        result['archived_url'] = existing
+        result['message'] = "Already archived"
+        return result
+    
+    # Submit to archive.is
+    print(f"Submitting {url} to Archive.is...")
+    
+    archive_submit_url = "https://archive.is/submit/"
+    
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+        'Content-Type': 'application/x-www-form-urlencoded',
+    }
+    
+    data = {
+        'url': url,
+        'anyway': 1
+    }
+    
+    try:
+        response = requests.post(
+            archive_submit_url, 
+            data=data, 
+            headers=headers, 
+            timeout=60,
+            allow_redirects=True
+        )
+        
+        # Check for the archived URL in response
+        if response.status_code == 200:
+            # The response URL should be the archived page
+            if "archive.is" in response.url or "archive.today" in response.url:
+                result['success'] = True
+                result['archived_url'] = response.url
+                result['message'] = "Successfully archived"
+            else:
+                # Sometimes archive.is returns a page with the archive link
+                # Try to extract it from the page content
+                if 'id="SHARE_LONGLINK"' in response.text:
+                    # Parse out the archive URL
+                    import re
+                    match = re.search(r'value="(https://archive\.[^"]+)"', response.text)
+                    if match:
+                        result['success'] = True
+                        result['archived_url'] = match.group(1)
+                        result['message'] = "Successfully archived"
+                    else:
+                        result['message'] = "Archived but couldn't extract URL"
+                else:
+                    result['message'] = "Submission sent - check archive.is manually"
+        else:
+            result['message'] = f"HTTP {response.status_code}"
+            
+    except requests.exceptions.Timeout:
+        result['message'] = "Request timed out - archive.is may be slow"
+    except Exception as e:
+        result['message'] = f"Error: {str(e)}"
+    
+    return result
+
+
+def archive_batch(urls, output_file=None):
+    """
+    Archive multiple URLs.
+    
+    Args:
+        urls: List of URLs to archive
+        output_file: Optional file to save results
+    
+    Returns:
+        List of results
+    """
+    results = []
+    total = len(urls)
+    
+    for i, url in enumerate(urls, 1):
+        url = url.strip()
+        if not url or url.startswith('#'):
+            continue
+            
+        print(f"\n[{i}/{total}] Processing: {url}")
+        result = archive_url(url)
+        results.append(result)
+        
+        if result['success']:
+            print(f"✅ Archived: {result['archived_url']}")
+        else:
+            print(f"❌ Failed: {result['message']}")
+    
+    # Save results if output file specified
+    if output_file:
+        with open(output_file, 'w', encoding='utf-8') as f:
+            f.write("# Archived URLs\n")
+            f.write("# Generated by archive_url.py\n\n")
+            
+            for r in results:
+                if r['success']:
+                    f.write(f"ORIGINAL: {r['original_url']}\n")
+                    f.write(f"ARCHIVED: {r['archived_url']}\n\n")
+                else:
+                    f.write(f"FAILED: {r['original_url']}\n")
+                    f.write(f"REASON: {r['message']}\n\n")
+        
+        print(f"\n📁 Results saved to: {output_file}")
+    
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Archive web pages to Archive.is for permanent preservation.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Archive a single URL
+  python archive_url.py --url "https://timesofindia.com/article/123"
+  
+  # Check if URL is already archived (don't create new)
+  python archive_url.py --url "https://example.com" --check-only
+  
+  # Archive multiple URLs from a file
+  python archive_url.py --batch "urls.txt" --output "archived.txt"
+  
+  # Output as JSON
+  python archive_url.py --url "https://example.com" --json
+
+Why Archive?
+  - News articles often get deleted or moved
+  - Paywalls can block access later
+  - Creates permanent proof of what was published
+  - Essential for journalism and research
+        """
+    )
+    
+    parser.add_argument("--url", "-u", help="URL to archive")
+    parser.add_argument("--check-only", "-c", action="store_true", 
+                        help="Only check if archived, don't create new archive")
+    parser.add_argument("--batch", "-b", help="File containing URLs to archive (one per line)")
+    parser.add_argument("--output", "-o", help="Output file for batch results")
+    parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")
+    
+    args = parser.parse_args()
+    
+    if not args.url and not args.batch:
+        parser.print_help()
+        print("\nError: Either --url or --batch is required")
+        sys.exit(1)
+    
+    if args.batch:
+        # Batch mode
+        try:
+            with open(args.batch, 'r', encoding='utf-8') as f:
+                urls = f.readlines()
+            results = archive_batch(urls, args.output)
+            
+            # Summary
+            success = sum(1 for r in results if r['success'])
+            print(f"\n📊 Summary: {success}/{len(results)} successfully archived")
+            
+        except FileNotFoundError:
+            print(f"Error: File not found: {args.batch}")
+            sys.exit(1)
+    
+    else:
+        # Single URL mode
+        if args.check_only:
+            print(f"Checking archive status for: {args.url}")
+            archived = check_existing_archive(args.url)
+            
+            if archived:
+                if args.json:
+                    print(json.dumps({'archived': True, 'url': archived}))
+                else:
+                    print(f"✅ Already archived: {archived}")
+            else:
+                if args.json:
+                    print(json.dumps({'archived': False, 'url': None}))
+                else:
+                    print("❌ Not archived yet")
+        else:
+            result = archive_url(args.url)
+            
+            if args.json:
+                print(json.dumps(result))
+            else:
+                if result['success']:
+                    print(f"\n✅ Successfully archived!")
+                    print(f"   Original: {result['original_url']}")
+                    print(f"   Archived: {result['archived_url']}")
+                else:
+                    print(f"\n❌ Archive failed: {result['message']}")
+                    print(f"   Try manually at: https://archive.is/")
+
+
+if __name__ == "__main__":
+    main()

package/tools/validators/link_checker.py

@@ -0,0 +1,45 @@
+import requests
+import sys
+import time
+from random import uniform
+
+def check_link(url):
+    # Add random delay to implement rate limiting
+    delay = uniform(1, 3)  # Random delay between 1-3 seconds
+    print(f"Rate limiting: Waiting {delay:.2f} seconds before checking {url}", file=sys.stderr)
+    time.sleep(delay)
+
+    try:
+        # More realistic User-Agent to appear like a regular browser
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1',
+        }
+
+        response = requests.head(url, headers=headers, timeout=5, allow_redirects=True)
+
+        if response.status_code == 200:
+            return True, "OK"
+        else:
+            # Retry with GET if HEAD fails (some servers block HEAD)
+            response = requests.get(url, headers=headers, timeout=5, stream=True)
+            if response.status_code == 200:
+                return True, "OK"
+            return False, f"Status Code: {response.status_code}"
+
+    except requests.exceptions.RequestException as e:
+        return False, f"Request error: {str(e)}"
+    except Exception as e:
+        return False, f"General error: {str(e)}"
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        url = sys.argv[1]
+        success, msg = check_link(url)
+        print(f"{'VALID' if success else 'INVALID'}: {msg}")
+    else:
+        print("Usage: python link_checker.py [URL]")

package/workflow_orchestrator.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+"""
+VideoNut Workflow Orchestrator
+Automates the agent workflow while maintaining manual control options
+"""
+import os
+import sys
+import subprocess
+import argparse
+import json
+from datetime import datetime
+import shutil
+
+class VideoNutOrchestrator:
+    def __init__(self, project_path):
+        self.project_path = project_path
+        self.checkpoint_file = os.path.join(project_path, ".workflow_checkpoint.json")
+        self.checkpoints = self.load_checkpoints()
+        
+    def load_checkpoints(self):
+        """Load workflow progress from checkpoint file"""
+        if os.path.exists(self.checkpoint_file):
+            with open(self.checkpoint_file, 'r') as f:
+                return json.load(f)
+        return {
+            "investigation_complete": False,
+            "scriptwriting_complete": False, 
+            "direction_complete": False,
+            "scavenging_complete": False,
+            "archiving_complete": False,
+            "last_step": "none"
+        }
+    
+    def save_checkpoints(self):
+        """Save workflow progress to checkpoint file"""
+        with open(self.checkpoint_file, 'w') as f:
+            json.dump(self.checkpoints, f, indent=2)
+    
+    def run_command(self, cmd, description):
+        """Execute a command with error handling"""
+        print(f"\n🚀 {description}")
+        print(f"Command: {cmd}")
+        
+        try:
+            result = subprocess.run(
+                cmd, 
+                shell=True, 
+                capture_output=True, 
+                text=True,
+                cwd=self.project_path
+            )
+            
+            if result.returncode == 0:
+                print(f"✅ {description} completed successfully")
+                return True
+            else:
+                print(f"❌ {description} failed:")
+                print(f"STDOUT: {result.stdout}")
+                print(f"STDERR: {result.stderr}")
+                return False
+                
+        except Exception as e:
+            print(f"❌ Error running {description}: {str(e)}")
+            return False
+    
+    def check_prerequisites(self):
+        """Check if required files exist for each step"""
+        files_needed = {
+            "investigation": [],
+            "scriptwriting": [os.path.join(self.project_path, "truth_dossier.md")],
+            "direction": [os.path.join(self.project_path, "narrative_script.md")],
+            "scavenging": [os.path.join(self.project_path, "master_script.md")],
+            "archiving": [os.path.join(self.project_path, "asset_manifest.md")]
+        }
+        
+        for step, required_files in files_needed.items():
+            for file_path in required_files:
+                if not os.path.exists(file_path):
+                    print(f"❌ Prerequisite missing for {step}: {file_path}")
+                    return False
+        return True
+    
+    def run_investigation(self):
+        """Run investigation phase"""
+        if self.checkpoints["investigation_complete"]:
+            print("⏭️ Investigation already completed, skipping...")
+            return True
+            
+        print("\n🔍 Starting Investigation Phase...")
+        # This would typically require user input, so we'll note this step
+        print("Please run: /investigator and complete the investigation")
+        print("When complete, mark investigation as done by creating truth_dossier.md")
+        
+        # For automation, we'd need to implement the actual investigator call
+        # For now, we'll just check if the output file exists
+        dossier_path = os.path.join(self.project_path, "truth_dossier.md")
+        if os.path.exists(dossier_path):
+            self.checkpoints["investigation_complete"] = True
+            self.checkpoints["last_step"] = "investigation"
+            self.save_checkpoints()
+            print("✅ Investigation step marked as complete")
+            return True
+        else:
+            print("❌ Investigation output not found. Please complete investigation manually.")
+            return False
+    
+    def run_scriptwriting(self):
+        """Run scriptwriting phase"""
+        if self.checkpoints["scriptwriting_complete"]:
+            print("⏭️ Scriptwriting already completed, skipping...")
+            return True
+            
+        print("\n✍️ Starting Scriptwriting Phase...")
+        
+        # Check if prerequisite exists
+        dossier_path = os.path.join(self.project_path, "truth_dossier.md")
+        if not os.path.exists(dossier_path):
+            print("❌ Prerequisite file missing: truth_dossier.md")
+            return False
+        
+        # Run the scriptwriter
+        cmd = f'python -c "import sys; sys.path.append(\'..\'); from tools.downloaders.caption_reader import *; exec(open(\'agents/creative/scriptwriter.md\').read())" 2>/dev/null || echo "Scriptwriter needs manual execution"'
+        # More realistic approach - just check if output exists
+        script_path = os.path.join(self.project_path, "narrative_script.md")
+        
+        if os.path.exists(script_path):
+            self.checkpoints["scriptwriting_complete"] = True
+            self.checkpoints["last_step"] = "scriptwriting"
+            self.save_checkpoints()
+            print("✅ Scriptwriting step marked as complete")
+            return True
+        else:
+            print("❌ Narrative script not found. Please run scriptwriter manually.")
+            return False
+    
+    def run_direction(self):
+        """Run direction phase"""
+        if self.checkpoints["direction_complete"]:
+            print("⏭️ Direction already completed, skipping...")
+            return True
+            
+        print("\n🎬 Starting Direction Phase...")
+        
+        # Check if prerequisite exists
+        script_path = os.path.join(self.project_path, "narrative_script.md")
+        if not os.path.exists(script_path):
+            print("❌ Prerequisite file missing: narrative_script.md")
+            return False
+        
+        # Check for output
+        master_script_path = os.path.join(self.project_path, "master_script.md")
+        if os.path.exists(master_script_path):
+            self.checkpoints["direction_complete"] = True
+            self.checkpoints["last_step"] = "direction"
+            self.save_checkpoints()
+            print("✅ Direction step marked as complete")
+            return True
+        else:
+            print("❌ Master script not found. Please run director manually.")
+            return False
+    
+    def run_scavenging(self):
+        """Run scavenging phase"""
+        if self.checkpoints["scavenging_complete"]:
+            print("⏭️ Scavenging already completed, skipping...")
+            return True
+            
+        print("\n🦅 Starting Scavenging Phase...")
+        
+        # Check if prerequisite exists
+        master_script_path = os.path.join(self.project_path, "master_script.md")
+        if not os.path.exists(master_script_path):
+            print("❌ Prerequisite file missing: master_script.md")
+            return False
+        
+        # Check for output
+        manifest_path = os.path.join(self.project_path, "asset_manifest.md")
+        if os.path.exists(manifest_path):
+            self.checkpoints["scavenging_complete"] = True
+            self.checkpoints["last_step"] = "scavenging"
+            self.save_checkpoints()
+            print("✅ Scavenging step marked as complete")
+            return True
+        else:
+            print("❌ Asset manifest not found. Please run scavenger manually.")
+            return False
+    
+    def run_archiving(self):
+        """Run archiving phase"""
+        if self.checkpoints["archiving_complete"]:
+            print("⏭️ Archiving already completed, skipping...")
+            return True
+            
+        print("\n💾 Starting Archiving Phase...")
+        
+        # Check if prerequisite exists
+        manifest_path = os.path.join(self.project_path, "asset_manifest.md")
+        if not os.path.exists(manifest_path):
+            print("❌ Prerequisite file missing: asset_manifest.md")
+            return False
+        
+        # Run archivist
+        cmd = f'python -c "import sys; sys.path.append(\'..\'); exec(open(\'agents/technical/archivist.md\').read())" 2>/dev/null || echo "Archivist needs manual execution"'
+        
+        # Check for output directory
+        assets_path = os.path.join(self.project_path, "assets")
+        if os.path.exists(assets_path):
+            self.checkpoints["archiving_complete"] = True
+            self.checkpoints["last_step"] = "archiving"
+            self.save_checkpoints()
+            print("✅ Archiving step marked as complete")
+            return True
+        else:
+            print("❌ Assets directory not found. Please run archivist manually.")
+            return False
+    
+    def run_full_workflow(self):
+        """Run the complete VideoNut workflow"""
+        print("🎬 Starting VideoNut Video Production Workflow")
+        print(f"Project Path: {self.project_path}")
+        print(f"Last completed step: {self.checkpoints['last_step']}")
+        
+        steps = [
+            ("Investigation", self.run_investigation),
+            ("Scriptwriting", self.run_scriptwriting), 
+            ("Direction", self.run_direction),
+            ("Scavenging", self.run_scavenging),
+            ("Archiving", self.run_archiving)
+        ]
+        
+        for step_name, step_func in steps:
+            print(f"\n--- {step_name.upper()} PHASE ---")
+            if not step_func():
+                print(f"❌ {step_name} phase failed. Workflow stopped.")
+                return False
+        
+        print("\n🎉 All workflow phases completed successfully!")
+        print("Your video assets are ready for editing!")
+        return True
+
+def main():
+    parser = argparse.ArgumentParser(description="VideoNut Workflow Orchestrator")
+    parser.add_argument("--project", required=True, help="Path to project directory")
+    parser.add_argument("--resume", action="store_true", help="Resume from last checkpoint")
+    parser.add_argument("--status", action="store_true", help="Show current workflow status")
+    parser.add_argument("--next", action="store_true", help="Show what to do next")
+    
+    args = parser.parse_args()
+    
+    if not os.path.exists(args.project):
+        print(f"❌ Project directory does not exist: {args.project}")
+        sys.exit(1)
+    
+    orchestrator = VideoNutOrchestrator(args.project)
+    
+    # Status command - show current progress
+    if args.status:
+        print("📊 VideoNut Workflow Status")
+        print("=" * 50)
+        print(f"Project: {args.project}")
+        print(f"Last step: {orchestrator.checkpoints['last_step']}")
+        print()
+        
+        status_icons = {True: "✅", False: "⏳"}
+        steps = [
+            ("Investigation", "investigation_complete", "truth_dossier.md"),
+            ("Scriptwriting", "scriptwriting_complete", "narrative_script.md"),
+            ("Direction", "direction_complete", "master_script.md"),
+            ("Scavenging", "scavenging_complete", "asset_manifest.md"),
+            ("Archiving", "archiving_complete", "assets/"),
+        ]
+        
+        for step_name, checkpoint_key, output_file in steps:
+            is_complete = orchestrator.checkpoints[checkpoint_key]
+            icon = status_icons[is_complete]
+            status = "Complete" if is_complete else "Pending"
+            print(f"  {icon} {step_name}: {status} → {output_file}")
+        
+        print()
+        sys.exit(0)
+    
+    # Next command - show what to do next
+    if args.next:
+        print("🎯 What to Do Next")
+        print("=" * 50)
+        
+        next_steps = {
+            "none": ("Investigation", "/investigator", "Create truth_dossier.md with research findings"),
+            "investigation": ("Scriptwriting", "/scriptwriter", "Create narrative_script.md from the dossier"),
+            "scriptwriting": ("Direction", "/director", "Create master_script.md with visual directions"),
+            "direction": ("Scavenging", "/scavenger", "Create asset_manifest.md with URLs"),
+            "scavenging": ("Archiving", "/archivist", "Download all assets to assets/ folder"),
+            "archiving": ("Complete", None, "🎉 All done! Your video assets are ready for editing."),
+        }
+        
+        last_step = orchestrator.checkpoints['last_step']
+        step_name, command, description = next_steps.get(last_step, next_steps["none"])
+        
+        print(f"📍 Current position: After '{last_step}'")
+        print(f"👉 Next step: {step_name}")
+        if command:
+            print(f"🔧 Command: {command}")
+        print(f"📝 What to do: {description}")
+        print()
+        
+        # Check for missing files
+        expected_files = {
+            "none": [],
+            "investigation": [("truth_dossier.md", "Run /investigator to create this file")],
+            "scriptwriting": [("narrative_script.md", "Run /scriptwriter to create this file")],
+            "direction": [("master_script.md", "Run /director to create this file")],
+            "scavenging": [("asset_manifest.md", "Run /scavenger to create this file")],
+        }
+        
+        for filename, hint in expected_files.get(last_step, []):
+            filepath = os.path.join(args.project, filename)
+            if not os.path.exists(filepath):
+                print(f"⚠️  Missing: {filename}")
+                print(f"   Fix: {hint}")
+        
+        sys.exit(0)
+    
+    if args.resume:
+        print("🔄 Resuming workflow from last checkpoint...")
+    
+    success = orchestrator.run_full_workflow()
+    
+    if success:
+        print(f"\n✅ Workflow completed! Checkpoint file: {orchestrator.checkpoint_file}")
+    else:
+        print(f"\n❌ Workflow failed. Checkpoint file: {orchestrator.checkpoint_file}")
+        print("\n💡 TIP: Run with --next to see what to do next")
+        print("💡 TIP: Run with --status to see full progress")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()