mcli-framework 7.1.3__py3-none-any.whl → 7.2.0__py3-none-any.whl

mcli/workflow/politician_trading/commands.py

@@ -1,1939 +0,0 @@
-"""
-CLI commands for politician trading workflow
-"""
-
-import asyncio
-import json
-import os
-import re
-from datetime import datetime, timedelta
-from pathlib import Path
-from typing import Any, Dict, List
-
-import click
-from rich.console import Console
-from rich.json import JSON
-from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn
-from rich.table import Table
-
-from mcli.lib.logger.logger import get_logger
-
-from .config import WorkflowConfig
-from .connectivity import (
-    SupabaseConnectivityValidator,
-    run_connectivity_validation,
-    run_continuous_monitoring,
-)
-from .database import PoliticianTradingDB
-from .monitoring import PoliticianTradingMonitor, run_health_check, run_stats_report
-from .workflow import (
-    PoliticianTradingWorkflow,
-    check_politician_trading_status,
-    run_politician_trading_collection,
-)
-
-logger = get_logger(__name__)
-console = Console()
-
-
-@click.group(name="politician-trading")
-def politician_trading_cli():
-    """Manage politician trading data collection workflow"""
-    pass
-
-
-@politician_trading_cli.command("run")
-@click.option("--full", is_flag=True, help="Run full data collection (default)")
-@click.option("--us-only", is_flag=True, help="Only collect US Congress data")
-@click.option("--eu-only", is_flag=True, help="Only collect EU Parliament data")
-def run_collection(full: bool, us_only: bool, eu_only: bool):
-    """Run politician trading data collection"""
-    console.print("🏛️ Starting Politician Trading Data Collection", style="bold cyan")
-
-    try:
-        if us_only:
-            console.print("Collecting US Congress data only...", style="yellow")
-            # Would implement US-only collection
-            result = asyncio.run(run_politician_trading_collection())
-        elif eu_only:
-            console.print("Collecting EU Parliament data only...", style="yellow")
-            # Would implement EU-only collection
-            result = asyncio.run(run_politician_trading_collection())
-        else:
-            console.print("Running full data collection...", style="green")
-            result = asyncio.run(run_politician_trading_collection())
-
-        # Display results
-        if result.get("status") == "completed":
-            console.print("✅ Collection completed successfully!", style="bold green")
-
-            # Create summary table
-            table = Table(title="Collection Summary")
-            table.add_column("Metric", style="cyan")
-            table.add_column("Value", style="green")
-
-            summary = result.get("summary", {})
-            table.add_row("New Disclosures", str(summary.get("total_new_disclosures", 0)))
-            table.add_row("Updated Disclosures", str(summary.get("total_updated_disclosures", 0)))
-            table.add_row("Errors", str(len(summary.get("errors", []))))
-            table.add_row(
-                "Duration",
-                _calculate_duration(result.get("started_at"), result.get("completed_at")),
-            )
-
-            console.print(table)
-
-            # Show job details
-            jobs = result.get("jobs", {})
-            for job_name, job_data in jobs.items():
-                job_panel = Panel(
-                    f"Status: {job_data.get('status', 'unknown')}\n"
-                    f"New: {job_data.get('new_disclosures', 0)} | "
-                    f"Updated: {job_data.get('updated_disclosures', 0)} | "
-                    f"Errors: {len(job_data.get('errors', []))}",
-                    title=f"📊 {job_name.upper()} Job",
-                    border_style="green",
-                )
-                console.print(job_panel)
-        else:
-            console.print("❌ Collection failed!", style="bold red")
-            if "error" in result:
-                console.print(f"Error: {result['error']}", style="red")
-
-    except Exception as e:
-        console.print(f"❌ Command failed: {e}", style="bold red")
-        logger.error(f"Collection command failed: {e}")
-
-
-@politician_trading_cli.command("status")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-def check_status(output_json: bool):
-    """Check current status of politician trading data collection"""
-    try:
-        status = asyncio.run(check_politician_trading_status())
-
-        if output_json:
-            console.print(JSON.from_data(status))
-            return
-
-        # Display formatted status
-        console.print("🏛️ Politician Trading Data Status", style="bold cyan")
-
-        # Overall status
-        if "error" in status:
-            console.print(f"❌ Status check failed: {status['error']}", style="red")
-            return
-
-        # Summary panel
-        summary_text = f"""Database Connection: {status.get('database_connection', 'unknown')}
-Configuration: {status.get('config_loaded', 'unknown')}
-Total Disclosures: {status.get('total_disclosures', 0):,}
-Today's New Records: {status.get('recent_disclosures_today', 0):,}
-Last Update: {status.get('timestamp', 'unknown')}"""
-
-        summary_panel = Panel(summary_text, title="📈 System Status", border_style="blue")
-        console.print(summary_panel)
-
-        # Recent jobs table
-        recent_jobs = status.get("recent_jobs", [])
-        if recent_jobs:
-            jobs_table = Table(title="Recent Jobs")
-            jobs_table.add_column("Job Type", style="cyan")
-            jobs_table.add_column("Status", style="green")
-            jobs_table.add_column("Started", style="yellow")
-            jobs_table.add_column("Records", justify="right", style="magenta")
-            jobs_table.add_column("Duration", style="blue")
-
-            for job in recent_jobs[:5]:  # Show last 5 jobs
-                status_style = (
-                    "green"
-                    if job.get("status") == "completed"
-                    else "red" if job.get("status") == "failed" else "yellow"
-                )
-
-                jobs_table.add_row(
-                    job.get("job_type", ""),
-                    f"[{status_style}]{job.get('status', '')}[/{status_style}]",
-                    _format_timestamp(job.get("started_at")),
-                    str(job.get("records_processed", 0)),
-                    _calculate_duration(job.get("started_at"), job.get("completed_at")),
-                )
-
-            console.print(jobs_table)
-
-    except Exception as e:
-        console.print(f"❌ Status check failed: {e}", style="bold red")
-        logger.error(f"Status command failed: {e}")
-
-
-@politician_trading_cli.command("setup")
-@click.option("--create-tables", is_flag=True, help="Create database tables")
-@click.option("--verify", is_flag=True, help="Verify configuration and connection")
-@click.option("--generate-schema", is_flag=True, help="Generate schema SQL file")
-@click.option("--output-dir", default=".", help="Directory to save generated files")
-def setup_workflow(create_tables: bool, verify: bool, generate_schema: bool, output_dir: str):
-    """Setup politician trading workflow"""
-    console.print("🔧 Setting up Politician Trading Workflow", style="bold blue")
-
-    try:
-        config = WorkflowConfig.default()
-        workflow = PoliticianTradingWorkflow(config)
-
-        if verify:
-            console.print("Verifying configuration and database connection...")
-
-            # Test database connection
-            try:
-                status = asyncio.run(workflow.run_quick_check())
-                if "error" not in status:
-                    console.print("✅ Database connection successful", style="green")
-                    console.print("✅ Configuration loaded", style="green")
-
-                    # Display config summary
-                    config_text = f"""Supabase URL: {config.supabase.url}
-Request Delay: {config.scraping.request_delay}s
-Max Retries: {config.scraping.max_retries}
-Timeout: {config.scraping.timeout}s"""
-
-                    config_panel = Panel(config_text, title="🔧 Configuration", border_style="blue")
-                    console.print(config_panel)
-                else:
-                    console.print(f"❌ Verification failed: {status['error']}", style="red")
-            except Exception as e:
-                console.print(f"❌ Verification failed: {e}", style="red")
-
-        if generate_schema:
-            console.print("📄 Generating database schema files...", style="blue")
-
-            # Generate schema file
-            import os
-            from pathlib import Path
-
-            output_path = Path(output_dir)
-            output_path.mkdir(exist_ok=True)
-
-            # Read the schema SQL from the module
-            schema_file = Path(__file__).parent / "schema.sql"
-            if schema_file.exists():
-                schema_content = schema_file.read_text()
-
-                # Write to output directory
-                output_schema_file = output_path / "politician_trading_schema.sql"
-                output_schema_file.write_text(schema_content)
-
-                console.print(
-                    f"✅ Schema SQL generated: {output_schema_file.absolute()}", style="green"
-                )
-
-                # Also generate a setup instructions file
-                instructions = f"""# Politician Trading Database Setup Instructions
-
-## Step 1: Create Database Schema
-
-1. Open your Supabase SQL editor: https://supabase.com/dashboard/project/{config.supabase.url.split('//')[1].split('.')[0]}/sql/new
-2. Copy and paste the contents of: {output_schema_file.absolute()}
-3. Execute the SQL to create all tables, indexes, and triggers
-
-## Step 2: Verify Setup
-
-Run the following command to verify everything is working:
-
-```bash
-politician-trading setup --verify
-```
-
-## Step 3: Test Connectivity
-
-```bash
-politician-trading connectivity
-```
-
-## Step 4: Run First Collection
-
-```bash
-politician-trading test-workflow --verbose
-```
-
-## Step 5: Setup Automated Collection (Optional)
-
-```bash
-politician-trading cron-job --create
-```
-
-## Database Tables Created
-
-- **politicians**: Stores politician information (US Congress, EU Parliament)
-- **trading_disclosures**: Individual trading transactions/disclosures  
-- **data_pull_jobs**: Job execution tracking and status
-- **data_sources**: Data source configuration and health
-
-## Troubleshooting
-
-If you encounter issues:
-
-1. Check connectivity: `politician-trading connectivity --json`
-2. View logs: `politician-trading health`
-3. Test workflow: `politician-trading test-workflow --verbose`
-"""
-
-                instructions_file = output_path / "SETUP_INSTRUCTIONS.md"
-                instructions_file.write_text(instructions)
-
-                console.print(
-                    f"✅ Setup instructions generated: {instructions_file.absolute()}",
-                    style="green",
-                )
-
-                # Display summary
-                console.print("\n📋 Generated Files:", style="bold")
-                console.print(f"  📄 Schema SQL: {output_schema_file.name}")
-                console.print(f"  📋 Instructions: {instructions_file.name}")
-                console.print(f"  📁 Location: {output_path.absolute()}")
-
-                console.print("\n🚀 Next Steps:", style="bold green")
-                console.print("1. Open Supabase SQL editor")
-                console.print(f"2. Execute SQL from: {output_schema_file.name}")
-                console.print("3. Run: politician-trading setup --verify")
-                console.print("4. Run: politician-trading test-workflow --verbose")
-
-            else:
-                console.print("❌ Schema template not found", style="red")
-
-        if create_tables:
-            console.print("Creating database tables...")
-            schema_ok = asyncio.run(workflow.db.ensure_schema())
-            if schema_ok:
-                console.print("✅ Database schema verified", style="green")
-            else:
-                console.print("⚠️ Database schema needs to be created manually", style="yellow")
-                console.print("💡 Run: politician-trading setup --generate-schema", style="blue")
-
-    except Exception as e:
-        console.print(f"❌ Setup failed: {e}", style="bold red")
-        logger.error(f"Setup command failed: {e}")
-
-
-@politician_trading_cli.command("cron-job")
-@click.option("--create", is_flag=True, help="Show how to create Supabase cron job")
-@click.option("--test", is_flag=True, help="Test the cron job function")
-def manage_cron_job(create: bool, test: bool):
-    """Manage Supabase cron job for automated data collection"""
-
-    if create:
-        console.print("🕒 Creating Supabase Cron Job", style="bold blue")
-
-        cron_sql = """
--- Create cron job for politician trading data collection
-SELECT cron.schedule(
-    'politician-trading-collection',
-    '0 */6 * * *',  -- Every 6 hours
-    $$
-    SELECT net.http_post(
-        url := 'https://your-function-url.supabase.co/functions/v1/politician-trading-collect',
-        headers := '{"Content-Type": "application/json", "Authorization": "Bearer YOUR_ANON_KEY"}'::jsonb,
-        body := '{}'::jsonb
-    ) as request_id;
-    $$
-);
-
--- Check cron job status
-SELECT * FROM cron.job;
-"""
-
-        console.print("Add this SQL to your Supabase SQL editor:", style="green")
-        console.print(Panel(cron_sql, title="📝 Cron Job SQL", border_style="green"))
-
-        console.print("\n📋 Next steps:", style="bold blue")
-        console.print("1. Create an Edge Function in Supabase for the collection endpoint")
-        console.print("2. Update the URL in the cron job SQL above")
-        console.print("3. Execute the SQL in your Supabase dashboard")
-        console.print("4. Monitor the job with: SELECT * FROM cron.job_run_details;")
-
-    if test:
-        console.print("🧪 Testing cron job function...", style="yellow")
-        try:
-            result = asyncio.run(run_politician_trading_collection())
-            console.print("✅ Cron job function test completed", style="green")
-            console.print(JSON.from_data(result))
-        except Exception as e:
-            console.print(f"❌ Cron job test failed: {e}", style="red")
-
-
-@politician_trading_cli.command("health")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-def check_health(output_json: bool):
-    """Check system health and status"""
-    try:
-        health = asyncio.run(run_health_check())
-
-        if output_json:
-            console.print(JSON.from_data(health))
-        else:
-            monitor = PoliticianTradingMonitor()
-            monitor.display_health_report(health)
-
-    except Exception as e:
-        console.print(f"❌ Health check failed: {e}", style="bold red")
-        logger.error(f"Health check command failed: {e}")
-
-
-@politician_trading_cli.command("stats")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-def show_stats(output_json: bool):
-    """Show detailed statistics"""
-    try:
-        stats = asyncio.run(run_stats_report())
-
-        if output_json:
-            console.print(JSON.from_data(stats))
-        else:
-            monitor = PoliticianTradingMonitor()
-            monitor.display_stats_report(stats)
-
-    except Exception as e:
-        console.print(f"❌ Stats generation failed: {e}", style="bold red")
-        logger.error(f"Stats command failed: {e}")
-
-
-@politician_trading_cli.command("monitor")
-@click.option("--interval", default=30, help="Check interval in seconds")
-@click.option("--count", default=0, help="Number of checks (0 = infinite)")
-def continuous_monitor(interval: int, count: int):
-    """Continuously monitor system health"""
-    console.print(f"🔄 Starting continuous monitoring (interval: {interval}s)", style="bold blue")
-
-    async def monitor_loop():
-        monitor = PoliticianTradingMonitor()
-        check_count = 0
-
-        while True:
-            try:
-                console.clear()
-                console.print(
-                    f"Check #{check_count + 1} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
-                    style="dim",
-                )
-
-                health = await monitor.get_system_health()
-                monitor.display_health_report(health)
-
-                check_count += 1
-                if count > 0 and check_count >= count:
-                    break
-
-                if count == 0 or check_count < count:
-                    console.print(
-                        f"\n⏱️ Next check in {interval} seconds... (Ctrl+C to stop)", style="dim"
-                    )
-                    await asyncio.sleep(interval)
-
-            except KeyboardInterrupt:
-                console.print("\n👋 Monitoring stopped by user", style="yellow")
-                break
-            except Exception as e:
-                console.print(f"❌ Monitor check failed: {e}", style="red")
-                await asyncio.sleep(interval)
-
-    try:
-        asyncio.run(monitor_loop())
-    except Exception as e:
-        console.print(f"❌ Monitoring failed: {e}", style="bold red")
-        logger.error(f"Monitor command failed: {e}")
-
-
-@politician_trading_cli.command("connectivity")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-@click.option("--continuous", is_flag=True, help="Run continuous monitoring")
-@click.option("--interval", default=30, help="Check interval in seconds (continuous mode)")
-@click.option("--duration", default=0, help="Duration in minutes (0 = infinite)")
-def check_connectivity(output_json: bool, continuous: bool, interval: int, duration: int):
-    """Test Supabase connectivity and database operations"""
-    if continuous:
-        console.print(f"🔄 Starting continuous connectivity monitoring", style="bold blue")
-        try:
-            asyncio.run(run_continuous_monitoring(interval, duration))
-        except Exception as e:
-            console.print(f"❌ Continuous monitoring failed: {e}", style="bold red")
-            logger.error(f"Continuous monitoring failed: {e}")
-    else:
-        try:
-            validation_result = asyncio.run(run_connectivity_validation())
-
-            if output_json:
-                console.print(JSON.from_data(validation_result))
-            else:
-                validator = SupabaseConnectivityValidator()
-                validator.display_connectivity_report(validation_result)
-
-        except Exception as e:
-            console.print(f"❌ Connectivity validation failed: {e}", style="bold red")
-            logger.error(f"Connectivity validation failed: {e}")
-
-
-@politician_trading_cli.command("test-workflow")
-@click.option("--verbose", "-v", is_flag=True, help="Verbose output")
-@click.option("--validate-writes", is_flag=True, help="Validate database writes")
-def test_full_workflow(verbose: bool, validate_writes: bool):
-    """Run a complete workflow test with live Supabase connectivity"""
-    console.print("🧪 Running Full Politician Trading Workflow Test", style="bold green")
-
-    async def run_test():
-        # First validate connectivity
-        console.print("\n🔗 Step 1: Validating Supabase connectivity...", style="blue")
-        validator = SupabaseConnectivityValidator()
-        connectivity_result = await validator.validate_connectivity()
-
-        if verbose:
-            validator.display_connectivity_report(connectivity_result)
-        else:
-            console.print(
-                f"Connectivity Score: {connectivity_result['connectivity_score']}%", style="cyan"
-            )
-
-        if connectivity_result["connectivity_score"] < 75:
-            console.print("⚠️ Connectivity issues detected. Workflow may fail.", style="yellow")
-
-        # Run the workflow
-        console.print("\n🏛️ Step 2: Running politician trading collection workflow...", style="blue")
-
-        try:
-            with console.status("[bold blue]Executing workflow...") as status:
-                workflow_result = await run_politician_trading_collection()
-
-            # Display workflow results
-            console.print("\n📊 Workflow Results:", style="bold")
-
-            if workflow_result.get("status") == "completed":
-                console.print("✅ Workflow completed successfully!", style="green")
-
-                summary = workflow_result.get("summary", {})
-                console.print(f"New Disclosures: {summary.get('total_new_disclosures', 0)}")
-                console.print(f"Updated Disclosures: {summary.get('total_updated_disclosures', 0)}")
-                console.print(f"Errors: {len(summary.get('errors', []))}")
-
-                if verbose and summary.get("errors"):
-                    console.print("\nErrors encountered:", style="red")
-                    for error in summary["errors"][:5]:  # Show first 5 errors
-                        console.print(f"  • {error}", style="dim red")
-
-            else:
-                console.print("❌ Workflow failed!", style="red")
-                if "error" in workflow_result:
-                    console.print(f"Error: {workflow_result['error']}", style="red")
-
-            # Validate writes if requested
-            if validate_writes:
-                console.print("\n🔍 Step 3: Validating database writes...", style="blue")
-                write_validation = await validator._test_write_operations()
-
-                if write_validation["success"]:
-                    console.print("✅ Database writes validated successfully", style="green")
-                else:
-                    console.print(
-                        f"❌ Database write validation failed: {write_validation.get('error', 'Unknown error')}",
-                        style="red",
-                    )
-
-            # Final connectivity check
-            console.print("\n🔗 Step 4: Post-workflow connectivity check...", style="blue")
-            final_connectivity = await validator.validate_connectivity()
-
-            console.print(
-                f"Final Connectivity Score: {final_connectivity['connectivity_score']}%",
-                style="cyan",
-            )
-
-            # Summary
-            console.print("\n📋 Test Summary:", style="bold")
-            workflow_status = (
-                "✅ PASSED" if workflow_result.get("status") == "completed" else "❌ FAILED"
-            )
-            connectivity_status = (
-                "✅ GOOD" if final_connectivity["connectivity_score"] >= 75 else "⚠️ DEGRADED"
-            )
-
-            console.print(f"Workflow: {workflow_status}")
-            console.print(f"Connectivity: {connectivity_status}")
-            console.print(
-                f"Duration: {workflow_result.get('started_at', '')} to {workflow_result.get('completed_at', '')}"
-            )
-
-            return {
-                "workflow_result": workflow_result,
-                "connectivity_result": final_connectivity,
-                "test_passed": workflow_result.get("status") == "completed"
-                and final_connectivity["connectivity_score"] >= 75,
-            }
-
-        except Exception as e:
-            console.print(f"❌ Workflow test failed: {e}", style="bold red")
-            if verbose:
-                console.print_exception()
-            return {"error": str(e), "test_passed": False}
-
-    try:
-        test_result = asyncio.run(run_test())
-
-        if test_result.get("test_passed"):
-            console.print("\n🎉 Full workflow test PASSED!", style="bold green")
-        else:
-            console.print("\n❌ Full workflow test FAILED!", style="bold red")
-
-    except Exception as e:
-        console.print(f"❌ Test execution failed: {e}", style="bold red")
-        logger.error(f"Test workflow command failed: {e}")
-
-
-@politician_trading_cli.command("schema")
-@click.option("--show-location", is_flag=True, help="Show schema file location")
-@click.option("--generate", is_flag=True, help="Generate schema files")
-@click.option("--output-dir", default=".", help="Output directory for generated files")
-def manage_schema(show_location: bool, generate: bool, output_dir: str):
-    """Manage database schema files"""
-
-    if show_location:
-        console.print("📁 Schema File Locations", style="bold blue")
-
-        from pathlib import Path
-
-        schema_file = Path(__file__).parent / "schema.sql"
-
-        console.print(f"Built-in Schema: {schema_file.absolute()}", style="cyan")
-        console.print(f"File size: {schema_file.stat().st_size} bytes", style="dim")
-        console.print(
-            f"Exists: {'✅ Yes' if schema_file.exists() else '❌ No'}",
-            style="green" if schema_file.exists() else "red",
-        )
-
-        # Show current working directory option
-        cwd_schema = Path.cwd() / "politician_trading_schema.sql"
-        console.print(f"\nCurrent directory: {cwd_schema.absolute()}", style="cyan")
-        console.print(
-            f"Exists: {'✅ Yes' if cwd_schema.exists() else '❌ No'}",
-            style="green" if cwd_schema.exists() else "dim",
-        )
-
-        if not cwd_schema.exists():
-            console.print("\n💡 To generate schema file here:", style="blue")
-            console.print("politician-trading schema --generate", style="yellow")
-
-    elif generate:
-        # Reuse the setup command logic
-        try:
-            import os
-            from pathlib import Path
-
-            console.print("📄 Generating database schema files...", style="blue")
-
-            output_path = Path(output_dir)
-            output_path.mkdir(exist_ok=True)
-
-            # Read the schema SQL from the module
-            schema_file = Path(__file__).parent / "schema.sql"
-            if schema_file.exists():
-                schema_content = schema_file.read_text()
-
-                # Write to output directory
-                output_schema_file = output_path / "politician_trading_schema.sql"
-                output_schema_file.write_text(schema_content)
-
-                console.print(
-                    f"✅ Schema SQL generated: {output_schema_file.absolute()}", style="green"
-                )
-
-                # Show file info
-                console.print(f"📊 File size: {output_schema_file.stat().st_size:,} bytes")
-                console.print(f"📅 Created: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-
-                # Count SQL statements
-                statements = len(
-                    [
-                        line
-                        for line in schema_content.split("\n")
-                        if line.strip().startswith(("CREATE", "INSERT", "SELECT"))
-                    ]
-                )
-                console.print(f"📝 SQL statements: {statements}")
-
-            else:
-                console.print("❌ Schema template not found", style="red")
-
-        except Exception as e:
-            console.print(f"❌ Schema generation failed: {e}", style="red")
-
-    else:
-        # Show schema information by default
-        console.print("🗂️ Politician Trading Database Schema", style="bold blue")
-
-        schema_info = [
-            (
-                "politicians",
-                "Stores politician information",
-                "UUID primary key, bioguide_id, role, party",
-            ),
-            (
-                "trading_disclosures",
-                "Individual trading transactions",
-                "References politicians, amount ranges, asset details",
-            ),
-            (
-                "data_pull_jobs",
-                "Job execution tracking",
-                "Status, timing, record counts, error details",
-            ),
-            (
-                "data_sources",
-                "Data source configuration",
-                "URLs, regions, health status, request config",
-            ),
-        ]
-
-        schema_table = Table(title="Database Tables")
-        schema_table.add_column("Table", style="cyan")
-        schema_table.add_column("Purpose", style="white")
-        schema_table.add_column("Key Features", style="yellow")
-
-        for table_name, purpose, features in schema_info:
-            schema_table.add_row(table_name, purpose, features)
-
-        console.print(schema_table)
-
-        console.print("\n🚀 Commands:", style="bold")
-        console.print("  --show-location    Show where schema files are located")
-        console.print("  --generate         Generate schema SQL file")
-        console.print("  --generate --output-dir DIR  Generate to specific directory")
-
-
-# Helper functions
-def _calculate_duration(start_time: str, end_time: str) -> str:
-    """Calculate duration between timestamps"""
-    if not start_time or not end_time:
-        return "Unknown"
-
-    try:
-        start = datetime.fromisoformat(start_time.replace("Z", "+00:00"))
-        end = datetime.fromisoformat(end_time.replace("Z", "+00:00"))
-        duration = end - start
-
-        total_seconds = int(duration.total_seconds())
-        hours = total_seconds // 3600
-        minutes = (total_seconds % 3600) // 60
-        seconds = total_seconds % 60
-
-        if hours > 0:
-            return f"{hours}h {minutes}m {seconds}s"
-        elif minutes > 0:
-            return f"{minutes}m {seconds}s"
-        else:
-            return f"{seconds}s"
-    except Exception:
-        return "Unknown"
-
-
-def _format_timestamp(timestamp: str) -> str:
-    """Format timestamp for display"""
-    if not timestamp:
-        return "Unknown"
-
-    try:
-        dt = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
-        return dt.strftime("%Y-%m-%d %H:%M")
-    except Exception:
-        return timestamp[:16] if len(timestamp) > 16 else timestamp
-
-
-def _format_asset_display(disclosure: Dict[str, Any]) -> str:
-    """Format asset display with proper ticker/name handling"""
-    asset_name = disclosure.get("asset_name", "Unknown Asset")
-    asset_ticker = disclosure.get("asset_ticker")
-
-    # If we have both ticker and name, show ticker first
-    if asset_ticker and asset_ticker.strip() and asset_ticker.lower() != "none":
-        return f"{asset_ticker} - {asset_name[:15]}"
-    # If we only have asset name, show just that
-    elif asset_name and asset_name.strip():
-        return asset_name[:20]
-    # Fallback
-    else:
-        return "Unknown Asset"
-
-
-@politician_trading_cli.command("data-sources")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-def view_data_sources(output_json: bool):
-    """View current data sources and their configurations"""
-    console = Console()
-
-    try:
-        from .config import WorkflowConfig
-        from .data_sources import ACTIVE_SOURCES, ALL_DATA_SOURCES, TOTAL_SOURCES
-
-        config = WorkflowConfig.default()
-        active_sources = config.scraping.get_active_sources()
-
-        # Group sources by category for display
-        data_sources = {}
-
-        for category, sources in ALL_DATA_SOURCES.items():
-            active_category_sources = [s for s in sources if s.status == "active"]
-            if active_category_sources:
-                data_sources[category] = {
-                    "name": {
-                        "us_federal": "US Federal Government",
-                        "us_states": "US State Governments",
-                        "eu_parliament": "EU Parliament",
-                        "eu_national": "EU National Parliaments",
-                        "third_party": "Third-Party Aggregators",
-                    }[category],
-                    "sources": active_category_sources,
-                    "count": len(active_category_sources),
-                    "status": "active",
-                    "description": {
-                        "us_federal": "Congressional and federal official financial disclosures",
-                        "us_states": "State legislature financial disclosure databases",
-                        "eu_parliament": "MEP financial interest and income declarations",
-                        "eu_national": "National parliament financial disclosure systems",
-                        "third_party": "Commercial aggregators and enhanced analysis platforms",
-                    }[category],
-                }
-
-        if output_json:
-            # For JSON output, convert DataSource objects to dictionaries
-            json_output = {}
-            for category, info in data_sources.items():
-                json_output[category] = {
-                    "name": info["name"],
-                    "description": info["description"],
-                    "count": info["count"],
-                    "status": info["status"],
-                    "sources": [
-                        {
-                            "name": source.name,
-                            "jurisdiction": source.jurisdiction,
-                            "institution": source.institution,
-                            "url": source.url,
-                            "disclosure_types": [dt.value for dt in source.disclosure_types],
-                            "access_method": source.access_method.value,
-                            "update_frequency": source.update_frequency,
-                            "threshold_amount": source.threshold_amount,
-                            "data_format": source.data_format,
-                            "notes": source.notes,
-                        }
-                        for source in info["sources"]
-                    ],
-                }
-            console.print(JSON.from_data(json_output))
-        else:
-            console.print(
-                f"📊 Comprehensive Political Trading Data Sources ({ACTIVE_SOURCES} active of {TOTAL_SOURCES} total)",
-                style="bold cyan",
-            )
-
-            for category_id, source_info in data_sources.items():
-                console.print(
-                    f"\n[bold blue]{source_info['name']}[/bold blue] ({source_info['count']} sources)"
-                )
-                console.print(f"   {source_info['description']}", style="dim")
-
-                # Create table for this category's sources
-                table = Table()
-                table.add_column("Source", style="cyan")
-                table.add_column("Jurisdiction", style="green")
-                table.add_column("Access", style="yellow")
-                table.add_column("Disclosure Types", style="magenta")
-                table.add_column("Threshold", style="blue")
-
-                for source in source_info["sources"]:
-                    # Format disclosure types
-                    types_display = ", ".join(
-                        [dt.value.replace("_", " ").title() for dt in source.disclosure_types]
-                    )
-
-                    # Format threshold
-                    threshold_display = (
-                        f"${source.threshold_amount:,}" if source.threshold_amount else "None"
-                    )
-
-                    table.add_row(
-                        source.name,
-                        source.jurisdiction,
-                        source.access_method.value.replace("_", " ").title(),
-                        types_display[:30] + ("..." if len(types_display) > 30 else ""),
-                        threshold_display,
-                    )
-
-                console.print(table)
-
-            console.print(
-                f"\n[dim]Total: {ACTIVE_SOURCES} active sources across {len(data_sources)} categories[/dim]"
-            )
-
-    except Exception as e:
-        if output_json:
-            console.print(JSON.from_data({"error": str(e)}))
-        else:
-            console.print(f"❌ Failed to load data sources: {e}", style="bold red")
-
-
-@politician_trading_cli.command("jobs")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-@click.option("--limit", default=10, help="Number of recent jobs to show")
-def view_jobs(output_json: bool, limit: int):
-    """View current and recent data collection jobs"""
-    console = Console()
-
-    try:
-
-        async def get_jobs():
-            from .config import WorkflowConfig
-            from .database import PoliticianTradingDB
-
-            config = WorkflowConfig.default()
-            db = PoliticianTradingDB(config)
-
-            # Get recent jobs
-            jobs_result = (
-                db.client.table("data_pull_jobs")
-                .select("*")
-                .order("started_at", desc=True)
-                .limit(limit)
-                .execute()
-            )
-
-            return jobs_result.data if jobs_result.data else []
-
-        jobs = asyncio.run(get_jobs())
-
-        if output_json:
-            console.print(JSON.from_data(jobs))
-        else:
-            console.print("🔄 Recent Data Collection Jobs", style="bold cyan")
-
-            if not jobs:
-                console.print("No jobs found", style="yellow")
-                return
-
-            jobs_table = Table()
-            jobs_table.add_column("Job ID", style="cyan")
-            jobs_table.add_column("Type", style="green")
-            jobs_table.add_column("Status", style="white")
-            jobs_table.add_column("Started", style="blue")
-            jobs_table.add_column("Duration", style="magenta")
-            jobs_table.add_column("Records", style="yellow")
-
-            for job in jobs:
-                status_color = {
-                    "completed": "green",
-                    "running": "yellow",
-                    "failed": "red",
-                    "pending": "blue",
-                }.get(job.get("status", "unknown"), "white")
-
-                # Calculate duration
-                started = job.get("started_at", "")
-                completed = job.get("completed_at", "")
-                duration = _format_duration_from_timestamps(started, completed)
-
-                # Format records
-                records_info = f"{job.get('records_new', 0)}n/{job.get('records_updated', 0)}u/{job.get('records_failed', 0)}f"
-
-                jobs_table.add_row(
-                    job.get("id", "")[:8] + "...",
-                    job.get("job_type", "unknown"),
-                    f"[{status_color}]{job.get('status', 'unknown')}[/{status_color}]",
-                    _format_timestamp(started),
-                    duration,
-                    records_info,
-                )
-
-            console.print(jobs_table)
-            console.print("\nLegend: Records = new/updated/failed", style="dim")
-
-    except Exception as e:
-        if output_json:
-            console.print(JSON.from_data({"error": str(e)}))
-        else:
-            console.print(f"❌ Failed to load jobs: {e}", style="bold red")
-            logger.error(f"Jobs view failed: {e}")
-
-
-def _format_duration_from_timestamps(started: str, completed: str) -> str:
-    """Calculate and format duration from timestamps"""
-    if not started:
-        return "Unknown"
-
-    try:
-        start_dt = datetime.fromisoformat(started.replace("Z", "+00:00"))
-
-        if completed:
-            end_dt = datetime.fromisoformat(completed.replace("Z", "+00:00"))
-            duration = end_dt - start_dt
-        else:
-            # Job still running
-            from datetime import timezone
-
-            duration = datetime.now(timezone.utc) - start_dt
-
-        return _format_duration_seconds(int(duration.total_seconds()))
-
-    except Exception:
-        return "Unknown"
-
-
-@politician_trading_cli.command("politicians")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-@click.option("--limit", default=20, help="Number of politicians to show")
-@click.option(
-    "--role", type=click.Choice(["us_house_rep", "us_senator", "eu_mep"]), help="Filter by role"
-)
-@click.option("--party", help="Filter by party")
-@click.option("--state", help="Filter by state/country")
-@click.option("--search", help="Search by name (first, last, or full name)")
-def view_politicians(output_json: bool, limit: int, role: str, party: str, state: str, search: str):
-    """View and search politicians in the database"""
-    console = Console()
-
-    try:
-
-        async def get_politicians():
-            from .config import WorkflowConfig
-            from .database import PoliticianTradingDB
-
-            config = WorkflowConfig.default()
-            db = PoliticianTradingDB(config)
-
-            # Build query
-            query = db.client.table("politicians").select("*")
-
-            # Apply filters
-            if role:
-                query = query.eq("role", role)
-            if party:
-                query = query.ilike("party", f"%{party}%")
-            if state:
-                query = query.ilike("state_or_country", f"%{state}%")
-            if search:
-                # Search across name fields
-                query = query.or_(
-                    f"first_name.ilike.%{search}%,last_name.ilike.%{search}%,full_name.ilike.%{search}%"
-                )
-
-            result = query.order("created_at", desc=True).limit(limit).execute()
-            return result.data if result.data else []
-
-        politicians = asyncio.run(get_politicians())
-
-        if output_json:
-            console.print(JSON.from_data(politicians))
-        else:
-            console.print("👥 Politicians Database", style="bold cyan")
-
-            if not politicians:
-                console.print("No politicians found", style="yellow")
-                return
-
-            politicians_table = Table()
-            politicians_table.add_column("Name", style="cyan", min_width=25)
-            politicians_table.add_column("Role", style="green")
-            politicians_table.add_column("Party", style="blue")
-            politicians_table.add_column("State/Country", style="magenta")
-            politicians_table.add_column("District", style="yellow")
-            politicians_table.add_column("Added", style="dim")
-
-            for pol in politicians:
-                role_display = {
-                    "us_house_rep": "🏛️ House Rep",
-                    "us_senator": "🏛️ Senator",
-                    "eu_mep": "🇪🇺 MEP",
-                }.get(pol.get("role", ""), pol.get("role", "Unknown"))
-
-                politicians_table.add_row(
-                    pol.get("full_name")
-                    or f"{pol.get('first_name', '')} {pol.get('last_name', '')}".strip(),
-                    role_display,
-                    pol.get("party", "") or "Independent",
-                    pol.get("state_or_country", ""),
-                    pol.get("district", "") or "At-Large",
-                    _format_timestamp(pol.get("created_at", "")),
-                )
-
-            console.print(politicians_table)
-            console.print(
-                f"\nShowing {len(politicians)} of {len(politicians)} politicians", style="dim"
-            )
-
-    except Exception as e:
-        if output_json:
-            console.print(JSON.from_data({"error": str(e)}))
-        else:
-            console.print(f"❌ Failed to load politicians: {e}", style="bold red")
-            logger.error(f"Politicians view failed: {e}")
-
-
-@politician_trading_cli.command("disclosures")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-@click.option("--limit", default=20, help="Number of disclosures to show")
-@click.option("--politician", help="Filter by politician name")
-@click.option("--asset", help="Filter by asset name or ticker")
-@click.option(
-    "--transaction-type",
-    type=click.Choice(["purchase", "sale", "exchange"]),
-    help="Filter by transaction type",
-)
-@click.option("--amount-min", type=float, help="Minimum transaction amount")
-@click.option("--amount-max", type=float, help="Maximum transaction amount")
-@click.option("--days", default=30, help="Show disclosures from last N days")
-@click.option("--details", is_flag=True, help="Show detailed information including raw data")
-def view_disclosures(
-    output_json: bool,
-    limit: int,
-    politician: str,
-    asset: str,
-    transaction_type: str,
-    amount_min: float,
-    amount_max: float,
-    days: int,
-    details: bool,
-):
-    """View and search trading disclosures in the database"""
-    console = Console()
-
-    try:
-
-        async def get_disclosures():
-            from datetime import datetime, timedelta, timezone
-
-            from .config import WorkflowConfig
-            from .database import PoliticianTradingDB
-
-            config = WorkflowConfig.default()
-            db = PoliticianTradingDB(config)
-
-            # Build query with join to get politician info
-            # Supabase uses foreign key relationships for joins
-            query = db.client.table("trading_disclosures").select("*, politicians!inner(*)")
-
-            # Date filter
-            if days > 0:
-                cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
-                query = query.gte("created_at", cutoff_date.isoformat())
-
-            # Apply filters
-            if politician:
-                # For nested relationships, we need a different approach
-                # Let's use a simpler filter on the main table for now
-                query = query.filter("politicians.full_name", "ilike", f"%{politician}%")
-
-            if asset:
-                query = query.or_(f"asset_name.ilike.%{asset}%,asset_ticker.ilike.%{asset}%")
-
-            if transaction_type:
-                query = query.eq("transaction_type", transaction_type)
-
-            if amount_min is not None:
-                query = query.gte("amount_range_min", amount_min)
-
-            if amount_max is not None:
-                query = query.lte("amount_range_max", amount_max)
-
-            result = query.order("transaction_date", desc=True).limit(limit).execute()
-            return result.data if result.data else []
-
-        disclosures = asyncio.run(get_disclosures())
-
-        if output_json:
-            console.print(JSON.from_data(disclosures))
-        else:
-            console.print("💰 Trading Disclosures Database", style="bold cyan")
-
-            if not disclosures:
-                console.print("No disclosures found", style="yellow")
-                return
-
-            if details:
-                # Detailed view
-                for i, disclosure in enumerate(disclosures):
-                    console.print(f"\n[bold cyan]Disclosure {i+1}[/bold cyan]")
-
-                    detail_table = Table()
-                    detail_table.add_column("Field", style="cyan")
-                    detail_table.add_column("Value", style="white")
-
-                    politician_info = disclosure.get("politicians", {})
-                    politician_name = (
-                        politician_info.get("full_name")
-                        or f"{politician_info.get('first_name', '')} {politician_info.get('last_name', '')}".strip()
-                    )
-
-                    detail_table.add_row(
-                        "Politician",
-                        f"{politician_name} ({politician_info.get('party', 'Unknown')})",
-                    )
-                    detail_table.add_row(
-                        "Asset",
-                        f"{disclosure.get('asset_name', 'Unknown')} ({disclosure.get('asset_ticker', 'N/A')})",
-                    )
-                    detail_table.add_row(
-                        "Transaction", disclosure.get("transaction_type", "Unknown").title()
-                    )
-                    detail_table.add_row(
-                        "Date", _format_timestamp(disclosure.get("transaction_date", ""))
-                    )
-                    detail_table.add_row(
-                        "Disclosure Date", _format_timestamp(disclosure.get("disclosure_date", ""))
-                    )
-
-                    # Amount formatting
-                    amount_min = disclosure.get("amount_range_min")
-                    amount_max = disclosure.get("amount_range_max")
-                    amount_exact = disclosure.get("amount_exact")
-
-                    if amount_exact:
-                        amount_str = f"${amount_exact:,.2f}"
-                    elif amount_min is not None and amount_max is not None:
-                        amount_str = f"${amount_min:,.0f} - ${amount_max:,.0f}"
-                    else:
-                        amount_str = "Unknown"
-
-                    detail_table.add_row("Amount", amount_str)
-                    detail_table.add_row("Source URL", disclosure.get("source_url", "N/A"))
-                    detail_table.add_row(
-                        "Added", _format_timestamp(disclosure.get("created_at", ""))
-                    )
-
-                    console.print(detail_table)
-            else:
-                # Compact table view
-                disclosures_table = Table()
-                disclosures_table.add_column("Politician", style="cyan", min_width=25)
-                disclosures_table.add_column("Asset", style="green")
-                disclosures_table.add_column("Type", style="blue")
-                disclosures_table.add_column("Amount", style="yellow")
-                disclosures_table.add_column("Date", style="magenta")
-                disclosures_table.add_column("Party", style="dim")
-
-                for disclosure in disclosures:
-                    politician_info = disclosure.get("politicians", {})
-                    politician_name = (
-                        politician_info.get("full_name")
-                        or f"{politician_info.get('first_name', '')} {politician_info.get('last_name', '')}".strip()
-                    )
-
-                    # Format amount
-                    amount_min = disclosure.get("amount_range_min")
-                    amount_max = disclosure.get("amount_range_max")
-                    amount_exact = disclosure.get("amount_exact")
-
-                    if amount_exact:
-                        amount_str = f"${amount_exact:,.0f}"
-                    elif amount_min is not None and amount_max is not None:
-                        amount_str = f"${amount_min:,.0f}-${amount_max:,.0f}"
-                    else:
-                        amount_str = "Unknown"
-
-                    # Transaction type with emoji
-                    trans_type = disclosure.get("transaction_type", "unknown")
-                    trans_emoji = {
-                        "purchase": "🟢 Buy",
-                        "sale": "🔴 Sell",
-                        "exchange": "🔄 Exchange",
-                    }.get(trans_type, "❓ " + trans_type.title())
-
-                    disclosures_table.add_row(
-                        politician_name[:35] + ("..." if len(politician_name) > 35 else ""),
-                        _format_asset_display(disclosure),
-                        trans_emoji,
-                        amount_str,
-                        _format_timestamp(disclosure.get("transaction_date", "")),
-                        politician_info.get("party", "")[:12],
-                    )
-
-                console.print(disclosures_table)
-
-            console.print(
-                f"\nShowing {len(disclosures)} disclosures from last {days} days", style="dim"
-            )
-
-    except Exception as e:
-        if output_json:
-            console.print(JSON.from_data({"error": str(e)}))
-        else:
-            console.print(f"❌ Failed to load disclosures: {e}", style="bold red")
-            logger.error(f"Disclosures view failed: {e}")
-
-
-@politician_trading_cli.command("verify")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-def verify_database(output_json: bool):
-    """Verify database integrity and show summary statistics"""
-    console = Console()
-
-    try:
-
-        async def verify_data():
-            from datetime import timedelta
-
-            from .config import WorkflowConfig
-            from .database import PoliticianTradingDB
-
-            config = WorkflowConfig.default()
-            db = PoliticianTradingDB(config)
-
-            verification = {
-                "timestamp": datetime.now().isoformat(),
-                "tables": {},
-                "integrity": {},
-                "summary": {},
-            }
-
-            # Check each table
-            tables_to_check = ["politicians", "trading_disclosures", "data_pull_jobs"]
-
-            for table_name in tables_to_check:
-                try:
-                    result = db.client.table(table_name).select("id").execute()
-                    count = len(result.data) if result.data else 0
-                    verification["tables"][table_name] = {
-                        "exists": True,
-                        "record_count": count,
-                        "status": "ok",
-                    }
-                except Exception as e:
-                    verification["tables"][table_name] = {
-                        "exists": False,
-                        "error": str(e),
-                        "status": "error",
-                    }
-
-            # Check referential integrity - simplified approach
-            try:
-                # Just verify we can query both tables
-                disclosures_result = db.client.table("trading_disclosures").select("id").execute()
-                politicians_result = db.client.table("politicians").select("id").execute()
-
-                disclosures_count = len(disclosures_result.data) if disclosures_result.data else 0
-                politicians_count = len(politicians_result.data) if politicians_result.data else 0
-
-                verification["integrity"] = {
-                    "disclosures_with_politicians": disclosures_count,
-                    "total_politicians": politicians_count,
-                    "status": "ok",
-                }
-            except Exception as e:
-                verification["integrity"] = {"error": str(e), "status": "error"}
-
-            # Summary statistics
-            try:
-                politicians_count = verification["tables"]["politicians"]["record_count"]
-                disclosures_count = verification["tables"]["trading_disclosures"]["record_count"]
-                jobs_count = verification["tables"]["data_pull_jobs"]["record_count"]
-
-                # Get recent activity
-                recent_jobs = (
-                    db.client.table("data_pull_jobs")
-                    .select("*")
-                    .gte("started_at", (datetime.now() - timedelta(days=7)).isoformat())
-                    .execute()
-                )
-
-                recent_jobs_count = len(recent_jobs.data) if recent_jobs.data else 0
-                successful_jobs = len(
-                    [j for j in (recent_jobs.data or []) if j.get("status") == "completed"]
-                )
-
-                verification["summary"] = {
-                    "total_politicians": politicians_count,
-                    "total_disclosures": disclosures_count,
-                    "total_jobs": jobs_count,
-                    "jobs_last_7_days": recent_jobs_count,
-                    "successful_jobs_last_7_days": successful_jobs,
-                    "success_rate_7_days": (
-                        (successful_jobs / recent_jobs_count * 100) if recent_jobs_count > 0 else 0
-                    ),
-                }
-
-            except Exception as e:
-                verification["summary"] = {"error": str(e)}
-
-            return verification
-
-        verification = asyncio.run(verify_data())
-
-        if output_json:
-            console.print(JSON.from_data(verification))
-        else:
-            console.print("🔍 Database Verification Report", style="bold cyan")
-
-            # Table status
-            tables_panel = Table(title="Table Status")
-            tables_panel.add_column("Table", style="cyan")
-            tables_panel.add_column("Status", style="white")
-            tables_panel.add_column("Records", justify="right", style="green")
-
-            for table_name, info in verification["tables"].items():
-                status_color = "green" if info["status"] == "ok" else "red"
-                status_text = f"[{status_color}]{info['status'].upper()}[/{status_color}]"
-                record_count = str(info.get("record_count", "N/A"))
-
-                tables_panel.add_row(table_name, status_text, record_count)
-
-            console.print(tables_panel)
-
-            # Integrity check
-            integrity_info = verification.get("integrity", {})
-            if integrity_info.get("status") == "ok":
-                console.print("✅ Data integrity check passed", style="green")
-                disc_count = integrity_info.get("disclosures_with_politicians", 0)
-                pol_count = integrity_info.get("total_politicians", 0)
-                console.print(
-                    f"   Disclosures: {disc_count}, Politicians: {pol_count}", style="dim"
-                )
-            else:
-                console.print("❌ Data integrity check failed", style="red")
-
-            # Summary
-            summary = verification.get("summary", {})
-            if "error" not in summary:
-                console.print("\n📊 Summary Statistics", style="bold blue")
-                console.print(f"Politicians: {summary.get('total_politicians', 0)}")
-                console.print(f"Trading Disclosures: {summary.get('total_disclosures', 0)}")
-                console.print(f"Data Collection Jobs: {summary.get('total_jobs', 0)}")
-                console.print(
-                    f"Jobs (7 days): {summary.get('jobs_last_7_days', 0)} ({summary.get('successful_jobs_last_7_days', 0)} successful)"
-                )
-                console.print(f"Success Rate: {summary.get('success_rate_7_days', 0):.1f}%")
-
-    except Exception as e:
-        if output_json:
-            console.print(JSON.from_data({"error": str(e)}))
-        else:
-            console.print(f"❌ Verification failed: {e}", style="bold red")
-            logger.error(f"Database verification failed: {e}")
-
-
-@politician_trading_cli.group("cron")
-def cron_commands():
-    """Manage cron-based automated data collection"""
-    pass
-
-
-@cron_commands.command("run")
-@click.option(
-    "--type",
-    "collection_type",
-    default="full",
-    type=click.Choice(["full", "us", "eu", "quick"]),
-    help="Type of collection to run",
-)
-def cron_run(collection_type: str):
-    """Run scheduled data collection (designed for cron jobs)"""
-
-    async def run_cron_collection():
-        """Run the cron collection"""
-        from datetime import datetime
-
-        logger.info(f"Starting scheduled collection: {collection_type}")
-        console.print(f"🕐 Running {collection_type} data collection...", style="blue")
-
-        try:
-            workflow = PoliticianTradingWorkflow()
-
-            if collection_type == "full":
-                results = await run_politician_trading_collection()
-            elif collection_type == "us":
-                # US-only collection
-                us_results = await workflow._collect_us_congress_data()
-                ca_results = await workflow._collect_california_data()
-                us_states_results = await workflow._collect_us_states_data()
-
-                results = {
-                    "status": "completed",
-                    "started_at": datetime.utcnow().isoformat(),
-                    "completed_at": datetime.utcnow().isoformat(),
-                    "jobs": {
-                        "us_congress": us_results,
-                        "california": ca_results,
-                        "us_states": us_states_results,
-                    },
-                    "summary": {
-                        "total_new_disclosures": sum(
-                            [
-                                us_results.get("new_disclosures", 0),
-                                ca_results.get("new_disclosures", 0),
-                                us_states_results.get("new_disclosures", 0),
-                            ]
-                        )
-                    },
-                }
-            elif collection_type == "eu":
-                # EU-only collection
-                eu_results = await workflow._collect_eu_parliament_data()
-                eu_states_results = await workflow._collect_eu_member_states_data()
-                uk_results = await workflow._collect_uk_parliament_data()
-
-                results = {
-                    "status": "completed",
-                    "started_at": datetime.utcnow().isoformat(),
-                    "completed_at": datetime.utcnow().isoformat(),
-                    "jobs": {
-                        "eu_parliament": eu_results,
-                        "eu_member_states": eu_states_results,
-                        "uk_parliament": uk_results,
-                    },
-                    "summary": {
-                        "total_new_disclosures": sum(
-                            [
-                                eu_results.get("new_disclosures", 0),
-                                eu_states_results.get("new_disclosures", 0),
-                                uk_results.get("new_disclosures", 0),
-                            ]
-                        )
-                    },
-                }
-            elif collection_type == "quick":
-                # Quick status check
-                status = await workflow.run_quick_check()
-                results = {
-                    "status": "completed",
-                    "type": "quick_check",
-                    "results": status,
-                    "summary": {"total_new_disclosures": 0},
-                }
-
-            # Log results
-            summary = results.get("summary", {})
-            logger.info(
-                f"Cron collection completed - New: {summary.get('total_new_disclosures', 0)}"
-            )
-
-            console.print(f"✅ {collection_type.title()} collection completed", style="green")
-            console.print(
-                f"New disclosures: {summary.get('total_new_disclosures', 0)}", style="cyan"
-            )
-
-            return results
-
-        except Exception as e:
-            logger.error(f"Cron collection failed: {e}")
-            console.print(f"❌ Collection failed: {e}", style="red")
-            return {"status": "failed", "error": str(e)}
-
-    asyncio.run(run_cron_collection())
-
-
-@cron_commands.command("setup")
-def cron_setup():
-    """Show cron setup instructions"""
-    console.print("🕐 CRON SETUP INSTRUCTIONS", style="bold cyan")
-    console.print("Add these lines to your crontab (run: crontab -e)", style="dim")
-
-    # Get current working directory for the cron commands
-    repo_path = Path(__file__).parent.parent.parent.parent.parent
-
-    instructions = f"""
-# Full collection every 6 hours  
-0 */6 * * * cd {repo_path} && source .venv/bin/activate && mcli politician-trading cron run --type full >> /tmp/politician_cron.log 2>&1
-
-# US collection every 4 hours
-0 */4 * * * cd {repo_path} && source .venv/bin/activate && mcli politician-trading cron run --type us >> /tmp/politician_cron.log 2>&1
-
-# EU collection every 8 hours  
-0 */8 * * * cd {repo_path} && source .venv/bin/activate && mcli politician-trading cron run --type eu >> /tmp/politician_cron.log 2>&1
-
-# Quick health check daily at 9 AM
-0 9 * * * cd {repo_path} && source .venv/bin/activate && mcli politician-trading cron run --type quick >> /tmp/politician_cron.log 2>&1
-"""
-
-    console.print(Panel(instructions, title="Crontab Entries", border_style="blue"))
-
-    console.print("\n💡 Tips:", style="bold yellow")
-    console.print("• Start with just one cron job to test", style="dim")
-    console.print("• Check logs at /tmp/politician_cron.log", style="dim")
-    console.print("• Use 'mcli politician-trading monitor' to check results", style="dim")
-
-
-@politician_trading_cli.command("monitor")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-def monitor_system(output_json: bool):
-    """Monitor system status, jobs, and database"""
-
-    async def run_monitor():
-        """Run the monitoring"""
-        try:
-            config = WorkflowConfig.default()
-            db = PoliticianTradingDB(config)
-            workflow = PoliticianTradingWorkflow(config)
-
-            # Get system health
-            await db.ensure_schema()
-            quick_status = await workflow.run_quick_check()
-
-            # Get job history
-            job_status = await db.get_job_status()
-            recent_jobs = job_status.get("recent_jobs", [])
-
-            # Analyze job statistics
-            status_counts = {"completed": 0, "running": 0, "failed": 0, "pending": 0}
-            job_types = {}
-            latest_by_type = {}
-
-            for job in recent_jobs:
-                status = job.get("status", "unknown")
-                job_type = job.get("job_type", "unknown")
-                started_at = job.get("started_at", "")
-
-                if status in status_counts:
-                    status_counts[status] += 1
-                job_types[job_type] = job_types.get(job_type, 0) + 1
-
-                if job_type not in latest_by_type or started_at > latest_by_type[job_type].get(
-                    "started_at", ""
-                ):
-                    latest_by_type[job_type] = job
-
-            # Get scraper availability
-            try:
-                from . import scrapers
-
-                scraper_status = {
-                    "UK Parliament API": scrapers.UK_SCRAPER_AVAILABLE,
-                    "California NetFile": scrapers.CALIFORNIA_SCRAPER_AVAILABLE,
-                    "EU Member States": scrapers.EU_MEMBER_STATES_SCRAPER_AVAILABLE,
-                    "US States Ethics": scrapers.US_STATES_SCRAPER_AVAILABLE,
-                }
-                available_scrapers = sum(scraper_status.values())
-            except:
-                scraper_status = {}
-                available_scrapers = 0
-
-            monitor_data = {
-                "system_health": {
-                    "database_connection": quick_status.get("database_connection", "unknown"),
-                    "config_loaded": quick_status.get("config_loaded", "unknown"),
-                    "timestamp": quick_status.get("timestamp", datetime.now().isoformat()),
-                },
-                "job_statistics": {
-                    "total_recent_jobs": len(recent_jobs),
-                    "status_counts": status_counts,
-                    "job_types": job_types,
-                },
-                "latest_jobs": latest_by_type,
-                "scraper_availability": {
-                    "available_count": available_scrapers,
-                    "total_count": len(scraper_status),
-                    "scrapers": scraper_status,
-                },
-            }
-
-            return monitor_data
-
-        except Exception as e:
-            logger.error(f"Monitoring failed: {e}")
-            return {"error": str(e)}
-
-    monitor_data = asyncio.run(run_monitor())
-
-    if output_json:
-        console.print(JSON.from_data(monitor_data))
-    else:
-        console.print("🔍 SYSTEM MONITOR", style="bold cyan")
-
-        # System Health
-        health = monitor_data.get("system_health", {})
-        health_table = Table(title="System Health")
-        health_table.add_column("Component", style="cyan")
-        health_table.add_column("Status", style="white")
-
-        db_status = health["database_connection"]
-        db_color = "green" if db_status == "ok" else "red"
-        health_table.add_row("Database", f"[{db_color}]{db_status.upper()}[/{db_color}]")
-
-        config_status = health["config_loaded"]
-        config_color = "green" if config_status == "ok" else "red"
-        health_table.add_row(
-            "Configuration", f"[{config_color}]{config_status.upper()}[/{config_color}]"
-        )
-
-        console.print(health_table)
-
-        # Job Statistics
-        job_stats = monitor_data.get("job_statistics", {})
-        console.print(
-            f"\n📊 Job Statistics (Total: {job_stats.get('total_recent_jobs', 0)})",
-            style="bold blue",
-        )
-
-        status_counts = job_stats.get("status_counts", {})
-        for status, count in status_counts.items():
-            if count > 0:
-                icon = {"completed": "✅", "running": "🔄", "failed": "❌", "pending": "⏳"}[status]
-                console.print(f"{icon} {status.title()}: {count}")
-
-        # Latest Jobs by Type
-        console.print(f"\n📋 Latest Jobs by Source", style="bold blue")
-        latest_jobs = monitor_data.get("latest_jobs", {})
-
-        for job_type, job in sorted(latest_jobs.items()):
-            status = job.get("status", "unknown")
-            icon = {"completed": "✅", "running": "🔄", "failed": "❌", "pending": "⏳"}.get(
-                status, "❓"
-            )
-
-            source_name = job_type.replace("_", " ").title()
-            console.print(f"\n{icon} {source_name}")
-            console.print(f"   Status: {status}")
-            console.print(f"   Last run: {job.get('started_at', 'N/A')[:19]}")
-            console.print(
-                f"   Records: {job.get('records_processed', 0)} processed, {job.get('records_new', 0)} new"
-            )
-
-        # Scraper Availability
-        scraper_info = monitor_data.get("scraper_availability", {})
-        available = scraper_info.get("available_count", 0)
-        total = scraper_info.get("total_count", 0)
-
-        console.print(f"\n🌍 Scraper Availability: {available}/{total}", style="bold blue")
-
-        scrapers_status = scraper_info.get("scrapers", {})
-        for scraper_name, available in scrapers_status.items():
-            icon = "✅" if available else "❌"
-            status = "Available" if available else "Not Available"
-            console.print(f"{icon} {scraper_name}: {status}")
-
-
-@politician_trading_cli.command("read-data")
-@click.option("--limit", default=50, help="Number of recent records to show")
-@click.option("--days", default=7, help="Days back to look for data")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON")
-def read_recent_data(limit: int, days: int, output_json: bool):
-    """Read recent data from the database"""
-
-    async def read_data():
-        """Read recent data from database"""
-        try:
-            config = WorkflowConfig.default()
-            db = PoliticianTradingDB(config)
-
-            # Get job history
-            job_status = await db.get_job_status()
-            jobs = job_status.get("recent_jobs", [])
-
-            # Analyze data freshness
-            freshness = {}
-            for job in jobs:
-                job_type = job.get("job_type", "unknown")
-                if job.get("status") == "completed":
-                    completed_at = job.get("completed_at")
-                    if (
-                        job_type not in freshness
-                        or completed_at > freshness[job_type]["last_success"]
-                    ):
-                        # Check if recent (within threshold)
-                        is_recent = False
-                        if completed_at:
-                            try:
-                                timestamp = datetime.fromisoformat(
-                                    completed_at.replace("Z", "+00:00")
-                                )
-                                is_recent = (
-                                    datetime.now() - timestamp.replace(tzinfo=None)
-                                ) < timedelta(hours=24)
-                            except:
-                                pass
-
-                        freshness[job_type] = {
-                            "last_success": completed_at,
-                            "records_collected": job.get("records_new", 0),
-                            "status": "fresh" if is_recent else "stale",
-                        }
-
-            return {
-                "recent_jobs": jobs[:limit],
-                "data_freshness": freshness,
-                "summary": {
-                    "total_jobs": len(jobs),
-                    "job_types": len(set(job.get("job_type") for job in jobs)),
-                    "fresh_sources": len([v for v in freshness.values() if v["status"] == "fresh"]),
-                },
-            }
-
-        except Exception as e:
-            logger.error(f"Failed to read data: {e}")
-            return {"error": str(e)}
-
-    data = asyncio.run(read_data())
-
-    if output_json:
-        console.print(JSON.from_data(data))
-    else:
-        console.print("📊 RECENT DATA SUMMARY", style="bold cyan")
-
-        if "error" in data:
-            console.print(f"❌ Error: {data['error']}", style="red")
-            return
-
-        # Summary stats
-        summary = data.get("summary", {})
-        console.print(f"\n📈 Summary:", style="bold blue")
-        console.print(f"Total recent jobs: {summary.get('total_jobs', 0)}")
-        console.print(f"Active job types: {summary.get('job_types', 0)}")
-        console.print(f"Fresh data sources: {summary.get('fresh_sources', 0)}")
-
-        # Data freshness
-        freshness = data.get("data_freshness", {})
-        if freshness:
-            console.print(f"\n🕐 Data Freshness:", style="bold blue")
-            for source, info in freshness.items():
-                status_icon = "🟢" if info["status"] == "fresh" else "🟡"
-                source_name = source.replace("_", " ").title()
-                last_success = info["last_success"][:19] if info["last_success"] else "Never"
-                console.print(f"{status_icon} {source_name}: {last_success}")
-
-        # Recent jobs
-        recent_jobs = data.get("recent_jobs", [])[:10]  # Show top 10
-        if recent_jobs:
-            console.print(f"\n📋 Recent Jobs (showing {len(recent_jobs)}):", style="bold blue")
-            for job in recent_jobs:
-                status_icon = {
-                    "completed": "✅",
-                    "running": "🔄",
-                    "failed": "❌",
-                    "pending": "⏳",
-                }.get(job.get("status"), "❓")
-                job_type = job.get("job_type", "unknown").replace("_", " ").title()
-                started_at = job.get("started_at", "N/A")[:19]
-                console.print(f"{status_icon} {job_type}: {started_at}")
-
-
-@politician_trading_cli.command("config-real-data")
-@click.option("--enable", is_flag=True, help="Enable real data collection")
-@click.option("--restore", is_flag=True, help="Restore sample data mode")
-@click.option("--status", is_flag=True, help="Show current configuration status")
-def configure_real_data(enable: bool, restore: bool, status: bool):
-    """Configure real vs sample data collection"""
-
-    if status or not (enable or restore):
-        # Show current status
-        console.print("🔧 DATA COLLECTION CONFIGURATION", style="bold cyan")
-
-        console.print("\n📋 Current Status:", style="bold blue")
-        console.print("• Sample data mode: Currently DISABLED", style="green")
-        console.print("• Real API calls: Currently ACTIVE", style="green")
-        console.print("• Database writes: Currently WORKING", style="green")
-
-        console.print("\n🎯 Data Source Readiness:", style="bold blue")
-        readiness_info = [
-            ("UK Parliament API", "✅ Active - Real API with full transaction data", "green"),
-            ("US House/Senate", "✅ Active - Real disclosure database access", "green"),
-            ("EU Parliament", "✅ Active - Real MEP profile scraping", "green"),
-            ("California NetFile", "⚠️  Limited - Complex forms require careful handling", "yellow"),
-            ("EU Member States", "⚠️  Limited - Country-specific implementations needed", "yellow"),
-        ]
-
-        for source, info, color in readiness_info:
-            console.print(f"{info}", style=color)
-
-        console.print("\n💡 Commands:", style="bold blue")
-        console.print("mcli politician-trading config-real-data --enable   # Enable real data")
-        console.print("mcli politician-trading config-real-data --restore  # Restore sample mode")
-
-        return
-
-    # Get scraper files
-    src_dir = Path(__file__).parent
-    scraper_files = [
-        "scrapers_uk.py",
-        "scrapers_california.py",
-        "scrapers_eu.py",
-        "scrapers_us_states.py",
-    ]
-
-    if restore:
-        console.print("🔄 RESTORING SAMPLE DATA MODE", style="bold yellow")
-
-        restored = 0
-        for file_name in scraper_files:
-            file_path = src_dir / file_name
-            backup_path = Path(str(file_path) + ".backup")
-
-            if backup_path.exists():
-                # Restore from backup
-                try:
-                    backup_content = backup_path.read_text()
-                    file_path.write_text(backup_content)
-                    restored += 1
-                    console.print(f"✅ Restored {file_name} from backup", style="green")
-                except Exception as e:
-                    console.print(f"❌ Failed to restore {file_name}: {e}", style="red")
-            else:
-                console.print(f"ℹ️  No backup found for {file_name}", style="dim")
-
-        console.print(f"\n🎯 Restored {restored} files to sample mode", style="green")
-
-    elif enable:
-        console.print("🚀 ENABLING REAL DATA COLLECTION", style="bold green")
-
-        with Progress(
-            SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console
-        ) as progress:
-            task = progress.add_task("Configuring scrapers...", total=len(scraper_files))
-
-            modifications_made = 0
-
-            for file_name in scraper_files:
-                progress.update(task, description=f"Processing {file_name}...")
-
-                file_path = src_dir / file_name
-
-                if not file_path.exists():
-                    progress.advance(task)
-                    continue
-
-                try:
-                    # Read file content
-                    content = file_path.read_text()
-                    original_content = content
-
-                    # Remove sample flags
-                    content = re.sub(r'"sample":\s*True', '"sample": False', content)
-                    content = re.sub(r"'sample':\s*True", "'sample': False", content)
-
-                    # Enable actual processing
-                    content = re.sub(
-                        r"# This would implement actual (.+?) scraping",
-                        r'logger.info("Processing real \1 data")',
-                        content,
-                    )
-
-                    if content != original_content:
-                        # Backup original
-                        backup_path = str(file_path) + ".backup"
-                        Path(backup_path).write_text(original_content)
-
-                        # Write modified content
-                        file_path.write_text(content)
-                        modifications_made += 1
-
-                except Exception as e:
-                    console.print(f"❌ Error processing {file_name}: {e}", style="red")
-
-                progress.advance(task)
-
-        console.print(f"\n✅ Real data configuration complete!", style="bold green")
-        console.print(f"Modified {modifications_made} scraper files", style="green")
-
-        if modifications_made > 0:
-            console.print(f"\n⚠️  Important Next Steps:", style="bold yellow")
-            console.print("1. Test with UK Parliament first (most reliable)", style="dim")
-            console.print("2. Monitor API rate limits carefully", style="dim")
-            console.print("3. Check logs for parsing errors", style="dim")
-            console.print("4. Use --restore flag if issues occur", style="dim")
-
-            console.print(f"\n🧪 Test Commands:", style="bold blue")
-            console.print("mcli politician-trading cron run --type quick  # Quick test")
-            console.print("mcli politician-trading monitor                # Check results")
-
-
-# Export the CLI group for registration
-cli = politician_trading_cli