mcli-framework 7.1.1__py3-none-any.whl → 7.1.2__py3-none-any.whl

mcli/workflow/politician_trading/commands.py

@@ -8,25 +8,30 @@ import os
 import re
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Dict, Any, List
+from typing import Any, Dict, List
 
 import click
 from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
 from rich.json import JSON
+from rich.panel import Panel
 from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.table import Table
 
 from mcli.lib.logger.logger import get_logger
+
+from .config import WorkflowConfig
+from .connectivity import (
+    SupabaseConnectivityValidator,
+    run_connectivity_validation,
+    run_continuous_monitoring,
+)
+from .database import PoliticianTradingDB
+from .monitoring import PoliticianTradingMonitor, run_health_check, run_stats_report
 from .workflow import (
     PoliticianTradingWorkflow,
-    run_politician_trading_collection,
     check_politician_trading_status,
+    run_politician_trading_collection,
 )
-from .config import WorkflowConfig
-from .database import PoliticianTradingDB
-from .monitoring import PoliticianTradingMonitor, run_health_check, run_stats_report
-from .connectivity import SupabaseConnectivityValidator, run_connectivity_validation, run_continuous_monitoring
 
 logger = get_logger(__name__)
 console = Console()
@@ -200,25 +205,27 @@ Timeout: {config.scraping.timeout}s"""
 
         if generate_schema:
             console.print("📄 Generating database schema files...", style="blue")
-            
+
             # Generate schema file
             import os
             from pathlib import Path
-            
+
             output_path = Path(output_dir)
             output_path.mkdir(exist_ok=True)
-            
+
             # Read the schema SQL from the module
             schema_file = Path(__file__).parent / "schema.sql"
             if schema_file.exists():
                 schema_content = schema_file.read_text()
-                
+
                 # Write to output directory
                 output_schema_file = output_path / "politician_trading_schema.sql"
                 output_schema_file.write_text(schema_content)
-                
-                console.print(f"✅ Schema SQL generated: {output_schema_file.absolute()}", style="green")
-                
+
+                console.print(
+                    f"✅ Schema SQL generated: {output_schema_file.absolute()}", style="green"
+                )
+
                 # Also generate a setup instructions file
                 instructions = f"""# Politician Trading Database Setup Instructions
 
@@ -269,24 +276,27 @@ If you encounter issues:
 2. View logs: `politician-trading health`
 3. Test workflow: `politician-trading test-workflow --verbose`
 """
-                
+
                 instructions_file = output_path / "SETUP_INSTRUCTIONS.md"
                 instructions_file.write_text(instructions)
-                
-                console.print(f"✅ Setup instructions generated: {instructions_file.absolute()}", style="green")
-                
+
+                console.print(
+                    f"✅ Setup instructions generated: {instructions_file.absolute()}",
+                    style="green",
+                )
+
                 # Display summary
                 console.print("\n📋 Generated Files:", style="bold")
                 console.print(f"  📄 Schema SQL: {output_schema_file.name}")
                 console.print(f"  📋 Instructions: {instructions_file.name}")
                 console.print(f"  📁 Location: {output_path.absolute()}")
-                
+
                 console.print("\n🚀 Next Steps:", style="bold green")
                 console.print("1. Open Supabase SQL editor")
                 console.print(f"2. Execute SQL from: {output_schema_file.name}")
                 console.print("3. Run: politician-trading setup --verify")
                 console.print("4. Run: politician-trading test-workflow --verbose")
-                
+
             else:
                 console.print("❌ Schema template not found", style="red")
 
@@ -449,13 +459,13 @@ def check_connectivity(output_json: bool, continuous: bool, interval: int, durat
     else:
         try:
             validation_result = asyncio.run(run_connectivity_validation())
-            
+
             if output_json:
                 console.print(JSON.from_data(validation_result))
             else:
                 validator = SupabaseConnectivityValidator()
                 validator.display_connectivity_report(validation_result)
-                
+
         except Exception as e:
             console.print(f"❌ Connectivity validation failed: {e}", style="bold red")
             logger.error(f"Connectivity validation failed: {e}")
@@ -467,94 +477,109 @@ def check_connectivity(output_json: bool, continuous: bool, interval: int, durat
 def test_full_workflow(verbose: bool, validate_writes: bool):
     """Run a complete workflow test with live Supabase connectivity"""
     console.print("🧪 Running Full Politician Trading Workflow Test", style="bold green")
-    
+
     async def run_test():
         # First validate connectivity
         console.print("\n🔗 Step 1: Validating Supabase connectivity...", style="blue")
         validator = SupabaseConnectivityValidator()
         connectivity_result = await validator.validate_connectivity()
-        
+
         if verbose:
             validator.display_connectivity_report(connectivity_result)
         else:
-            console.print(f"Connectivity Score: {connectivity_result['connectivity_score']}%", style="cyan")
-            
-        if connectivity_result['connectivity_score'] < 75:
+            console.print(
+                f"Connectivity Score: {connectivity_result['connectivity_score']}%", style="cyan"
+            )
+
+        if connectivity_result["connectivity_score"] < 75:
             console.print("⚠️ Connectivity issues detected. Workflow may fail.", style="yellow")
-        
+
         # Run the workflow
         console.print("\n🏛️ Step 2: Running politician trading collection workflow...", style="blue")
-        
+
         try:
             with console.status("[bold blue]Executing workflow...") as status:
                 workflow_result = await run_politician_trading_collection()
-            
+
             # Display workflow results
             console.print("\n📊 Workflow Results:", style="bold")
-            
+
             if workflow_result.get("status") == "completed":
                 console.print("✅ Workflow completed successfully!", style="green")
-                
+
                 summary = workflow_result.get("summary", {})
                 console.print(f"New Disclosures: {summary.get('total_new_disclosures', 0)}")
                 console.print(f"Updated Disclosures: {summary.get('total_updated_disclosures', 0)}")
                 console.print(f"Errors: {len(summary.get('errors', []))}")
-                
+
                 if verbose and summary.get("errors"):
                     console.print("\nErrors encountered:", style="red")
                     for error in summary["errors"][:5]:  # Show first 5 errors
                         console.print(f"  • {error}", style="dim red")
-                
+
             else:
                 console.print("❌ Workflow failed!", style="red")
                 if "error" in workflow_result:
                     console.print(f"Error: {workflow_result['error']}", style="red")
-            
+
             # Validate writes if requested
             if validate_writes:
                 console.print("\n🔍 Step 3: Validating database writes...", style="blue")
                 write_validation = await validator._test_write_operations()
-                
+
                 if write_validation["success"]:
                     console.print("✅ Database writes validated successfully", style="green")
                 else:
-                    console.print(f"❌ Database write validation failed: {write_validation.get('error', 'Unknown error')}", style="red")
-            
+                    console.print(
+                        f"❌ Database write validation failed: {write_validation.get('error', 'Unknown error')}",
+                        style="red",
+                    )
+
             # Final connectivity check
             console.print("\n🔗 Step 4: Post-workflow connectivity check...", style="blue")
             final_connectivity = await validator.validate_connectivity()
-            
-            console.print(f"Final Connectivity Score: {final_connectivity['connectivity_score']}%", style="cyan")
-            
+
+            console.print(
+                f"Final Connectivity Score: {final_connectivity['connectivity_score']}%",
+                style="cyan",
+            )
+
             # Summary
             console.print("\n📋 Test Summary:", style="bold")
-            workflow_status = "✅ PASSED" if workflow_result.get("status") == "completed" else "❌ FAILED"
-            connectivity_status = "✅ GOOD" if final_connectivity['connectivity_score'] >= 75 else "⚠️ DEGRADED"
-            
+            workflow_status = (
+                "✅ PASSED" if workflow_result.get("status") == "completed" else "❌ FAILED"
+            )
+            connectivity_status = (
+                "✅ GOOD" if final_connectivity["connectivity_score"] >= 75 else "⚠️ DEGRADED"
+            )
+
             console.print(f"Workflow: {workflow_status}")
             console.print(f"Connectivity: {connectivity_status}")
-            console.print(f"Duration: {workflow_result.get('started_at', '')} to {workflow_result.get('completed_at', '')}")
-            
+            console.print(
+                f"Duration: {workflow_result.get('started_at', '')} to {workflow_result.get('completed_at', '')}"
+            )
+
             return {
                 "workflow_result": workflow_result,
                 "connectivity_result": final_connectivity,
-                "test_passed": workflow_result.get("status") == "completed" and final_connectivity['connectivity_score'] >= 75
+                "test_passed": workflow_result.get("status") == "completed"
+                and final_connectivity["connectivity_score"] >= 75,
             }
-            
+
         except Exception as e:
             console.print(f"❌ Workflow test failed: {e}", style="bold red")
             if verbose:
                 console.print_exception()
             return {"error": str(e), "test_passed": False}
-    
+
     try:
         test_result = asyncio.run(run_test())
-        
+
         if test_result.get("test_passed"):
             console.print("\n🎉 Full workflow test PASSED!", style="bold green")
         else:
             console.print("\n❌ Full workflow test FAILED!", style="bold red")
-            
+
     except Exception as e:
         console.print(f"❌ Test execution failed: {e}", style="bold red")
         logger.error(f"Test workflow command failed: {e}")
@@ -566,83 +591,114 @@ def test_full_workflow(verbose: bool, validate_writes: bool):
 @click.option("--output-dir", default=".", help="Output directory for generated files")
 def manage_schema(show_location: bool, generate: bool, output_dir: str):
     """Manage database schema files"""
-    
+
     if show_location:
         console.print("📁 Schema File Locations", style="bold blue")
-        
+
         from pathlib import Path
+
         schema_file = Path(__file__).parent / "schema.sql"
-        
+
         console.print(f"Built-in Schema: {schema_file.absolute()}", style="cyan")
         console.print(f"File size: {schema_file.stat().st_size} bytes", style="dim")
-        console.print(f"Exists: {'✅ Yes' if schema_file.exists() else '❌ No'}", style="green" if schema_file.exists() else "red")
-        
+        console.print(
+            f"Exists: {'✅ Yes' if schema_file.exists() else '❌ No'}",
+            style="green" if schema_file.exists() else "red",
+        )
+
         # Show current working directory option
         cwd_schema = Path.cwd() / "politician_trading_schema.sql"
         console.print(f"\nCurrent directory: {cwd_schema.absolute()}", style="cyan")
-        console.print(f"Exists: {'✅ Yes' if cwd_schema.exists() else '❌ No'}", style="green" if cwd_schema.exists() else "dim")
-        
+        console.print(
+            f"Exists: {'✅ Yes' if cwd_schema.exists() else '❌ No'}",
+            style="green" if cwd_schema.exists() else "dim",
+        )
+
         if not cwd_schema.exists():
             console.print("\n💡 To generate schema file here:", style="blue")
             console.print("politician-trading schema --generate", style="yellow")
-    
+
     elif generate:
         # Reuse the setup command logic
         try:
-            from pathlib import Path
             import os
-            
+            from pathlib import Path
+
             console.print("📄 Generating database schema files...", style="blue")
-            
+
             output_path = Path(output_dir)
             output_path.mkdir(exist_ok=True)
-            
+
             # Read the schema SQL from the module
             schema_file = Path(__file__).parent / "schema.sql"
             if schema_file.exists():
                 schema_content = schema_file.read_text()
-                
+
                 # Write to output directory
                 output_schema_file = output_path / "politician_trading_schema.sql"
                 output_schema_file.write_text(schema_content)
-                
-                console.print(f"✅ Schema SQL generated: {output_schema_file.absolute()}", style="green")
-                
+
+                console.print(
+                    f"✅ Schema SQL generated: {output_schema_file.absolute()}", style="green"
+                )
+
                 # Show file info
                 console.print(f"📊 File size: {output_schema_file.stat().st_size:,} bytes")
                 console.print(f"📅 Created: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-                
+
                 # Count SQL statements
-                statements = len([line for line in schema_content.split('\n') if line.strip().startswith(('CREATE', 'INSERT', 'SELECT'))])
+                statements = len(
+                    [
+                        line
+                        for line in schema_content.split("\n")
+                        if line.strip().startswith(("CREATE", "INSERT", "SELECT"))
+                    ]
+                )
                 console.print(f"📝 SQL statements: {statements}")
-                
+
             else:
                 console.print("❌ Schema template not found", style="red")
-                
+
         except Exception as e:
             console.print(f"❌ Schema generation failed: {e}", style="red")
-    
+
     else:
         # Show schema information by default
         console.print("🗂️ Politician Trading Database Schema", style="bold blue")
-        
+
         schema_info = [
-            ("politicians", "Stores politician information", "UUID primary key, bioguide_id, role, party"),
-            ("trading_disclosures", "Individual trading transactions", "References politicians, amount ranges, asset details"),  
-            ("data_pull_jobs", "Job execution tracking", "Status, timing, record counts, error details"),
-            ("data_sources", "Data source configuration", "URLs, regions, health status, request config")
+            (
+                "politicians",
+                "Stores politician information",
+                "UUID primary key, bioguide_id, role, party",
+            ),
+            (
+                "trading_disclosures",
+                "Individual trading transactions",
+                "References politicians, amount ranges, asset details",
+            ),
+            (
+                "data_pull_jobs",
+                "Job execution tracking",
+                "Status, timing, record counts, error details",
+            ),
+            (
+                "data_sources",
+                "Data source configuration",
+                "URLs, regions, health status, request config",
+            ),
         ]
-        
+
         schema_table = Table(title="Database Tables")
         schema_table.add_column("Table", style="cyan")
         schema_table.add_column("Purpose", style="white")
         schema_table.add_column("Key Features", style="yellow")
-        
+
         for table_name, purpose, features in schema_info:
             schema_table.add_row(table_name, purpose, features)
-        
+
         console.print(schema_table)
-        
+
         console.print("\n🚀 Commands:", style="bold")
         console.print("  --show-location    Show where schema files are located")
         console.print("  --generate         Generate schema SQL file")
@@ -689,11 +745,11 @@ def _format_timestamp(timestamp: str) -> str:
 
 def _format_asset_display(disclosure: Dict[str, Any]) -> str:
     """Format asset display with proper ticker/name handling"""
-    asset_name = disclosure.get('asset_name', 'Unknown Asset')
-    asset_ticker = disclosure.get('asset_ticker')
-    
+    asset_name = disclosure.get("asset_name", "Unknown Asset")
+    asset_ticker = disclosure.get("asset_ticker")
+
     # If we have both ticker and name, show ticker first
-    if asset_ticker and asset_ticker.strip() and asset_ticker.lower() != 'none':
+    if asset_ticker and asset_ticker.strip() and asset_ticker.lower() != "none":
         return f"{asset_ticker} - {asset_name[:15]}"
     # If we only have asset name, show just that
     elif asset_name and asset_name.strip():
@@ -708,27 +764,27 @@ def _format_asset_display(disclosure: Dict[str, Any]) -> str:
 def view_data_sources(output_json: bool):
     """View current data sources and their configurations"""
     console = Console()
-    
+
     try:
         from .config import WorkflowConfig
-        from .data_sources import ALL_DATA_SOURCES, TOTAL_SOURCES, ACTIVE_SOURCES
-        
+        from .data_sources import ACTIVE_SOURCES, ALL_DATA_SOURCES, TOTAL_SOURCES
+
         config = WorkflowConfig.default()
         active_sources = config.scraping.get_active_sources()
-        
+
         # Group sources by category for display
         data_sources = {}
-        
+
         for category, sources in ALL_DATA_SOURCES.items():
             active_category_sources = [s for s in sources if s.status == "active"]
             if active_category_sources:
                 data_sources[category] = {
                     "name": {
                         "us_federal": "US Federal Government",
-                        "us_states": "US State Governments", 
+                        "us_states": "US State Governments",
                         "eu_parliament": "EU Parliament",
                         "eu_national": "EU National Parliaments",
-                        "third_party": "Third-Party Aggregators"
+                        "third_party": "Third-Party Aggregators",
                     }[category],
                     "sources": active_category_sources,
                     "count": len(active_category_sources),
@@ -736,12 +792,12 @@ def view_data_sources(output_json: bool):
                     "description": {
                         "us_federal": "Congressional and federal official financial disclosures",
                         "us_states": "State legislature financial disclosure databases",
-                        "eu_parliament": "MEP financial interest and income declarations", 
+                        "eu_parliament": "MEP financial interest and income declarations",
                         "eu_national": "National parliament financial disclosure systems",
-                        "third_party": "Commercial aggregators and enhanced analysis platforms"
-                    }[category]
+                        "third_party": "Commercial aggregators and enhanced analysis platforms",
+                    }[category],
                 }
-        
+
         if output_json:
             # For JSON output, convert DataSource objects to dictionaries
             json_output = {}
@@ -762,19 +818,24 @@ def view_data_sources(output_json: bool):
                             "update_frequency": source.update_frequency,
                             "threshold_amount": source.threshold_amount,
                             "data_format": source.data_format,
-                            "notes": source.notes
+                            "notes": source.notes,
                         }
                         for source in info["sources"]
-                    ]
+                    ],
                 }
             console.print(JSON.from_data(json_output))
         else:
-            console.print(f"📊 Comprehensive Political Trading Data Sources ({ACTIVE_SOURCES} active of {TOTAL_SOURCES} total)", style="bold cyan")
-            
+            console.print(
+                f"📊 Comprehensive Political Trading Data Sources ({ACTIVE_SOURCES} active of {TOTAL_SOURCES} total)",
+                style="bold cyan",
+            )
+
             for category_id, source_info in data_sources.items():
-                console.print(f"\n[bold blue]{source_info['name']}[/bold blue] ({source_info['count']} sources)")
+                console.print(
+                    f"\n[bold blue]{source_info['name']}[/bold blue] ({source_info['count']} sources)"
+                )
                 console.print(f"   {source_info['description']}", style="dim")
-                
+
                 # Create table for this category's sources
                 table = Table()
                 table.add_column("Source", style="cyan")
@@ -782,32 +843,32 @@ def view_data_sources(output_json: bool):
                 table.add_column("Access", style="yellow")
                 table.add_column("Disclosure Types", style="magenta")
                 table.add_column("Threshold", style="blue")
-                
+
                 for source in source_info["sources"]:
                     # Format disclosure types
-                    types_display = ", ".join([
-                        dt.value.replace("_", " ").title() 
-                        for dt in source.disclosure_types
-                    ])
-                    
+                    types_display = ", ".join(
+                        [dt.value.replace("_", " ").title() for dt in source.disclosure_types]
+                    )
+
                     # Format threshold
                     threshold_display = (
-                        f"${source.threshold_amount:,}" if source.threshold_amount 
-                        else "None"
+                        f"${source.threshold_amount:,}" if source.threshold_amount else "None"
                     )
-                    
+
                     table.add_row(
                         source.name,
                         source.jurisdiction,
                         source.access_method.value.replace("_", " ").title(),
                         types_display[:30] + ("..." if len(types_display) > 30 else ""),
-                        threshold_display
+                        threshold_display,
                     )
-                
+
                 console.print(table)
-            
-            console.print(f"\n[dim]Total: {ACTIVE_SOURCES} active sources across {len(data_sources)} categories[/dim]")
-                
+
+            console.print(
+                f"\n[dim]Total: {ACTIVE_SOURCES} active sources across {len(data_sources)} categories[/dim]"
+            )
+
     except Exception as e:
         if output_json:
             console.print(JSON.from_data({"error": str(e)}))
@@ -821,15 +882,16 @@ def view_data_sources(output_json: bool):
 def view_jobs(output_json: bool, limit: int):
     """View current and recent data collection jobs"""
     console = Console()
-    
+
     try:
+
         async def get_jobs():
-            from .database import PoliticianTradingDB
             from .config import WorkflowConfig
-            
+            from .database import PoliticianTradingDB
+
             config = WorkflowConfig.default()
             db = PoliticianTradingDB(config)
-            
+
             # Get recent jobs
             jobs_result = (
                 db.client.table("data_pull_jobs")
@@ -838,20 +900,20 @@ def view_jobs(output_json: bool, limit: int):
                 .limit(limit)
                 .execute()
             )
-            
+
             return jobs_result.data if jobs_result.data else []
-            
+
         jobs = asyncio.run(get_jobs())
-        
+
         if output_json:
             console.print(JSON.from_data(jobs))
         else:
             console.print("🔄 Recent Data Collection Jobs", style="bold cyan")
-            
+
             if not jobs:
                 console.print("No jobs found", style="yellow")
                 return
-            
+
             jobs_table = Table()
             jobs_table.add_column("Job ID", style="cyan")
             jobs_table.add_column("Type", style="green")
@@ -859,35 +921,35 @@ def view_jobs(output_json: bool, limit: int):
             jobs_table.add_column("Started", style="blue")
             jobs_table.add_column("Duration", style="magenta")
             jobs_table.add_column("Records", style="yellow")
-            
+
             for job in jobs:
                 status_color = {
                     "completed": "green",
-                    "running": "yellow", 
+                    "running": "yellow",
                     "failed": "red",
-                    "pending": "blue"
+                    "pending": "blue",
                 }.get(job.get("status", "unknown"), "white")
-                
+
                 # Calculate duration
                 started = job.get("started_at", "")
                 completed = job.get("completed_at", "")
                 duration = _format_duration_from_timestamps(started, completed)
-                
+
                 # Format records
                 records_info = f"{job.get('records_new', 0)}n/{job.get('records_updated', 0)}u/{job.get('records_failed', 0)}f"
-                
+
                 jobs_table.add_row(
                     job.get("id", "")[:8] + "...",
                     job.get("job_type", "unknown"),
                     f"[{status_color}]{job.get('status', 'unknown')}[/{status_color}]",
                     _format_timestamp(started),
                     duration,
-                    records_info
+                    records_info,
                 )
-            
+
             console.print(jobs_table)
             console.print("\nLegend: Records = new/updated/failed", style="dim")
-                
+
     except Exception as e:
         if output_json:
             console.print(JSON.from_data({"error": str(e)}))
@@ -900,20 +962,21 @@ def _format_duration_from_timestamps(started: str, completed: str) -> str:
     """Calculate and format duration from timestamps"""
     if not started:
         return "Unknown"
-    
+
     try:
         start_dt = datetime.fromisoformat(started.replace("Z", "+00:00"))
-        
+
         if completed:
             end_dt = datetime.fromisoformat(completed.replace("Z", "+00:00"))
             duration = end_dt - start_dt
         else:
             # Job still running
             from datetime import timezone
+
             duration = datetime.now(timezone.utc) - start_dt
-            
+
         return _format_duration_seconds(int(duration.total_seconds()))
-        
+
     except Exception:
         return "Unknown"
 
@@ -921,25 +984,28 @@ def _format_duration_from_timestamps(started: str, completed: str) -> str:
 @politician_trading_cli.command("politicians")
 @click.option("--json", "output_json", is_flag=True, help="Output as JSON")
 @click.option("--limit", default=20, help="Number of politicians to show")
-@click.option("--role", type=click.Choice(['us_house_rep', 'us_senator', 'eu_mep']), help="Filter by role")
+@click.option(
+    "--role", type=click.Choice(["us_house_rep", "us_senator", "eu_mep"]), help="Filter by role"
+)
 @click.option("--party", help="Filter by party")
 @click.option("--state", help="Filter by state/country")
 @click.option("--search", help="Search by name (first, last, or full name)")
 def view_politicians(output_json: bool, limit: int, role: str, party: str, state: str, search: str):
     """View and search politicians in the database"""
     console = Console()
-    
+
     try:
+
         async def get_politicians():
-            from .database import PoliticianTradingDB
             from .config import WorkflowConfig
-            
+            from .database import PoliticianTradingDB
+
             config = WorkflowConfig.default()
             db = PoliticianTradingDB(config)
-            
+
             # Build query
             query = db.client.table("politicians").select("*")
-            
+
             # Apply filters
             if role:
                 query = query.eq("role", role)
@@ -949,22 +1015,24 @@ def view_politicians(output_json: bool, limit: int, role: str, party: str, state
                 query = query.ilike("state_or_country", f"%{state}%")
             if search:
                 # Search across name fields
-                query = query.or_(f"first_name.ilike.%{search}%,last_name.ilike.%{search}%,full_name.ilike.%{search}%")
-            
+                query = query.or_(
+                    f"first_name.ilike.%{search}%,last_name.ilike.%{search}%,full_name.ilike.%{search}%"
+                )
+
             result = query.order("created_at", desc=True).limit(limit).execute()
             return result.data if result.data else []
-            
+
         politicians = asyncio.run(get_politicians())
-        
+
         if output_json:
             console.print(JSON.from_data(politicians))
         else:
             console.print("👥 Politicians Database", style="bold cyan")
-            
+
             if not politicians:
                 console.print("No politicians found", style="yellow")
                 return
-            
+
             politicians_table = Table()
             politicians_table.add_column("Name", style="cyan", min_width=25)
             politicians_table.add_column("Role", style="green")
@@ -972,26 +1040,29 @@ def view_politicians(output_json: bool, limit: int, role: str, party: str, state
             politicians_table.add_column("State/Country", style="magenta")
             politicians_table.add_column("District", style="yellow")
             politicians_table.add_column("Added", style="dim")
-            
+
             for pol in politicians:
                 role_display = {
                     "us_house_rep": "🏛️ House Rep",
-                    "us_senator": "🏛️ Senator", 
-                    "eu_mep": "🇪🇺 MEP"
+                    "us_senator": "🏛️ Senator",
+                    "eu_mep": "🇪🇺 MEP",
                 }.get(pol.get("role", ""), pol.get("role", "Unknown"))
-                
+
                 politicians_table.add_row(
-                    pol.get("full_name") or f"{pol.get('first_name', '')} {pol.get('last_name', '')}".strip(),
+                    pol.get("full_name")
+                    or f"{pol.get('first_name', '')} {pol.get('last_name', '')}".strip(),
                     role_display,
                     pol.get("party", "") or "Independent",
                     pol.get("state_or_country", ""),
                     pol.get("district", "") or "At-Large",
-                    _format_timestamp(pol.get("created_at", ""))
+                    _format_timestamp(pol.get("created_at", "")),
                 )
-            
+
             console.print(politicians_table)
-            console.print(f"\nShowing {len(politicians)} of {len(politicians)} politicians", style="dim")
-                
+            console.print(
+                f"\nShowing {len(politicians)} of {len(politicians)} politicians", style="dim"
+            )
+
     except Exception as e:
         if output_json:
             console.print(JSON.from_data({"error": str(e)}))
@@ -1000,109 +1071,137 @@ def view_politicians(output_json: bool, limit: int, role: str, party: str, state
             logger.error(f"Politicians view failed: {e}")
 
 
-@politician_trading_cli.command("disclosures")  
+@politician_trading_cli.command("disclosures")
 @click.option("--json", "output_json", is_flag=True, help="Output as JSON")
 @click.option("--limit", default=20, help="Number of disclosures to show")
 @click.option("--politician", help="Filter by politician name")
 @click.option("--asset", help="Filter by asset name or ticker")
-@click.option("--transaction-type", type=click.Choice(['purchase', 'sale', 'exchange']), help="Filter by transaction type")
+@click.option(
+    "--transaction-type",
+    type=click.Choice(["purchase", "sale", "exchange"]),
+    help="Filter by transaction type",
+)
 @click.option("--amount-min", type=float, help="Minimum transaction amount")
 @click.option("--amount-max", type=float, help="Maximum transaction amount")
 @click.option("--days", default=30, help="Show disclosures from last N days")
 @click.option("--details", is_flag=True, help="Show detailed information including raw data")
-def view_disclosures(output_json: bool, limit: int, politician: str, asset: str, 
-                    transaction_type: str, amount_min: float, amount_max: float, 
-                    days: int, details: bool):
+def view_disclosures(
+    output_json: bool,
+    limit: int,
+    politician: str,
+    asset: str,
+    transaction_type: str,
+    amount_min: float,
+    amount_max: float,
+    days: int,
+    details: bool,
+):
     """View and search trading disclosures in the database"""
     console = Console()
-    
+
     try:
+
         async def get_disclosures():
-            from .database import PoliticianTradingDB
-            from .config import WorkflowConfig
             from datetime import datetime, timedelta, timezone
-            
+
+            from .config import WorkflowConfig
+            from .database import PoliticianTradingDB
+
             config = WorkflowConfig.default()
             db = PoliticianTradingDB(config)
-            
-            # Build query with join to get politician info  
+
+            # Build query with join to get politician info
             # Supabase uses foreign key relationships for joins
-            query = (
-                db.client.table("trading_disclosures")
-                .select("*, politicians!inner(*)")
-            )
-            
+            query = db.client.table("trading_disclosures").select("*, politicians!inner(*)")
+
             # Date filter
             if days > 0:
                 cutoff_date = datetime.now(timezone.utc) - timedelta(days=days)
                 query = query.gte("created_at", cutoff_date.isoformat())
-            
+
             # Apply filters
             if politician:
                 # For nested relationships, we need a different approach
                 # Let's use a simpler filter on the main table for now
                 query = query.filter("politicians.full_name", "ilike", f"%{politician}%")
-            
+
             if asset:
                 query = query.or_(f"asset_name.ilike.%{asset}%,asset_ticker.ilike.%{asset}%")
-            
+
             if transaction_type:
                 query = query.eq("transaction_type", transaction_type)
-            
+
             if amount_min is not None:
                 query = query.gte("amount_range_min", amount_min)
-            
+
             if amount_max is not None:
                 query = query.lte("amount_range_max", amount_max)
-            
+
             result = query.order("transaction_date", desc=True).limit(limit).execute()
             return result.data if result.data else []
-            
+
         disclosures = asyncio.run(get_disclosures())
-        
+
         if output_json:
             console.print(JSON.from_data(disclosures))
         else:
             console.print("💰 Trading Disclosures Database", style="bold cyan")
-            
+
             if not disclosures:
                 console.print("No disclosures found", style="yellow")
                 return
-            
+
             if details:
                 # Detailed view
                 for i, disclosure in enumerate(disclosures):
                     console.print(f"\n[bold cyan]Disclosure {i+1}[/bold cyan]")
-                    
+
                     detail_table = Table()
                     detail_table.add_column("Field", style="cyan")
                     detail_table.add_column("Value", style="white")
-                    
+
                     politician_info = disclosure.get("politicians", {})
-                    politician_name = politician_info.get("full_name") or f"{politician_info.get('first_name', '')} {politician_info.get('last_name', '')}".strip()
-                    
-                    detail_table.add_row("Politician", f"{politician_name} ({politician_info.get('party', 'Unknown')})")
-                    detail_table.add_row("Asset", f"{disclosure.get('asset_name', 'Unknown')} ({disclosure.get('asset_ticker', 'N/A')})")
-                    detail_table.add_row("Transaction", disclosure.get('transaction_type', 'Unknown').title())
-                    detail_table.add_row("Date", _format_timestamp(disclosure.get('transaction_date', '')))
-                    detail_table.add_row("Disclosure Date", _format_timestamp(disclosure.get('disclosure_date', '')))
-                    
+                    politician_name = (
+                        politician_info.get("full_name")
+                        or f"{politician_info.get('first_name', '')} {politician_info.get('last_name', '')}".strip()
+                    )
+
+                    detail_table.add_row(
+                        "Politician",
+                        f"{politician_name} ({politician_info.get('party', 'Unknown')})",
+                    )
+                    detail_table.add_row(
+                        "Asset",
+                        f"{disclosure.get('asset_name', 'Unknown')} ({disclosure.get('asset_ticker', 'N/A')})",
+                    )
+                    detail_table.add_row(
+                        "Transaction", disclosure.get("transaction_type", "Unknown").title()
+                    )
+                    detail_table.add_row(
+                        "Date", _format_timestamp(disclosure.get("transaction_date", ""))
+                    )
+                    detail_table.add_row(
+                        "Disclosure Date", _format_timestamp(disclosure.get("disclosure_date", ""))
+                    )
+
                     # Amount formatting
-                    amount_min = disclosure.get('amount_range_min')
-                    amount_max = disclosure.get('amount_range_max')
-                    amount_exact = disclosure.get('amount_exact')
-                    
+                    amount_min = disclosure.get("amount_range_min")
+                    amount_max = disclosure.get("amount_range_max")
+                    amount_exact = disclosure.get("amount_exact")
+
                     if amount_exact:
                         amount_str = f"${amount_exact:,.2f}"
                     elif amount_min is not None and amount_max is not None:
                         amount_str = f"${amount_min:,.0f} - ${amount_max:,.0f}"
                     else:
                         amount_str = "Unknown"
-                    
+
                     detail_table.add_row("Amount", amount_str)
-                    detail_table.add_row("Source URL", disclosure.get('source_url', 'N/A'))
-                    detail_table.add_row("Added", _format_timestamp(disclosure.get('created_at', '')))
-                    
+                    detail_table.add_row("Source URL", disclosure.get("source_url", "N/A"))
+                    detail_table.add_row(
+                        "Added", _format_timestamp(disclosure.get("created_at", ""))
+                    )
+
                     console.print(detail_table)
             else:
                 # Compact table view
@@ -1113,40 +1212,49 @@ def view_disclosures(output_json: bool, limit: int, politician: str, asset: str,
                 disclosures_table.add_column("Amount", style="yellow")
                 disclosures_table.add_column("Date", style="magenta")
                 disclosures_table.add_column("Party", style="dim")
-                
+
                 for disclosure in disclosures:
                     politician_info = disclosure.get("politicians", {})
-                    politician_name = politician_info.get("full_name") or f"{politician_info.get('first_name', '')} {politician_info.get('last_name', '')}".strip()
-                    
+                    politician_name = (
+                        politician_info.get("full_name")
+                        or f"{politician_info.get('first_name', '')} {politician_info.get('last_name', '')}".strip()
+                    )
+
                     # Format amount
-                    amount_min = disclosure.get('amount_range_min')
-                    amount_max = disclosure.get('amount_range_max') 
-                    amount_exact = disclosure.get('amount_exact')
-                    
+                    amount_min = disclosure.get("amount_range_min")
+                    amount_max = disclosure.get("amount_range_max")
+                    amount_exact = disclosure.get("amount_exact")
+
                     if amount_exact:
                         amount_str = f"${amount_exact:,.0f}"
                     elif amount_min is not None and amount_max is not None:
                         amount_str = f"${amount_min:,.0f}-${amount_max:,.0f}"
                     else:
                         amount_str = "Unknown"
-                    
+
                     # Transaction type with emoji
-                    trans_type = disclosure.get('transaction_type', 'unknown')
-                    trans_emoji = {"purchase": "🟢 Buy", "sale": "🔴 Sell", "exchange": "🔄 Exchange"}.get(trans_type, "❓ " + trans_type.title())
-                    
+                    trans_type = disclosure.get("transaction_type", "unknown")
+                    trans_emoji = {
+                        "purchase": "🟢 Buy",
+                        "sale": "🔴 Sell",
+                        "exchange": "🔄 Exchange",
+                    }.get(trans_type, "❓ " + trans_type.title())
+
                     disclosures_table.add_row(
                         politician_name[:35] + ("..." if len(politician_name) > 35 else ""),
                         _format_asset_display(disclosure),
                         trans_emoji,
                         amount_str,
-                        _format_timestamp(disclosure.get('transaction_date', '')),
-                        politician_info.get('party', '')[:12]
+                        _format_timestamp(disclosure.get("transaction_date", "")),
+                        politician_info.get("party", "")[:12],
                     )
-                
+
                 console.print(disclosures_table)
-            
-            console.print(f"\nShowing {len(disclosures)} disclosures from last {days} days", style="dim")
-                
+
+            console.print(
+                f"\nShowing {len(disclosures)} disclosures from last {days} days", style="dim"
+            )
+
     except Exception as e:
         if output_json:
             console.print(JSON.from_data({"error": str(e)}))
@@ -1160,26 +1268,28 @@ def view_disclosures(output_json: bool, limit: int, politician: str, asset: str,
 def verify_database(output_json: bool):
     """Verify database integrity and show summary statistics"""
     console = Console()
-    
+
     try:
+
         async def verify_data():
-            from .database import PoliticianTradingDB
-            from .config import WorkflowConfig
             from datetime import timedelta
-            
+
+            from .config import WorkflowConfig
+            from .database import PoliticianTradingDB
+
             config = WorkflowConfig.default()
             db = PoliticianTradingDB(config)
-            
+
             verification = {
                 "timestamp": datetime.now().isoformat(),
                 "tables": {},
                 "integrity": {},
-                "summary": {}
+                "summary": {},
             }
-            
+
             # Check each table
             tables_to_check = ["politicians", "trading_disclosures", "data_pull_jobs"]
-            
+
             for table_name in tables_to_check:
                 try:
                     result = db.client.table(table_name).select("id").execute()
@@ -1187,41 +1297,38 @@ def verify_database(output_json: bool):
                     verification["tables"][table_name] = {
                         "exists": True,
                         "record_count": count,
-                        "status": "ok"
+                        "status": "ok",
                     }
                 except Exception as e:
                     verification["tables"][table_name] = {
                         "exists": False,
                         "error": str(e),
-                        "status": "error"
+                        "status": "error",
                     }
-            
+
             # Check referential integrity - simplified approach
             try:
                 # Just verify we can query both tables
                 disclosures_result = db.client.table("trading_disclosures").select("id").execute()
                 politicians_result = db.client.table("politicians").select("id").execute()
-                
+
                 disclosures_count = len(disclosures_result.data) if disclosures_result.data else 0
                 politicians_count = len(politicians_result.data) if politicians_result.data else 0
-                
+
                 verification["integrity"] = {
                     "disclosures_with_politicians": disclosures_count,
                     "total_politicians": politicians_count,
-                    "status": "ok"
+                    "status": "ok",
                 }
             except Exception as e:
-                verification["integrity"] = {
-                    "error": str(e),
-                    "status": "error"
-                }
-            
+                verification["integrity"] = {"error": str(e), "status": "error"}
+
             # Summary statistics
             try:
                 politicians_count = verification["tables"]["politicians"]["record_count"]
                 disclosures_count = verification["tables"]["trading_disclosures"]["record_count"]
                 jobs_count = verification["tables"]["data_pull_jobs"]["record_count"]
-                
+
                 # Get recent activity
                 recent_jobs = (
                     db.client.table("data_pull_jobs")
@@ -1229,56 +1336,62 @@ def verify_database(output_json: bool):
                     .gte("started_at", (datetime.now() - timedelta(days=7)).isoformat())
                     .execute()
                 )
-                
+
                 recent_jobs_count = len(recent_jobs.data) if recent_jobs.data else 0
-                successful_jobs = len([j for j in (recent_jobs.data or []) if j.get("status") == "completed"])
-                
+                successful_jobs = len(
+                    [j for j in (recent_jobs.data or []) if j.get("status") == "completed"]
+                )
+
                 verification["summary"] = {
                     "total_politicians": politicians_count,
                     "total_disclosures": disclosures_count,
                     "total_jobs": jobs_count,
                     "jobs_last_7_days": recent_jobs_count,
                     "successful_jobs_last_7_days": successful_jobs,
-                    "success_rate_7_days": (successful_jobs / recent_jobs_count * 100) if recent_jobs_count > 0 else 0
+                    "success_rate_7_days": (
+                        (successful_jobs / recent_jobs_count * 100) if recent_jobs_count > 0 else 0
+                    ),
                 }
-                
+
             except Exception as e:
                 verification["summary"] = {"error": str(e)}
-            
+
             return verification
-            
+
         verification = asyncio.run(verify_data())
-        
+
         if output_json:
             console.print(JSON.from_data(verification))
         else:
             console.print("🔍 Database Verification Report", style="bold cyan")
-            
+
             # Table status
             tables_panel = Table(title="Table Status")
             tables_panel.add_column("Table", style="cyan")
             tables_panel.add_column("Status", style="white")
             tables_panel.add_column("Records", justify="right", style="green")
-            
+
             for table_name, info in verification["tables"].items():
                 status_color = "green" if info["status"] == "ok" else "red"
                 status_text = f"[{status_color}]{info['status'].upper()}[/{status_color}]"
                 record_count = str(info.get("record_count", "N/A"))
-                
+
                 tables_panel.add_row(table_name, status_text, record_count)
-            
+
             console.print(tables_panel)
-            
+
             # Integrity check
             integrity_info = verification.get("integrity", {})
             if integrity_info.get("status") == "ok":
                 console.print("✅ Data integrity check passed", style="green")
                 disc_count = integrity_info.get("disclosures_with_politicians", 0)
                 pol_count = integrity_info.get("total_politicians", 0)
-                console.print(f"   Disclosures: {disc_count}, Politicians: {pol_count}", style="dim")
+                console.print(
+                    f"   Disclosures: {disc_count}, Politicians: {pol_count}", style="dim"
+                )
             else:
                 console.print("❌ Data integrity check failed", style="red")
-            
+
             # Summary
             summary = verification.get("summary", {})
             if "error" not in summary:
@@ -1286,9 +1399,11 @@ def verify_database(output_json: bool):
                 console.print(f"Politicians: {summary.get('total_politicians', 0)}")
                 console.print(f"Trading Disclosures: {summary.get('total_disclosures', 0)}")
                 console.print(f"Data Collection Jobs: {summary.get('total_jobs', 0)}")
-                console.print(f"Jobs (7 days): {summary.get('jobs_last_7_days', 0)} ({summary.get('successful_jobs_last_7_days', 0)} successful)")
+                console.print(
+                    f"Jobs (7 days): {summary.get('jobs_last_7_days', 0)} ({summary.get('successful_jobs_last_7_days', 0)} successful)"
+                )
                 console.print(f"Success Rate: {summary.get('success_rate_7_days', 0):.1f}%")
-                
+
     except Exception as e:
         if output_json:
             console.print(JSON.from_data({"error": str(e)}))
@@ -1304,30 +1419,34 @@ def cron_commands():
 
 
 @cron_commands.command("run")
-@click.option("--type", "collection_type", default="full", 
-              type=click.Choice(["full", "us", "eu", "quick"]), 
-              help="Type of collection to run")
+@click.option(
+    "--type",
+    "collection_type",
+    default="full",
+    type=click.Choice(["full", "us", "eu", "quick"]),
+    help="Type of collection to run",
+)
 def cron_run(collection_type: str):
     """Run scheduled data collection (designed for cron jobs)"""
-    
+
     async def run_cron_collection():
         """Run the cron collection"""
         from datetime import datetime
-        
+
         logger.info(f"Starting scheduled collection: {collection_type}")
         console.print(f"🕐 Running {collection_type} data collection...", style="blue")
-        
+
         try:
             workflow = PoliticianTradingWorkflow()
-            
+
             if collection_type == "full":
                 results = await run_politician_trading_collection()
             elif collection_type == "us":
                 # US-only collection
                 us_results = await workflow._collect_us_congress_data()
-                ca_results = await workflow._collect_california_data() 
+                ca_results = await workflow._collect_california_data()
                 us_states_results = await workflow._collect_us_states_data()
-                
+
                 results = {
                     "status": "completed",
                     "started_at": datetime.utcnow().isoformat(),
@@ -1335,22 +1454,24 @@ def cron_run(collection_type: str):
                     "jobs": {
                         "us_congress": us_results,
                         "california": ca_results,
-                        "us_states": us_states_results
+                        "us_states": us_states_results,
                     },
                     "summary": {
-                        "total_new_disclosures": sum([
-                            us_results.get("new_disclosures", 0),
-                            ca_results.get("new_disclosures", 0), 
-                            us_states_results.get("new_disclosures", 0)
-                        ])
-                    }
+                        "total_new_disclosures": sum(
+                            [
+                                us_results.get("new_disclosures", 0),
+                                ca_results.get("new_disclosures", 0),
+                                us_states_results.get("new_disclosures", 0),
+                            ]
+                        )
+                    },
                 }
             elif collection_type == "eu":
                 # EU-only collection
                 eu_results = await workflow._collect_eu_parliament_data()
                 eu_states_results = await workflow._collect_eu_member_states_data()
                 uk_results = await workflow._collect_uk_parliament_data()
-                
+
                 results = {
                     "status": "completed",
                     "started_at": datetime.utcnow().isoformat(),
@@ -1358,15 +1479,17 @@ def cron_run(collection_type: str):
                     "jobs": {
                         "eu_parliament": eu_results,
                         "eu_member_states": eu_states_results,
-                        "uk_parliament": uk_results
+                        "uk_parliament": uk_results,
                     },
                     "summary": {
-                        "total_new_disclosures": sum([
-                            eu_results.get("new_disclosures", 0),
-                            eu_states_results.get("new_disclosures", 0),
-                            uk_results.get("new_disclosures", 0)
-                        ])
-                    }
+                        "total_new_disclosures": sum(
+                            [
+                                eu_results.get("new_disclosures", 0),
+                                eu_states_results.get("new_disclosures", 0),
+                                uk_results.get("new_disclosures", 0),
+                            ]
+                        )
+                    },
                 }
             elif collection_type == "quick":
                 # Quick status check
@@ -1375,23 +1498,27 @@ def cron_run(collection_type: str):
                     "status": "completed",
                     "type": "quick_check",
                     "results": status,
-                    "summary": {"total_new_disclosures": 0}
+                    "summary": {"total_new_disclosures": 0},
                 }
-            
+
             # Log results
-            summary = results.get('summary', {})
-            logger.info(f"Cron collection completed - New: {summary.get('total_new_disclosures', 0)}")
-            
+            summary = results.get("summary", {})
+            logger.info(
+                f"Cron collection completed - New: {summary.get('total_new_disclosures', 0)}"
+            )
+
             console.print(f"✅ {collection_type.title()} collection completed", style="green")
-            console.print(f"New disclosures: {summary.get('total_new_disclosures', 0)}", style="cyan")
-            
+            console.print(
+                f"New disclosures: {summary.get('total_new_disclosures', 0)}", style="cyan"
+            )
+
             return results
-            
+
         except Exception as e:
             logger.error(f"Cron collection failed: {e}")
             console.print(f"❌ Collection failed: {e}", style="red")
             return {"status": "failed", "error": str(e)}
-    
+
     asyncio.run(run_cron_collection())
 
 
@@ -1400,10 +1527,10 @@ def cron_setup():
     """Show cron setup instructions"""
     console.print("🕐 CRON SETUP INSTRUCTIONS", style="bold cyan")
     console.print("Add these lines to your crontab (run: crontab -e)", style="dim")
-    
+
     # Get current working directory for the cron commands
     repo_path = Path(__file__).parent.parent.parent.parent.parent
-    
+
     instructions = f"""
 # Full collection every 6 hours  
 0 */6 * * * cd {repo_path} && source .venv/bin/activate && mcli politician-trading cron run --type full >> /tmp/politician_cron.log 2>&1
@@ -1417,9 +1544,9 @@ def cron_setup():
 # Quick health check daily at 9 AM
 0 9 * * * cd {repo_path} && source .venv/bin/activate && mcli politician-trading cron run --type quick >> /tmp/politician_cron.log 2>&1
 """
-    
+
     console.print(Panel(instructions, title="Crontab Entries", border_style="blue"))
-    
+
     console.print("\n💡 Tips:", style="bold yellow")
     console.print("• Start with just one cron job to test", style="dim")
     console.print("• Check logs at /tmp/politician_cron.log", style="dim")
@@ -1430,136 +1557,148 @@ def cron_setup():
 @click.option("--json", "output_json", is_flag=True, help="Output as JSON")
 def monitor_system(output_json: bool):
     """Monitor system status, jobs, and database"""
-    
+
     async def run_monitor():
         """Run the monitoring"""
         try:
             config = WorkflowConfig.default()
             db = PoliticianTradingDB(config)
             workflow = PoliticianTradingWorkflow(config)
-            
+
             # Get system health
             await db.ensure_schema()
             quick_status = await workflow.run_quick_check()
-            
+
             # Get job history
             job_status = await db.get_job_status()
-            recent_jobs = job_status.get('recent_jobs', [])
-            
+            recent_jobs = job_status.get("recent_jobs", [])
+
             # Analyze job statistics
-            status_counts = {'completed': 0, 'running': 0, 'failed': 0, 'pending': 0}
+            status_counts = {"completed": 0, "running": 0, "failed": 0, "pending": 0}
             job_types = {}
             latest_by_type = {}
-            
+
             for job in recent_jobs:
-                status = job.get('status', 'unknown')
-                job_type = job.get('job_type', 'unknown')
-                started_at = job.get('started_at', '')
-                
+                status = job.get("status", "unknown")
+                job_type = job.get("job_type", "unknown")
+                started_at = job.get("started_at", "")
+
                 if status in status_counts:
                     status_counts[status] += 1
                 job_types[job_type] = job_types.get(job_type, 0) + 1
-                
-                if job_type not in latest_by_type or started_at > latest_by_type[job_type].get('started_at', ''):
+
+                if job_type not in latest_by_type or started_at > latest_by_type[job_type].get(
+                    "started_at", ""
+                ):
                     latest_by_type[job_type] = job
-            
+
             # Get scraper availability
             try:
                 from . import scrapers
+
                 scraper_status = {
-                    'UK Parliament API': scrapers.UK_SCRAPER_AVAILABLE,
-                    'California NetFile': scrapers.CALIFORNIA_SCRAPER_AVAILABLE,
-                    'EU Member States': scrapers.EU_MEMBER_STATES_SCRAPER_AVAILABLE,
-                    'US States Ethics': scrapers.US_STATES_SCRAPER_AVAILABLE,
+                    "UK Parliament API": scrapers.UK_SCRAPER_AVAILABLE,
+                    "California NetFile": scrapers.CALIFORNIA_SCRAPER_AVAILABLE,
+                    "EU Member States": scrapers.EU_MEMBER_STATES_SCRAPER_AVAILABLE,
+                    "US States Ethics": scrapers.US_STATES_SCRAPER_AVAILABLE,
                 }
                 available_scrapers = sum(scraper_status.values())
             except:
                 scraper_status = {}
                 available_scrapers = 0
-            
+
             monitor_data = {
                 "system_health": {
-                    "database_connection": quick_status.get('database_connection', 'unknown'),
-                    "config_loaded": quick_status.get('config_loaded', 'unknown'),
-                    "timestamp": quick_status.get('timestamp', datetime.now().isoformat())
+                    "database_connection": quick_status.get("database_connection", "unknown"),
+                    "config_loaded": quick_status.get("config_loaded", "unknown"),
+                    "timestamp": quick_status.get("timestamp", datetime.now().isoformat()),
                 },
                 "job_statistics": {
                     "total_recent_jobs": len(recent_jobs),
                     "status_counts": status_counts,
-                    "job_types": job_types
+                    "job_types": job_types,
                 },
                 "latest_jobs": latest_by_type,
                 "scraper_availability": {
                     "available_count": available_scrapers,
                     "total_count": len(scraper_status),
-                    "scrapers": scraper_status
-                }
+                    "scrapers": scraper_status,
+                },
             }
-            
+
             return monitor_data
-            
+
         except Exception as e:
             logger.error(f"Monitoring failed: {e}")
             return {"error": str(e)}
-    
+
     monitor_data = asyncio.run(run_monitor())
-    
+
     if output_json:
         console.print(JSON.from_data(monitor_data))
     else:
         console.print("🔍 SYSTEM MONITOR", style="bold cyan")
-        
+
         # System Health
-        health = monitor_data.get('system_health', {})
+        health = monitor_data.get("system_health", {})
         health_table = Table(title="System Health")
         health_table.add_column("Component", style="cyan")
         health_table.add_column("Status", style="white")
-        
-        db_status = health['database_connection']
+
+        db_status = health["database_connection"]
         db_color = "green" if db_status == "ok" else "red"
         health_table.add_row("Database", f"[{db_color}]{db_status.upper()}[/{db_color}]")
-        
-        config_status = health['config_loaded'] 
+
+        config_status = health["config_loaded"]
         config_color = "green" if config_status == "ok" else "red"
-        health_table.add_row("Configuration", f"[{config_color}]{config_status.upper()}[/{config_color}]")
-        
+        health_table.add_row(
+            "Configuration", f"[{config_color}]{config_status.upper()}[/{config_color}]"
+        )
+
         console.print(health_table)
-        
+
         # Job Statistics
-        job_stats = monitor_data.get('job_statistics', {})
-        console.print(f"\n📊 Job Statistics (Total: {job_stats.get('total_recent_jobs', 0)})", style="bold blue")
-        
-        status_counts = job_stats.get('status_counts', {})
+        job_stats = monitor_data.get("job_statistics", {})
+        console.print(
+            f"\n📊 Job Statistics (Total: {job_stats.get('total_recent_jobs', 0)})",
+            style="bold blue",
+        )
+
+        status_counts = job_stats.get("status_counts", {})
         for status, count in status_counts.items():
             if count > 0:
-                icon = {'completed': '✅', 'running': '🔄', 'failed': '❌', 'pending': '⏳'}[status]
+                icon = {"completed": "✅", "running": "🔄", "failed": "❌", "pending": "⏳"}[status]
                 console.print(f"{icon} {status.title()}: {count}")
-        
+
         # Latest Jobs by Type
         console.print(f"\n📋 Latest Jobs by Source", style="bold blue")
-        latest_jobs = monitor_data.get('latest_jobs', {})
-        
+        latest_jobs = monitor_data.get("latest_jobs", {})
+
         for job_type, job in sorted(latest_jobs.items()):
-            status = job.get('status', 'unknown')
-            icon = {'completed': '✅', 'running': '🔄', 'failed': '❌', 'pending': '⏳'}.get(status, '❓')
-            
-            source_name = job_type.replace('_', ' ').title()
+            status = job.get("status", "unknown")
+            icon = {"completed": "✅", "running": "🔄", "failed": "❌", "pending": "⏳"}.get(
+                status, "❓"
+            )
+
+            source_name = job_type.replace("_", " ").title()
             console.print(f"\n{icon} {source_name}")
             console.print(f"   Status: {status}")
             console.print(f"   Last run: {job.get('started_at', 'N/A')[:19]}")
-            console.print(f"   Records: {job.get('records_processed', 0)} processed, {job.get('records_new', 0)} new")
-        
+            console.print(
+                f"   Records: {job.get('records_processed', 0)} processed, {job.get('records_new', 0)} new"
+            )
+
         # Scraper Availability
-        scraper_info = monitor_data.get('scraper_availability', {})
-        available = scraper_info.get('available_count', 0)
-        total = scraper_info.get('total_count', 0)
-        
+        scraper_info = monitor_data.get("scraper_availability", {})
+        available = scraper_info.get("available_count", 0)
+        total = scraper_info.get("total_count", 0)
+
         console.print(f"\n🌍 Scraper Availability: {available}/{total}", style="bold blue")
-        
-        scrapers_status = scraper_info.get('scrapers', {})
+
+        scrapers_status = scraper_info.get("scrapers", {})
         for scraper_name, available in scrapers_status.items():
-            icon = '✅' if available else '❌'
-            status = 'Available' if available else 'Not Available'
+            icon = "✅" if available else "❌"
+            status = "Available" if available else "Not Available"
             console.print(f"{icon} {scraper_name}: {status}")
 
 
@@ -1569,143 +1708,155 @@ def monitor_system(output_json: bool):
 @click.option("--json", "output_json", is_flag=True, help="Output as JSON")
 def read_recent_data(limit: int, days: int, output_json: bool):
     """Read recent data from the database"""
-    
+
     async def read_data():
         """Read recent data from database"""
         try:
             config = WorkflowConfig.default()
             db = PoliticianTradingDB(config)
-            
+
             # Get job history
             job_status = await db.get_job_status()
-            jobs = job_status.get('recent_jobs', [])
-            
+            jobs = job_status.get("recent_jobs", [])
+
             # Analyze data freshness
             freshness = {}
             for job in jobs:
-                job_type = job.get('job_type', 'unknown')
-                if job.get('status') == 'completed':
-                    completed_at = job.get('completed_at')
-                    if job_type not in freshness or completed_at > freshness[job_type]['last_success']:
+                job_type = job.get("job_type", "unknown")
+                if job.get("status") == "completed":
+                    completed_at = job.get("completed_at")
+                    if (
+                        job_type not in freshness
+                        or completed_at > freshness[job_type]["last_success"]
+                    ):
                         # Check if recent (within threshold)
                         is_recent = False
                         if completed_at:
                             try:
-                                timestamp = datetime.fromisoformat(completed_at.replace('Z', '+00:00'))
-                                is_recent = (datetime.now() - timestamp.replace(tzinfo=None)) < timedelta(hours=24)
+                                timestamp = datetime.fromisoformat(
+                                    completed_at.replace("Z", "+00:00")
+                                )
+                                is_recent = (
+                                    datetime.now() - timestamp.replace(tzinfo=None)
+                                ) < timedelta(hours=24)
                             except:
                                 pass
-                        
+
                         freshness[job_type] = {
-                            'last_success': completed_at,
-                            'records_collected': job.get('records_new', 0),
-                            'status': 'fresh' if is_recent else 'stale'
+                            "last_success": completed_at,
+                            "records_collected": job.get("records_new", 0),
+                            "status": "fresh" if is_recent else "stale",
                         }
-            
+
             return {
                 "recent_jobs": jobs[:limit],
                 "data_freshness": freshness,
                 "summary": {
                     "total_jobs": len(jobs),
-                    "job_types": len(set(job.get('job_type') for job in jobs)),
-                    "fresh_sources": len([v for v in freshness.values() if v['status'] == 'fresh'])
-                }
+                    "job_types": len(set(job.get("job_type") for job in jobs)),
+                    "fresh_sources": len([v for v in freshness.values() if v["status"] == "fresh"]),
+                },
             }
-            
+
         except Exception as e:
             logger.error(f"Failed to read data: {e}")
             return {"error": str(e)}
-    
+
     data = asyncio.run(read_data())
-    
+
     if output_json:
         console.print(JSON.from_data(data))
     else:
         console.print("📊 RECENT DATA SUMMARY", style="bold cyan")
-        
+
         if "error" in data:
             console.print(f"❌ Error: {data['error']}", style="red")
             return
-        
+
         # Summary stats
-        summary = data.get('summary', {})
+        summary = data.get("summary", {})
         console.print(f"\n📈 Summary:", style="bold blue")
         console.print(f"Total recent jobs: {summary.get('total_jobs', 0)}")
         console.print(f"Active job types: {summary.get('job_types', 0)}")
         console.print(f"Fresh data sources: {summary.get('fresh_sources', 0)}")
-        
+
         # Data freshness
-        freshness = data.get('data_freshness', {})
+        freshness = data.get("data_freshness", {})
         if freshness:
             console.print(f"\n🕐 Data Freshness:", style="bold blue")
             for source, info in freshness.items():
-                status_icon = '🟢' if info['status'] == 'fresh' else '🟡'
-                source_name = source.replace('_', ' ').title()
-                last_success = info['last_success'][:19] if info['last_success'] else 'Never'
+                status_icon = "🟢" if info["status"] == "fresh" else "🟡"
+                source_name = source.replace("_", " ").title()
+                last_success = info["last_success"][:19] if info["last_success"] else "Never"
                 console.print(f"{status_icon} {source_name}: {last_success}")
-        
+
         # Recent jobs
-        recent_jobs = data.get('recent_jobs', [])[:10]  # Show top 10
+        recent_jobs = data.get("recent_jobs", [])[:10]  # Show top 10
         if recent_jobs:
-            console.print(f"\n📋 Recent Jobs (showing {len(recent_jobs)}):", style="bold blue") 
+            console.print(f"\n📋 Recent Jobs (showing {len(recent_jobs)}):", style="bold blue")
             for job in recent_jobs:
-                status_icon = {'completed': '✅', 'running': '🔄', 'failed': '❌', 'pending': '⏳'}.get(job.get('status'), '❓')
-                job_type = job.get('job_type', 'unknown').replace('_', ' ').title()
-                started_at = job.get('started_at', 'N/A')[:19]
+                status_icon = {
+                    "completed": "✅",
+                    "running": "🔄",
+                    "failed": "❌",
+                    "pending": "⏳",
+                }.get(job.get("status"), "❓")
+                job_type = job.get("job_type", "unknown").replace("_", " ").title()
+                started_at = job.get("started_at", "N/A")[:19]
                 console.print(f"{status_icon} {job_type}: {started_at}")
 
 
-@politician_trading_cli.command("config-real-data")  
+@politician_trading_cli.command("config-real-data")
 @click.option("--enable", is_flag=True, help="Enable real data collection")
 @click.option("--restore", is_flag=True, help="Restore sample data mode")
 @click.option("--status", is_flag=True, help="Show current configuration status")
 def configure_real_data(enable: bool, restore: bool, status: bool):
     """Configure real vs sample data collection"""
-    
+
     if status or not (enable or restore):
         # Show current status
         console.print("🔧 DATA COLLECTION CONFIGURATION", style="bold cyan")
-        
+
         console.print("\n📋 Current Status:", style="bold blue")
         console.print("• Sample data mode: Currently DISABLED", style="green")
         console.print("• Real API calls: Currently ACTIVE", style="green")
         console.print("• Database writes: Currently WORKING", style="green")
-        
+
         console.print("\n🎯 Data Source Readiness:", style="bold blue")
         readiness_info = [
             ("UK Parliament API", "✅ Active - Real API with full transaction data", "green"),
-            ("US House/Senate", "✅ Active - Real disclosure database access", "green"), 
+            ("US House/Senate", "✅ Active - Real disclosure database access", "green"),
             ("EU Parliament", "✅ Active - Real MEP profile scraping", "green"),
             ("California NetFile", "⚠️  Limited - Complex forms require careful handling", "yellow"),
-            ("EU Member States", "⚠️  Limited - Country-specific implementations needed", "yellow")
+            ("EU Member States", "⚠️  Limited - Country-specific implementations needed", "yellow"),
         ]
-        
+
         for source, info, color in readiness_info:
             console.print(f"{info}", style=color)
-        
+
         console.print("\n💡 Commands:", style="bold blue")
         console.print("mcli politician-trading config-real-data --enable   # Enable real data")
         console.print("mcli politician-trading config-real-data --restore  # Restore sample mode")
-        
+
         return
-    
+
     # Get scraper files
     src_dir = Path(__file__).parent
     scraper_files = [
         "scrapers_uk.py",
         "scrapers_california.py",
-        "scrapers_eu.py", 
-        "scrapers_us_states.py"
+        "scrapers_eu.py",
+        "scrapers_us_states.py",
     ]
-    
+
     if restore:
         console.print("🔄 RESTORING SAMPLE DATA MODE", style="bold yellow")
-        
+
         restored = 0
         for file_name in scraper_files:
             file_path = src_dir / file_name
             backup_path = Path(str(file_path) + ".backup")
-            
+
             if backup_path.exists():
                 # Restore from backup
                 try:
@@ -1717,70 +1868,68 @@ def configure_real_data(enable: bool, restore: bool, status: bool):
                     console.print(f"❌ Failed to restore {file_name}: {e}", style="red")
             else:
                 console.print(f"ℹ️  No backup found for {file_name}", style="dim")
-        
+
         console.print(f"\n🎯 Restored {restored} files to sample mode", style="green")
-        
+
     elif enable:
         console.print("🚀 ENABLING REAL DATA COLLECTION", style="bold green")
-        
+
         with Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            console=console
+            SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console
         ) as progress:
             task = progress.add_task("Configuring scrapers...", total=len(scraper_files))
-            
+
             modifications_made = 0
-            
+
             for file_name in scraper_files:
                 progress.update(task, description=f"Processing {file_name}...")
-                
+
                 file_path = src_dir / file_name
-                
+
                 if not file_path.exists():
                     progress.advance(task)
                     continue
-                
+
                 try:
                     # Read file content
                     content = file_path.read_text()
                     original_content = content
-                    
+
                     # Remove sample flags
                     content = re.sub(r'"sample":\s*True', '"sample": False', content)
                     content = re.sub(r"'sample':\s*True", "'sample': False", content)
-                    
+
                     # Enable actual processing
                     content = re.sub(
-                        r'# This would implement actual (.+?) scraping',
+                        r"# This would implement actual (.+?) scraping",
                         r'logger.info("Processing real \1 data")',
-                        content
+                        content,
                     )
-                    
+
                     if content != original_content:
                         # Backup original
                         backup_path = str(file_path) + ".backup"
                         Path(backup_path).write_text(original_content)
-                        
+
                         # Write modified content
                         file_path.write_text(content)
                         modifications_made += 1
-                
+
                 except Exception as e:
                     console.print(f"❌ Error processing {file_name}: {e}", style="red")
-                
+
                 progress.advance(task)
-        
+
         console.print(f"\n✅ Real data configuration complete!", style="bold green")
         console.print(f"Modified {modifications_made} scraper files", style="green")
-        
+
         if modifications_made > 0:
             console.print(f"\n⚠️  Important Next Steps:", style="bold yellow")
             console.print("1. Test with UK Parliament first (most reliable)", style="dim")
             console.print("2. Monitor API rate limits carefully", style="dim")
             console.print("3. Check logs for parsing errors", style="dim")
             console.print("4. Use --restore flag if issues occur", style="dim")
-            
+
             console.print(f"\n🧪 Test Commands:", style="bold blue")
             console.print("mcli politician-trading cron run --type quick  # Quick test")
             console.print("mcli politician-trading monitor                # Check results")