mcli-framework 7.10.1__py3-none-any.whl → 7.11.0__py3-none-any.whl

mcli/workflow/politician_trading/database.py

@@ -1,412 +0,0 @@
-"""
-Database client and schema management for politician trading data
-"""
-
-import asyncio
-import logging
-from datetime import datetime
-from typing import Any, Dict, List, Optional
-from uuid import uuid4
-
-from postgrest.exceptions import APIError
-from supabase import Client, create_client
-
-from .config import WorkflowConfig
-from .models import DataPullJob, DataSource, Politician, TradingDisclosure
-
-logger = logging.getLogger(__name__)
-
-
-class PoliticianTradingDB:
-    """Database client for politician trading data"""
-
-    def __init__(self, config: WorkflowConfig):
-        self.config = config
-        self.client: Optional[Client] = None
-        self._init_client()
-
-    def _init_client(self):
-        """Initialize Supabase client"""
-        try:
-            self.client = create_client(self.config.supabase.url, self.config.supabase.key)
-            logger.info("Supabase client initialized successfully")
-        except Exception as e:
-            logger.error(f"Failed to initialize Supabase client: {e}")
-            raise
-
-    async def ensure_schema(self) -> bool:
-        """Ensure database schema exists"""
-        try:
-            # Check if tables exist by trying to query them
-            await self._check_table_exists("politicians")
-            await self._check_table_exists("trading_disclosures")
-            await self._check_table_exists("data_pull_jobs")
-            await self._check_table_exists("data_sources")
-            logger.info("Database schema verified")
-            return True
-        except Exception as e:
-            logger.error(f"Schema check failed: {e}")
-            logger.info("You'll need to create the database schema manually")
-            return False
-
-    async def _check_table_exists(self, table_name: str):
-        """Check if table exists"""
-        try:
-            result = self.client.table(table_name).select("*").limit(1).execute()
-            return True
-        except Exception as e:
-            logger.warning(f"Table {table_name} may not exist: {e}")
-            raise
-
-    # Politician management
-    async def get_politician(self, politician_id: str) -> Optional[Politician]:
-        """Get politician by ID"""
-        try:
-            result = self.client.table("politicians").select("*").eq("id", politician_id).execute()
-            if result.data:
-                return self._dict_to_politician(result.data[0])
-            return None
-        except Exception as e:
-            logger.error(f"Failed to get politician {politician_id}: {e}")
-            return None
-
-    async def find_politician_by_name(
-        self, first_name: str, last_name: str
-    ) -> Optional[Politician]:
-        """Find politician by name"""
-        try:
-            result = (
-                self.client.table("politicians")
-                .select("*")
-                .eq("first_name", first_name)
-                .eq("last_name", last_name)
-                .execute()
-            )
-            if result.data:
-                return self._dict_to_politician(result.data[0])
-            return None
-        except Exception as e:
-            logger.error(f"Failed to find politician {first_name} {last_name}: {e}")
-            return None
-
-    async def upsert_politician(self, politician: Politician) -> str:
-        """Insert or update politician"""
-        try:
-            # First, try to find an existing politician
-            existing = await self.find_politician_by_name(
-                politician.first_name, politician.last_name
-            )
-
-            if existing:
-                # Update existing politician (but don't change ID)
-                politician_dict = self._politician_to_dict(politician)
-                politician_dict["id"] = existing.id  # Keep existing ID
-                politician_dict["updated_at"] = datetime.utcnow().isoformat()
-
-                result = (
-                    self.client.table("politicians")
-                    .update(politician_dict)
-                    .eq("id", existing.id)
-                    .execute()
-                )
-
-                if result.data:
-                    return result.data[0]["id"]
-                return existing.id
-            else:
-                # Insert new politician
-                politician_dict = self._politician_to_dict(politician)
-                if not politician_dict.get("id"):
-                    politician_dict["id"] = str(uuid4())
-
-                politician_dict["created_at"] = datetime.utcnow().isoformat()
-                politician_dict["updated_at"] = datetime.utcnow().isoformat()
-
-                result = self.client.table("politicians").insert(politician_dict).execute()
-
-                if result.data:
-                    return result.data[0]["id"]
-                return politician_dict["id"]
-
-        except Exception as e:
-            logger.error(f"Failed to upsert politician: {e}")
-            # For debugging: log the politician data that caused the error
-            logger.error(f"Politician data: {politician.first_name} {politician.last_name}")
-            return ""  # Return empty string instead of raising to prevent cascade failures
-
-    # Trading disclosure management
-    async def get_disclosure(self, disclosure_id: str) -> Optional[TradingDisclosure]:
-        """Get trading disclosure by ID"""
-        try:
-            result = (
-                self.client.table("trading_disclosures")
-                .select("*")
-                .eq("id", disclosure_id)
-                .execute()
-            )
-            if result.data:
-                return self._dict_to_disclosure(result.data[0])
-            return None
-        except Exception as e:
-            logger.error(f"Failed to get disclosure {disclosure_id}: {e}")
-            return None
-
-    async def find_disclosure_by_transaction(
-        self, politician_id: str, transaction_date: datetime, asset_name: str, transaction_type: str
-    ) -> Optional[TradingDisclosure]:
-        """Find existing disclosure by transaction details"""
-        try:
-            result = (
-                self.client.table("trading_disclosures")
-                .select("*")
-                .eq("politician_id", politician_id)
-                .eq("transaction_date", transaction_date.isoformat())
-                .eq("asset_name", asset_name)
-                .eq("transaction_type", transaction_type)
-                .execute()
-            )
-            if result.data:
-                return self._dict_to_disclosure(result.data[0])
-            return None
-        except Exception as e:
-            logger.error(f"Failed to find disclosure: {e}")
-            return None
-
-    async def insert_disclosure(self, disclosure: TradingDisclosure) -> str:
-        """Insert new trading disclosure"""
-        try:
-            disclosure_dict = self._disclosure_to_dict(disclosure)
-            if not disclosure_dict.get("id"):
-                disclosure_dict["id"] = str(uuid4())
-
-            result = self.client.table("trading_disclosures").insert(disclosure_dict).execute()
-            if result.data:
-                return result.data[0]["id"]
-            return disclosure_dict["id"]
-        except Exception as e:
-            logger.error(f"Failed to insert disclosure: {e}")
-            raise
-
-    async def update_disclosure(self, disclosure: TradingDisclosure) -> bool:
-        """Update existing trading disclosure"""
-        try:
-            disclosure_dict = self._disclosure_to_dict(disclosure)
-            disclosure_dict["updated_at"] = datetime.utcnow().isoformat()
-
-            result = (
-                self.client.table("trading_disclosures")
-                .update(disclosure_dict)
-                .eq("id", disclosure.id)
-                .execute()
-            )
-            return len(result.data) > 0
-        except Exception as e:
-            logger.error(f"Failed to update disclosure: {e}")
-            return False
-
-    async def get_recent_disclosures(self, limit: int = 100) -> List[TradingDisclosure]:
-        """Get recent trading disclosures"""
-        try:
-            result = (
-                self.client.table("trading_disclosures")
-                .select("*")
-                .order("disclosure_date", desc=True)
-                .limit(limit)
-                .execute()
-            )
-            return [self._dict_to_disclosure(d) for d in result.data]
-        except Exception as e:
-            logger.error(f"Failed to get recent disclosures: {e}")
-            return []
-
-    # Data pull job management
-    async def create_data_pull_job(self, job_type: str, config_snapshot: Dict[str, Any]) -> str:
-        """Create new data pull job"""
-        try:
-            job = DataPullJob(
-                id=str(uuid4()),
-                job_type=job_type,
-                status="pending",
-                config_snapshot=config_snapshot,
-                started_at=datetime.utcnow(),
-            )
-
-            job_dict = self._job_to_dict(job)
-            result = self.client.table("data_pull_jobs").insert(job_dict).execute()
-            if result.data:
-                return result.data[0]["id"]
-            return job.id
-        except Exception as e:
-            logger.error(f"Failed to create data pull job: {e}")
-            raise
-
-    async def update_data_pull_job(self, job: DataPullJob) -> bool:
-        """Update data pull job"""
-        try:
-            job_dict = self._job_to_dict(job)
-            result = self.client.table("data_pull_jobs").update(job_dict).eq("id", job.id).execute()
-            return len(result.data) > 0
-        except Exception as e:
-            logger.error(f"Failed to update data pull job: {e}")
-            return False
-
-    async def get_job_status(self) -> Dict[str, Any]:
-        """Get current job status summary"""
-        try:
-            # Get recent jobs
-            result = (
-                self.client.table("data_pull_jobs")
-                .select("*")
-                .order("created_at", desc=True)
-                .limit(10)
-                .execute()
-            )
-
-            jobs = result.data
-
-            # Calculate summary statistics
-            total_disclosures = (
-                self.client.table("trading_disclosures").select("id", count="exact").execute()
-            ).count
-
-            recent_disclosures = (
-                self.client.table("trading_disclosures")
-                .select("id", count="exact")
-                .gte(
-                    "created_at",
-                    (datetime.utcnow().replace(hour=0, minute=0, second=0)).isoformat(),
-                )
-                .execute()
-            ).count
-
-            return {
-                "total_disclosures": total_disclosures,
-                "recent_disclosures_today": recent_disclosures,
-                "recent_jobs": jobs,
-                "last_update": datetime.utcnow().isoformat(),
-            }
-        except Exception as e:
-            logger.error(f"Failed to get job status: {e}")
-            return {"error": str(e)}
-
-    # Helper methods for data conversion
-    def _politician_to_dict(self, politician: Politician) -> Dict[str, Any]:
-        """Convert Politician to dictionary"""
-        return {
-            "id": politician.id,
-            "first_name": politician.first_name,
-            "last_name": politician.last_name,
-            "full_name": politician.full_name,
-            "role": politician.role.value if politician.role else None,
-            "party": politician.party,
-            "state_or_country": politician.state_or_country,
-            "district": politician.district,
-            "term_start": politician.term_start.isoformat() if politician.term_start else None,
-            "term_end": politician.term_end.isoformat() if politician.term_end else None,
-            "bioguide_id": politician.bioguide_id,
-            "eu_id": politician.eu_id,
-            "created_at": politician.created_at.isoformat(),
-            "updated_at": politician.updated_at.isoformat(),
-        }
-
-    def _dict_to_politician(self, data: Dict[str, Any]) -> Politician:
-        """Convert dictionary to Politician"""
-        from .models import PoliticianRole
-
-        return Politician(
-            id=data.get("id"),
-            first_name=data.get("first_name", ""),
-            last_name=data.get("last_name", ""),
-            full_name=data.get("full_name", ""),
-            role=PoliticianRole(data.get("role", "us_house_representative")),
-            party=data.get("party", ""),
-            state_or_country=data.get("state_or_country", ""),
-            district=data.get("district"),
-            term_start=(
-                datetime.fromisoformat(data["term_start"]) if data.get("term_start") else None
-            ),
-            term_end=datetime.fromisoformat(data["term_end"]) if data.get("term_end") else None,
-            bioguide_id=data.get("bioguide_id"),
-            eu_id=data.get("eu_id"),
-            created_at=datetime.fromisoformat(data["created_at"]),
-            updated_at=datetime.fromisoformat(data["updated_at"]),
-        )
-
-    def _disclosure_to_dict(self, disclosure: TradingDisclosure) -> Dict[str, Any]:
-        """Convert TradingDisclosure to dictionary"""
-        return {
-            "id": disclosure.id,
-            "politician_id": disclosure.politician_id,
-            "transaction_date": disclosure.transaction_date.isoformat(),
-            "disclosure_date": disclosure.disclosure_date.isoformat(),
-            "transaction_type": (
-                disclosure.transaction_type.value if disclosure.transaction_type else None
-            ),
-            "asset_name": disclosure.asset_name,
-            "asset_ticker": disclosure.asset_ticker,
-            "asset_type": disclosure.asset_type,
-            "amount_range_min": (
-                float(disclosure.amount_range_min) if disclosure.amount_range_min else None
-            ),
-            "amount_range_max": (
-                float(disclosure.amount_range_max) if disclosure.amount_range_max else None
-            ),
-            "amount_exact": float(disclosure.amount_exact) if disclosure.amount_exact else None,
-            "source_url": disclosure.source_url,
-            "source_document_id": disclosure.source_document_id,
-            "raw_data": disclosure.raw_data,
-            "status": disclosure.status.value if disclosure.status else None,
-            "processing_notes": disclosure.processing_notes,
-            "created_at": disclosure.created_at.isoformat(),
-            "updated_at": disclosure.updated_at.isoformat(),
-        }
-
-    def _dict_to_disclosure(self, data: Dict[str, Any]) -> TradingDisclosure:
-        """Convert dictionary to TradingDisclosure"""
-        from decimal import Decimal
-
-        from .models import DisclosureStatus, TransactionType
-
-        return TradingDisclosure(
-            id=data.get("id"),
-            politician_id=data.get("politician_id", ""),
-            transaction_date=datetime.fromisoformat(data["transaction_date"]),
-            disclosure_date=datetime.fromisoformat(data["disclosure_date"]),
-            transaction_type=TransactionType(data.get("transaction_type", "purchase")),
-            asset_name=data.get("asset_name", ""),
-            asset_ticker=data.get("asset_ticker"),
-            asset_type=data.get("asset_type", ""),
-            amount_range_min=(
-                Decimal(str(data["amount_range_min"])) if data.get("amount_range_min") else None
-            ),
-            amount_range_max=(
-                Decimal(str(data["amount_range_max"])) if data.get("amount_range_max") else None
-            ),
-            amount_exact=Decimal(str(data["amount_exact"])) if data.get("amount_exact") else None,
-            source_url=data.get("source_url", ""),
-            source_document_id=data.get("source_document_id"),
-            raw_data=data.get("raw_data", {}),
-            status=DisclosureStatus(data.get("status", "pending")),
-            processing_notes=data.get("processing_notes", ""),
-            created_at=datetime.fromisoformat(data["created_at"]),
-            updated_at=datetime.fromisoformat(data["updated_at"]),
-        )
-
-    def _job_to_dict(self, job: DataPullJob) -> Dict[str, Any]:
-        """Convert DataPullJob to dictionary"""
-        return {
-            "id": job.id,
-            "job_type": job.job_type,
-            "status": job.status,
-            "started_at": job.started_at.isoformat() if job.started_at else None,
-            "completed_at": job.completed_at.isoformat() if job.completed_at else None,
-            "records_found": job.records_found,
-            "records_processed": job.records_processed,
-            "records_new": job.records_new,
-            "records_updated": job.records_updated,
-            "records_failed": job.records_failed,
-            "error_message": job.error_message,
-            "error_details": job.error_details,
-            "config_snapshot": job.config_snapshot,
-            "created_at": job.created_at.isoformat(),
-        }

mcli/workflow/politician_trading/demo.py

@@ -1,249 +0,0 @@
-"""
-Demonstration script showing politician trading workflow execution and data creation
-"""
-
-import asyncio
-import json
-import uuid
-from datetime import datetime
-
-from rich.console import Console
-from rich.json import JSON
-from rich.panel import Panel
-from rich.table import Table
-
-from .connectivity import SupabaseConnectivityValidator
-from .monitoring import run_health_check, run_stats_report
-from .workflow import run_politician_trading_collection
-
-console = Console()
-
-
-async def demonstrate_workflow_execution():
-    """Comprehensive demonstration of the workflow execution"""
-
-    console.print("🏛️ Politician Trading Data Collection - Full Demonstration", style="bold cyan")
-    console.print("=" * 80, style="dim")
-
-    # Step 1: Show what would happen with connectivity validation
-    console.print("\n📋 STEP 1: Supabase Connectivity Validation", style="bold blue")
-    console.print("This step validates database connectivity and operations...")
-
-    validator = SupabaseConnectivityValidator()
-
-    # Show the types of tests that would run
-    tests_info = [
-        ("Basic Connection", "Tests fundamental database connectivity"),
-        ("Read Operations", "Validates ability to query all required tables"),
-        ("Write Operations", "Creates, updates, and deletes test records"),
-        ("Table Access", "Verifies schema and table structure"),
-        ("Job Tracking", "Tests job status and history tracking"),
-        ("Real-time Sync", "Validates immediate write/read consistency"),
-    ]
-
-    test_table = Table(title="Connectivity Tests")
-    test_table.add_column("Test", style="cyan")
-    test_table.add_column("Description", style="white")
-
-    for test_name, description in tests_info:
-        test_table.add_row(test_name, description)
-
-    console.print(test_table)
-
-    # Step 2: Show database schema that would be created
-    console.print("\n📋 STEP 2: Database Schema Requirements", style="bold blue")
-
-    schema_info = [
-        (
-            "politicians",
-            "Stores politician information (US Congress, EU Parliament)",
-            "~1000 records",
-        ),
-        ("trading_disclosures", "Individual trading transactions/disclosures", "~50,000+ records"),
-        ("data_pull_jobs", "Job execution tracking and status", "~100 records"),
-        ("data_sources", "Data source configuration and health", "~10 records"),
-    ]
-
-    schema_table = Table(title="Database Tables")
-    schema_table.add_column("Table", style="cyan")
-    schema_table.add_column("Purpose", style="white")
-    schema_table.add_column("Expected Size", style="yellow")
-
-    for table_name, purpose, size in schema_info:
-        schema_table.add_row(table_name, purpose, size)
-
-    console.print(schema_table)
-
-    # Step 3: Demonstrate workflow execution
-    console.print("\n📋 STEP 3: Workflow Execution Simulation", style="bold blue")
-    console.print("Running the politician trading collection workflow...")
-
-    try:
-        # This will attempt to run the workflow (may fail due to schema)
-        workflow_result = await run_politician_trading_collection()
-
-        # Show what the result structure looks like
-        console.print("\n🔍 Workflow Result Structure:", style="bold")
-
-        # Create a mock successful result to show what it would look like
-        mock_successful_result = {
-            "started_at": "2024-09-02T09:00:00.000Z",
-            "completed_at": "2024-09-02T09:05:30.150Z",
-            "status": "completed",
-            "jobs": {
-                "us_congress": {
-                    "job_id": "job_12345",
-                    "status": "completed",
-                    "new_disclosures": 15,
-                    "updated_disclosures": 3,
-                    "errors": [],
-                },
-                "eu_parliament": {
-                    "job_id": "job_12346",
-                    "status": "completed",
-                    "new_disclosures": 8,
-                    "updated_disclosures": 1,
-                    "errors": [],
-                },
-            },
-            "summary": {"total_new_disclosures": 23, "total_updated_disclosures": 4, "errors": []},
-        }
-
-        console.print(JSON.from_data(mock_successful_result))
-
-        # Show the actual result we got
-        console.print("\n🔍 Actual Workflow Result:", style="bold")
-        console.print(JSON.from_data(workflow_result))
-
-    except Exception as e:
-        console.print(f"\n⚠️ Workflow execution encountered expected issues: {e}", style="yellow")
-        console.print("This is normal when database schema hasn't been created yet.", style="dim")
-
-    # Step 4: Show what data would be created
-    console.print("\n📋 STEP 4: Sample Data That Would Be Created", style="bold blue")
-
-    # Sample politician records
-    console.print("\n👥 Sample Politician Records:", style="bold")
-    sample_politicians = [
-        {
-            "full_name": "Nancy Pelosi",
-            "role": "us_house_representative",
-            "party": "Democratic",
-            "state_or_country": "CA",
-            "district": "5",
-            "bioguide_id": "P000197",
-        },
-        {
-            "full_name": "Ted Cruz",
-            "role": "us_senator",
-            "party": "Republican",
-            "state_or_country": "TX",
-            "bioguide_id": "C001098",
-        },
-    ]
-
-    for politician in sample_politicians:
-        console.print(JSON.from_data(politician))
-
-    # Sample trading disclosures
-    console.print("\n💰 Sample Trading Disclosure Records:", style="bold")
-    sample_disclosures = [
-        {
-            "politician_id": str(uuid.uuid4()),
-            "transaction_date": "2024-08-15T00:00:00Z",
-            "disclosure_date": "2024-08-20T00:00:00Z",
-            "transaction_type": "purchase",
-            "asset_name": "Apple Inc.",
-            "asset_ticker": "AAPL",
-            "asset_type": "stock",
-            "amount_range_min": 15001.00,
-            "amount_range_max": 50000.00,
-            "source_url": "https://disclosures-clerk.house.gov",
-            "status": "processed",
-        },
-        {
-            "politician_id": "pol_2",
-            "transaction_date": "2024-08-10T00:00:00Z",
-            "disclosure_date": "2024-08-25T00:00:00Z",
-            "transaction_type": "sale",
-            "asset_name": "Microsoft Corporation",
-            "asset_ticker": "MSFT",
-            "asset_type": "stock",
-            "amount_range_min": 1001.00,
-            "amount_range_max": 15000.00,
-            "source_url": "https://efdsearch.senate.gov",
-            "status": "processed",
-        },
-    ]
-
-    for disclosure in sample_disclosures:
-        console.print(JSON.from_data(disclosure))
-
-    # Step 5: Show job tracking
-    console.print("\n📋 STEP 5: Job Tracking and Monitoring", style="bold blue")
-
-    sample_job_record = {
-        "id": "job_12345",
-        "job_type": "us_congress",
-        "status": "completed",
-        "started_at": "2024-09-02T09:00:00Z",
-        "completed_at": "2024-09-02T09:03:45Z",
-        "records_found": 20,
-        "records_processed": 18,
-        "records_new": 15,
-        "records_updated": 3,
-        "records_failed": 2,
-        "config_snapshot": {
-            "supabase_url": "https://uljsqvwkomdrlnofmlad.supabase.co",
-            "request_delay": 1.0,
-            "max_retries": 3,
-        },
-    }
-
-    console.print("📊 Sample Job Record:", style="bold")
-    console.print(JSON.from_data(sample_job_record))
-
-    # Step 6: Show CLI commands for management
-    console.print("\n📋 STEP 6: CLI Commands for Management", style="bold blue")
-
-    commands_info = [
-        ("politician-trading setup --create-tables", "Create database schema"),
-        ("politician-trading connectivity", "Test Supabase connectivity"),
-        ("politician-trading run", "Execute data collection"),
-        ("politician-trading status", "Check system status"),
-        ("politician-trading health", "System health monitoring"),
-        ("politician-trading stats", "View detailed statistics"),
-        ("politician-trading test-workflow -v", "Run full workflow test"),
-        ("politician-trading connectivity --continuous", "Continuous monitoring"),
-        ("politician-trading cron-job --create", "Setup automated scheduling"),
-    ]
-
-    commands_table = Table(title="Available CLI Commands")
-    commands_table.add_column("Command", style="cyan")
-    commands_table.add_column("Description", style="white")
-
-    for command, description in commands_info:
-        commands_table.add_row(command, description)
-
-    console.print(commands_table)
-
-    # Summary
-    console.print("\n📋 SUMMARY", style="bold green")
-    console.print("✅ Workflow validates Supabase connectivity with 6 comprehensive tests")
-    console.print("✅ Creates and manages 4 database tables with proper indexing")
-    console.print("✅ Scrapes data from US Congress and EU Parliament sources")
-    console.print("✅ Tracks job execution with detailed status and metrics")
-    console.print("✅ Provides comprehensive CLI for management and monitoring")
-    console.print("✅ Supports automated scheduling via Supabase cron jobs")
-    console.print("✅ Includes real-time monitoring and health checks")
-
-    console.print("\n🚀 Next Steps to Deploy:", style="bold blue")
-    console.print("1. Execute the schema.sql in your Supabase SQL editor")
-    console.print("2. Run: politician-trading setup --verify")
-    console.print("3. Run: politician-trading connectivity")
-    console.print("4. Run: politician-trading test-workflow --verbose")
-    console.print("5. Setup cron job: politician-trading cron-job --create")
-
-
-if __name__ == "__main__":
-    asyncio.run(demonstrate_workflow_execution())