mcli-framework 7.1.3__py3-none-any.whl → 7.2.0__py3-none-any.whl

mcli/workflow/politician_trading/seed_database.py

@@ -0,0 +1,539 @@
+"""
+Database Seeding Script for Politician Trading Data
+
+This script provides functionality to seed the Supabase database with politician
+trading data from multiple sources, creating a comprehensive data bank that can
+be iteratively updated.
+
+Usage:
+    python -m mcli.workflow.politician_trading.seed_database --sources all
+    python -m mcli.workflow.politician_trading.seed_database --sources propublica
+    python -m mcli.workflow.politician_trading.seed_database --test-run
+"""
+
+import argparse
+import logging
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional
+from uuid import UUID
+
+from supabase import create_client, Client
+
+# Load environment variables from .env file
+try:
+    from dotenv import load_dotenv
+    # Look for .env in project root
+    env_path = Path(__file__).parent.parent.parent.parent.parent / ".env"
+    if env_path.exists():
+        load_dotenv(env_path)
+        logger = logging.getLogger(__name__)
+        logger.info(f"Loaded environment variables from {env_path}")
+except ImportError:
+    # python-dotenv not installed, try loading from .streamlit/secrets.toml
+    pass
+
+from .data_sources import ALL_DATA_SOURCES, AccessMethod, DataSource
+from .models import Politician, TradingDisclosure
+from .scrapers_free_sources import FreeDataFetcher
+
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler("/tmp/seed_database.log")
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Database Connection
+# =============================================================================
+
+
+def get_supabase_client() -> Client:
+    """Get Supabase client from environment variables"""
+    url = os.getenv("SUPABASE_URL")
+    key = os.getenv("SUPABASE_SERVICE_ROLE_KEY") or os.getenv("SUPABASE_KEY")
+
+    if not url or not key:
+        raise ValueError(
+            "SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY (or SUPABASE_KEY) "
+            "environment variables must be set"
+        )
+
+    return create_client(url, key)
+
+
+# =============================================================================
+# Data Pull Job Tracking
+# =============================================================================
+
+
+def create_data_pull_job(
+    client: Client,
+    job_type: str,
+    config: Optional[Dict] = None
+) -> UUID:
+    """
+    Create a new data pull job record
+
+    Args:
+        client: Supabase client
+        job_type: Type of job (e.g., "propublica", "stocknear", "seed_all")
+        config: Optional configuration snapshot
+
+    Returns:
+        Job ID
+    """
+    try:
+        result = client.table("data_pull_jobs").insert({
+            "job_type": job_type,
+            "status": "running",
+            "started_at": datetime.now().isoformat(),
+            "config_snapshot": config or {}
+        }).execute()
+
+        job_id = result.data[0]["id"]
+        logger.info(f"Created data pull job: {job_id} (type: {job_type})")
+        return UUID(job_id)
+
+    except Exception as e:
+        logger.error(f"Error creating data pull job: {e}")
+        raise
+
+
+def update_data_pull_job(
+    client: Client,
+    job_id: UUID,
+    status: str,
+    stats: Optional[Dict] = None,
+    error: Optional[str] = None
+):
+    """
+    Update data pull job with results
+
+    Args:
+        client: Supabase client
+        job_id: Job ID to update
+        status: Job status ("completed", "failed", "running")
+        stats: Optional statistics (records_found, records_new, etc.)
+        error: Optional error message if failed
+    """
+    try:
+        update_data = {
+            "status": status,
+            "completed_at": datetime.now().isoformat()
+        }
+
+        if stats:
+            update_data.update(stats)
+
+        if error:
+            update_data["error_message"] = error
+
+        client.table("data_pull_jobs").update(update_data).eq("id", str(job_id)).execute()
+
+        logger.info(f"Updated job {job_id}: status={status}")
+
+    except Exception as e:
+        logger.error(f"Error updating data pull job: {e}")
+
+
+# =============================================================================
+# Politician Upsert Logic
+# =============================================================================
+
+
+def upsert_politicians(
+    client: Client,
+    politicians: List[Politician]
+) -> Dict[str, UUID]:
+    """
+    Upsert politicians to database, returning mapping of bioguide_id -> UUID
+
+    Args:
+        client: Supabase client
+        politicians: List of Politician objects
+
+    Returns:
+        Dictionary mapping bioguide_id to politician UUID
+    """
+    politician_map = {}
+    new_count = 0
+    updated_count = 0
+
+    for politician in politicians:
+        try:
+            # Convert to database format
+            pol_data = {
+                "first_name": politician.first_name,
+                "last_name": politician.last_name,
+                "full_name": politician.full_name,
+                "role": politician.role,
+                "party": politician.party,
+                "state_or_country": politician.state_or_country,
+                "district": politician.district,
+                "bioguide_id": politician.bioguide_id,
+            }
+
+            # Try to find existing politician
+            if politician.bioguide_id:
+                # Query by bioguide_id if available
+                existing = client.table("politicians").select("id").eq(
+                    "bioguide_id", politician.bioguide_id
+                ).execute()
+            else:
+                # Query by unique constraint fields (first_name, last_name, role, state_or_country)
+                existing = client.table("politicians").select("id").eq(
+                    "first_name", politician.first_name
+                ).eq(
+                    "last_name", politician.last_name
+                ).eq(
+                    "role", politician.role
+                ).eq(
+                    "state_or_country", politician.state_or_country
+                ).execute()
+
+            if existing.data:
+                # Update existing
+                pol_id = UUID(existing.data[0]["id"])
+                client.table("politicians").update(pol_data).eq("id", str(pol_id)).execute()
+                updated_count += 1
+            else:
+                # Insert new
+                result = client.table("politicians").insert(pol_data).execute()
+                pol_id = UUID(result.data[0]["id"])
+                new_count += 1
+
+            # Store mapping - use bioguide_id if available, otherwise use full_name
+            if politician.bioguide_id:
+                politician_map[politician.bioguide_id] = pol_id
+            elif politician.full_name:
+                # For sources without bioguide_id (e.g., Senate Stock Watcher), use full_name
+                politician_map[politician.full_name] = pol_id
+
+        except Exception as e:
+            logger.error(f"Error upserting politician {politician.full_name}: {e}")
+            continue
+
+    logger.info(f"Upserted {len(politicians)} politicians ({new_count} new, {updated_count} updated)")
+
+    return politician_map
+
+
+# =============================================================================
+# Trading Disclosure Upsert Logic
+# =============================================================================
+
+
+def upsert_trading_disclosures(
+    client: Client,
+    disclosures: List[TradingDisclosure],
+    politician_map: Dict[str, UUID]
+) -> Dict[str, int]:
+    """
+    Upsert trading disclosures to database
+
+    Args:
+        client: Supabase client
+        disclosures: List of TradingDisclosure objects
+        politician_map: Mapping of bioguide_id to politician UUID
+
+    Returns:
+        Statistics dictionary with counts
+    """
+    new_count = 0
+    updated_count = 0
+    skipped_count = 0
+
+    for disclosure in disclosures:
+        try:
+            # Get politician ID
+            pol_id = politician_map.get(disclosure.politician_bioguide_id)
+            if not pol_id:
+                logger.warning(
+                    f"Skipping disclosure - politician not found: "
+                    f"{disclosure.politician_bioguide_id}"
+                )
+                skipped_count += 1
+                continue
+
+            # Convert to database format
+            disclosure_data = {
+                "politician_id": str(pol_id),
+                "transaction_date": disclosure.transaction_date.isoformat(),
+                "disclosure_date": disclosure.disclosure_date.isoformat(),
+                "transaction_type": disclosure.transaction_type,
+                "asset_name": disclosure.asset_name,
+                "asset_ticker": disclosure.asset_ticker,
+                "asset_type": disclosure.asset_type,
+                "amount_range_min": disclosure.amount_range_min,
+                "amount_range_max": disclosure.amount_range_max,
+                "amount_exact": disclosure.amount_exact,
+                "source_url": disclosure.source_url,
+                "raw_data": disclosure.raw_data,
+                "status": "processed",
+            }
+
+            # Check for existing disclosure (using unique constraint)
+            existing = client.table("trading_disclosures").select("id").eq(
+                "politician_id", str(pol_id)
+            ).eq(
+                "transaction_date", disclosure.transaction_date.isoformat()
+            ).eq(
+                "asset_name", disclosure.asset_name
+            ).eq(
+                "transaction_type", disclosure.transaction_type
+            ).eq(
+                "disclosure_date", disclosure.disclosure_date.isoformat()
+            ).execute()
+
+            if existing.data:
+                # Update existing
+                disc_id = existing.data[0]["id"]
+                client.table("trading_disclosures").update(disclosure_data).eq(
+                    "id", disc_id
+                ).execute()
+                updated_count += 1
+            else:
+                # Insert new
+                client.table("trading_disclosures").insert(disclosure_data).execute()
+                new_count += 1
+
+        except Exception as e:
+            logger.error(f"Error upserting disclosure: {e}")
+            skipped_count += 1
+            continue
+
+    logger.info(
+        f"Upserted {len(disclosures)} disclosures "
+        f"({new_count} new, {updated_count} updated, {skipped_count} skipped)"
+    )
+
+    return {
+        "records_found": len(disclosures),
+        "records_new": new_count,
+        "records_updated": updated_count,
+        "records_failed": skipped_count,
+    }
+
+
+# =============================================================================
+# Source-Specific Seeding Functions
+# =============================================================================
+
+
+def seed_from_senate_watcher(
+    client: Client,
+    test_run: bool = False,
+    recent_only: bool = False,
+    days: int = 90
+) -> Dict[str, int]:
+    """
+    Seed database from Senate Stock Watcher GitHub dataset
+
+    Args:
+        client: Supabase client
+        test_run: If True, only fetch but don't insert to DB
+        recent_only: If True, only fetch recent transactions
+        days: Number of days to look back if recent_only=True
+
+    Returns:
+        Statistics dictionary
+    """
+    logger.info("=" * 80)
+    logger.info("SEEDING FROM SENATE STOCK WATCHER (GitHub)")
+    logger.info("=" * 80)
+
+    # Create job record
+    job_id = create_data_pull_job(client, "senate_watcher_seed", {
+        "recent_only": recent_only,
+        "days": days
+    })
+
+    try:
+        # Initialize fetcher
+        fetcher = FreeDataFetcher()
+
+        # Fetch data
+        data = fetcher.fetch_from_senate_watcher(
+            recent_only=recent_only,
+            days=days
+        )
+
+        politicians = data["politicians"]
+        disclosures = data["disclosures"]
+
+        logger.info(f"Fetched {len(politicians)} politicians, {len(disclosures)} disclosures")
+
+        if test_run:
+            logger.info("TEST RUN - Not inserting to database")
+            logger.info(f"Sample politician: {politicians[0] if politicians else 'None'}")
+            logger.info(f"Sample disclosure: {disclosures[0] if disclosures else 'None'}")
+            update_data_pull_job(client, job_id, "completed", {
+                "records_found": len(politicians) + len(disclosures),
+                "records_new": 0,
+                "records_updated": 0,
+            })
+            return {"records_found": len(politicians) + len(disclosures)}
+
+        # Upsert politicians
+        politician_map = upsert_politicians(client, politicians)
+
+        # Upsert disclosures
+        disclosure_stats = upsert_trading_disclosures(client, disclosures, politician_map)
+
+        # Update job record
+        update_data_pull_job(client, job_id, "completed", disclosure_stats)
+
+        return disclosure_stats
+
+    except Exception as e:
+        logger.error(f"Error seeding from Senate Stock Watcher: {e}")
+        update_data_pull_job(client, job_id, "failed", error=str(e))
+        raise
+
+
+def seed_from_all_sources(
+    client: Client,
+    test_run: bool = False
+) -> Dict[str, Dict[str, int]]:
+    """
+    Seed database from all available sources
+
+    Args:
+        client: Supabase client
+        test_run: If True, only fetch but don't insert to DB
+
+    Returns:
+        Dictionary mapping source name to statistics
+    """
+    logger.info("=" * 80)
+    logger.info("SEEDING FROM ALL SOURCES")
+    logger.info("=" * 80)
+
+    results = {}
+
+    # Senate Stock Watcher (free GitHub dataset - no API key needed!)
+    try:
+        logger.info("\n📡 Senate Stock Watcher (GitHub)")
+        results["senate_watcher"] = seed_from_senate_watcher(client, test_run)
+    except Exception as e:
+        logger.error(f"Senate Stock Watcher seeding failed: {e}")
+        results["senate_watcher"] = {"error": str(e)}
+
+    # TODO: Add other sources as implemented
+    # - Finnhub (requires free API key from finnhub.io)
+    # - SEC Edgar (free, no API key, but need to implement Form 4 parsing)
+    # - StockNear (requires JavaScript rendering)
+    # - QuiverQuant (requires premium subscription)
+
+    logger.info("\n" + "=" * 80)
+    logger.info("SEEDING SUMMARY")
+    logger.info("=" * 80)
+
+    for source, stats in results.items():
+        logger.info(f"\n{source}:")
+        if "error" in stats:
+            logger.error(f"  ❌ Failed: {stats['error']}")
+        else:
+            logger.info(f"  ✅ Found: {stats.get('records_found', 0)}")
+            logger.info(f"  ➕ New: {stats.get('records_new', 0)}")
+            logger.info(f"  🔄 Updated: {stats.get('records_updated', 0)}")
+            logger.info(f"  ⚠️  Failed: {stats.get('records_failed', 0)}")
+
+    return results
+
+
+# =============================================================================
+# CLI Interface
+# =============================================================================
+
+
+def main():
+    """Main CLI entry point"""
+    parser = argparse.ArgumentParser(
+        description="Seed politician trading database from multiple sources"
+    )
+
+    parser.add_argument(
+        "--sources",
+        choices=["all", "senate", "finnhub", "sec-edgar"],
+        default="all",
+        help="Which data sources to seed from (default: all)"
+    )
+
+    parser.add_argument(
+        "--recent-only",
+        action="store_true",
+        help="Only fetch recent transactions (last 90 days)"
+    )
+
+    parser.add_argument(
+        "--days",
+        type=int,
+        default=90,
+        help="Number of days to look back when using --recent-only (default: 90)"
+    )
+
+    parser.add_argument(
+        "--test-run",
+        action="store_true",
+        help="Fetch data but don't insert to database (for testing)"
+    )
+
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging"
+    )
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    # Get Supabase client
+    try:
+        client = get_supabase_client()
+        logger.info("✅ Connected to Supabase")
+    except Exception as e:
+        logger.error(f"❌ Failed to connect to Supabase: {e}")
+        sys.exit(1)
+
+    # Run seeding
+    try:
+        if args.sources == "senate":
+            seed_from_senate_watcher(
+                client,
+                test_run=args.test_run,
+                recent_only=args.recent_only,
+                days=args.days
+            )
+        elif args.sources == "all":
+            seed_from_all_sources(client, args.test_run)
+        else:
+            logger.error(f"Source '{args.sources}' not yet implemented")
+            logger.info("Available sources: all, senate")
+            logger.info("Coming soon: finnhub, sec-edgar")
+            sys.exit(1)
+
+        logger.info("\n✅ Seeding completed successfully!")
+
+    except KeyboardInterrupt:
+        logger.info("\n⚠️  Seeding interrupted by user")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"\n❌ Seeding failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

mcli/workflow/workflow.py

@@ -1,37 +1,18 @@
-import click
+"""
+Workflow command group for mcli.
+
+All workflow commands are now loaded from portable JSON files in ~/.mcli/commands/
+This provides a clean, maintainable way to manage workflow commands.
+"""
 
-from .daemon.api_daemon import api_daemon
-from .daemon.commands import daemon
-from .dashboard.dashboard_cmd import dashboard
-from .file.file import file
-from .git_commit.commands import git_commit_cli
-from .politician_trading.commands import politician_trading_cli
-from .scheduler.commands import scheduler
-from .sync.sync_cmd import sync
-from .videos.videos import videos
+import click
 
 
 @click.group(name="workflow")
 def workflow():
-    """Workflow commands"""
+    """Workflow commands for automation, video processing, and daemon management"""
     pass
 
 
-# Add subcommands
-def register_workflow_commands():
-    workflow.add_command(file)
-    workflow.add_command(videos)
-    workflow.add_command(daemon)
-    workflow.add_command(api_daemon)
-    workflow.add_command(dashboard)
-    workflow.add_command(git_commit_cli)
-    workflow.add_command(scheduler)
-    workflow.add_command(sync)
-    workflow.add_command(politician_trading_cli)
-
-
-register_workflow_commands()
-
-
 if __name__ == "__main__":
     workflow()

{mcli_framework-7.1.3.dist-info → mcli_framework-7.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcli-framework
-Version: 7.1.3
+Version: 7.2.0
 Summary: 🚀 High-performance CLI framework with Rust extensions, AI chat, and stunning visuals
 Author-email: Luis Fernandez de la Vara <luis@lefv.io>
 Maintainer-email: Luis Fernandez de la Vara <luis@lefv.io>