mcli-framework 7.5.1__py3-none-any.whl → 7.6.1__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_free_sources.py

@@ -7,17 +7,16 @@ This module contains scrapers for free, publicly available politician trading da
 - SEC Edgar Insider Trading API
 """
 
+import logging
 import os
 import time
 from datetime import datetime, timedelta
 from typing import Dict, List, Optional
-import logging
 
 import requests
 
 from .models import Politician, TradingDisclosure
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,9 +35,7 @@ class SenateStockWatcherScraper:
 
     def __init__(self):
         self.session = requests.Session()
-        self.session.headers.update({
-            "User-Agent": "PoliticianTradingTracker/1.0"
-        })
+        self.session.headers.update({"User-Agent": "PoliticianTradingTracker/1.0"})
 
     def fetch_all_transactions(self) -> List[Dict]:
         """
@@ -156,9 +153,7 @@ class SenateStockWatcherScraper:
         return politicians
 
     def convert_to_disclosures(
-        self,
-        transactions: List[Dict],
-        politician_lookup: Optional[Dict[str, str]] = None
+        self, transactions: List[Dict], politician_lookup: Optional[Dict[str, str]] = None
     ) -> List[TradingDisclosure]:
         """
         Convert transaction data to TradingDisclosure objects
@@ -187,7 +182,11 @@ class SenateStockWatcherScraper:
                     transaction_date = datetime.now()
 
                 try:
-                    disclosure_date = datetime.strptime(disclosure_date_str, "%m/%d/%Y") if disclosure_date_str else transaction_date
+                    disclosure_date = (
+                        datetime.strptime(disclosure_date_str, "%m/%d/%Y")
+                        if disclosure_date_str
+                        else transaction_date
+                    )
                 except ValueError:
                     disclosure_date = transaction_date
 
@@ -285,10 +284,7 @@ class FinnhubCongressionalAPI:
         self.session = requests.Session()
 
     def get_congressional_trading(
-        self,
-        symbol: str,
-        from_date: Optional[str] = None,
-        to_date: Optional[str] = None
+        self, symbol: str, from_date: Optional[str] = None, to_date: Optional[str] = None
     ) -> List[Dict]:
         """
         Get congressional trading for a specific stock symbol
@@ -303,10 +299,7 @@ class FinnhubCongressionalAPI:
         """
         try:
             url = f"{self.BASE_URL}/stock/congressional-trading"
-            params = {
-                "symbol": symbol,
-                "token": self.api_key
-            }
+            params = {"symbol": symbol, "token": self.api_key}
 
             if from_date:
                 params["from"] = from_date
@@ -349,11 +342,13 @@ class SECEdgarInsiderAPI:
     def __init__(self):
         self.session = requests.Session()
         # SEC requires a User-Agent header
-        self.session.headers.update({
-            "User-Agent": "PoliticianTradingTracker/1.0 (contact@example.com)",
-            "Accept-Encoding": "gzip, deflate",
-            "Host": "data.sec.gov"
-        })
+        self.session.headers.update(
+            {
+                "User-Agent": "PoliticianTradingTracker/1.0 (contact@example.com)",
+                "Accept-Encoding": "gzip, deflate",
+                "Host": "data.sec.gov",
+            }
+        )
 
     def get_company_submissions(self, cik: str) -> Dict:
         """
@@ -419,12 +414,18 @@ class SECEdgarInsiderAPI:
 
         for i, form in enumerate(forms):
             if form == "4":  # Form 4 is insider transaction report
-                form4_transactions.append({
-                    "accessionNumber": accession_numbers[i] if i < len(accession_numbers) else None,
-                    "filingDate": filing_dates[i] if i < len(filing_dates) else None,
-                    "primaryDocument": primary_documents[i] if i < len(primary_documents) else None,
-                    "cik": cik
-                })
+                form4_transactions.append(
+                    {
+                        "accessionNumber": (
+                            accession_numbers[i] if i < len(accession_numbers) else None
+                        ),
+                        "filingDate": filing_dates[i] if i < len(filing_dates) else None,
+                        "primaryDocument": (
+                            primary_documents[i] if i < len(primary_documents) else None
+                        ),
+                        "cik": cik,
+                    }
+                )
 
         logger.info(f"Found {len(form4_transactions)} Form 4 filings for CIK {cik}")
         return form4_transactions
@@ -440,10 +441,7 @@ class FreeDataFetcher:
     Unified interface for fetching politician trading data from free sources
     """
 
-    def __init__(
-        self,
-        finnhub_api_key: Optional[str] = None
-    ):
+    def __init__(self, finnhub_api_key: Optional[str] = None):
         """
         Initialize fetcher with optional API keys
 
@@ -461,9 +459,7 @@ class FreeDataFetcher:
                 logger.warning(f"Finnhub API not initialized: {e}")
 
     def fetch_from_senate_watcher(
-        self,
-        recent_only: bool = False,
-        days: int = 90
+        self, recent_only: bool = False, days: int = 90
     ) -> Dict[str, List]:
         """
         Fetch data from Senate Stock Watcher GitHub dataset
@@ -498,10 +494,7 @@ class FreeDataFetcher:
             f"{len(disclosures)} disclosures from Senate Stock Watcher"
         )
 
-        return {
-            "politicians": politicians,
-            "disclosures": disclosures
-        }
+        return {"politicians": politicians, "disclosures": disclosures}
 
 
 # =============================================================================

mcli/workflow/politician_trading/scrapers_third_party.py

@@ -9,18 +9,17 @@ politician trading activity:
 - ProPublica Congress API
 """
 
+import logging
 import os
 import time
 from datetime import datetime, timedelta
 from typing import Dict, List, Optional
-import logging
 
 import requests
 from bs4 import BeautifulSoup
 
 from .models import Politician, TradingDisclosure
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,9 +35,9 @@ class StockNearScraper:
 
     def __init__(self):
         self.session = requests.Session()
-        self.session.headers.update({
-            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
-        })
+        self.session.headers.update(
+            {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"}
+        )
 
     def fetch_politicians_list(self) -> List[Dict]:
         """Fetch list of politicians tracked by StockNear"""
@@ -46,7 +45,7 @@ class StockNearScraper:
             response = self.session.get(self.BASE_URL, timeout=30)
             response.raise_for_status()
 
-            soup = BeautifulSoup(response.content, 'html.parser')
+            soup = BeautifulSoup(response.content, "html.parser")
 
             # StockNear loads data via JavaScript - would need Selenium or API access
             # For now, return structure for manual data entry or API integration
@@ -80,18 +79,17 @@ class ProPublicaAPI:
     def __init__(self, api_key: Optional[str] = None):
         self.api_key = api_key or os.getenv("PROPUBLICA_API_KEY")
         if not self.api_key:
-            raise ValueError("ProPublica API key required. Set PROPUBLICA_API_KEY environment variable.")
+            raise ValueError(
+                "ProPublica API key required. Set PROPUBLICA_API_KEY environment variable."
+            )
 
         self.session = requests.Session()
-        self.session.headers.update({
-            "X-API-Key": self.api_key,
-            "User-Agent": "PoliticianTradingTracker/1.0"
-        })
+        self.session.headers.update(
+            {"X-API-Key": self.api_key, "User-Agent": "PoliticianTradingTracker/1.0"}
+        )
 
     def get_member_financial_disclosures(
-        self,
-        member_id: str,
-        congress: int = 118  # 118th Congress (2023-2025)
+        self, member_id: str, congress: int = 118  # 118th Congress (2023-2025)
     ) -> List[Dict]:
         """
         Get financial disclosures for a specific member of Congress
@@ -128,11 +126,7 @@ class ProPublicaAPI:
             logger.error(f"Error fetching ProPublica financial disclosures: {e}")
             return []
 
-    def get_recent_stock_transactions(
-        self,
-        congress: int = 118,
-        offset: int = 0
-    ) -> List[Dict]:
+    def get_recent_stock_transactions(self, congress: int = 118, offset: int = 0) -> List[Dict]:
         """
         Get recent stock transactions by members of Congress
 
@@ -144,7 +138,9 @@ class ProPublicaAPI:
             List of stock transactions
         """
         try:
-            url = f"{self.BASE_URL}/{congress}/house/members/financial-disclosures/transactions.json"
+            url = (
+                f"{self.BASE_URL}/{congress}/house/members/financial-disclosures/transactions.json"
+            )
             params = {"offset": offset}
 
             response = self.session.get(url, params=params, timeout=30)
@@ -160,9 +156,7 @@ class ProPublicaAPI:
             return []
 
     def list_current_members(
-        self,
-        chamber: str = "house",  # "house" or "senate"
-        congress: int = 118
+        self, chamber: str = "house", congress: int = 118  # "house" or "senate"
     ) -> List[Dict]:
         """
         Get list of current members of Congress
@@ -220,9 +214,7 @@ class ThirdPartyDataFetcher:
         self.stocknear = StockNearScraper()
 
     def fetch_from_propublica(
-        self,
-        fetch_members: bool = True,
-        fetch_transactions: bool = True
+        self, fetch_members: bool = True, fetch_transactions: bool = True
     ) -> Dict[str, List]:
         """
         Fetch data from ProPublica Congress API
@@ -262,16 +254,9 @@ class ThirdPartyDataFetcher:
             f"{len(disclosures)} disclosures from ProPublica"
         )
 
-        return {
-            "politicians": politicians,
-            "disclosures": disclosures
-        }
+        return {"politicians": politicians, "disclosures": disclosures}
 
-    def _convert_propublica_members(
-        self,
-        members: List[Dict],
-        chamber: str
-    ) -> List[Politician]:
+    def _convert_propublica_members(self, members: List[Dict], chamber: str) -> List[Politician]:
         """Convert ProPublica member data to Politician objects"""
         politicians = []
 
@@ -294,10 +279,7 @@ class ThirdPartyDataFetcher:
 
         return politicians
 
-    def _convert_propublica_transactions(
-        self,
-        transactions: List[Dict]
-    ) -> List[TradingDisclosure]:
+    def _convert_propublica_transactions(self, transactions: List[Dict]) -> List[TradingDisclosure]:
         """Convert ProPublica transaction data to TradingDisclosure objects"""
         disclosures = []
 

mcli/workflow/politician_trading/seed_database.py

@@ -20,11 +20,12 @@ from pathlib import Path
 from typing import Dict, List, Optional
 from uuid import UUID
 
-from supabase import create_client, Client
+from supabase import Client, create_client
 
 # Load environment variables from .env file
 try:
     from dotenv import load_dotenv
+
     # Look for .env in project root
     env_path = Path(__file__).parent.parent.parent.parent.parent / ".env"
     if env_path.exists():
@@ -39,15 +40,11 @@ from .data_sources import ALL_DATA_SOURCES, AccessMethod, DataSource
 from .models import Politician, TradingDisclosure
 from .scrapers_free_sources import FreeDataFetcher
 
-
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler("/tmp/seed_database.log")
-    ]
+    handlers=[logging.StreamHandler(sys.stdout), logging.FileHandler("/tmp/seed_database.log")],
 )
 logger = logging.getLogger(__name__)
 
@@ -76,11 +73,7 @@ def get_supabase_client() -> Client:
 # =============================================================================
 
 
-def create_data_pull_job(
-    client: Client,
-    job_type: str,
-    config: Optional[Dict] = None
-) -> UUID:
+def create_data_pull_job(client: Client, job_type: str, config: Optional[Dict] = None) -> UUID:
     """
     Create a new data pull job record
 
@@ -93,12 +86,18 @@ def create_data_pull_job(
         Job ID
     """
     try:
-        result = client.table("data_pull_jobs").insert({
-            "job_type": job_type,
-            "status": "running",
-            "started_at": datetime.now().isoformat(),
-            "config_snapshot": config or {}
-        }).execute()
+        result = (
+            client.table("data_pull_jobs")
+            .insert(
+                {
+                    "job_type": job_type,
+                    "status": "running",
+                    "started_at": datetime.now().isoformat(),
+                    "config_snapshot": config or {},
+                }
+            )
+            .execute()
+        )
 
         job_id = result.data[0]["id"]
         logger.info(f"Created data pull job: {job_id} (type: {job_type})")
@@ -114,7 +113,7 @@ def update_data_pull_job(
     job_id: UUID,
     status: str,
     stats: Optional[Dict] = None,
-    error: Optional[str] = None
+    error: Optional[str] = None,
 ):
     """
     Update data pull job with results
@@ -127,10 +126,7 @@ def update_data_pull_job(
         error: Optional error message if failed
     """
     try:
-        update_data = {
-            "status": status,
-            "completed_at": datetime.now().isoformat()
-        }
+        update_data = {"status": status, "completed_at": datetime.now().isoformat()}
 
         if stats:
             update_data.update(stats)
@@ -151,10 +147,7 @@ def update_data_pull_job(
 # =============================================================================
 
 
-def upsert_politicians(
-    client: Client,
-    politicians: List[Politician]
-) -> Dict[str, UUID]:
+def upsert_politicians(client: Client, politicians: List[Politician]) -> Dict[str, UUID]:
     """
     Upsert politicians to database, returning mapping of bioguide_id -> UUID
 
@@ -186,20 +179,23 @@ def upsert_politicians(
             # Try to find existing politician
             if politician.bioguide_id:
                 # Query by bioguide_id if available
-                existing = client.table("politicians").select("id").eq(
-                    "bioguide_id", politician.bioguide_id
-                ).execute()
+                existing = (
+                    client.table("politicians")
+                    .select("id")
+                    .eq("bioguide_id", politician.bioguide_id)
+                    .execute()
+                )
             else:
                 # Query by unique constraint fields (first_name, last_name, role, state_or_country)
-                existing = client.table("politicians").select("id").eq(
-                    "first_name", politician.first_name
-                ).eq(
-                    "last_name", politician.last_name
-                ).eq(
-                    "role", politician.role
-                ).eq(
-                    "state_or_country", politician.state_or_country
-                ).execute()
+                existing = (
+                    client.table("politicians")
+                    .select("id")
+                    .eq("first_name", politician.first_name)
+                    .eq("last_name", politician.last_name)
+                    .eq("role", politician.role)
+                    .eq("state_or_country", politician.state_or_country)
+                    .execute()
+                )
 
             if existing.data:
                 # Update existing
@@ -223,7 +219,9 @@ def upsert_politicians(
             logger.error(f"Error upserting politician {politician.full_name}: {e}")
             continue
 
-    logger.info(f"Upserted {len(politicians)} politicians ({new_count} new, {updated_count} updated)")
+    logger.info(
+        f"Upserted {len(politicians)} politicians ({new_count} new, {updated_count} updated)"
+    )
 
     return politician_map
 
@@ -234,9 +232,7 @@ def upsert_politicians(
 
 
 def upsert_trading_disclosures(
-    client: Client,
-    disclosures: List[TradingDisclosure],
-    politician_map: Dict[str, UUID]
+    client: Client, disclosures: List[TradingDisclosure], politician_map: Dict[str, UUID]
 ) -> Dict[str, int]:
     """
     Upsert trading disclosures to database
@@ -283,17 +279,16 @@ def upsert_trading_disclosures(
             }
 
             # Check for existing disclosure (using unique constraint)
-            existing = client.table("trading_disclosures").select("id").eq(
-                "politician_id", str(pol_id)
-            ).eq(
-                "transaction_date", disclosure.transaction_date.isoformat()
-            ).eq(
-                "asset_name", disclosure.asset_name
-            ).eq(
-                "transaction_type", disclosure.transaction_type
-            ).eq(
-                "disclosure_date", disclosure.disclosure_date.isoformat()
-            ).execute()
+            existing = (
+                client.table("trading_disclosures")
+                .select("id")
+                .eq("politician_id", str(pol_id))
+                .eq("transaction_date", disclosure.transaction_date.isoformat())
+                .eq("asset_name", disclosure.asset_name)
+                .eq("transaction_type", disclosure.transaction_type)
+                .eq("disclosure_date", disclosure.disclosure_date.isoformat())
+                .execute()
+            )
 
             if existing.data:
                 # Update existing
@@ -331,10 +326,7 @@ def upsert_trading_disclosures(
 
 
 def seed_from_senate_watcher(
-    client: Client,
-    test_run: bool = False,
-    recent_only: bool = False,
-    days: int = 90
+    client: Client, test_run: bool = False, recent_only: bool = False, days: int = 90
 ) -> Dict[str, int]:
     """
     Seed database from Senate Stock Watcher GitHub dataset
@@ -353,20 +345,16 @@ def seed_from_senate_watcher(
     logger.info("=" * 80)
 
     # Create job record
-    job_id = create_data_pull_job(client, "senate_watcher_seed", {
-        "recent_only": recent_only,
-        "days": days
-    })
+    job_id = create_data_pull_job(
+        client, "senate_watcher_seed", {"recent_only": recent_only, "days": days}
+    )
 
     try:
         # Initialize fetcher
         fetcher = FreeDataFetcher()
 
         # Fetch data
-        data = fetcher.fetch_from_senate_watcher(
-            recent_only=recent_only,
-            days=days
-        )
+        data = fetcher.fetch_from_senate_watcher(recent_only=recent_only, days=days)
 
         politicians = data["politicians"]
         disclosures = data["disclosures"]
@@ -377,11 +365,16 @@ def seed_from_senate_watcher(
             logger.info("TEST RUN - Not inserting to database")
             logger.info(f"Sample politician: {politicians[0] if politicians else 'None'}")
             logger.info(f"Sample disclosure: {disclosures[0] if disclosures else 'None'}")
-            update_data_pull_job(client, job_id, "completed", {
-                "records_found": len(politicians) + len(disclosures),
-                "records_new": 0,
-                "records_updated": 0,
-            })
+            update_data_pull_job(
+                client,
+                job_id,
+                "completed",
+                {
+                    "records_found": len(politicians) + len(disclosures),
+                    "records_new": 0,
+                    "records_updated": 0,
+                },
+            )
             return {"records_found": len(politicians) + len(disclosures)}
 
         # Upsert politicians
@@ -401,10 +394,7 @@ def seed_from_senate_watcher(
         raise
 
 
-def seed_from_all_sources(
-    client: Client,
-    test_run: bool = False
-) -> Dict[str, Dict[str, int]]:
+def seed_from_all_sources(client: Client, test_run: bool = False) -> Dict[str, Dict[str, int]]:
     """
     Seed database from all available sources
 
@@ -467,33 +457,27 @@ def main():
         "--sources",
         choices=["all", "senate", "finnhub", "sec-edgar"],
         default="all",
-        help="Which data sources to seed from (default: all)"
+        help="Which data sources to seed from (default: all)",
     )
 
     parser.add_argument(
-        "--recent-only",
-        action="store_true",
-        help="Only fetch recent transactions (last 90 days)"
+        "--recent-only", action="store_true", help="Only fetch recent transactions (last 90 days)"
     )
 
     parser.add_argument(
         "--days",
         type=int,
         default=90,
-        help="Number of days to look back when using --recent-only (default: 90)"
+        help="Number of days to look back when using --recent-only (default: 90)",
     )
 
     parser.add_argument(
         "--test-run",
         action="store_true",
-        help="Fetch data but don't insert to database (for testing)"
+        help="Fetch data but don't insert to database (for testing)",
     )
 
-    parser.add_argument(
-        "--verbose",
-        action="store_true",
-        help="Enable verbose logging"
-    )
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
 
     args = parser.parse_args()
 
@@ -512,10 +496,7 @@ def main():
     try:
         if args.sources == "senate":
             seed_from_senate_watcher(
-                client,
-                test_run=args.test_run,
-                recent_only=args.recent_only,
-                days=args.days
+                client, test_run=args.test_run, recent_only=args.recent_only, days=args.days
             )
         elif args.sources == "all":
             seed_from_all_sources(client, args.test_run)

{mcli_framework-7.5.1.dist-info → mcli_framework-7.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcli-framework
-Version: 7.5.1
+Version: 7.6.1
 Summary: 🚀 High-performance CLI framework with Rust extensions, AI chat, and stunning visuals
 Author-email: Luis Fernandez de la Vara <luis@lefv.io>
 Maintainer-email: Luis Fernandez de la Vara <luis@lefv.io>