mcli-framework 7.1.1__py3-none-any.whl → 7.1.3__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_us_states.py

@@ -7,42 +7,48 @@ beyond federal Congress data.
 
 import asyncio
 import logging
-from datetime import datetime, timedelta
-from typing import List, Dict, Any, Optional
-import aiohttp
 import re
+from datetime import datetime, timedelta
 from decimal import Decimal
+from typing import Any, Dict, List, Optional
+
+import aiohttp
 
+from .models import Politician, PoliticianRole, TradingDisclosure, TransactionType
 from .scrapers import BaseScraper
-from .models import TradingDisclosure, Politician, PoliticianRole, TransactionType
 
 logger = logging.getLogger(__name__)
 
 
 class TexasEthicsCommissionScraper(BaseScraper):
     """Scraper for Texas Ethics Commission financial disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://www.ethics.state.tx.us"
         self.session: Optional[aiohttp.ClientSession] = None
-        
+
     async def scrape_texas_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Texas state official financial disclosures"""
         logger.info("Starting Texas Ethics Commission disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Texas officials file personal financial statements
             # PFS (Personal Financial Statement) requirements
-            
+
             # Sample Texas politicians
             texas_politicians = [
-                "Greg Abbott", "Dan Patrick", "Dade Phelan", "Ken Paxton",
-                "Glenn Hegar", "Sid Miller", "George P. Bush"
+                "Greg Abbott",
+                "Dan Patrick",
+                "Dade Phelan",
+                "Ken Paxton",
+                "Glenn Hegar",
+                "Sid Miller",
+                "George P. Bush",
             ]
-            
+
             for politician in texas_politicians[:3]:  # Create sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -59,40 +65,45 @@ class TexasEthicsCommissionScraper(BaseScraper):
                         "state": "Texas",
                         "form_type": "PFS",
                         "politician_name": politician,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Texas Ethics Commission data: {e}")
-        
+
         return disclosures
 
 
 class NewYorkJCOPEScraper(BaseScraper):
     """Scraper for New York JCOPE (Joint Commission on Public Ethics) disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://www.jcope.ny.gov"
-        
+
     async def scrape_new_york_disclosures(self) -> List[TradingDisclosure]:
         """Scrape New York state official financial disclosures"""
         logger.info("Starting New York JCOPE disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # New York officials file annual financial disclosure statements
             # JCOPE oversees ethics and disclosure requirements
-            
+
             # Sample New York politicians
             ny_politicians = [
-                "Kathy Hochul", "Antonio Delgado", "Carl Heastie", "Andrea Stewart-Cousins",
-                "Letitia James", "Thomas DiNapoli", "Adrienne Harris"
+                "Kathy Hochul",
+                "Antonio Delgado",
+                "Carl Heastie",
+                "Andrea Stewart-Cousins",
+                "Letitia James",
+                "Thomas DiNapoli",
+                "Adrienne Harris",
             ]
-            
+
             for politician in ny_politicians[:2]:  # Create sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -109,40 +120,45 @@ class NewYorkJCOPEScraper(BaseScraper):
                         "state": "New York",
                         "authority": "JCOPE",
                         "politician_name": politician,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape New York JCOPE data: {e}")
-        
+
         return disclosures
 
 
 class FloridaCommissionEthicsScraper(BaseScraper):
     """Scraper for Florida Commission on Ethics disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://www.ethics.state.fl.us"
-        
+
     async def scrape_florida_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Florida state official financial disclosures"""
         logger.info("Starting Florida Commission on Ethics disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Florida has comprehensive financial disclosure requirements
             # Form 6 for full public disclosure
-            
+
             # Sample Florida politicians
             fl_politicians = [
-                "Ron DeSantis", "Jeanette Nuñez", "Ashley Moody", "Jimmy Patronis",
-                "Nikki Fried", "Paul Renner", "Kathleen Passidomo"
+                "Ron DeSantis",
+                "Jeanette Nuñez",
+                "Ashley Moody",
+                "Jimmy Patronis",
+                "Nikki Fried",
+                "Paul Renner",
+                "Kathleen Passidomo",
             ]
-            
+
             for politician in fl_politicians[:2]:  # Create sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -159,40 +175,45 @@ class FloridaCommissionEthicsScraper(BaseScraper):
                         "state": "Florida",
                         "form_type": "Form_6",
                         "politician_name": politician,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Florida Ethics Commission data: {e}")
-        
+
         return disclosures
 
 
 class IllinoisEthicsScraper(BaseScraper):
     """Scraper for Illinois state ethics disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://ethics.illinois.gov"
-        
+
     async def scrape_illinois_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Illinois state official financial disclosures"""
         logger.info("Starting Illinois ethics disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Illinois requires statement of economic interests
             # Filed with Illinois Secretary of State
-            
+
             # Sample Illinois politicians
             il_politicians = [
-                "J.B. Pritzker", "Juliana Stratton", "Kwame Raoul", "Susana Mendoza",
-                "Mike Frerichs", "Jesse White", "Emanuel Chris Welch"
+                "J.B. Pritzker",
+                "Juliana Stratton",
+                "Kwame Raoul",
+                "Susana Mendoza",
+                "Mike Frerichs",
+                "Jesse White",
+                "Emanuel Chris Welch",
             ]
-            
+
             for politician in il_politicians[:2]:  # Create sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -209,40 +230,45 @@ class IllinoisEthicsScraper(BaseScraper):
                         "state": "Illinois",
                         "form_type": "Statement_of_Economic_Interests",
                         "politician_name": politician,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Illinois ethics data: {e}")
-        
+
         return disclosures
 
 
 class PennsylvaniaEthicsScraper(BaseScraper):
     """Scraper for Pennsylvania State Ethics Commission disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://www.ethics.pa.gov"
-        
+
     async def scrape_pennsylvania_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Pennsylvania state official financial disclosures"""
         logger.info("Starting Pennsylvania Ethics Commission disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Pennsylvania requires statements of financial interests
             # Filed with State Ethics Commission
-            
+
             # Sample Pennsylvania politicians
             pa_politicians = [
-                "Josh Shapiro", "Austin Davis", "Michelle Henry", "Stacy Garrity",
-                "Al Schmidt", "Russell Redding", "Bryan Cutler"
+                "Josh Shapiro",
+                "Austin Davis",
+                "Michelle Henry",
+                "Stacy Garrity",
+                "Al Schmidt",
+                "Russell Redding",
+                "Bryan Cutler",
             ]
-            
+
             for politician in pa_politicians[:2]:  # Create sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -259,40 +285,45 @@ class PennsylvaniaEthicsScraper(BaseScraper):
                         "state": "Pennsylvania",
                         "commission": "State_Ethics_Commission",
                         "politician_name": politician,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Pennsylvania ethics data: {e}")
-        
+
         return disclosures
 
 
 class MassachusettsEthicsCommissionScraper(BaseScraper):
     """Scraper for Massachusetts State Ethics Commission disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://www.mass.gov/orgs/state-ethics-commission"
-        
+
     async def scrape_massachusetts_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Massachusetts state official financial disclosures"""
         logger.info("Starting Massachusetts Ethics Commission disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Massachusetts requires statements of financial interests
             # Filed annually by state officials
-            
+
             # Sample Massachusetts politicians
             ma_politicians = [
-                "Maura Healey", "Kim Driscoll", "Andrea Campbell", "Deb Goldberg",
-                "Ron Mariano", "Karen Spilka", "William Galvin"
+                "Maura Healey",
+                "Kim Driscoll",
+                "Andrea Campbell",
+                "Deb Goldberg",
+                "Ron Mariano",
+                "Karen Spilka",
+                "William Galvin",
             ]
-            
+
             for politician in ma_politicians[:2]:  # Create sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -308,37 +339,37 @@ class MassachusettsEthicsCommissionScraper(BaseScraper):
                         "source": "massachusetts_ethics",
                         "state": "Massachusetts",
                         "politician_name": politician,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Massachusetts ethics data: {e}")
-        
+
         return disclosures
 
 
 class USStatesScraper(BaseScraper):
     """Consolidated scraper for multiple US states"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.scrapers = [
             TexasEthicsCommissionScraper(config),
             NewYorkJCOPEScraper(config),
-            FloridaCommissionEthicsScraper(config), 
+            FloridaCommissionEthicsScraper(config),
             IllinoisEthicsScraper(config),
             PennsylvaniaEthicsScraper(config),
             MassachusettsEthicsCommissionScraper(config),
         ]
-    
+
     async def scrape_all_us_states(self) -> List[TradingDisclosure]:
         """Scrape financial disclosures from all configured US states"""
         logger.info("Starting comprehensive US states financial disclosures collection")
-        
+
         all_disclosures = []
-        
+
         for scraper in self.scrapers:
             try:
                 async with scraper:
@@ -356,16 +387,18 @@ class USStatesScraper(BaseScraper):
                         disclosures = await scraper.scrape_massachusetts_disclosures()
                     else:
                         continue
-                    
+
                     all_disclosures.extend(disclosures)
-                    logger.info(f"Collected {len(disclosures)} disclosures from {scraper.__class__.__name__}")
-                    
+                    logger.info(
+                        f"Collected {len(disclosures)} disclosures from {scraper.__class__.__name__}"
+                    )
+
                     # Rate limiting between different state scrapers
                     await asyncio.sleep(self.config.request_delay * 2)
-                    
+
             except Exception as e:
                 logger.error(f"Failed to scrape {scraper.__class__.__name__}: {e}")
-        
+
         logger.info(f"Total US states disclosures collected: {len(all_disclosures)}")
         return all_disclosures
 
@@ -417,22 +450,22 @@ async def run_massachusetts_collection(config) -> List[TradingDisclosure]:
 # Example usage for testing
 if __name__ == "__main__":
     from .config import WorkflowConfig
-    
+
     async def main():
         config = WorkflowConfig.default()
         disclosures = await run_us_states_collection(config.scraping)
         print(f"Collected {len(disclosures)} US state financial disclosures")
-        
+
         # Group by state
         by_state = {}
         for disclosure in disclosures:
-            state = disclosure.raw_data.get('state', 'Unknown')
+            state = disclosure.raw_data.get("state", "Unknown")
             if state not in by_state:
                 by_state[state] = []
             by_state[state].append(disclosure)
-        
+
         print("\\nBreakdown by state:")
         for state, state_disclosures in by_state.items():
             print(f"- {state}: {len(state_disclosures)} disclosures")
-    
-    asyncio.run(main())
+
+    asyncio.run(main())