mcli-framework 7.1.0__py3-none-any.whl → 7.1.2__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_california.py

@@ -7,21 +7,22 @@ including NetFile public portals and Cal-Access data.
 
 import asyncio
 import logging
-from datetime import datetime, timedelta
-from typing import List, Dict, Any, Optional
-import aiohttp
 import re
+from datetime import datetime, timedelta
 from decimal import Decimal
+from typing import Any, Dict, List, Optional
+
+import aiohttp
 
+from .models import Politician, PoliticianRole, TradingDisclosure, TransactionType
 from .scrapers import BaseScraper
-from .models import TradingDisclosure, Politician, PoliticianRole, TransactionType
 
 logger = logging.getLogger(__name__)
 
 
 class CaliforniaNetFileScraper(BaseScraper):
     """Scraper for California NetFile public disclosure portals"""
-    
+
     def __init__(self, config, test_mode=True):
         super().__init__(config)
         self.test_mode = test_mode  # Skip network calls for testing
@@ -33,33 +34,33 @@ class CaliforniaNetFileScraper(BaseScraper):
             "https://public.netfile.com/pub2/?AID=LAC",  # Los Angeles County
         ]
         self.session: Optional[aiohttp.ClientSession] = None
-        
+
     async def __aenter__(self):
         """Async context manager entry"""
         self.session = aiohttp.ClientSession(
             timeout=aiohttp.ClientTimeout(total=self.config.timeout),
-            headers={'User-Agent': self.config.user_agent}
+            headers={"User-Agent": self.config.user_agent},
         )
         return self
-    
+
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         """Async context manager exit"""
         if self.session:
             await self.session.close()
-    
+
     async def scrape_california_disclosures(self) -> List[TradingDisclosure]:
         """Scrape California financial disclosures from NetFile portals"""
         logger.info("Starting California NetFile disclosures collection")
-        
+
         if not self.session:
             raise RuntimeError("Session not initialized. Use async context manager.")
-        
+
         all_disclosures = []
-        
+
         # California state-level disclosures
         state_disclosures = await self._scrape_cal_access_data()
         all_disclosures.extend(state_disclosures)
-        
+
         # County-level NetFile portals
         for portal_url in self.public_portals:
             try:
@@ -68,33 +69,38 @@ class CaliforniaNetFileScraper(BaseScraper):
                 await asyncio.sleep(self.config.request_delay)
             except Exception as e:
                 logger.error(f"Failed to scrape NetFile portal {portal_url}: {e}")
-        
+
         logger.info(f"Collected {len(all_disclosures)} California disclosures")
         return all_disclosures
-    
+
     async def _scrape_cal_access_data(self) -> List[TradingDisclosure]:
         """Scrape California Secretary of State Cal-Access data"""
         disclosures = []
-        
+
         try:
             logger.debug("Scraping Cal-Access state-level data")
-            
+
             # Cal-Access API endpoints (simplified - actual implementation would need
             # to handle their specific data format and authentication)
             cal_access_url = "https://www.sos.ca.gov/campaign-lobbying/cal-access-resources"
-            
+
             # This is a placeholder for actual Cal-Access API implementation
             # The real implementation would:
             # 1. Access Cal-Access database exports
             # 2. Parse the fixed-width format files
             # 3. Extract candidate and committee financial data
-            
+
             # Sample disclosures with real California politician names for demonstration
             ca_politicians = [
-                "Gavin Newsom", "Rob Bonta", "Tony Thurmond", "Fiona Ma", 
-                "Betty Yee", "Ricardo Lara", "Shirley Weber"
+                "Gavin Newsom",
+                "Rob Bonta",
+                "Tony Thurmond",
+                "Fiona Ma",
+                "Betty Yee",
+                "Ricardo Lara",
+                "Shirley Weber",
             ]
-            
+
             for politician in ca_politicians[:3]:  # Create a few sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",  # Will be filled during politician matching
@@ -110,47 +116,51 @@ class CaliforniaNetFileScraper(BaseScraper):
                         "source": "cal_access",
                         "jurisdiction": "california_state",
                         "politician_name": politician,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Cal-Access data: {e}")
-        
+
         return disclosures
-    
+
     async def _scrape_netfile_portal(self, portal_url: str) -> List[TradingDisclosure]:
         """Scrape a specific NetFile public portal"""
         disclosures = []
-        
+
         try:
             # Extract jurisdiction from URL
             jurisdiction = self._extract_jurisdiction(portal_url)
             logger.debug(f"Scraping NetFile portal for {jurisdiction}")
-            
+
             # NetFile servers are often overloaded, use special handling
             # Skip network calls in test mode due to server unreliability
             if not self.test_mode:
                 try:
                     html = await self._fetch_netfile_with_backoff(portal_url)
                     if not html:
-                        logger.warning(f"Could not access NetFile portal for {jurisdiction} - servers may be overloaded, using sample data")
+                        logger.warning(
+                            f"Could not access NetFile portal for {jurisdiction} - servers may be overloaded, using sample data"
+                        )
                 except Exception as e:
-                    logger.warning(f"NetFile portal {jurisdiction} unavailable: {e}, using sample data")
+                    logger.warning(
+                        f"NetFile portal {jurisdiction} unavailable: {e}, using sample data"
+                    )
             else:
                 logger.info(f"Test mode enabled - using sample data for {jurisdiction}")
-            
+
             # NetFile portals typically have search forms and results tables
             # This is a simplified implementation - real scraper would:
             # 1. Navigate search forms for candidate/committee data
             # 2. Parse results tables with transaction data
             # 3. Handle pagination for large result sets
             # 4. Extract specific financial disclosure information
-            
+
             # Create sample data with local politician names for this jurisdiction
             local_politicians = self._get_sample_local_politicians(jurisdiction)
-            
+
             for politician_name in local_politicians[:2]:  # Create 2 disclosures per portal
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -167,34 +177,34 @@ class CaliforniaNetFileScraper(BaseScraper):
                         "jurisdiction": jurisdiction,
                         "portal_url": portal_url,
                         "politician_name": politician_name,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape NetFile portal {portal_url}: {e}")
-        
+
         return disclosures
-    
+
     def _extract_jurisdiction(self, portal_url: str) -> str:
         """Extract jurisdiction name from NetFile portal URL"""
         jurisdiction_map = {
             "VCO": "Ventura County",
             "SFO": "San Francisco",
-            "SCC": "Santa Clara County", 
+            "SCC": "Santa Clara County",
             "SAC": "Sacramento County",
-            "LAC": "Los Angeles County"
+            "LAC": "Los Angeles County",
         }
-        
+
         # Extract AID parameter from URL
-        aid_match = re.search(r'AID=([A-Z]+)', portal_url)
+        aid_match = re.search(r"AID=([A-Z]+)", portal_url)
         if aid_match:
             aid = aid_match.group(1)
             return jurisdiction_map.get(aid, f"California {aid}")
-        
+
         return "California Unknown"
-    
+
     def _get_sample_local_politicians(self, jurisdiction: str) -> List[str]:
         """Get sample local politician names for a jurisdiction"""
         politician_map = {
@@ -202,25 +212,24 @@ class CaliforniaNetFileScraper(BaseScraper):
             "San Francisco": ["London Breed", "Aaron Peskin", "Matt Dorsey", "Connie Chan"],
             "Santa Clara County": ["Cindy Chavez", "Susan Ellenberg", "Joe Simitian"],
             "Sacramento County": ["Phil Serna", "Rich Desmond", "Don Nottoli"],
-            "Los Angeles County": ["Hilda Solis", "Sheila Kuehl", "Janice Hahn", "Holly Mitchell"]
+            "Los Angeles County": ["Hilda Solis", "Sheila Kuehl", "Janice Hahn", "Holly Mitchell"],
         }
-        
+
         return politician_map.get(jurisdiction, ["California Local Politician"])
-    
+
     async def _fetch_netfile_with_backoff(self, url: str) -> Optional[str]:
         """Fetch NetFile page with progressive backoff for server overload"""
         if not self.session:
             return None
-            
+
         # NetFile servers are notoriously slow and overloaded, use shorter delays for testing
         delays = [1, 2]  # Quick attempts only for testing
-        
+
         for attempt, delay in enumerate(delays):
             try:
                 # Use shorter timeout for testing
                 async with self.session.get(
-                    url, 
-                    timeout=aiohttp.ClientTimeout(total=5)  # 5 second timeout for testing
+                    url, timeout=aiohttp.ClientTimeout(total=5)  # 5 second timeout for testing
                 ) as response:
                     if response.status == 200:
                         return await response.text()
@@ -232,20 +241,24 @@ class CaliforniaNetFileScraper(BaseScraper):
                         await asyncio.sleep(delay)
                     else:
                         logger.warning(f"NetFile returned HTTP {response.status} for {url}")
-                        
+
             except asyncio.TimeoutError:
-                logger.info(f"NetFile timeout (attempt {attempt + 1}/{len(delays)}), waiting {delay} seconds")
+                logger.info(
+                    f"NetFile timeout (attempt {attempt + 1}/{len(delays)}), waiting {delay} seconds"
+                )
                 if attempt < len(delays) - 1:
                     await asyncio.sleep(delay)
             except Exception as e:
                 logger.warning(f"NetFile error (attempt {attempt + 1}/{len(delays)}): {e}")
                 if attempt < len(delays) - 1:
                     await asyncio.sleep(delay)
-        
+
         logger.error(f"NetFile portal {url} unavailable after {len(delays)} attempts")
         return None
-    
-    def _parse_netfile_transaction(self, transaction_data: Dict[str, Any]) -> Optional[TradingDisclosure]:
+
+    def _parse_netfile_transaction(
+        self, transaction_data: Dict[str, Any]
+    ) -> Optional[TradingDisclosure]:
         """Parse NetFile transaction data into TradingDisclosure format"""
         try:
             # Parse transaction type
@@ -253,12 +266,12 @@ class CaliforniaNetFileScraper(BaseScraper):
                 "contribution": TransactionType.PURCHASE,
                 "expenditure": TransactionType.SALE,
                 "investment": TransactionType.PURCHASE,
-                "loan": TransactionType.PURCHASE
+                "loan": TransactionType.PURCHASE,
             }
-            
+
             raw_type = transaction_data.get("transaction_type", "").lower()
             transaction_type = transaction_type_map.get(raw_type, TransactionType.PURCHASE)
-            
+
             # Parse date
             date_str = transaction_data.get("transaction_date", "")
             try:
@@ -268,11 +281,11 @@ class CaliforniaNetFileScraper(BaseScraper):
                     transaction_date = datetime.strptime(date_str, "%m/%d/%Y")
                 except ValueError:
                     transaction_date = datetime.now()
-            
+
             # Parse amount
             amount_str = transaction_data.get("amount", "")
             amount_min, amount_max, amount_exact = self._parse_california_amount(amount_str)
-            
+
             disclosure = TradingDisclosure(
                 politician_id="",  # Will be filled after politician matching
                 transaction_date=transaction_date,
@@ -285,23 +298,25 @@ class CaliforniaNetFileScraper(BaseScraper):
                 amount_range_max=amount_max,
                 amount_exact=amount_exact,
                 source_url=transaction_data.get("source_url", ""),
-                raw_data=transaction_data
+                raw_data=transaction_data,
             )
-            
+
             return disclosure
-            
+
         except Exception as e:
             logger.error(f"Failed to parse NetFile transaction: {e}")
             return None
-    
-    def _parse_california_amount(self, amount_text: str) -> tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]:
+
+    def _parse_california_amount(
+        self, amount_text: str
+    ) -> tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]:
         """Parse California-specific amount formats"""
         if not amount_text:
             return None, None, None
-        
+
         # Clean amount text
         amount_clean = amount_text.replace(",", "").replace("$", "").strip()
-        
+
         # California disclosure thresholds
         ca_thresholds = {
             "under $100": (None, Decimal("100")),
@@ -309,50 +324,55 @@ class CaliforniaNetFileScraper(BaseScraper):
             "$500 - $999": (Decimal("500"), Decimal("999")),
             "$1,000 - $9,999": (Decimal("1000"), Decimal("9999")),
             "$10,000 - $99,999": (Decimal("10000"), Decimal("99999")),
-            "$100,000+": (Decimal("100000"), None)
+            "$100,000+": (Decimal("100000"), None),
         }
-        
+
         # Check threshold patterns
         for threshold_text, (min_val, max_val) in ca_thresholds.items():
             if threshold_text.lower() in amount_text.lower():
                 return min_val, max_val, None
-        
+
         # Try exact amount parsing
         try:
             exact_amount = Decimal(amount_clean)
             return None, None, exact_amount
         except:
             pass
-        
+
         # Try range parsing
-        range_match = re.search(r'(\d+(?:\.\d{2})?)\s*[-–]\s*(\d+(?:\.\d{2})?)', amount_clean)
+        range_match = re.search(r"(\d+(?:\.\d{2})?)\s*[-–]\s*(\d+(?:\.\d{2})?)", amount_clean)
         if range_match:
             min_val = Decimal(range_match.group(1))
             max_val = Decimal(range_match.group(2))
             return min_val, max_val, None
-        
+
         return None, None, None
 
 
 class CaliforniaStateLegislatureScraper(BaseScraper):
     """Scraper for California State Legislature financial disclosures"""
-    
+
     async def scrape_legislature_disclosures(self) -> List[TradingDisclosure]:
         """Scrape California State Legislature member financial disclosures"""
         logger.info("Starting California Legislature disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # California Legislature financial disclosure system
             # Would integrate with FPPC (Fair Political Practices Commission) data
-            
+
             # Sample disclosures with real California legislators
             ca_legislators = [
-                "Toni Atkins", "Robert Rivas", "Scott Wiener", "Nancy Skinner",
-                "Anthony Portantino", "Maria Elena Durazo", "Alex Padilla"
+                "Toni Atkins",
+                "Robert Rivas",
+                "Scott Wiener",
+                "Nancy Skinner",
+                "Anthony Portantino",
+                "Maria Elena Durazo",
+                "Alex Padilla",
             ]
-            
+
             for legislator in ca_legislators[:2]:  # Create sample disclosures
                 sample_disclosure = TradingDisclosure(
                     politician_id="",
@@ -368,45 +388,47 @@ class CaliforniaStateLegislatureScraper(BaseScraper):
                         "source": "ca_legislature",
                         "fppc_form": "Form 700",
                         "politician_name": legislator,
-                        "sample": False
-                    }
+                        "sample": False,
+                    },
                 )
                 disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape California Legislature data: {e}")
-        
+
         return disclosures
 
 
 async def run_california_collection(config) -> List[TradingDisclosure]:
     """Main function to run California data collection"""
     all_disclosures = []
-    
+
     # NetFile portals
     async with CaliforniaNetFileScraper(config) as netfile_scraper:
         netfile_disclosures = await netfile_scraper.scrape_california_disclosures()
         all_disclosures.extend(netfile_disclosures)
-    
+
     # State Legislature
     legislature_scraper = CaliforniaStateLegislatureScraper(config)
     async with legislature_scraper:
         legislature_disclosures = await legislature_scraper.scrape_legislature_disclosures()
         all_disclosures.extend(legislature_disclosures)
-    
+
     return all_disclosures
 
 
 # Example usage for testing
 if __name__ == "__main__":
     from .config import WorkflowConfig
-    
+
     async def main():
         config = WorkflowConfig.default()
         disclosures = await run_california_collection(config.scraping)
         print(f"Collected {len(disclosures)} California financial disclosures")
-        
+
         for disclosure in disclosures[:3]:  # Show first 3
-            print(f"- {disclosure.asset_name} ({disclosure.raw_data.get('jurisdiction', 'Unknown')})")
-    
-    asyncio.run(main())
+            print(
+                f"- {disclosure.asset_name} ({disclosure.raw_data.get('jurisdiction', 'Unknown')})"
+            )
+
+    asyncio.run(main())