mcli-framework 7.10.1__py3-none-any.whl → 7.10.2__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_california.py

@@ -1,434 +0,0 @@
-"""
-California NetFile and Secretary of State scraper for political financial disclosures
-
-This module implements scrapers for California's campaign finance disclosure systems,
-including NetFile public portals and Cal-Access data.
-"""
-
-import asyncio
-import logging
-import re
-from datetime import datetime, timedelta
-from decimal import Decimal
-from typing import Any, Dict, List, Optional
-
-import aiohttp
-
-from .models import Politician, PoliticianRole, TradingDisclosure, TransactionType
-from .scrapers import BaseScraper
-
-logger = logging.getLogger(__name__)
-
-
-class CaliforniaNetFileScraper(BaseScraper):
-    """Scraper for California NetFile public disclosure portals"""
-
-    def __init__(self, config, test_mode=True):
-        super().__init__(config)
-        self.test_mode = test_mode  # Skip network calls for testing
-        self.public_portals = [
-            "https://public.netfile.com/pub2/?AID=VCO",  # Ventura County
-            "https://public.netfile.com/pub2/?AID=SFO",  # San Francisco
-            "https://public.netfile.com/pub2/?AID=SCC",  # Santa Clara County
-            "https://public.netfile.com/pub2/?AID=SAC",  # Sacramento County
-            "https://public.netfile.com/pub2/?AID=LAC",  # Los Angeles County
-        ]
-        self.session: Optional[aiohttp.ClientSession] = None
-
-    async def __aenter__(self):
-        """Async context manager entry"""
-        self.session = aiohttp.ClientSession(
-            timeout=aiohttp.ClientTimeout(total=self.config.timeout),
-            headers={"User-Agent": self.config.user_agent},
-        )
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        """Async context manager exit"""
-        if self.session:
-            await self.session.close()
-
-    async def scrape_california_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape California financial disclosures from NetFile portals"""
-        logger.info("Starting California NetFile disclosures collection")
-
-        if not self.session:
-            raise RuntimeError("Session not initialized. Use async context manager.")
-
-        all_disclosures = []
-
-        # California state-level disclosures
-        state_disclosures = await self._scrape_cal_access_data()
-        all_disclosures.extend(state_disclosures)
-
-        # County-level NetFile portals
-        for portal_url in self.public_portals:
-            try:
-                county_disclosures = await self._scrape_netfile_portal(portal_url)
-                all_disclosures.extend(county_disclosures)
-                await asyncio.sleep(self.config.request_delay)
-            except Exception as e:
-                logger.error(f"Failed to scrape NetFile portal {portal_url}: {e}")
-
-        logger.info(f"Collected {len(all_disclosures)} California disclosures")
-        return all_disclosures
-
-    async def _scrape_cal_access_data(self) -> List[TradingDisclosure]:
-        """Scrape California Secretary of State Cal-Access data"""
-        disclosures = []
-
-        try:
-            logger.debug("Scraping Cal-Access state-level data")
-
-            # Cal-Access API endpoints (simplified - actual implementation would need
-            # to handle their specific data format and authentication)
-            cal_access_url = "https://www.sos.ca.gov/campaign-lobbying/cal-access-resources"
-
-            # This is a placeholder for actual Cal-Access API implementation
-            # The real implementation would:
-            # 1. Access Cal-Access database exports
-            # 2. Parse the fixed-width format files
-            # 3. Extract candidate and committee financial data
-
-            # Sample disclosures with real California politician names for demonstration
-            ca_politicians = [
-                "Gavin Newsom",
-                "Rob Bonta",
-                "Tony Thurmond",
-                "Fiona Ma",
-                "Betty Yee",
-                "Ricardo Lara",
-                "Shirley Weber",
-            ]
-
-            for politician in ca_politicians[:3]:  # Create a few sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",  # Will be filled during politician matching
-                    transaction_date=datetime.now() - timedelta(days=30),
-                    disclosure_date=datetime.now() - timedelta(days=15),
-                    transaction_type=TransactionType.PURCHASE,
-                    asset_name=f"California State Investment Fund",
-                    asset_type="investment",
-                    amount_range_min=Decimal("1000"),
-                    amount_range_max=Decimal("10000"),
-                    source_url=cal_access_url,
-                    raw_data={
-                        "source": "cal_access",
-                        "jurisdiction": "california_state",
-                        "politician_name": politician,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Cal-Access data: {e}")
-
-        return disclosures
-
-    async def _scrape_netfile_portal(self, portal_url: str) -> List[TradingDisclosure]:
-        """Scrape a specific NetFile public portal"""
-        disclosures = []
-
-        try:
-            # Extract jurisdiction from URL
-            jurisdiction = self._extract_jurisdiction(portal_url)
-            logger.debug(f"Scraping NetFile portal for {jurisdiction}")
-
-            # NetFile servers are often overloaded, use special handling
-            # Skip network calls in test mode due to server unreliability
-            if not self.test_mode:
-                try:
-                    html = await self._fetch_netfile_with_backoff(portal_url)
-                    if not html:
-                        logger.warning(
-                            f"Could not access NetFile portal for {jurisdiction} - servers may be overloaded, using sample data"
-                        )
-                except Exception as e:
-                    logger.warning(
-                        f"NetFile portal {jurisdiction} unavailable: {e}, using sample data"
-                    )
-            else:
-                logger.info(f"Test mode enabled - using sample data for {jurisdiction}")
-
-            # NetFile portals typically have search forms and results tables
-            # This is a simplified implementation - real scraper would:
-            # 1. Navigate search forms for candidate/committee data
-            # 2. Parse results tables with transaction data
-            # 3. Handle pagination for large result sets
-            # 4. Extract specific financial disclosure information
-
-            # Create sample data with local politician names for this jurisdiction
-            local_politicians = self._get_sample_local_politicians(jurisdiction)
-
-            for politician_name in local_politicians[:2]:  # Create 2 disclosures per portal
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=45),
-                    disclosure_date=datetime.now() - timedelta(days=20),
-                    transaction_type=TransactionType.SALE,
-                    asset_name=f"Municipal Investment - {jurisdiction}",
-                    asset_type="municipal_investment",
-                    amount_range_min=Decimal("5000"),
-                    amount_range_max=Decimal("25000"),
-                    source_url=portal_url,
-                    raw_data={
-                        "source": "netfile_portal",
-                        "jurisdiction": jurisdiction,
-                        "portal_url": portal_url,
-                        "politician_name": politician_name,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape NetFile portal {portal_url}: {e}")
-
-        return disclosures
-
-    def _extract_jurisdiction(self, portal_url: str) -> str:
-        """Extract jurisdiction name from NetFile portal URL"""
-        jurisdiction_map = {
-            "VCO": "Ventura County",
-            "SFO": "San Francisco",
-            "SCC": "Santa Clara County",
-            "SAC": "Sacramento County",
-            "LAC": "Los Angeles County",
-        }
-
-        # Extract AID parameter from URL
-        aid_match = re.search(r"AID=([A-Z]+)", portal_url)
-        if aid_match:
-            aid = aid_match.group(1)
-            return jurisdiction_map.get(aid, f"California {aid}")
-
-        return "California Unknown"
-
-    def _get_sample_local_politicians(self, jurisdiction: str) -> List[str]:
-        """Get sample local politician names for a jurisdiction"""
-        politician_map = {
-            "Ventura County": ["Matt LaVere", "Carmen Ramirez", "Jeff Gorell"],
-            "San Francisco": ["London Breed", "Aaron Peskin", "Matt Dorsey", "Connie Chan"],
-            "Santa Clara County": ["Cindy Chavez", "Susan Ellenberg", "Joe Simitian"],
-            "Sacramento County": ["Phil Serna", "Rich Desmond", "Don Nottoli"],
-            "Los Angeles County": ["Hilda Solis", "Sheila Kuehl", "Janice Hahn", "Holly Mitchell"],
-        }
-
-        return politician_map.get(jurisdiction, ["California Local Politician"])
-
-    async def _fetch_netfile_with_backoff(self, url: str) -> Optional[str]:
-        """Fetch NetFile page with progressive backoff for server overload"""
-        if not self.session:
-            return None
-
-        # NetFile servers are notoriously slow and overloaded, use shorter delays for testing
-        delays = [1, 2]  # Quick attempts only for testing
-
-        for attempt, delay in enumerate(delays):
-            try:
-                # Use shorter timeout for testing
-                async with self.session.get(
-                    url, timeout=aiohttp.ClientTimeout(total=5)  # 5 second timeout for testing
-                ) as response:
-                    if response.status == 200:
-                        return await response.text()
-                    elif response.status == 429:  # Rate limited
-                        logger.info(f"NetFile rate limited, waiting {delay * 2} seconds")
-                        await asyncio.sleep(delay * 2)
-                    elif response.status in [503, 504]:  # Server overloaded
-                        logger.info(f"NetFile server overloaded, waiting {delay} seconds")
-                        await asyncio.sleep(delay)
-                    else:
-                        logger.warning(f"NetFile returned HTTP {response.status} for {url}")
-
-            except asyncio.TimeoutError:
-                logger.info(
-                    f"NetFile timeout (attempt {attempt + 1}/{len(delays)}), waiting {delay} seconds"
-                )
-                if attempt < len(delays) - 1:
-                    await asyncio.sleep(delay)
-            except Exception as e:
-                logger.warning(f"NetFile error (attempt {attempt + 1}/{len(delays)}): {e}")
-                if attempt < len(delays) - 1:
-                    await asyncio.sleep(delay)
-
-        logger.error(f"NetFile portal {url} unavailable after {len(delays)} attempts")
-        return None
-
-    def _parse_netfile_transaction(
-        self, transaction_data: Dict[str, Any]
-    ) -> Optional[TradingDisclosure]:
-        """Parse NetFile transaction data into TradingDisclosure format"""
-        try:
-            # Parse transaction type
-            transaction_type_map = {
-                "contribution": TransactionType.PURCHASE,
-                "expenditure": TransactionType.SALE,
-                "investment": TransactionType.PURCHASE,
-                "loan": TransactionType.PURCHASE,
-            }
-
-            raw_type = transaction_data.get("transaction_type", "").lower()
-            transaction_type = transaction_type_map.get(raw_type, TransactionType.PURCHASE)
-
-            # Parse date
-            date_str = transaction_data.get("transaction_date", "")
-            try:
-                transaction_date = datetime.strptime(date_str, "%Y-%m-%d")
-            except ValueError:
-                try:
-                    transaction_date = datetime.strptime(date_str, "%m/%d/%Y")
-                except ValueError:
-                    transaction_date = datetime.now()
-
-            # Parse amount
-            amount_str = transaction_data.get("amount", "")
-            amount_min, amount_max, amount_exact = self._parse_california_amount(amount_str)
-
-            disclosure = TradingDisclosure(
-                politician_id="",  # Will be filled after politician matching
-                transaction_date=transaction_date,
-                disclosure_date=datetime.now(),
-                transaction_type=transaction_type,
-                asset_name=transaction_data.get("description", ""),
-                asset_ticker=None,
-                asset_type="california_disclosure",
-                amount_range_min=amount_min,
-                amount_range_max=amount_max,
-                amount_exact=amount_exact,
-                source_url=transaction_data.get("source_url", ""),
-                raw_data=transaction_data,
-            )
-
-            return disclosure
-
-        except Exception as e:
-            logger.error(f"Failed to parse NetFile transaction: {e}")
-            return None
-
-    def _parse_california_amount(
-        self, amount_text: str
-    ) -> tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]:
-        """Parse California-specific amount formats"""
-        if not amount_text:
-            return None, None, None
-
-        # Clean amount text
-        amount_clean = amount_text.replace(",", "").replace("$", "").strip()
-
-        # California disclosure thresholds
-        ca_thresholds = {
-            "under $100": (None, Decimal("100")),
-            "$100 - $499": (Decimal("100"), Decimal("499")),
-            "$500 - $999": (Decimal("500"), Decimal("999")),
-            "$1,000 - $9,999": (Decimal("1000"), Decimal("9999")),
-            "$10,000 - $99,999": (Decimal("10000"), Decimal("99999")),
-            "$100,000+": (Decimal("100000"), None),
-        }
-
-        # Check threshold patterns
-        for threshold_text, (min_val, max_val) in ca_thresholds.items():
-            if threshold_text.lower() in amount_text.lower():
-                return min_val, max_val, None
-
-        # Try exact amount parsing
-        try:
-            exact_amount = Decimal(amount_clean)
-            return None, None, exact_amount
-        except:
-            pass
-
-        # Try range parsing
-        range_match = re.search(r"(\d+(?:\.\d{2})?)\s*[-–]\s*(\d+(?:\.\d{2})?)", amount_clean)
-        if range_match:
-            min_val = Decimal(range_match.group(1))
-            max_val = Decimal(range_match.group(2))
-            return min_val, max_val, None
-
-        return None, None, None
-
-
-class CaliforniaStateLegislatureScraper(BaseScraper):
-    """Scraper for California State Legislature financial disclosures"""
-
-    async def scrape_legislature_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape California State Legislature member financial disclosures"""
-        logger.info("Starting California Legislature disclosures collection")
-
-        disclosures = []
-
-        try:
-            # California Legislature financial disclosure system
-            # Would integrate with FPPC (Fair Political Practices Commission) data
-
-            # Sample disclosures with real California legislators
-            ca_legislators = [
-                "Toni Atkins",
-                "Robert Rivas",
-                "Scott Wiener",
-                "Nancy Skinner",
-                "Anthony Portantino",
-                "Maria Elena Durazo",
-                "Alex Padilla",
-            ]
-
-            for legislator in ca_legislators[:2]:  # Create sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=60),
-                    disclosure_date=datetime.now() - timedelta(days=30),
-                    transaction_type=TransactionType.PURCHASE,
-                    asset_name="California Legislature Investment",
-                    asset_type="legislative_investment",
-                    amount_range_min=Decimal("10000"),
-                    amount_range_max=Decimal("100000"),
-                    source_url="https://www.fppc.ca.gov/",
-                    raw_data={
-                        "source": "ca_legislature",
-                        "fppc_form": "Form 700",
-                        "politician_name": legislator,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape California Legislature data: {e}")
-
-        return disclosures
-
-
-async def run_california_collection(config) -> List[TradingDisclosure]:
-    """Main function to run California data collection"""
-    all_disclosures = []
-
-    # NetFile portals
-    async with CaliforniaNetFileScraper(config) as netfile_scraper:
-        netfile_disclosures = await netfile_scraper.scrape_california_disclosures()
-        all_disclosures.extend(netfile_disclosures)
-
-    # State Legislature
-    legislature_scraper = CaliforniaStateLegislatureScraper(config)
-    async with legislature_scraper:
-        legislature_disclosures = await legislature_scraper.scrape_legislature_disclosures()
-        all_disclosures.extend(legislature_disclosures)
-
-    return all_disclosures
-
-
-# Example usage for testing
-if __name__ == "__main__":
-    from .config import WorkflowConfig
-
-    async def main():
-        config = WorkflowConfig.default()
-        disclosures = await run_california_collection(config.scraping)
-        print(f"Collected {len(disclosures)} California financial disclosures")
-
-        for disclosure in disclosures[:3]:  # Show first 3
-            print(
-                f"- {disclosure.asset_name} ({disclosure.raw_data.get('jurisdiction', 'Unknown')})"
-            )
-
-    asyncio.run(main())