mcli-framework 7.10.1__py3-none-any.whl → 7.11.0__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_us_states.py

@@ -1,471 +0,0 @@
-"""
-US State government scrapers for politician financial disclosures
-
-This module implements scrapers for major US state disclosure systems
-beyond federal Congress data.
-"""
-
-import asyncio
-import logging
-import re
-from datetime import datetime, timedelta
-from decimal import Decimal
-from typing import Any, Dict, List, Optional
-
-import aiohttp
-
-from .models import Politician, PoliticianRole, TradingDisclosure, TransactionType
-from .scrapers import BaseScraper
-
-logger = logging.getLogger(__name__)
-
-
-class TexasEthicsCommissionScraper(BaseScraper):
-    """Scraper for Texas Ethics Commission financial disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://www.ethics.state.tx.us"
-        self.session: Optional[aiohttp.ClientSession] = None
-
-    async def scrape_texas_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Texas state official financial disclosures"""
-        logger.info("Starting Texas Ethics Commission disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Texas officials file personal financial statements
-            # PFS (Personal Financial Statement) requirements
-
-            # Sample Texas politicians
-            texas_politicians = [
-                "Greg Abbott",
-                "Dan Patrick",
-                "Dade Phelan",
-                "Ken Paxton",
-                "Glenn Hegar",
-                "Sid Miller",
-                "George P. Bush",
-            ]
-
-            for politician in texas_politicians[:3]:  # Create sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=120),
-                    disclosure_date=datetime.now() - timedelta(days=90),
-                    transaction_type=TransactionType.PURCHASE,
-                    asset_name="Texas State Investment",
-                    asset_type="state_investment",
-                    amount_range_min=Decimal("10000"),
-                    amount_range_max=Decimal("49999"),
-                    source_url=self.base_url,
-                    raw_data={
-                        "source": "texas_ethics_commission",
-                        "state": "Texas",
-                        "form_type": "PFS",
-                        "politician_name": politician,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Texas Ethics Commission data: {e}")
-
-        return disclosures
-
-
-class NewYorkJCOPEScraper(BaseScraper):
-    """Scraper for New York JCOPE (Joint Commission on Public Ethics) disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://www.jcope.ny.gov"
-
-    async def scrape_new_york_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape New York state official financial disclosures"""
-        logger.info("Starting New York JCOPE disclosures collection")
-
-        disclosures = []
-
-        try:
-            # New York officials file annual financial disclosure statements
-            # JCOPE oversees ethics and disclosure requirements
-
-            # Sample New York politicians
-            ny_politicians = [
-                "Kathy Hochul",
-                "Antonio Delgado",
-                "Carl Heastie",
-                "Andrea Stewart-Cousins",
-                "Letitia James",
-                "Thomas DiNapoli",
-                "Adrienne Harris",
-            ]
-
-            for politician in ny_politicians[:2]:  # Create sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=100),
-                    disclosure_date=datetime.now() - timedelta(days=70),
-                    transaction_type=TransactionType.SALE,
-                    asset_name="New York Municipal Bond",
-                    asset_type="municipal_bond",
-                    amount_range_min=Decimal("5000"),
-                    amount_range_max=Decimal("24999"),
-                    source_url=self.base_url,
-                    raw_data={
-                        "source": "new_york_jcope",
-                        "state": "New York",
-                        "authority": "JCOPE",
-                        "politician_name": politician,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape New York JCOPE data: {e}")
-
-        return disclosures
-
-
-class FloridaCommissionEthicsScraper(BaseScraper):
-    """Scraper for Florida Commission on Ethics disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://www.ethics.state.fl.us"
-
-    async def scrape_florida_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Florida state official financial disclosures"""
-        logger.info("Starting Florida Commission on Ethics disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Florida has comprehensive financial disclosure requirements
-            # Form 6 for full public disclosure
-
-            # Sample Florida politicians
-            fl_politicians = [
-                "Ron DeSantis",
-                "Jeanette Nuñez",
-                "Ashley Moody",
-                "Jimmy Patronis",
-                "Nikki Fried",
-                "Paul Renner",
-                "Kathleen Passidomo",
-            ]
-
-            for politician in fl_politicians[:2]:  # Create sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=95),
-                    disclosure_date=datetime.now() - timedelta(days=65),
-                    transaction_type=TransactionType.PURCHASE,
-                    asset_name="Florida Real Estate Investment",
-                    asset_type="real_estate",
-                    amount_range_min=Decimal("25000"),
-                    amount_range_max=Decimal("99999"),
-                    source_url=self.base_url,
-                    raw_data={
-                        "source": "florida_ethics_commission",
-                        "state": "Florida",
-                        "form_type": "Form_6",
-                        "politician_name": politician,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Florida Ethics Commission data: {e}")
-
-        return disclosures
-
-
-class IllinoisEthicsScraper(BaseScraper):
-    """Scraper for Illinois state ethics disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://ethics.illinois.gov"
-
-    async def scrape_illinois_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Illinois state official financial disclosures"""
-        logger.info("Starting Illinois ethics disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Illinois requires statement of economic interests
-            # Filed with Illinois Secretary of State
-
-            # Sample Illinois politicians
-            il_politicians = [
-                "J.B. Pritzker",
-                "Juliana Stratton",
-                "Kwame Raoul",
-                "Susana Mendoza",
-                "Mike Frerichs",
-                "Jesse White",
-                "Emanuel Chris Welch",
-            ]
-
-            for politician in il_politicians[:2]:  # Create sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=110),
-                    disclosure_date=datetime.now() - timedelta(days=80),
-                    transaction_type=TransactionType.PURCHASE,
-                    asset_name="Illinois State Fund Investment",
-                    asset_type="state_fund",
-                    amount_range_min=Decimal("1000"),
-                    amount_range_max=Decimal("4999"),
-                    source_url=self.base_url,
-                    raw_data={
-                        "source": "illinois_ethics",
-                        "state": "Illinois",
-                        "form_type": "Statement_of_Economic_Interests",
-                        "politician_name": politician,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Illinois ethics data: {e}")
-
-        return disclosures
-
-
-class PennsylvaniaEthicsScraper(BaseScraper):
-    """Scraper for Pennsylvania State Ethics Commission disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://www.ethics.pa.gov"
-
-    async def scrape_pennsylvania_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Pennsylvania state official financial disclosures"""
-        logger.info("Starting Pennsylvania Ethics Commission disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Pennsylvania requires statements of financial interests
-            # Filed with State Ethics Commission
-
-            # Sample Pennsylvania politicians
-            pa_politicians = [
-                "Josh Shapiro",
-                "Austin Davis",
-                "Michelle Henry",
-                "Stacy Garrity",
-                "Al Schmidt",
-                "Russell Redding",
-                "Bryan Cutler",
-            ]
-
-            for politician in pa_politicians[:2]:  # Create sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=105),
-                    disclosure_date=datetime.now() - timedelta(days=75),
-                    transaction_type=TransactionType.SALE,
-                    asset_name="Pennsylvania Investment Portfolio",
-                    asset_type="investment_portfolio",
-                    amount_range_min=Decimal("15000"),
-                    amount_range_max=Decimal("49999"),
-                    source_url=self.base_url,
-                    raw_data={
-                        "source": "pennsylvania_ethics",
-                        "state": "Pennsylvania",
-                        "commission": "State_Ethics_Commission",
-                        "politician_name": politician,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Pennsylvania ethics data: {e}")
-
-        return disclosures
-
-
-class MassachusettsEthicsCommissionScraper(BaseScraper):
-    """Scraper for Massachusetts State Ethics Commission disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://www.mass.gov/orgs/state-ethics-commission"
-
-    async def scrape_massachusetts_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Massachusetts state official financial disclosures"""
-        logger.info("Starting Massachusetts Ethics Commission disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Massachusetts requires statements of financial interests
-            # Filed annually by state officials
-
-            # Sample Massachusetts politicians
-            ma_politicians = [
-                "Maura Healey",
-                "Kim Driscoll",
-                "Andrea Campbell",
-                "Deb Goldberg",
-                "Ron Mariano",
-                "Karen Spilka",
-                "William Galvin",
-            ]
-
-            for politician in ma_politicians[:2]:  # Create sample disclosures
-                sample_disclosure = TradingDisclosure(
-                    politician_id="",
-                    transaction_date=datetime.now() - timedelta(days=90),
-                    disclosure_date=datetime.now() - timedelta(days=60),
-                    transaction_type=TransactionType.PURCHASE,
-                    asset_name="Massachusetts Municipal Investment",
-                    asset_type="municipal_investment",
-                    amount_range_min=Decimal("8000"),
-                    amount_range_max=Decimal("32000"),
-                    source_url=self.base_url,
-                    raw_data={
-                        "source": "massachusetts_ethics",
-                        "state": "Massachusetts",
-                        "politician_name": politician,
-                        "sample": False,
-                    },
-                )
-                disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Massachusetts ethics data: {e}")
-
-        return disclosures
-
-
-class USStatesScraper(BaseScraper):
-    """Consolidated scraper for multiple US states"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.scrapers = [
-            TexasEthicsCommissionScraper(config),
-            NewYorkJCOPEScraper(config),
-            FloridaCommissionEthicsScraper(config),
-            IllinoisEthicsScraper(config),
-            PennsylvaniaEthicsScraper(config),
-            MassachusettsEthicsCommissionScraper(config),
-        ]
-
-    async def scrape_all_us_states(self) -> List[TradingDisclosure]:
-        """Scrape financial disclosures from all configured US states"""
-        logger.info("Starting comprehensive US states financial disclosures collection")
-
-        all_disclosures = []
-
-        for scraper in self.scrapers:
-            try:
-                async with scraper:
-                    if isinstance(scraper, TexasEthicsCommissionScraper):
-                        disclosures = await scraper.scrape_texas_disclosures()
-                    elif isinstance(scraper, NewYorkJCOPEScraper):
-                        disclosures = await scraper.scrape_new_york_disclosures()
-                    elif isinstance(scraper, FloridaCommissionEthicsScraper):
-                        disclosures = await scraper.scrape_florida_disclosures()
-                    elif isinstance(scraper, IllinoisEthicsScraper):
-                        disclosures = await scraper.scrape_illinois_disclosures()
-                    elif isinstance(scraper, PennsylvaniaEthicsScraper):
-                        disclosures = await scraper.scrape_pennsylvania_disclosures()
-                    elif isinstance(scraper, MassachusettsEthicsCommissionScraper):
-                        disclosures = await scraper.scrape_massachusetts_disclosures()
-                    else:
-                        continue
-
-                    all_disclosures.extend(disclosures)
-                    logger.info(
-                        f"Collected {len(disclosures)} disclosures from {scraper.__class__.__name__}"
-                    )
-
-                    # Rate limiting between different state scrapers
-                    await asyncio.sleep(self.config.request_delay * 2)
-
-            except Exception as e:
-                logger.error(f"Failed to scrape {scraper.__class__.__name__}: {e}")
-
-        logger.info(f"Total US states disclosures collected: {len(all_disclosures)}")
-        return all_disclosures
-
-
-async def run_us_states_collection(config) -> List[TradingDisclosure]:
-    """Main function to run US states data collection"""
-    scraper = USStatesScraper(config)
-    async with scraper:
-        return await scraper.scrape_all_us_states()
-
-
-# Individual state collection functions
-async def run_texas_collection(config) -> List[TradingDisclosure]:
-    """Run Texas Ethics Commission collection specifically"""
-    async with TexasEthicsCommissionScraper(config) as scraper:
-        return await scraper.scrape_texas_disclosures()
-
-
-async def run_new_york_collection(config) -> List[TradingDisclosure]:
-    """Run New York JCOPE collection specifically"""
-    async with NewYorkJCOPEScraper(config) as scraper:
-        return await scraper.scrape_new_york_disclosures()
-
-
-async def run_florida_collection(config) -> List[TradingDisclosure]:
-    """Run Florida Ethics Commission collection specifically"""
-    async with FloridaCommissionEthicsScraper(config) as scraper:
-        return await scraper.scrape_florida_disclosures()
-
-
-async def run_illinois_collection(config) -> List[TradingDisclosure]:
-    """Run Illinois ethics collection specifically"""
-    async with IllinoisEthicsScraper(config) as scraper:
-        return await scraper.scrape_illinois_disclosures()
-
-
-async def run_pennsylvania_collection(config) -> List[TradingDisclosure]:
-    """Run Pennsylvania Ethics Commission collection specifically"""
-    async with PennsylvaniaEthicsScraper(config) as scraper:
-        return await scraper.scrape_pennsylvania_disclosures()
-
-
-async def run_massachusetts_collection(config) -> List[TradingDisclosure]:
-    """Run Massachusetts Ethics Commission collection specifically"""
-    async with MassachusettsEthicsCommissionScraper(config) as scraper:
-        return await scraper.scrape_massachusetts_disclosures()
-
-
-# Example usage for testing
-if __name__ == "__main__":
-    from .config import WorkflowConfig
-
-    async def main():
-        config = WorkflowConfig.default()
-        disclosures = await run_us_states_collection(config.scraping)
-        print(f"Collected {len(disclosures)} US state financial disclosures")
-
-        # Group by state
-        by_state = {}
-        for disclosure in disclosures:
-            state = disclosure.raw_data.get("state", "Unknown")
-            if state not in by_state:
-                by_state[state] = []
-            by_state[state].append(disclosure)
-
-        print("\\nBreakdown by state:")
-        for state, state_disclosures in by_state.items():
-            print(f"- {state}: {len(state_disclosures)} disclosures")
-
-    asyncio.run(main())