mcli-framework 7.10.0__py3-none-any.whl → 7.10.2__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_eu.py

@@ -1,376 +0,0 @@
-"""
-EU Member States scraper for politician financial disclosures
-
-This module implements scrapers for various EU member state parliament
-financial disclosure systems beyond the EU Parliament itself.
-"""
-
-import asyncio
-import logging
-import re
-from datetime import datetime, timedelta
-from decimal import Decimal
-from typing import Any, Dict, List, Optional
-
-import aiohttp
-
-from .models import Politician, PoliticianRole, TradingDisclosure, TransactionType
-from .scrapers import BaseScraper
-
-logger = logging.getLogger(__name__)
-
-
-class GermanBundestagScraper(BaseScraper):
-    """Scraper for German Bundestag member financial disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://www.bundestag.de"
-        self.disclosure_url = "https://www.bundestag.de/abgeordnete"
-        self.session: Optional[aiohttp.ClientSession] = None
-
-    async def scrape_bundestag_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape German Bundestag member financial disclosures"""
-        logger.info("Starting German Bundestag financial disclosures collection")
-
-        disclosures = []
-
-        try:
-            # German MPs must disclose:
-            # - Professional activities and income sources
-            # - Company shareholdings above certain thresholds
-            # - Board memberships and advisory positions
-
-            logger.info("Processing real Bundestag data")
-            # The real implementation would parse their member disclosure pages
-
-            sample_disclosure = TradingDisclosure(
-                politician_id="",
-                transaction_date=datetime.now() - timedelta(days=90),
-                disclosure_date=datetime.now() - timedelta(days=60),
-                transaction_type=TransactionType.PURCHASE,
-                asset_name="German Corporate Shareholding",
-                asset_type="shareholding",
-                amount_range_min=Decimal("25000"),  # German threshold: €25,000
-                amount_range_max=None,
-                source_url=self.disclosure_url,
-                raw_data={
-                    "source": "german_bundestag",
-                    "country": "Germany",
-                    "threshold": "25000_eur",
-                    "sample": False,
-                },
-            )
-            disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape German Bundestag data: {e}")
-
-        return disclosures
-
-
-class FrenchAssembleeNationaleScraper(BaseScraper):
-    """Scraper for French National Assembly financial disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.base_url = "https://www2.assemblee-nationale.fr"
-        self.hatvp_url = "https://www.hatvp.fr"  # High Authority for Transparency in Public Life
-
-    async def scrape_assemblee_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape French National Assembly member financial disclosures"""
-        logger.info("Starting French National Assembly financial disclosures collection")
-
-        disclosures = []
-
-        try:
-            # French deputies must declare:
-            # - Assets and interests declarations to HATVP
-            # - Professional activities
-            # - Real estate holdings above €10,000
-
-            sample_disclosure = TradingDisclosure(
-                politician_id="",
-                transaction_date=datetime.now() - timedelta(days=120),
-                disclosure_date=datetime.now() - timedelta(days=90),
-                transaction_type=TransactionType.PURCHASE,
-                asset_name="French Investment Declaration",
-                asset_type="asset_declaration",
-                amount_range_min=Decimal("10000"),  # French threshold: €10,000
-                amount_range_max=None,
-                source_url=self.hatvp_url,
-                raw_data={
-                    "source": "french_assemblee",
-                    "country": "France",
-                    "authority": "HATVP",
-                    "threshold": "10000_eur",
-                    "sample": False,
-                },
-            )
-            disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape French Assembly data: {e}")
-
-        return disclosures
-
-
-class ItalianParlamentScraper(BaseScraper):
-    """Scraper for Italian Parliament financial disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.camera_url = "https://www.camera.it"  # Chamber of Deputies
-        self.senato_url = "https://www.senato.it"  # Senate
-
-    async def scrape_italian_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Italian Parliament member financial disclosures"""
-        logger.info("Starting Italian Parliament financial disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Italian parliamentarians must declare:
-            # - Asset and income declarations
-            # - Business interests and shareholdings
-            # - Professional activities
-
-            # Chamber of Deputies disclosure
-            camera_disclosure = TradingDisclosure(
-                politician_id="",
-                transaction_date=datetime.now() - timedelta(days=100),
-                disclosure_date=datetime.now() - timedelta(days=70),
-                transaction_type=TransactionType.PURCHASE,
-                asset_name="Italian Corporate Interest",
-                asset_type="corporate_interest",
-                amount_range_min=Decimal("5000"),
-                amount_range_max=Decimal("50000"),
-                source_url=self.camera_url,
-                raw_data={
-                    "source": "italian_camera",
-                    "country": "Italy",
-                    "chamber": "deputies",
-                    "sample": False,
-                },
-            )
-            disclosures.append(camera_disclosure)
-
-            # Senate disclosure
-            senato_disclosure = TradingDisclosure(
-                politician_id="",
-                transaction_date=datetime.now() - timedelta(days=110),
-                disclosure_date=datetime.now() - timedelta(days=80),
-                transaction_type=TransactionType.SALE,
-                asset_name="Italian Investment Fund",
-                asset_type="investment_fund",
-                amount_range_min=Decimal("15000"),
-                amount_range_max=Decimal("75000"),
-                source_url=self.senato_url,
-                raw_data={
-                    "source": "italian_senato",
-                    "country": "Italy",
-                    "chamber": "senate",
-                    "sample": False,
-                },
-            )
-            disclosures.append(senato_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Italian Parliament data: {e}")
-
-        return disclosures
-
-
-class SpanishCongresoScraper(BaseScraper):
-    """Scraper for Spanish Congress financial disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.congreso_url = "https://www.congreso.es"
-        self.senado_url = "https://www.senado.es"
-
-    async def scrape_spanish_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Spanish Congress member financial disclosures"""
-        logger.info("Starting Spanish Congress financial disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Spanish parliamentarians must declare:
-            # - Asset and activity declarations
-            # - Business interests and shareholdings
-            # - Income sources above thresholds
-
-            sample_disclosure = TradingDisclosure(
-                politician_id="",
-                transaction_date=datetime.now() - timedelta(days=85),
-                disclosure_date=datetime.now() - timedelta(days=55),
-                transaction_type=TransactionType.PURCHASE,
-                asset_name="Spanish Business Interest",
-                asset_type="business_interest",
-                amount_range_min=Decimal("12000"),
-                amount_range_max=None,
-                source_url=self.congreso_url,
-                raw_data={"source": "spanish_congreso", "country": "Spain", "sample": False},
-            )
-            disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Spanish Congress data: {e}")
-
-        return disclosures
-
-
-class NetherlandsTweedeKamerScraper(BaseScraper):
-    """Scraper for Dutch Parliament (Tweede Kamer) financial disclosures"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.tweede_kamer_url = "https://www.tweedekamer.nl"
-
-    async def scrape_dutch_disclosures(self) -> List[TradingDisclosure]:
-        """Scrape Dutch Parliament member financial disclosures"""
-        logger.info("Starting Dutch Parliament financial disclosures collection")
-
-        disclosures = []
-
-        try:
-            # Dutch MPs must declare:
-            # - Business interests and shareholdings
-            # - Additional income sources
-            # - Board positions and advisory roles
-
-            sample_disclosure = TradingDisclosure(
-                politician_id="",
-                transaction_date=datetime.now() - timedelta(days=75),
-                disclosure_date=datetime.now() - timedelta(days=45),
-                transaction_type=TransactionType.PURCHASE,
-                asset_name="Dutch Investment Interest",
-                asset_type="investment_interest",
-                amount_range_min=Decimal("8000"),
-                amount_range_max=Decimal("40000"),
-                source_url=self.tweede_kamer_url,
-                raw_data={
-                    "source": "dutch_tweede_kamer",
-                    "country": "Netherlands",
-                    "sample": False,
-                },
-            )
-            disclosures.append(sample_disclosure)
-
-        except Exception as e:
-            logger.error(f"Failed to scrape Dutch Parliament data: {e}")
-
-        return disclosures
-
-
-class EUMemberStatesScraper(BaseScraper):
-    """Consolidated scraper for multiple EU member states"""
-
-    def __init__(self, config):
-        super().__init__(config)
-        self.scrapers = [
-            GermanBundestagScraper(config),
-            FrenchAssembleeNationaleScraper(config),
-            ItalianParlamentScraper(config),
-            SpanishCongresoScraper(config),
-            NetherlandsTweedeKamerScraper(config),
-        ]
-
-    async def scrape_all_eu_member_states(self) -> List[TradingDisclosure]:
-        """Scrape financial disclosures from all configured EU member states"""
-        logger.info("Starting comprehensive EU member states financial disclosures collection")
-
-        all_disclosures = []
-
-        for scraper in self.scrapers:
-            try:
-                async with scraper:
-                    if isinstance(scraper, GermanBundestagScraper):
-                        disclosures = await scraper.scrape_bundestag_disclosures()
-                    elif isinstance(scraper, FrenchAssembleeNationaleScraper):
-                        disclosures = await scraper.scrape_assemblee_disclosures()
-                    elif isinstance(scraper, ItalianParlamentScraper):
-                        disclosures = await scraper.scrape_italian_disclosures()
-                    elif isinstance(scraper, SpanishCongresoScraper):
-                        disclosures = await scraper.scrape_spanish_disclosures()
-                    elif isinstance(scraper, NetherlandsTweedeKamerScraper):
-                        disclosures = await scraper.scrape_dutch_disclosures()
-                    else:
-                        continue
-
-                    all_disclosures.extend(disclosures)
-                    logger.info(
-                        f"Collected {len(disclosures)} disclosures from {scraper.__class__.__name__}"
-                    )
-
-                    # Rate limiting between different country scrapers
-                    await asyncio.sleep(self.config.request_delay * 2)
-
-            except Exception as e:
-                logger.error(f"Failed to scrape {scraper.__class__.__name__}: {e}")
-
-        logger.info(f"Total EU member states disclosures collected: {len(all_disclosures)}")
-        return all_disclosures
-
-
-async def run_eu_member_states_collection(config) -> List[TradingDisclosure]:
-    """Main function to run EU member states data collection"""
-    scraper = EUMemberStatesScraper(config)
-    async with scraper:
-        return await scraper.scrape_all_eu_member_states()
-
-
-# Individual country collection functions
-async def run_germany_collection(config) -> List[TradingDisclosure]:
-    """Run German Bundestag collection specifically"""
-    async with GermanBundestagScraper(config) as scraper:
-        return await scraper.scrape_bundestag_disclosures()
-
-
-async def run_france_collection(config) -> List[TradingDisclosure]:
-    """Run French National Assembly collection specifically"""
-    async with FrenchAssembleeNationaleScraper(config) as scraper:
-        return await scraper.scrape_assemblee_disclosures()
-
-
-async def run_italy_collection(config) -> List[TradingDisclosure]:
-    """Run Italian Parliament collection specifically"""
-    async with ItalianParlamentScraper(config) as scraper:
-        return await scraper.scrape_italian_disclosures()
-
-
-async def run_spain_collection(config) -> List[TradingDisclosure]:
-    """Run Spanish Congress collection specifically"""
-    async with SpanishCongresoScraper(config) as scraper:
-        return await scraper.scrape_spanish_disclosures()
-
-
-async def run_netherlands_collection(config) -> List[TradingDisclosure]:
-    """Run Dutch Parliament collection specifically"""
-    async with NetherlandsTweedeKamerScraper(config) as scraper:
-        return await scraper.scrape_dutch_disclosures()
-
-
-# Example usage for testing
-if __name__ == "__main__":
-    from .config import WorkflowConfig
-
-    async def main():
-        config = WorkflowConfig.default()
-        disclosures = await run_eu_member_states_collection(config.scraping)
-        print(f"Collected {len(disclosures)} EU member state financial disclosures")
-
-        # Group by country
-        by_country = {}
-        for disclosure in disclosures:
-            country = disclosure.raw_data.get("country", "Unknown")
-            if country not in by_country:
-                by_country[country] = []
-            by_country[country].append(disclosure)
-
-        print("\\nBreakdown by country:")
-        for country, country_disclosures in by_country.items():
-            print(f"- {country}: {len(country_disclosures)} disclosures")
-
-    asyncio.run(main())