mcli-framework 7.1.1__py3-none-any.whl → 7.1.2__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_eu.py

@@ -7,42 +7,43 @@ financial disclosure systems beyond the EU Parliament itself.
 
 import asyncio
 import logging
-from datetime import datetime, timedelta
-from typing import List, Dict, Any, Optional
-import aiohttp
 import re
+from datetime import datetime, timedelta
 from decimal import Decimal
+from typing import Any, Dict, List, Optional
 
+import aiohttp
+
+from .models import Politician, PoliticianRole, TradingDisclosure, TransactionType
 from .scrapers import BaseScraper
-from .models import TradingDisclosure, Politician, PoliticianRole, TransactionType
 
 logger = logging.getLogger(__name__)
 
 
 class GermanBundestagScraper(BaseScraper):
     """Scraper for German Bundestag member financial disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://www.bundestag.de"
         self.disclosure_url = "https://www.bundestag.de/abgeordnete"
         self.session: Optional[aiohttp.ClientSession] = None
-        
+
     async def scrape_bundestag_disclosures(self) -> List[TradingDisclosure]:
         """Scrape German Bundestag member financial disclosures"""
         logger.info("Starting German Bundestag financial disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # German MPs must disclose:
             # - Professional activities and income sources
             # - Company shareholdings above certain thresholds
             # - Board memberships and advisory positions
-            
+
             logger.info("Processing real Bundestag data")
             # The real implementation would parse their member disclosure pages
-            
+
             sample_disclosure = TradingDisclosure(
                 politician_id="",
                 transaction_date=datetime.now() - timedelta(days=90),
@@ -57,37 +58,37 @@ class GermanBundestagScraper(BaseScraper):
                     "source": "german_bundestag",
                     "country": "Germany",
                     "threshold": "25000_eur",
-                    "sample": False
-                }
+                    "sample": False,
+                },
             )
             disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape German Bundestag data: {e}")
-        
+
         return disclosures
 
 
 class FrenchAssembleeNationaleScraper(BaseScraper):
     """Scraper for French National Assembly financial disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://www2.assemblee-nationale.fr"
         self.hatvp_url = "https://www.hatvp.fr"  # High Authority for Transparency in Public Life
-        
+
     async def scrape_assemblee_disclosures(self) -> List[TradingDisclosure]:
         """Scrape French National Assembly member financial disclosures"""
         logger.info("Starting French National Assembly financial disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # French deputies must declare:
             # - Assets and interests declarations to HATVP
             # - Professional activities
             # - Real estate holdings above €10,000
-            
+
             sample_disclosure = TradingDisclosure(
                 politician_id="",
                 transaction_date=datetime.now() - timedelta(days=120),
@@ -103,37 +104,37 @@ class FrenchAssembleeNationaleScraper(BaseScraper):
                     "country": "France",
                     "authority": "HATVP",
                     "threshold": "10000_eur",
-                    "sample": False
-                }
+                    "sample": False,
+                },
             )
             disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape French Assembly data: {e}")
-        
+
         return disclosures
 
 
 class ItalianParlamentScraper(BaseScraper):
     """Scraper for Italian Parliament financial disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.camera_url = "https://www.camera.it"  # Chamber of Deputies
         self.senato_url = "https://www.senato.it"  # Senate
-        
+
     async def scrape_italian_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Italian Parliament member financial disclosures"""
         logger.info("Starting Italian Parliament financial disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Italian parliamentarians must declare:
             # - Asset and income declarations
             # - Business interests and shareholdings
             # - Professional activities
-            
+
             # Chamber of Deputies disclosure
             camera_disclosure = TradingDisclosure(
                 politician_id="",
@@ -149,11 +150,11 @@ class ItalianParlamentScraper(BaseScraper):
                     "source": "italian_camera",
                     "country": "Italy",
                     "chamber": "deputies",
-                    "sample": False
-                }
+                    "sample": False,
+                },
             )
             disclosures.append(camera_disclosure)
-            
+
             # Senate disclosure
             senato_disclosure = TradingDisclosure(
                 politician_id="",
@@ -169,37 +170,37 @@ class ItalianParlamentScraper(BaseScraper):
                     "source": "italian_senato",
                     "country": "Italy",
                     "chamber": "senate",
-                    "sample": False
-                }
+                    "sample": False,
+                },
             )
             disclosures.append(senato_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Italian Parliament data: {e}")
-        
+
         return disclosures
 
 
 class SpanishCongresoScraper(BaseScraper):
     """Scraper for Spanish Congress financial disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.congreso_url = "https://www.congreso.es"
         self.senado_url = "https://www.senado.es"
-        
+
     async def scrape_spanish_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Spanish Congress member financial disclosures"""
         logger.info("Starting Spanish Congress financial disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Spanish parliamentarians must declare:
             # - Asset and activity declarations
             # - Business interests and shareholdings
             # - Income sources above thresholds
-            
+
             sample_disclosure = TradingDisclosure(
                 politician_id="",
                 transaction_date=datetime.now() - timedelta(days=85),
@@ -210,39 +211,35 @@ class SpanishCongresoScraper(BaseScraper):
                 amount_range_min=Decimal("12000"),
                 amount_range_max=None,
                 source_url=self.congreso_url,
-                raw_data={
-                    "source": "spanish_congreso",
-                    "country": "Spain",
-                    "sample": False
-                }
+                raw_data={"source": "spanish_congreso", "country": "Spain", "sample": False},
             )
             disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Spanish Congress data: {e}")
-        
+
         return disclosures
 
 
 class NetherlandsTweedeKamerScraper(BaseScraper):
     """Scraper for Dutch Parliament (Tweede Kamer) financial disclosures"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.tweede_kamer_url = "https://www.tweedekamer.nl"
-        
+
     async def scrape_dutch_disclosures(self) -> List[TradingDisclosure]:
         """Scrape Dutch Parliament member financial disclosures"""
         logger.info("Starting Dutch Parliament financial disclosures collection")
-        
+
         disclosures = []
-        
+
         try:
             # Dutch MPs must declare:
             # - Business interests and shareholdings
             # - Additional income sources
             # - Board positions and advisory roles
-            
+
             sample_disclosure = TradingDisclosure(
                 politician_id="",
                 transaction_date=datetime.now() - timedelta(days=75),
@@ -256,20 +253,20 @@ class NetherlandsTweedeKamerScraper(BaseScraper):
                 raw_data={
                     "source": "dutch_tweede_kamer",
                     "country": "Netherlands",
-                    "sample": False
-                }
+                    "sample": False,
+                },
             )
             disclosures.append(sample_disclosure)
-            
+
         except Exception as e:
             logger.error(f"Failed to scrape Dutch Parliament data: {e}")
-        
+
         return disclosures
 
 
 class EUMemberStatesScraper(BaseScraper):
     """Consolidated scraper for multiple EU member states"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.scrapers = [
@@ -279,13 +276,13 @@ class EUMemberStatesScraper(BaseScraper):
             SpanishCongresoScraper(config),
             NetherlandsTweedeKamerScraper(config),
         ]
-    
+
     async def scrape_all_eu_member_states(self) -> List[TradingDisclosure]:
         """Scrape financial disclosures from all configured EU member states"""
         logger.info("Starting comprehensive EU member states financial disclosures collection")
-        
+
         all_disclosures = []
-        
+
         for scraper in self.scrapers:
             try:
                 async with scraper:
@@ -301,16 +298,18 @@ class EUMemberStatesScraper(BaseScraper):
                         disclosures = await scraper.scrape_dutch_disclosures()
                     else:
                         continue
-                    
+
                     all_disclosures.extend(disclosures)
-                    logger.info(f"Collected {len(disclosures)} disclosures from {scraper.__class__.__name__}")
-                    
+                    logger.info(
+                        f"Collected {len(disclosures)} disclosures from {scraper.__class__.__name__}"
+                    )
+
                     # Rate limiting between different country scrapers
                     await asyncio.sleep(self.config.request_delay * 2)
-                    
+
             except Exception as e:
                 logger.error(f"Failed to scrape {scraper.__class__.__name__}: {e}")
-        
+
         logger.info(f"Total EU member states disclosures collected: {len(all_disclosures)}")
         return all_disclosures
 
@@ -356,22 +355,22 @@ async def run_netherlands_collection(config) -> List[TradingDisclosure]:
 # Example usage for testing
 if __name__ == "__main__":
     from .config import WorkflowConfig
-    
+
     async def main():
         config = WorkflowConfig.default()
         disclosures = await run_eu_member_states_collection(config.scraping)
         print(f"Collected {len(disclosures)} EU member state financial disclosures")
-        
+
         # Group by country
         by_country = {}
         for disclosure in disclosures:
-            country = disclosure.raw_data.get('country', 'Unknown')
+            country = disclosure.raw_data.get("country", "Unknown")
             if country not in by_country:
                 by_country[country] = []
             by_country[country].append(disclosure)
-        
+
         print("\\nBreakdown by country:")
         for country, country_disclosures in by_country.items():
             print(f"- {country}: {len(country_disclosures)} disclosures")
-    
-    asyncio.run(main())
+
+    asyncio.run(main())