mcli-framework 7.1.0__py3-none-any.whl → 7.1.2__py3-none-any.whl

mcli/workflow/politician_trading/scrapers_uk.py

@@ -8,169 +8,186 @@ to collect MP financial disclosure data.
 import asyncio
 import logging
 from datetime import datetime
-from typing import List, Dict, Any, Optional
+from typing import Any, Dict, List, Optional
+
 import aiohttp
 
+from .models import Politician, PoliticianRole, TradingDisclosure, TransactionType
 from .scrapers import BaseScraper
-from .models import TradingDisclosure, Politician, PoliticianRole, TransactionType
 
 logger = logging.getLogger(__name__)
 
 
 class UKParliamentScraper(BaseScraper):
     """Scraper for UK Parliament Register of Interests API"""
-    
+
     def __init__(self, config):
         super().__init__(config)
         self.base_url = "https://interests-api.parliament.uk/api/v1"
         self.session: Optional[aiohttp.ClientSession] = None
-        
+
     async def __aenter__(self):
         """Async context manager entry"""
         self.session = aiohttp.ClientSession(
             timeout=aiohttp.ClientTimeout(total=self.config.timeout),
-            headers={'User-Agent': self.config.user_agent}
+            headers={"User-Agent": self.config.user_agent},
         )
         return self
-    
+
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         """Async context manager exit"""
         if self.session:
             await self.session.close()
-    
+
     async def fetch_members_interests(self) -> List[TradingDisclosure]:
         """Fetch all MP financial interests from the API"""
         logger.info("Starting UK Parliament financial interests collection")
-        
+
         if not self.session:
             raise RuntimeError("Session not initialized. Use async context manager.")
-        
+
         disclosures = []
-        
+
         try:
             # First, get all interest categories to understand what types of interests exist
             categories = await self._fetch_categories()
             logger.info(f"Found {len(categories)} interest categories")
-            
+
             # Get all interests for financial/investment categories
             financial_categories = self._filter_financial_categories(categories)
-            
+
             for category in financial_categories:
                 category_disclosures = await self._fetch_interests_by_category(category)
                 disclosures.extend(category_disclosures)
-                
+
                 # Rate limiting
                 await asyncio.sleep(self.config.request_delay)
-            
+
             logger.info(f"Collected {len(disclosures)} UK Parliament financial interests")
             return disclosures
-            
+
         except Exception as e:
             logger.error(f"Failed to fetch UK Parliament interests: {e}")
             raise
-    
+
     async def _fetch_categories(self) -> List[Dict[str, Any]]:
         """Fetch all interest categories from the API"""
         url = f"{self.base_url}/Categories"
         params = {"Take": 100}  # Get up to 100 categories
-        
+
         async with self.session.get(url, params=params) as response:
             response.raise_for_status()
             data = await response.json()
             return data.get("items", [])
-    
-    def _filter_financial_categories(self, categories: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+    def _filter_financial_categories(
+        self, categories: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
         """Filter categories to include only financial/investment related ones"""
         financial_keywords = [
-            "shareholding", "share", "investment", "financial", "company",
-            "directorship", "employment", "remuneration", "sponsorship",
-            "gift", "benefit", "land", "property"
+            "shareholding",
+            "share",
+            "investment",
+            "financial",
+            "company",
+            "directorship",
+            "employment",
+            "remuneration",
+            "sponsorship",
+            "gift",
+            "benefit",
+            "land",
+            "property",
         ]
-        
+
         financial_categories = []
         for category in categories:
             category_name = category.get("name", "").lower()
             if any(keyword in category_name for keyword in financial_keywords):
                 financial_categories.append(category)
                 logger.debug(f"Including financial category: {category.get('name')}")
-        
+
         return financial_categories
-    
-    async def _fetch_interests_by_category(self, category: Dict[str, Any]) -> List[TradingDisclosure]:
+
+    async def _fetch_interests_by_category(
+        self, category: Dict[str, Any]
+    ) -> List[TradingDisclosure]:
         """Fetch interests for a specific category"""
         category_id = category.get("id")
         category_name = category.get("name")
-        
+
         logger.debug(f"Fetching interests for category: {category_name} (ID: {category_id})")
-        
+
         disclosures = []
         skip = 0
         take = 50
-        
+
         while True:
             url = f"{self.base_url}/Interests"
-            params = {
-                "categoryId": category_id,
-                "Skip": skip,
-                "Take": take
-            }
-            
+            params = {"categoryId": category_id, "Skip": skip, "Take": take}
+
             try:
                 async with self.session.get(url, params=params) as response:
                     response.raise_for_status()
                     data = await response.json()
-                    
+
                     interests = data.get("items", [])
                     if not interests:
                         break
-                    
+
                     for interest in interests:
                         disclosure = await self._parse_uk_interest(interest, category_name)
                         if disclosure:
                             disclosures.append(disclosure)
-                    
+
                     skip += take
-                    
+
                     # If we got fewer results than requested, we're done
                     if len(interests) < take:
                         break
-                        
+
             except Exception as e:
                 logger.error(f"Failed to fetch interests for category {category_name}: {e}")
                 break
-        
+
         logger.debug(f"Found {len(disclosures)} interests in category: {category_name}")
         return disclosures
-    
-    async def _parse_uk_interest(self, interest: Dict[str, Any], category_name: str) -> Optional[TradingDisclosure]:
+
+    async def _parse_uk_interest(
+        self, interest: Dict[str, Any], category_name: str
+    ) -> Optional[TradingDisclosure]:
         """Parse a UK Parliament interest into a TradingDisclosure"""
         try:
             # Extract member information from the new API structure
             member_data = interest.get("member")
             if not member_data:
                 return None
-            
+
             member_id = member_data.get("id")
             politician_name = member_data.get("nameDisplayAs", "")
-            
+
             # Get interest details
             interest_id = interest.get("id")
             description = interest.get("summary", "")
             registered_date = interest.get("registrationDate")
-            
+
             # Parse dates
-            transaction_date = self._parse_date(registered_date) if registered_date else datetime.now()
+            transaction_date = (
+                self._parse_date(registered_date) if registered_date else datetime.now()
+            )
             disclosure_date = transaction_date  # UK system doesn't separate these
-            
+
             # Determine transaction type from description
             transaction_type = self._infer_transaction_type(description, category_name)
-            
+
             # Extract asset information from fields and description
-            asset_name, asset_ticker = self._extract_asset_info_from_fields(interest, description, category_name)
-            
+            asset_name, asset_ticker = self._extract_asset_info_from_fields(
+                interest, description, category_name
+            )
+
             # Extract amount information (if available)
             amount_min, amount_max, amount_exact = self._extract_amount_info(description)
-            
+
             disclosure = TradingDisclosure(
                 id=f"uk_parliament_{interest_id}",
                 politician_id="",  # Will be filled during politician matching
@@ -191,29 +208,29 @@ class UKParliamentScraper(BaseScraper):
                     "category_name": category_name,
                     "registered_date": registered_date,
                     "source": "uk_parliament_api",
-                    "politician_name": politician_name
-                }
+                    "politician_name": politician_name,
+                },
             )
-            
+
             return disclosure
-            
+
         except Exception as e:
             logger.error(f"Failed to parse UK interest: {e}")
             return None
-    
+
     async def _fetch_mp_name(self, member_id: int) -> str:
         """Fetch MP name from the Parliament API using member ID"""
         if not self.session:
             return ""
-            
+
         try:
             # Try the Members endpoint to get MP details
             member_url = f"{self.base_url}/Members/{member_id}"
-            
+
             async with self.session.get(member_url) as response:
                 if response.status == 200:
                     data = await response.json()
-                    
+
                     # Extract name from the response
                     name = data.get("name", "")
                     if not name:
@@ -225,19 +242,21 @@ class UKParliamentScraper(BaseScraper):
                         last_name = data.get("nameFull", "") or data.get("nameFamily", "")
                         if first_name and last_name:
                             name = f"{first_name} {last_name}"
-                    
+
                     if name:
                         logger.debug(f"Found MP name for ID {member_id}: {name}")
                         return name.strip()
-                        
+
                 else:
-                    logger.debug(f"Could not fetch MP details for ID {member_id}: HTTP {response.status}")
-                    
+                    logger.debug(
+                        f"Could not fetch MP details for ID {member_id}: HTTP {response.status}"
+                    )
+
         except Exception as e:
             logger.debug(f"Failed to fetch MP name for ID {member_id}: {e}")
-            
+
         return ""
-    
+
     def _parse_date(self, date_str: str) -> datetime:
         """Parse UK Parliament API date format"""
         try:
@@ -245,12 +264,12 @@ class UKParliamentScraper(BaseScraper):
             return datetime.fromisoformat(date_str.replace("Z", "+00:00"))
         except Exception:
             return datetime.now()
-    
+
     def _infer_transaction_type(self, description: str, category_name: str) -> TransactionType:
         """Infer transaction type from description and category"""
         description_lower = description.lower()
         category_lower = category_name.lower()
-        
+
         # UK Parliament disclosures are mostly about holdings, not transactions
         # But we can infer some information
         if any(word in description_lower for word in ["sold", "disposed", "divested"]):
@@ -261,29 +280,33 @@ class UKParliamentScraper(BaseScraper):
             return TransactionType.PURCHASE  # Assume shareholding disclosure is a purchase
         else:
             return TransactionType.PURCHASE  # Default assumption
-    
-    def _extract_asset_info_from_fields(self, interest: Dict[str, Any], description: str, category_name: str) -> tuple[str, Optional[str]]:
+
+    def _extract_asset_info_from_fields(
+        self, interest: Dict[str, Any], description: str, category_name: str
+    ) -> tuple[str, Optional[str]]:
         """Extract asset name and ticker from interest fields"""
         # Look for OrganisationName in fields
         fields = interest.get("fields", [])
         organization_name = None
-        
+
         for field in fields:
             if field.get("name") == "OrganisationName":
                 organization_name = field.get("value")
                 break
-        
+
         # Use organization name if available, otherwise fall back to description
         if organization_name:
             return organization_name, None
         else:
             return self._extract_asset_info(description, category_name)
-    
-    def _extract_asset_info(self, description: str, category_name: str) -> tuple[str, Optional[str]]:
+
+    def _extract_asset_info(
+        self, description: str, category_name: str
+    ) -> tuple[str, Optional[str]]:
         """Extract asset name and ticker from description"""
         # UK descriptions often contain company names
         # This is a simplified extraction - could be enhanced with NLP
-        
+
         if "shareholding" in category_name.lower():
             # Try to extract company name from shareholding descriptions
             # Format often like: "Shareholding in [Company Name] Ltd"
@@ -292,38 +315,41 @@ class UKParliamentScraper(BaseScraper):
                 if len(parts) > 1:
                     asset_name = parts[1].strip().rstrip(".")
                     return asset_name, None
-        
+
         # Fallback: use description as asset name
         return description[:100], None  # Truncate to reasonable length
-    
-    def _extract_amount_info(self, description: str) -> tuple[Optional[float], Optional[float], Optional[float]]:
+
+    def _extract_amount_info(
+        self, description: str
+    ) -> tuple[Optional[float], Optional[float], Optional[float]]:
         """Extract amount information from description"""
         # UK Parliament disclosures often don't include specific amounts
         # They use threshold categories (£70,000+, etc.)
-        
+
         description_lower = description.lower()
-        
+
         # Look for UK threshold amounts
         if "£70,000" in description_lower or "70000" in description_lower:
             return 70000.0, None, None
         elif "£" in description_lower:
             # Try to extract specific amounts
             import re
-            amount_pattern = r'£([\d,]+)'
+
+            amount_pattern = r"£([\d,]+)"
             matches = re.findall(amount_pattern, description)
             if matches:
                 try:
-                    amount = float(matches[0].replace(',', ''))
+                    amount = float(matches[0].replace(",", ""))
                     return amount, None, amount
                 except ValueError:
                     pass
-        
+
         return None, None, None
-    
+
     async def get_politicians(self) -> List[Politician]:
         """Fetch current MPs from the Members API"""
         logger.info("Fetching current UK MPs")
-        
+
         # For now, return empty list - would need Members API integration
         # This would require calling https://members-api.parliament.uk/
         return []
@@ -338,13 +364,15 @@ async def run_uk_parliament_collection(config) -> List[TradingDisclosure]:
 # Example usage for testing
 if __name__ == "__main__":
     from .config import WorkflowConfig
-    
+
     async def main():
         config = WorkflowConfig.default()
         disclosures = await run_uk_parliament_collection(config.scraping)
         print(f"Collected {len(disclosures)} UK Parliament financial interests")
-        
+
         for disclosure in disclosures[:3]:  # Show first 3
-            print(f"- {disclosure.asset_name} by {disclosure.raw_data.get('politician_name', 'Unknown')}")
-    
-    asyncio.run(main())
+            print(
+                f"- {disclosure.asset_name} by {disclosure.raw_data.get('politician_name', 'Unknown')}"
+            )
+
+    asyncio.run(main())