mcli-framework 7.1.0__py3-none-any.whl → 7.1.2__py3-none-any.whl

mcli/workflow/politician_trading/config.py

@@ -73,24 +73,24 @@ class ScrapingConfig:
     def get_active_sources(self):
         """Get all active data sources based on configuration"""
         from .data_sources import ALL_DATA_SOURCES
-        
+
         active_sources = []
-        
+
         if self.enable_us_federal:
             active_sources.extend(ALL_DATA_SOURCES["us_federal"])
-        
+
         if self.enable_us_states:
             active_sources.extend(ALL_DATA_SOURCES["us_states"])
-            
+
         if self.enable_eu_parliament:
             active_sources.extend(ALL_DATA_SOURCES["eu_parliament"])
-            
+
         if self.enable_eu_national:
             active_sources.extend(ALL_DATA_SOURCES["eu_national"])
-            
+
         if self.enable_third_party:
             active_sources.extend(ALL_DATA_SOURCES["third_party"])
-        
+
         # Filter to only active status sources
         return [source for source in active_sources if source.status == "active"]
 

mcli/workflow/politician_trading/connectivity.py

@@ -4,19 +4,19 @@ Continuous Supabase connectivity validation and monitoring
 
 import asyncio
 import json
+import logging
 import time
 from datetime import datetime, timedelta
-from typing import Dict, Any, Optional
-import logging
+from typing import Any, Dict, Optional
 
 from rich.console import Console
 from rich.live import Live
-from rich.table import Table
 from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn
+from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+from rich.table import Table
 
-from .database import PoliticianTradingDB
 from .config import WorkflowConfig
+from .database import PoliticianTradingDB
 
 logger = logging.getLogger(__name__)
 console = Console()
@@ -100,13 +100,14 @@ class SupabaseConnectivityValidator:
         try:
             # Test basic REST API connectivity instead of RPC
             import httpx
+
             async with httpx.AsyncClient() as client:
                 response = await client.get(
-                    self.config.supabase.url + '/rest/v1/',
-                    headers={'apikey': self.config.supabase.key},
-                    timeout=30.0
+                    self.config.supabase.url + "/rest/v1/",
+                    headers={"apikey": self.config.supabase.key},
+                    timeout=30.0,
                 )
-                
+
                 if response.status_code == 200:
                     return {
                         "success": True,
@@ -115,8 +116,8 @@ class SupabaseConnectivityValidator:
                     }
                 else:
                     return {
-                        "success": False, 
-                        "error": f"HTTP {response.status_code}: {response.text[:100]}"
+                        "success": False,
+                        "error": f"HTTP {response.status_code}: {response.text[:100]}",
                     }
         except Exception as e:
             return {"success": False, "error": str(e)}
@@ -127,7 +128,7 @@ class SupabaseConnectivityValidator:
             # Try reading from multiple tables
             tables_to_test = [
                 "politicians",
-                "trading_disclosures", 
+                "trading_disclosures",
                 "data_pull_jobs",
                 "data_sources",
             ]
@@ -148,14 +149,14 @@ class SupabaseConnectivityValidator:
 
             accessible_count = sum(1 for status in read_results.values() if status == "accessible")
             missing_count = sum(1 for status in read_results.values() if status == "table_missing")
-            
+
             if schema_missing and accessible_count == 0:
                 return {
                     "success": False,
                     "tables_tested": read_results,
                     "accessible_tables": accessible_count,
                     "missing_tables": missing_count,
-                    "message": "Database schema not set up. Run 'mcli workflow politician-trading setup --generate-schema' to get setup instructions."
+                    "message": "Database schema not set up. Run 'mcli workflow politician-trading setup --generate-schema' to get setup instructions.",
                 }
             else:
                 success = accessible_count > 0
@@ -195,15 +196,15 @@ class SupabaseConnectivityValidator:
                     return {
                         "success": False,
                         "error": "Table 'data_pull_jobs' not found",
-                        "message": "Database schema not set up. Run schema setup first."
+                        "message": "Database schema not set up. Run schema setup first.",
                     }
                 else:
                     raise e
 
             # Get the inserted record ID
             if insert_result.data and len(insert_result.data) > 0:
-                inserted_id = insert_result.data[0]['id']
-                
+                inserted_id = insert_result.data[0]["id"]
+
                 # Update the record
                 update_result = (
                     self.db.client.table("data_pull_jobs")
@@ -322,26 +323,27 @@ class SupabaseConnectivityValidator:
             test_source_id = f"rt_test_{int(time.time())}"
 
             # Insert
-            insert_result = self.db.client.table("data_sources").insert(
-                {
-                    "name": "Real-time Test Source",
-                    "url": "https://test.example.com",
-                    "source_type": "test",
-                    "region": "test",
-                    "is_active": True,
-                    "created_at": timestamp,
-                }
-            ).execute()
+            insert_result = (
+                self.db.client.table("data_sources")
+                .insert(
+                    {
+                        "name": "Real-time Test Source",
+                        "url": "https://test.example.com",
+                        "source_type": "test",
+                        "region": "test",
+                        "is_active": True,
+                        "created_at": timestamp,
+                    }
+                )
+                .execute()
+            )
 
             if insert_result.data and len(insert_result.data) > 0:
-                inserted_id = insert_result.data[0]['id']
-                
+                inserted_id = insert_result.data[0]["id"]
+
                 # Immediate read-back
                 result = (
-                    self.db.client.table("data_sources")
-                    .select("*")
-                    .eq("id", inserted_id)
-                    .execute()
+                    self.db.client.table("data_sources").select("*").eq("id", inserted_id).execute()
                 )
 
                 # Clean up

mcli/workflow/politician_trading/data_sources.py

@@ -8,43 +8,50 @@ Based on 2025 research of available public databases and APIs.
 """
 
 from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Literal
 from enum import Enum
+from typing import Dict, List, Literal, Optional
+
 
 class DisclosureType(Enum):
     """Types of financial disclosures available"""
-    STOCK_TRANSACTIONS = "stock_transactions"      # Individual buy/sell transactions
-    FINANCIAL_INTERESTS = "financial_interests"    # General financial interests/assets
-    ASSET_DECLARATIONS = "asset_declarations"      # Property, investments, etc.
-    INCOME_SOURCES = "income_sources"             # Outside income sources
-    CONFLICT_INTERESTS = "conflict_interests"      # Potential conflicts of interest
+
+    STOCK_TRANSACTIONS = "stock_transactions"  # Individual buy/sell transactions
+    FINANCIAL_INTERESTS = "financial_interests"  # General financial interests/assets
+    ASSET_DECLARATIONS = "asset_declarations"  # Property, investments, etc.
+    INCOME_SOURCES = "income_sources"  # Outside income sources
+    CONFLICT_INTERESTS = "conflict_interests"  # Potential conflicts of interest
+
 
 class AccessMethod(Enum):
     """How data can be accessed"""
-    WEB_SCRAPING = "web_scraping"        # HTML scraping required
-    API = "api"                          # JSON/XML API available
-    PDF_PARSING = "pdf_parsing"          # PDF documents to parse
+
+    WEB_SCRAPING = "web_scraping"  # HTML scraping required
+    API = "api"  # JSON/XML API available
+    PDF_PARSING = "pdf_parsing"  # PDF documents to parse
     MANUAL_DOWNLOAD = "manual_download"  # Manual download required
-    DATABASE_QUERY = "database_query"   # Direct database access
+    DATABASE_QUERY = "database_query"  # Direct database access
+
 
 @dataclass
 class DataSource:
     """Configuration for a single data source"""
+
     name: str
-    jurisdiction: str                              # e.g., "US-Federal", "US-CA", "EU", "DE"
-    institution: str                               # e.g., "House", "Senate", "Bundestag"
+    jurisdiction: str  # e.g., "US-Federal", "US-CA", "EU", "DE"
+    institution: str  # e.g., "House", "Senate", "Bundestag"
     url: str
     disclosure_types: List[DisclosureType]
     access_method: AccessMethod
-    update_frequency: str                          # e.g., "daily", "weekly", "monthly"
-    threshold_amount: Optional[int] = None         # Minimum disclosure amount in USD
-    data_format: str = "html"                      # html, json, xml, pdf
+    update_frequency: str  # e.g., "daily", "weekly", "monthly"
+    threshold_amount: Optional[int] = None  # Minimum disclosure amount in USD
+    data_format: str = "html"  # html, json, xml, pdf
     api_key_required: bool = False
     rate_limits: Optional[str] = None
     historical_data_available: bool = True
     notes: Optional[str] = None
     status: Literal["active", "inactive", "testing", "planned"] = "active"
 
+
 # =============================================================================
 # US FEDERAL SOURCES
 # =============================================================================
@@ -62,12 +69,11 @@ US_FEDERAL_SOURCES = [
         data_format="html",
         historical_data_available=True,
         notes="STOCK Act requires prompt disclosure of transactions >$1,000. 8-year archive available.",
-        status="active"
+        status="active",
     ),
-    
     DataSource(
         name="US Senate Financial Disclosures",
-        jurisdiction="US-Federal", 
+        jurisdiction="US-Federal",
         institution="Senate",
         url="https://efd.senate.gov",
         disclosure_types=[DisclosureType.STOCK_TRANSACTIONS, DisclosureType.ASSET_DECLARATIONS],
@@ -77,9 +83,8 @@ US_FEDERAL_SOURCES = [
         data_format="html",
         historical_data_available=True,
         notes="Filing threshold $150,160 for 2025. 6-year retention after leaving office.",
-        status="active"
+        status="active",
     ),
-    
     DataSource(
         name="Office of Government Ethics",
         jurisdiction="US-Federal",
@@ -91,8 +96,8 @@ US_FEDERAL_SOURCES = [
         data_format="pdf",
         historical_data_available=True,
         notes="Executive branch officials, judges, and senior staff disclosures",
-        status="active"
-    )
+        status="active",
+    ),
 ]
 
 # =============================================================================
@@ -113,23 +118,21 @@ US_STATE_SOURCES = [
         data_format="json",
         api_key_required=False,
         notes="Fair Political Practices Commission Form 700. NetFile API available.",
-        status="active"
+        status="active",
     ),
-    
     # New York
     DataSource(
         name="New York State Financial Disclosure",
         jurisdiction="US-NY",
-        institution="State Legislature", 
+        institution="State Legislature",
         url="https://ethics.ny.gov/financial-disclosure-statements-elected-officials",
         disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.INCOME_SOURCES],
         access_method=AccessMethod.PDF_PARSING,
         update_frequency="Annually (May 15 deadline)",
         data_format="pdf",
         notes="Commission on Ethics and Lobbying in Government",
-        status="active"
+        status="active",
     ),
-    
     # Florida
     DataSource(
         name="Florida Financial Disclosure",
@@ -141,22 +144,20 @@ US_STATE_SOURCES = [
         update_frequency="Annually (July 1 deadline, grace period until Sept 1)",
         data_format="html",
         notes="All elected state and local public officers required to file",
-        status="active"
+        status="active",
     ),
-    
     # Texas
     DataSource(
         name="Texas Ethics Commission",
-        jurisdiction="US-TX", 
+        jurisdiction="US-TX",
         institution="State Legislature",
         url="https://www.ethics.state.tx.us/search/cf/",
         disclosure_types=[DisclosureType.FINANCIAL_INTERESTS],
         access_method=AccessMethod.WEB_SCRAPING,
         update_frequency="Annually",
         data_format="html",
-        status="active"
+        status="active",
     ),
-    
     # Michigan
     DataSource(
         name="Michigan Personal Financial Disclosure",
@@ -166,10 +167,10 @@ US_STATE_SOURCES = [
         disclosure_types=[DisclosureType.FINANCIAL_INTERESTS],
         access_method=AccessMethod.WEB_SCRAPING,
         update_frequency="Annually",
-        data_format="html", 
+        data_format="html",
         notes="Candidates for Governor, Lt. Gov, SoS, AG, and Legislature required",
-        status="active"
-    )
+        status="active",
+    ),
 ]
 
 # =============================================================================
@@ -188,9 +189,8 @@ EU_PARLIAMENT_SOURCES = [
         threshold_amount=5000,  # €5,000+ outside income must be declared
         data_format="pdf",
         notes="Individual MEP pages have declarations. Third-party aggregation by EU Integrity Watch.",
-        status="active"
+        status="active",
     ),
-    
     DataSource(
         name="EU Integrity Watch",
         jurisdiction="EU",
@@ -201,8 +201,8 @@ EU_PARLIAMENT_SOURCES = [
         update_frequency="Updated after MEP declarations",
         data_format="html",
         notes="Automated extraction from Parliament PDFs. Interactive database available.",
-        status="active"
-    )
+        status="active",
+    ),
 ]
 
 # =============================================================================
@@ -222,13 +222,12 @@ EU_NATIONAL_SOURCES = [
         threshold_amount=None,  # 5% company ownership threshold (down from 25% in 2021)
         data_format="html",
         notes="Transparency Act 2021. Company ownership >5%, tougher bribery laws (1-10 years prison).",
-        status="active"
+        status="active",
     ),
-    
     # France
     DataSource(
         name="French Parliament Financial Declarations",
-        jurisdiction="FR", 
+        jurisdiction="FR",
         institution="National Assembly & Senate",
         url="https://www.hatvp.fr/",  # High Authority for Transparency in Public Life
         disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
@@ -236,12 +235,11 @@ EU_NATIONAL_SOURCES = [
         update_frequency="Annually",
         data_format="html",
         notes="HATVP publishes declarations. Asset declarations for MEPs since 2019. Penalties: 3 years prison + €45,000 fine.",
-        status="active"
+        status="active",
     ),
-    
     # United Kingdom
     DataSource(
-        name="UK Parliament Register of Members' Financial Interests", 
+        name="UK Parliament Register of Members' Financial Interests",
         jurisdiction="UK",
         institution="House of Commons",
         url="https://www.parliament.uk/mps-lords-and-offices/standards-and-financial-interests/parliamentary-commissioner-for-standards/registers-of-interests/register-of-members-financial-interests/",
@@ -252,22 +250,20 @@ EU_NATIONAL_SOURCES = [
         data_format="json",
         api_key_required=False,
         notes="Open Parliament Licence API available. Register updated bi-weekly.",
-        status="active"
+        status="active",
     ),
-    
     DataSource(
         name="UK House of Lords Register of Interests",
         jurisdiction="UK",
-        institution="House of Lords", 
+        institution="House of Lords",
         url="https://members.parliament.uk/members/lords/interests/register-of-lords-interests",
         disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.INCOME_SOURCES],
         access_method=AccessMethod.WEB_SCRAPING,
         update_frequency="Updated regularly",
         data_format="html",
         notes="More detailed shareholding disclosure than Commons. Searchable database.",
-        status="active"
+        status="active",
     ),
-    
     # Spain
     DataSource(
         name="Spanish Parliament Transparency Portal",
@@ -279,9 +275,8 @@ EU_NATIONAL_SOURCES = [
         update_frequency="Updated as required",
         data_format="html",
         notes="Deputies and senators publish institutional agendas with interest representatives. No lobbyist register.",
-        status="active"
+        status="active",
     ),
-    
     # Italy
     DataSource(
         name="Italian Parliament Financial Declarations",
@@ -293,8 +288,8 @@ EU_NATIONAL_SOURCES = [
         update_frequency="Per legislative term",
         data_format="html",
         notes="Individual member pages contain declarations. Limited standardization.",
-        status="testing"
-    )
+        status="testing",
+    ),
 ]
 
 # =============================================================================
@@ -314,12 +309,11 @@ THIRD_PARTY_SOURCES = [
         api_key_required=True,
         rate_limits="1000 requests/day",
         notes="Center for Responsive Politics aggregation of federal disclosures.",
-        status="active"
+        status="active",
     ),
-    
     DataSource(
         name="LegiStorm Financial Disclosures",
-        jurisdiction="US-Federal", 
+        jurisdiction="US-Federal",
         institution="Third-party aggregator",
         url="https://www.legistorm.com/financial_disclosure.html",
         disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.STOCK_TRANSACTIONS],
@@ -327,13 +321,12 @@ THIRD_PARTY_SOURCES = [
         update_frequency="Real-time from government sources",
         data_format="html",
         notes="Subscription service with enhanced search and analysis tools.",
-        status="active"
+        status="active",
     ),
-    
     DataSource(
         name="QuiverQuant Congressional Trading",
         jurisdiction="US-Federal",
-        institution="Third-party aggregator", 
+        institution="Third-party aggregator",
         url="https://api.quiverquant.com/beta/live/congresstrading",
         disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
         access_method=AccessMethod.API,
@@ -342,8 +335,8 @@ THIRD_PARTY_SOURCES = [
         api_key_required=True,
         rate_limits="Varies by subscription",
         notes="Financial data company focusing on congressional stock trades.",
-        status="active"
-    )
+        status="active",
+    ),
 ]
 
 # =============================================================================
@@ -352,19 +345,19 @@ THIRD_PARTY_SOURCES = [
 
 ALL_DATA_SOURCES = {
     "us_federal": US_FEDERAL_SOURCES,
-    "us_states": US_STATE_SOURCES, 
+    "us_states": US_STATE_SOURCES,
     "eu_parliament": EU_PARLIAMENT_SOURCES,
     "eu_national": EU_NATIONAL_SOURCES,
-    "third_party": THIRD_PARTY_SOURCES
+    "third_party": THIRD_PARTY_SOURCES,
 }
 
 # Summary statistics
 TOTAL_SOURCES = sum(len(sources) for sources in ALL_DATA_SOURCES.values())
 ACTIVE_SOURCES = sum(
-    len([s for s in sources if s.status == "active"]) 
-    for sources in ALL_DATA_SOURCES.values()
+    len([s for s in sources if s.status == "active"]) for sources in ALL_DATA_SOURCES.values()
 )
 
+
 def get_sources_by_jurisdiction(jurisdiction: str) -> List[DataSource]:
     """Get all sources for a specific jurisdiction (e.g., 'US-CA', 'DE', 'EU')"""
     all_sources = []
@@ -372,6 +365,7 @@ def get_sources_by_jurisdiction(jurisdiction: str) -> List[DataSource]:
         all_sources.extend([s for s in source_group if s.jurisdiction == jurisdiction])
     return all_sources
 
+
 def get_sources_by_type(disclosure_type: DisclosureType) -> List[DataSource]:
     """Get all sources that provide a specific type of disclosure"""
     all_sources = []
@@ -379,6 +373,7 @@ def get_sources_by_type(disclosure_type: DisclosureType) -> List[DataSource]:
         all_sources.extend([s for s in source_group if disclosure_type in s.disclosure_types])
     return all_sources
 
+
 def get_api_sources() -> List[DataSource]:
     """Get all sources that provide API access"""
     all_sources = []
@@ -386,10 +381,16 @@ def get_api_sources() -> List[DataSource]:
         all_sources.extend([s for s in source_group if s.access_method == AccessMethod.API])
     return all_sources
 
+
 # Export for use in workflow configuration
 __all__ = [
-    'DataSource', 'DisclosureType', 'AccessMethod',
-    'ALL_DATA_SOURCES', 'get_sources_by_jurisdiction', 
-    'get_sources_by_type', 'get_api_sources',
-    'TOTAL_SOURCES', 'ACTIVE_SOURCES'
-]
+    "DataSource",
+    "DisclosureType",
+    "AccessMethod",
+    "ALL_DATA_SOURCES",
+    "get_sources_by_jurisdiction",
+    "get_sources_by_type",
+    "get_api_sources",
+    "TOTAL_SOURCES",
+    "ACTIVE_SOURCES",
+]

mcli/workflow/politician_trading/database.py

@@ -5,14 +5,14 @@ Database client and schema management for politician trading data
 import asyncio
 import logging
 from datetime import datetime
-from typing import List, Optional, Dict, Any
+from typing import Any, Dict, List, Optional
 from uuid import uuid4
 
-from supabase import create_client, Client
 from postgrest.exceptions import APIError
+from supabase import Client, create_client
 
 from .config import WorkflowConfig
-from .models import Politician, TradingDisclosure, DataPullJob, DataSource
+from .models import DataPullJob, DataSource, Politician, TradingDisclosure
 
 logger = logging.getLogger(__name__)
 
@@ -96,17 +96,20 @@ class PoliticianTradingDB:
             existing = await self.find_politician_by_name(
                 politician.first_name, politician.last_name
             )
-            
+
             if existing:
                 # Update existing politician (but don't change ID)
                 politician_dict = self._politician_to_dict(politician)
                 politician_dict["id"] = existing.id  # Keep existing ID
                 politician_dict["updated_at"] = datetime.utcnow().isoformat()
-                
-                result = self.client.table("politicians").update(
-                    politician_dict
-                ).eq("id", existing.id).execute()
-                
+
+                result = (
+                    self.client.table("politicians")
+                    .update(politician_dict)
+                    .eq("id", existing.id)
+                    .execute()
+                )
+
                 if result.data:
                     return result.data[0]["id"]
                 return existing.id
@@ -115,18 +118,16 @@ class PoliticianTradingDB:
                 politician_dict = self._politician_to_dict(politician)
                 if not politician_dict.get("id"):
                     politician_dict["id"] = str(uuid4())
-                    
+
                 politician_dict["created_at"] = datetime.utcnow().isoformat()
                 politician_dict["updated_at"] = datetime.utcnow().isoformat()
 
-                result = self.client.table("politicians").insert(
-                    politician_dict
-                ).execute()
-                
+                result = self.client.table("politicians").insert(politician_dict).execute()
+
                 if result.data:
                     return result.data[0]["id"]
                 return politician_dict["id"]
-                
+
         except Exception as e:
             logger.error(f"Failed to upsert politician: {e}")
             # For debugging: log the politician data that caused the error
@@ -362,9 +363,10 @@ class PoliticianTradingDB:
 
     def _dict_to_disclosure(self, data: Dict[str, Any]) -> TradingDisclosure:
         """Convert dictionary to TradingDisclosure"""
-        from .models import TransactionType, DisclosureStatus
         from decimal import Decimal
 
+        from .models import DisclosureStatus, TransactionType
+
         return TradingDisclosure(
             id=data.get("id"),
             politician_id=data.get("politician_id", ""),

mcli/workflow/politician_trading/demo.py

@@ -6,14 +6,15 @@ import asyncio
 import json
 import uuid
 from datetime import datetime
+
 from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
 from rich.json import JSON
+from rich.panel import Panel
+from rich.table import Table
 
-from .workflow import run_politician_trading_collection
 from .connectivity import SupabaseConnectivityValidator
 from .monitoring import run_health_check, run_stats_report
+from .workflow import run_politician_trading_collection
 
 console = Console()