mcli-framework 7.1.3__py3-none-any.whl → 7.2.0__py3-none-any.whl

mcli/workflow/politician_trading/data_sources.py

@@ -297,6 +297,20 @@ EU_NATIONAL_SOURCES = [
 # =============================================================================
 
 THIRD_PARTY_SOURCES = [
+    DataSource(
+        name="Senate Stock Watcher (GitHub)",
+        jurisdiction="US-Federal",
+        institution="Third-party aggregator",
+        url="https://github.com/timothycarambat/senate-stock-watcher-data",
+        disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Continuously updated from Senate filings",
+        data_format="json",
+        api_key_required=False,
+        rate_limits="GitHub rate limits",
+        notes="FREE! Automated aggregation of Senate PTR filings. JSON dataset updated continuously. All historical data available in all_transactions.json. No API key required!",
+        status="active",
+    ),
     DataSource(
         name="OpenSecrets Personal Finances",
         jurisdiction="US-Federal",
@@ -327,6 +341,20 @@ THIRD_PARTY_SOURCES = [
         name="QuiverQuant Congressional Trading",
         jurisdiction="US-Federal",
         institution="Third-party aggregator",
+        url="https://www.quiverquant.com/congresstrading/",
+        disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.WEB_SCRAPING,
+        update_frequency="Real-time",
+        data_format="html",
+        api_key_required=False,
+        rate_limits="Web scraping rate limits apply",
+        notes="Financial data company focusing on congressional stock trades. Web interface with trade details, filing dates, and performance metrics. Premium API available.",
+        status="active",
+    ),
+    DataSource(
+        name="QuiverQuant API",
+        jurisdiction="US-Federal",
+        institution="Third-party aggregator",
         url="https://api.quiverquant.com/beta/live/congresstrading",
         disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
         access_method=AccessMethod.API,
@@ -334,7 +362,236 @@ THIRD_PARTY_SOURCES = [
         data_format="json",
         api_key_required=True,
         rate_limits="Varies by subscription",
-        notes="Financial data company focusing on congressional stock trades.",
+        notes="Premium API for QuiverQuant congressional trading data. Requires subscription.",
+        status="active",
+    ),
+    DataSource(
+        name="StockNear Politicians",
+        jurisdiction="US-Federal",
+        institution="Third-party aggregator",
+        url="https://stocknear.com/politicians",
+        disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.WEB_SCRAPING,
+        update_frequency="Real-time",
+        data_format="html",
+        api_key_required=False,
+        notes="Tracks 299 politicians with trade counts, districts, last trade dates, and party affiliation. Pro subscription for unlimited access.",
+        status="active",
+    ),
+    DataSource(
+        name="Barchart Politician Insider Trading",
+        jurisdiction="US-Federal",
+        institution="Third-party aggregator",
+        url="https://www.barchart.com/investing-ideas/politician-insider-trading",
+        disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.WEB_SCRAPING,
+        update_frequency="Updated within 45 days of transaction",
+        data_format="html",
+        threshold_amount=None,
+        notes="Tracks House and Senate trades from last 60 days. Includes buy/sell counts and transaction totals. CSV export available.",
+        status="active",
+    ),
+    DataSource(
+        name="ProPublica Congress API",
+        jurisdiction="US-Federal",
+        institution="Third-party aggregator",
+        url="https://api.propublica.org/congress/v1",
+        disclosure_types=[DisclosureType.STOCK_TRANSACTIONS, DisclosureType.FINANCIAL_INTERESTS],
+        access_method=AccessMethod.API,
+        update_frequency="Daily",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="5000 requests/day (free tier)",
+        notes="DEPRECATED: ProPublica Congress API is no longer available as of 2025. Use Senate Stock Watcher or Finnhub instead.",
+        status="inactive",
+    ),
+    DataSource(
+        name="Finnhub Congressional Trading",
+        jurisdiction="US-Federal",
+        institution="Third-party aggregator",
+        url="https://finnhub.io/docs/api/congressional-trading",
+        disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="30 requests/second (free tier)",
+        notes="FREE API key available at finnhub.io. Provides congressional trading data by stock symbol. Response includes representative name, transaction date/type, and amount ranges.",
+        status="active",
+    ),
+    DataSource(
+        name="SEC Edgar Insider Trading",
+        jurisdiction="US-Federal",
+        institution="Official government source",
+        url="https://data.sec.gov",
+        disclosure_types=[DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time",
+        data_format="json",
+        api_key_required=False,
+        rate_limits="10 requests/second",
+        notes="FREE! Official SEC data. Access company submissions and Form 4 insider trading filings via data.sec.gov/submissions/CIK##########.json. Requires User-Agent header.",
+        status="active",
+    ),
+]
+
+# =============================================================================
+# CORPORATE REGISTRY & FINANCIAL DISCLOSURE SOURCES
+# =============================================================================
+
+CORPORATE_REGISTRY_SOURCES = [
+    DataSource(
+        name="UK Companies House REST API",
+        jurisdiction="UK",
+        institution="Companies House (UK company registry)",
+        url="https://api.companieshouse.gov.uk/",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="600 requests per 5 minutes per key",
+        notes="HTTP Basic Auth using API key as username. Endpoints: /company/{company_number}, /company/{company_number}/filing-history, /officers, /persons-with-significant-control, /search/companies. Some filings/accounts documents are metadata only, not full financial statement parsing. Docs: https://developer.company-information.service.gov.uk/",
+        status="active",
+    ),
+    DataSource(
+        name="UK Companies House Streaming API",
+        jurisdiction="UK",
+        institution="Companies House (UK company registry)",
+        url="https://stream.companieshouse.gov.uk/",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time (streaming)",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="Streaming connection",
+        notes="Streaming API for real-time company changes. Requires stream key obtained via registration. Streams: company information, filing history, insolvency, charges. Delivers JSON events as changes occur. Useful for real-time updates vs polling REST API. Docs: https://www.api.gov.uk/ch/companies-house-streaming/",
+        status="active",
+    ),
+    DataSource(
+        name="GetEDGE API (ASIC Australia)",
+        jurisdiction="Australia",
+        institution="ASIC (Australian Securities and Investments Commission)",
+        url="https://getedge.com.au/docs/api",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="Varies by subscription",
+        notes="Token (API key) authentication. 60-character API key via account portal. Endpoints: Company Registration, Name Change, Business Name Registration, Registry Agent Services, Document production. Oriented to registry/incorporation/document services rather than full financial disclosure data. Requires 'software provider' / digital agent status for some endpoints. Docs: https://getedge.com.au/docs/api",
+        status="active",
+    ),
+    DataSource(
+        name="Info-Financière API (France)",
+        jurisdiction="France",
+        institution="AMF (Autorité des marchés financiers)",
+        url="https://info-financiere.gouv.fr/api/v1/console",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time",
+        data_format="json",
+        api_key_required=False,
+        rate_limits="10,000 API calls per IP per day",
+        notes="FREE! Open access (accès libre) via OpenData service for publicly listed/regulated disclosures. Returns metadata + original documents (PDF, HTML, XML) plus links. Documents are in issuer's original language and format - not always fully parsed. Some regulatory constraints on personal data/redaction may apply. Docs: https://www.data.gouv.fr/dataservices/api-info-financiere/",
+        status="active",
+    ),
+    DataSource(
+        name="Hong Kong Companies Registry e-Monitor API",
+        jurisdiction="Hong Kong",
+        institution="Companies Registry (Hong Kong)",
+        url="https://www.cr.gov.hk/en/electronic/e-servicesportal/",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time (notifications)",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="N/A (notification-based)",
+        notes="Notification/subscription API - not full search/document retrieval. Users must register and subscribe to 'Other Companies' service (HK$17/year per company) to receive notifications via API. Notifications are JSON via HTTPS POST to subscriber's endpoint. Payload includes change data and encrypted API key header for verification. API endpoint must support HTTPS and validate certificate. Docs: https://www.cr.gov.hk/en/electronic/e-servicesportal/faq/e-monitor.htm",
+        status="active",
+    ),
+    DataSource(
+        name="Hong Kong Companies Registry (General)",
+        jurisdiction="Hong Kong",
+        institution="Companies Registry (Hong Kong)",
+        url="https://www.cr.gov.hk/en/electronic/e-servicesportal/",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.WEB_SCRAPING,
+        update_frequency="Real-time",
+        data_format="html",
+        api_key_required=True,
+        rate_limits="Portal-based",
+        notes="Requires login/account. Public search services via portal. Not a fully open API. Outputs may be HTML, images, document scans; limited structured data. Corporate registry made more restrictive: directors' residential address/identity data partly redacted; only limited shareholder details publicly accessible. No full open API - requires portal access or purchase.",
+        status="planned",
+    ),
+    DataSource(
+        name="OpenCorporates API",
+        jurisdiction="Global",
+        institution="Third-party aggregator (multi-jurisdiction)",
+        url="https://api.opencorporates.com/v0.4/",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Daily",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="Varies by tier (free + paid)",
+        notes="Global multi-jurisdiction aggregator. Endpoints: /companies/{jurisdiction}/{company_id}, /companies/search, /officers, /filings, /events. Supports pagination. Query parameters: q, jurisdiction_code, company_number, per_page, page, order. Depth of filings/events data depends on jurisdiction and data source - many 'filings' may just be metadata or pointers to documents rather than full statements. Rate limits apply per key. Docs: https://api.opencorporates.com/documentation/API-Reference",
+        status="active",
+    ),
+    DataSource(
+        name="Transparent Data - Company Registers API",
+        jurisdiction="EU/Europe",
+        institution="Third-party aggregator (EU registry metadata)",
+        url="https://apidoc.transparentdata.pl/company_registers_api.html",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Daily",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="Varies by subscription",
+        notes="EU/Europe registry aggregator. Covers registry/legal metadata rather than full financial statements or regulatory filings. Likely JSON REST style with parameters for jurisdiction, company registration number, etc. Docs: https://apidoc.transparentdata.pl/company_registers_api.html",
+        status="active",
+    ),
+    DataSource(
+        name="XBRL/ESEF/UKSEF via filings.xbrl.org",
+        jurisdiction="EU/UK/Ukraine",
+        institution="XBRL International (standardized financial reporting)",
+        url="https://filings.xbrl.org/",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.ASSET_DECLARATIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Daily",
+        data_format="json",
+        api_key_required=False,
+        rate_limits="None specified",
+        notes="FREE! JSON:API compliant responses. Filtering via query parameters (filter[...]), pagination, sorting. Covers EU/UK/Ukraine filings. Some jurisdictions' filings missing (e.g., Germany, Ireland) as of current state. Endpoints: /filings, /entities, /validation_messages. Docs: https://filings.xbrl.org/docs/api",
+        status="active",
+    ),
+    DataSource(
+        name="XBRL US API",
+        jurisdiction="USA",
+        institution="XBRL US (financial data standardization)",
+        url="https://github.com/xbrlus/xbrl-api",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time (~15 min latency from SEC)",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="Varies by tier",
+        notes="FREE API key available. JSON/REST endpoints for company, filing, facts. Fact-level retrieval mapping XBRL tags to numeric values. Latency ~15 minutes behind SEC updates. Best for programmatic fact extraction from SEC filings. Docs: https://github.com/xbrlus/xbrl-api",
+        status="active",
+    ),
+    DataSource(
+        name="XBRLAnalyst API",
+        jurisdiction="USA",
+        institution="Third-party aggregator (SEC filings)",
+        url="https://www.finddynamics.com/",
+        disclosure_types=[DisclosureType.FINANCIAL_INTERESTS, DisclosureType.STOCK_TRANSACTIONS],
+        access_method=AccessMethod.API,
+        update_frequency="Real-time",
+        data_format="json",
+        api_key_required=True,
+        rate_limits="Varies by subscription",
+        notes="Endpoints for firms, filings, statements, metrics. JSON (default) or XML (via format parameter). Free (limited) access for non-registered users for core metrics; full access for subscribers. Focused on US public companies (SEC filings).",
         status="active",
     ),
 ]
@@ -349,6 +606,7 @@ ALL_DATA_SOURCES = {
     "eu_parliament": EU_PARLIAMENT_SOURCES,
     "eu_national": EU_NATIONAL_SOURCES,
     "third_party": THIRD_PARTY_SOURCES,
+    "corporate_registry": CORPORATE_REGISTRY_SOURCES,
 }
 
 # Summary statistics

mcli/workflow/politician_trading/models.py

@@ -64,7 +64,7 @@ class Politician:
     first_name: str = ""
     last_name: str = ""
     full_name: str = ""
-    role: PoliticianRole = PoliticianRole.US_HOUSE_REP
+    role: str = "House"  # Can be string or PoliticianRole enum
     party: str = ""
     state_or_country: str = ""
     district: Optional[str] = None
@@ -85,6 +85,7 @@ class TradingDisclosure:
 
     id: Optional[str] = None
     politician_id: str = ""
+    politician_bioguide_id: Optional[str] = None  # For lookups before politician_id is assigned
 
     # Transaction details
     transaction_date: datetime = field(default_factory=datetime.utcnow)
@@ -163,3 +164,160 @@ class DataSource:
 
     created_at: datetime = field(default_factory=datetime.utcnow)
     updated_at: datetime = field(default_factory=datetime.utcnow)
+
+
+# =============================================================================
+# Corporate Registry Models
+# =============================================================================
+
+
+@dataclass
+class Company:
+    """Corporate registry company information"""
+
+    id: Optional[str] = None
+    company_number: str = ""  # Registration number in jurisdiction
+    company_name: str = ""
+    jurisdiction: str = ""  # Country/region code (e.g., "GB", "US", "FR")
+
+    # Company details
+    company_type: Optional[str] = None
+    status: str = "active"  # active, dissolved, liquidation, etc.
+    incorporation_date: Optional[datetime] = None
+    registered_address: Optional[str] = None
+
+    # Business information
+    sic_codes: List[str] = field(default_factory=list)  # Standard Industrial Classification
+    nature_of_business: Optional[str] = None
+
+    # Source information
+    source: str = ""  # "uk_companies_house", "opencorporates", etc.
+    source_url: Optional[str] = None
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+
+    created_at: datetime = field(default_factory=datetime.utcnow)
+    updated_at: datetime = field(default_factory=datetime.utcnow)
+
+
+@dataclass
+class CompanyOfficer:
+    """Company officer/director information"""
+
+    id: Optional[str] = None
+    company_id: str = ""  # Foreign key to Company
+
+    # Officer details
+    name: str = ""
+    officer_role: str = ""  # director, secretary, etc.
+    appointed_on: Optional[datetime] = None
+    resigned_on: Optional[datetime] = None
+
+    # Personal details (may be limited by privacy laws)
+    nationality: Optional[str] = None
+    occupation: Optional[str] = None
+    country_of_residence: Optional[str] = None
+    date_of_birth: Optional[datetime] = None  # Often only month/year available
+
+    # Address (often redacted for privacy)
+    address: Optional[str] = None
+
+    # Source information
+    source: str = ""
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+
+    created_at: datetime = field(default_factory=datetime.utcnow)
+    updated_at: datetime = field(default_factory=datetime.utcnow)
+
+
+@dataclass
+class PersonWithSignificantControl:
+    """Person with significant control (PSC) - UK Companies House"""
+
+    id: Optional[str] = None
+    company_id: str = ""  # Foreign key to Company
+
+    # PSC details
+    name: str = ""
+    kind: str = ""  # individual-person-with-significant-control, corporate-entity-person-with-significant-control, etc.
+
+    # Control nature
+    natures_of_control: List[str] = field(default_factory=list)  # ownership-of-shares-75-to-100-percent, etc.
+    notified_on: Optional[datetime] = None
+
+    # Personal details (may be redacted)
+    nationality: Optional[str] = None
+    country_of_residence: Optional[str] = None
+    date_of_birth: Optional[datetime] = None  # Usually only month/year
+
+    # Address
+    address: Optional[str] = None
+
+    # Source information
+    source: str = ""
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+
+    created_at: datetime = field(default_factory=datetime.utcnow)
+    updated_at: datetime = field(default_factory=datetime.utcnow)
+
+
+@dataclass
+class FinancialPublication:
+    """Financial publication/disclosure (e.g., France Info-Financière)"""
+
+    id: Optional[str] = None
+    publication_id: str = ""  # Source publication ID
+
+    # Publication details
+    title: str = ""
+    publication_type: str = ""  # prospectus, annual-report, regulatory-filing, etc.
+    publication_date: datetime = field(default_factory=datetime.utcnow)
+
+    # Issuer/company
+    issuer_name: Optional[str] = None
+    issuer_id: Optional[str] = None  # LEI, ISIN, or other identifier
+    company_id: Optional[str] = None  # Foreign key to Company (if linked)
+
+    # Document information
+    document_url: Optional[str] = None
+    document_format: Optional[str] = None  # pdf, html, xml
+    language: Optional[str] = None
+
+    # Source information
+    source: str = ""  # "info_financiere", "xbrl_filings", etc.
+    jurisdiction: str = ""
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+
+    created_at: datetime = field(default_factory=datetime.utcnow)
+    updated_at: datetime = field(default_factory=datetime.utcnow)
+
+
+@dataclass
+class XBRLFiling:
+    """XBRL financial statement filing"""
+
+    id: Optional[str] = None
+    filing_id: str = ""  # Source filing ID
+
+    # Filing details
+    entity_name: str = ""
+    entity_id: Optional[str] = None  # LEI or other identifier
+    company_id: Optional[str] = None  # Foreign key to Company (if linked)
+
+    # Filing information
+    filing_date: datetime = field(default_factory=datetime.utcnow)
+    period_start: Optional[datetime] = None
+    period_end: Optional[datetime] = None
+    fiscal_year: Optional[int] = None
+    fiscal_period: Optional[str] = None  # Q1, Q2, FY, etc.
+
+    # Document
+    document_url: Optional[str] = None
+    taxonomy: Optional[str] = None  # ESEF, UKSEF, US-GAAP, etc.
+
+    # Source information
+    source: str = ""  # "xbrl_filings", "xbrl_us", etc.
+    jurisdiction: str = ""
+    raw_data: Dict[str, Any] = field(default_factory=dict)
+
+    created_at: datetime = field(default_factory=datetime.utcnow)
+    updated_at: datetime = field(default_factory=datetime.utcnow)