mcli-framework 7.6.0__py3-none-any.whl → 7.6.2__py3-none-any.whl

mcli/ml/dashboard/pages/scrapers_and_logs.py

@@ -32,25 +32,22 @@ logger = logging.getLogger(__name__)
 def show_scrapers_and_logs():
     """Main function for scrapers and logs page"""
     st.header("🔍 Data Scrapers & System Logs")
-    
+
     # Add a simple test to ensure the page is rendering
     st.info("📋 Page loaded successfully - Scrapers & Logs functionality is available")
 
-    st.markdown("""
+    st.markdown(
+        """
     **Features:**
     - 🚀 Manual data scraping from corporate registries
     - 📊 Real-time scraper logs and job status
     - 📝 System logs viewer
     - 📈 Job history and statistics
-    """)
+    """
+    )
 
     # Create tabs
-    tabs = st.tabs([
-        "🚀 Manual Scraping",
-        "📊 Scraper Logs",
-        "📝 System Logs",
-        "📈 Job History"
-    ])
+    tabs = st.tabs(["🚀 Manual Scraping", "📊 Scraper Logs", "📝 System Logs", "📈 Job History"])
 
     with tabs[0]:
         show_manual_scraping()
@@ -69,10 +66,12 @@ def show_manual_scraping():
     """Manual scraping interface"""
     st.subheader("🚀 Manual Data Scraping")
 
-    st.markdown("""
+    st.markdown(
+        """
     Manually trigger data scraping jobs from various sources.
     Select a source, configure parameters, and run the scraper.
-    """)
+    """
+    )
 
     # Source selection
     source_type = st.selectbox(
@@ -85,7 +84,7 @@ def show_manual_scraping():
             "XBRL US",
             "Senate Stock Watcher (GitHub)",
         ],
-        help="Choose which data source to scrape"
+        help="Choose which data source to scrape",
     )
 
     # Source-specific configuration
@@ -108,17 +107,21 @@ def show_uk_companies_house_scraper():
     st.markdown("### UK Companies House Configuration")
 
     # Check API key
-    api_key = os.getenv("UK_COMPANIES_HOUSE_API_KEY") or st.secrets.get("UK_COMPANIES_HOUSE_API_KEY", "")
+    api_key = os.getenv("UK_COMPANIES_HOUSE_API_KEY") or st.secrets.get(
+        "UK_COMPANIES_HOUSE_API_KEY", ""
+    )
 
     if not api_key:
         st.error("❌ UK Companies House API key not configured")
-        st.info("""
+        st.info(
+            """
         To use this scraper, set `UK_COMPANIES_HOUSE_API_KEY` in:
         - Streamlit Cloud: Settings → Secrets
         - Local: .streamlit/secrets.toml or environment variable
 
         Get free API key: https://developer.company-information.service.gov.uk/
-        """)
+        """
+        )
         return
 
     st.success("✅ API key configured")
@@ -128,16 +131,14 @@ def show_uk_companies_house_scraper():
 
     with col1:
         company_query = st.text_input(
-            "Company Name",
-            value="Tesco",
-            help="Company name to search for"
+            "Company Name", value="Tesco", help="Company name to search for"
         )
         max_results = st.number_input(
             "Max Results",
             min_value=1,
             max_value=100,
             value=10,
-            help="Maximum number of companies to fetch"
+            help="Maximum number of companies to fetch",
         )
 
     with col2:
@@ -148,24 +149,18 @@ def show_uk_companies_house_scraper():
     # Run scraper
     if st.button("🚀 Run UK Companies House Scraper", type="primary"):
         run_uk_companies_house_scraper(
-            company_query,
-            max_results,
-            fetch_officers,
-            fetch_psc,
-            save_to_db
+            company_query, max_results, fetch_officers, fetch_psc, save_to_db
         )
 
 
 def run_uk_companies_house_scraper(
-    query: str,
-    max_results: int,
-    fetch_officers: bool,
-    fetch_psc: bool,
-    save_to_db: bool
+    query: str, max_results: int, fetch_officers: bool, fetch_psc: bool, save_to_db: bool
 ):
     """Execute UK Companies House scraper"""
     try:
-        from mcli.workflow.politician_trading.scrapers_corporate_registry import UKCompaniesHouseScraper
+        from mcli.workflow.politician_trading.scrapers_corporate_registry import (
+            UKCompaniesHouseScraper,
+        )
 
         # Create log capture
         log_stream = StringIO()
@@ -174,7 +169,9 @@ def run_uk_companies_house_scraper(
         formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
         handler.setFormatter(formatter)
 
-        scraper_logger = logging.getLogger("mcli.workflow.politician_trading.scrapers_corporate_registry")
+        scraper_logger = logging.getLogger(
+            "mcli.workflow.politician_trading.scrapers_corporate_registry"
+        )
         scraper_logger.addHandler(handler)
 
         # Create progress containers
@@ -220,11 +217,7 @@ def run_uk_companies_house_scraper(
             progress_bar.progress(30 + int((i + 1) / len(companies) * 50))
 
         # Display logs
-        log_container.text_area(
-            "Scraper Logs",
-            log_stream.getvalue(),
-            height=200
-        )
+        log_container.text_area("Scraper Logs", log_stream.getvalue(), height=200)
 
         # Display results
         with results_container:
@@ -237,40 +230,57 @@ def run_uk_companies_house_scraper(
 
             # Show companies
             st.markdown("#### Companies Found")
-            companies_df = pd.DataFrame([{
-                "Number": c.get("company_number"),
-                "Name": c.get("title"),
-                "Status": c.get("company_status"),
-                "Type": c.get("company_type"),
-                "Address": c.get("address_snippet", "")[:50]
-            } for c in companies])
+            companies_df = pd.DataFrame(
+                [
+                    {
+                        "Number": c.get("company_number"),
+                        "Name": c.get("title"),
+                        "Status": c.get("company_status"),
+                        "Type": c.get("company_type"),
+                        "Address": c.get("address_snippet", "")[:50],
+                    }
+                    for c in companies
+                ]
+            )
             st.dataframe(companies_df, use_container_width=True)
 
             # Show officers
             if all_officers:
                 st.markdown("#### Officers Found")
-                officers_df = pd.DataFrame([{
-                    "Name": o.get("name"),
-                    "Role": o.get("officer_role"),
-                    "Appointed": o.get("appointed_on", ""),
-                    "Nationality": o.get("nationality", ""),
-                    "Occupation": o.get("occupation", "")
-                } for o in all_officers[:50]])  # Limit to 50 for display
+                officers_df = pd.DataFrame(
+                    [
+                        {
+                            "Name": o.get("name"),
+                            "Role": o.get("officer_role"),
+                            "Appointed": o.get("appointed_on", ""),
+                            "Nationality": o.get("nationality", ""),
+                            "Occupation": o.get("occupation", ""),
+                        }
+                        for o in all_officers[:50]
+                    ]
+                )  # Limit to 50 for display
                 st.dataframe(officers_df, use_container_width=True)
 
             # Show PSC
             if all_psc:
                 st.markdown("#### Persons with Significant Control")
-                psc_df = pd.DataFrame([{
-                    "Name": p.get("name"),
-                    "Kind": p.get("kind", "").replace("-", " ").title(),
-                    "Control": ", ".join(p.get("natures_of_control", [])),
-                    "Nationality": p.get("nationality", ""),
-                } for p in all_psc[:50]])
+                psc_df = pd.DataFrame(
+                    [
+                        {
+                            "Name": p.get("name"),
+                            "Kind": p.get("kind", "").replace("-", " ").title(),
+                            "Control": ", ".join(p.get("natures_of_control", [])),
+                            "Nationality": p.get("nationality", ""),
+                        }
+                        for p in all_psc[:50]
+                    ]
+                )
                 st.dataframe(psc_df, use_container_width=True)
 
         progress_bar.progress(100)
-        status_container.success(f"✅ Scraping completed! Found {len(companies)} companies, {len(all_officers)} officers, {len(all_psc)} PSC")
+        status_container.success(
+            f"✅ Scraping completed! Found {len(companies)} companies, {len(all_officers)} officers, {len(all_psc)} PSC"
+        )
 
         # Save to database if requested
         if save_to_db:
@@ -279,6 +289,7 @@ def run_uk_companies_house_scraper(
     except Exception as e:
         st.error(f"❌ Error: {e}")
         import traceback
+
         st.code(traceback.format_exc())
 
 
@@ -293,25 +304,18 @@ def show_info_financiere_scraper():
 
     with col1:
         query = st.text_input(
-            "Search Query (optional)",
-            value="",
-            help="Company name, ISIN, or leave blank for all"
+            "Search Query (optional)", value="", help="Company name, ISIN, or leave blank for all"
         )
         days_back = st.number_input(
             "Days Back",
             min_value=1,
             max_value=365,
             value=30,
-            help="How many days of history to fetch"
+            help="How many days of history to fetch",
         )
 
     with col2:
-        max_results = st.number_input(
-            "Max Results",
-            min_value=1,
-            max_value=100,
-            value=20
-        )
+        max_results = st.number_input("Max Results", min_value=1, max_value=100, value=20)
         save_to_db = st.checkbox("Save to Database", value=False)
 
     # Run scraper
@@ -319,15 +323,12 @@ def show_info_financiere_scraper():
         run_info_financiere_scraper(query, days_back, max_results, save_to_db)
 
 
-def run_info_financiere_scraper(
-    query: str,
-    days_back: int,
-    max_results: int,
-    save_to_db: bool
-):
+def run_info_financiere_scraper(query: str, days_back: int, max_results: int, save_to_db: bool):
     """Execute Info-Financière scraper"""
     try:
-        from mcli.workflow.politician_trading.scrapers_corporate_registry import InfoFinanciereAPIScraper
+        from mcli.workflow.politician_trading.scrapers_corporate_registry import (
+            InfoFinanciereAPIScraper,
+        )
 
         status_container = st.empty()
         progress_bar = st.progress(0)
@@ -345,10 +346,7 @@ def run_info_financiere_scraper(
         # Search publications
         status_container.info(f"🔍 Searching publications ({from_date} to {to_date})...")
         publications = scraper.search_publications(
-            query=query or None,
-            from_date=from_date,
-            to_date=to_date,
-            per_page=max_results
+            query=query or None, from_date=from_date, to_date=to_date, per_page=max_results
         )
         progress_bar.progress(80)
 
@@ -363,12 +361,17 @@ def run_info_financiere_scraper(
             st.metric("Publications Found", len(publications))
 
             # Show publications
-            pubs_df = pd.DataFrame([{
-                "Date": p.get("publication_date", ""),
-                "Title": p.get("title", "")[:100],
-                "Type": p.get("publication_type", ""),
-                "Issuer": p.get("issuer_name", "")
-            } for p in publications])
+            pubs_df = pd.DataFrame(
+                [
+                    {
+                        "Date": p.get("publication_date", ""),
+                        "Title": p.get("title", "")[:100],
+                        "Type": p.get("publication_type", ""),
+                        "Issuer": p.get("issuer_name", ""),
+                    }
+                    for p in publications
+                ]
+            )
             st.dataframe(pubs_df, use_container_width=True)
 
         progress_bar.progress(100)
@@ -380,6 +383,7 @@ def run_info_financiere_scraper(
     except Exception as e:
         st.error(f"❌ Error: {e}")
         import traceback
+
         st.code(traceback.format_exc())
 
 
@@ -392,30 +396,23 @@ def show_opencorporates_scraper():
     if api_key:
         st.success("✅ API key configured")
     else:
-        st.info("ℹ️ No API key (free tier with rate limits). Get API key for better performance: https://opencorporates.com/api_accounts/new")
+        st.info(
+            "ℹ️ No API key (free tier with rate limits). Get API key for better performance: https://opencorporates.com/api_accounts/new"
+        )
 
     # Configuration
     col1, col2 = st.columns(2)
 
     with col1:
-        query = st.text_input(
-            "Company Name",
-            value="Apple",
-            help="Company name to search for"
-        )
+        query = st.text_input("Company Name", value="Apple", help="Company name to search for")
         jurisdiction = st.selectbox(
             "Jurisdiction (optional)",
             ["", "us_ca", "us_de", "us_ny", "gb", "de", "fr", "nl"],
-            help="Filter by jurisdiction code"
+            help="Filter by jurisdiction code",
         )
 
     with col2:
-        max_results = st.number_input(
-            "Max Results",
-            min_value=1,
-            max_value=100,
-            value=10
-        )
+        max_results = st.number_input("Max Results", min_value=1, max_value=100, value=10)
         save_to_db = st.checkbox("Save to Database", value=False)
 
     # Run scraper
@@ -423,15 +420,12 @@ def show_opencorporates_scraper():
         run_opencorporates_scraper(query, jurisdiction or None, max_results, save_to_db)
 
 
-def run_opencorporates_scraper(
-    query: str,
-    jurisdiction: str,
-    max_results: int,
-    save_to_db: bool
-):
+def run_opencorporates_scraper(query: str, jurisdiction: str, max_results: int, save_to_db: bool):
     """Execute OpenCorporates scraper"""
     try:
-        from mcli.workflow.politician_trading.scrapers_corporate_registry import OpenCorporatesScraper
+        from mcli.workflow.politician_trading.scrapers_corporate_registry import (
+            OpenCorporatesScraper,
+        )
 
         status_container = st.empty()
         progress_bar = st.progress(0)
@@ -445,9 +439,7 @@ def run_opencorporates_scraper(
         # Search companies
         status_container.info(f"🔍 Searching for '{query}'...")
         companies = scraper.search_companies(
-            query,
-            jurisdiction_code=jurisdiction,
-            per_page=max_results
+            query, jurisdiction_code=jurisdiction, per_page=max_results
         )
         progress_bar.progress(80)
 
@@ -462,13 +454,18 @@ def run_opencorporates_scraper(
             st.metric("Companies Found", len(companies))
 
             # Show companies
-            companies_df = pd.DataFrame([{
-                "Jurisdiction": c.get("company", {}).get("jurisdiction_code", ""),
-                "Number": c.get("company", {}).get("company_number", ""),
-                "Name": c.get("company", {}).get("name", ""),
-                "Status": c.get("company", {}).get("current_status", ""),
-                "Type": c.get("company", {}).get("company_type", "")
-            } for c in companies])
+            companies_df = pd.DataFrame(
+                [
+                    {
+                        "Jurisdiction": c.get("company", {}).get("jurisdiction_code", ""),
+                        "Number": c.get("company", {}).get("company_number", ""),
+                        "Name": c.get("company", {}).get("name", ""),
+                        "Status": c.get("company", {}).get("current_status", ""),
+                        "Type": c.get("company", {}).get("company_type", ""),
+                    }
+                    for c in companies
+                ]
+            )
             st.dataframe(companies_df, use_container_width=True)
 
         progress_bar.progress(100)
@@ -477,6 +474,7 @@ def run_opencorporates_scraper(
     except Exception as e:
         st.error(f"❌ Error: {e}")
         import traceback
+
         st.code(traceback.format_exc())
 
 
@@ -493,22 +491,12 @@ def show_xbrl_filings_scraper():
         country = st.selectbox(
             "Country (optional)",
             ["", "GB", "FR", "DE", "ES", "IT", "NL", "BE"],
-            help="Filter by country code"
-        )
-        days_back = st.number_input(
-            "Days Back",
-            min_value=1,
-            max_value=365,
-            value=30
+            help="Filter by country code",
         )
+        days_back = st.number_input("Days Back", min_value=1, max_value=365, value=30)
 
     with col2:
-        max_results = st.number_input(
-            "Max Results",
-            min_value=1,
-            max_value=500,
-            value=100
-        )
+        max_results = st.number_input("Max Results", min_value=1, max_value=500, value=100)
         save_to_db = st.checkbox("Save to Database", value=False)
 
     # Run scraper
@@ -516,12 +504,7 @@ def show_xbrl_filings_scraper():
         run_xbrl_filings_scraper(country or None, days_back, max_results, save_to_db)
 
 
-def run_xbrl_filings_scraper(
-    country: str,
-    days_back: int,
-    max_results: int,
-    save_to_db: bool
-):
+def run_xbrl_filings_scraper(country: str, days_back: int, max_results: int, save_to_db: bool):
     """Execute XBRL Filings scraper"""
     try:
         from mcli.workflow.politician_trading.scrapers_corporate_registry import XBRLFilingsScraper
@@ -540,11 +523,7 @@ def run_xbrl_filings_scraper(
 
         # Get filings
         status_container.info(f"🔍 Fetching XBRL filings since {from_date}...")
-        filings = scraper.get_filings(
-            country=country,
-            from_date=from_date,
-            page_size=max_results
-        )
+        filings = scraper.get_filings(country=country, from_date=from_date, page_size=max_results)
         progress_bar.progress(80)
 
         # Display results
@@ -558,13 +537,18 @@ def run_xbrl_filings_scraper(
             st.metric("Filings Found", len(filings))
 
             # Show filings
-            filings_df = pd.DataFrame([{
-                "ID": f.get("id", ""),
-                "Country": f.get("attributes", {}).get("country", ""),
-                "Entity": f.get("attributes", {}).get("entity_name", "")[:50],
-                "Period": f.get("attributes", {}).get("period_end", ""),
-                "Date Added": f.get("attributes", {}).get("date_added", "")
-            } for f in filings])
+            filings_df = pd.DataFrame(
+                [
+                    {
+                        "ID": f.get("id", ""),
+                        "Country": f.get("attributes", {}).get("country", ""),
+                        "Entity": f.get("attributes", {}).get("entity_name", "")[:50],
+                        "Period": f.get("attributes", {}).get("period_end", ""),
+                        "Date Added": f.get("attributes", {}).get("date_added", ""),
+                    }
+                    for f in filings
+                ]
+            )
             st.dataframe(filings_df, use_container_width=True)
 
         progress_bar.progress(100)
@@ -573,6 +557,7 @@ def run_xbrl_filings_scraper(
     except Exception as e:
         st.error(f"❌ Error: {e}")
         import traceback
+
         st.code(traceback.format_exc())
 
 
@@ -584,13 +569,15 @@ def show_xbrl_us_scraper():
 
     if not api_key:
         st.error("❌ XBRL US API key not configured")
-        st.info("""
+        st.info(
+            """
         To use this scraper, set `XBRL_US_API_KEY` in:
         - Streamlit Cloud: Settings → Secrets
         - Local: .streamlit/secrets.toml or environment variable
 
         Get free API key: https://xbrl.us/home/use/xbrl-api/
-        """)
+        """
+        )
         return
 
     st.success("✅ API key configured")
@@ -600,18 +587,11 @@ def show_xbrl_us_scraper():
 
     with col1:
         query = st.text_input(
-            "Company Name or Ticker",
-            value="Tesla",
-            help="Search by company name or stock ticker"
+            "Company Name or Ticker", value="Tesla", help="Search by company name or stock ticker"
         )
 
     with col2:
-        max_results = st.number_input(
-            "Max Results",
-            min_value=1,
-            max_value=100,
-            value=10
-        )
+        max_results = st.number_input("Max Results", min_value=1, max_value=100, value=10)
         save_to_db = st.checkbox("Save to Database", value=False)
 
     # Run scraper
@@ -619,11 +599,7 @@ def show_xbrl_us_scraper():
         run_xbrl_us_scraper(query, max_results, save_to_db)
 
 
-def run_xbrl_us_scraper(
-    query: str,
-    max_results: int,
-    save_to_db: bool
-):
+def run_xbrl_us_scraper(query: str, max_results: int, save_to_db: bool):
     """Execute XBRL US scraper"""
     try:
         from mcli.workflow.politician_trading.scrapers_corporate_registry import XBRLUSScraper
@@ -653,12 +629,17 @@ def run_xbrl_us_scraper(
             st.metric("Entities Found", len(entities))
 
             # Show entities
-            entities_df = pd.DataFrame([{
-                "ID": e.get("entity", {}).get("id", ""),
-                "Name": e.get("entity", {}).get("name", ""),
-                "CIK": e.get("entity", {}).get("cik", ""),
-                "Ticker": e.get("entity", {}).get("ticker", "")
-            } for e in entities])
+            entities_df = pd.DataFrame(
+                [
+                    {
+                        "ID": e.get("entity", {}).get("id", ""),
+                        "Name": e.get("entity", {}).get("name", ""),
+                        "CIK": e.get("entity", {}).get("cik", ""),
+                        "Ticker": e.get("entity", {}).get("ticker", ""),
+                    }
+                    for e in entities
+                ]
+            )
             st.dataframe(entities_df, use_container_width=True)
 
         progress_bar.progress(100)
@@ -667,6 +648,7 @@ def run_xbrl_us_scraper(
     except Exception as e:
         st.error(f"❌ Error: {e}")
         import traceback
+
         st.code(traceback.format_exc())
 
 
@@ -682,11 +664,7 @@ def show_senate_watcher_scraper():
     with col1:
         recent_only = st.checkbox("Recent Only", value=True)
         days_back = st.number_input(
-            "Days Back (if recent)",
-            min_value=1,
-            max_value=365,
-            value=90,
-            disabled=not recent_only
+            "Days Back (if recent)", min_value=1, max_value=365, value=90, disabled=not recent_only
         )
 
     with col2:
@@ -697,11 +675,7 @@ def show_senate_watcher_scraper():
         run_senate_watcher_scraper(recent_only, days_back, save_to_db)
 
 
-def run_senate_watcher_scraper(
-    recent_only: bool,
-    days_back: int,
-    save_to_db: bool
-):
+def run_senate_watcher_scraper(recent_only: bool, days_back: int, save_to_db: bool):
     """Execute Senate Stock Watcher scraper"""
     try:
         from mcli.workflow.politician_trading.scrapers_free_sources import FreeDataFetcher
@@ -734,19 +708,30 @@ def run_senate_watcher_scraper(
             # Show disclosures
             if disclosures:
                 st.markdown("#### Recent Trading Disclosures")
-                disc_df = pd.DataFrame([{
-                    "Date": d.transaction_date.strftime("%Y-%m-%d") if hasattr(d.transaction_date, 'strftime') else str(d.transaction_date),
-                    "Ticker": d.asset_ticker or "—",
-                    "Asset": d.asset_name[:50],
-                    "Type": d.transaction_type,
-                    "Politician": d.politician_bioguide_id,
-                    "Min": f"${d.amount_range_min:,.0f}" if d.amount_range_min else "",
-                    "Max": f"${d.amount_range_max:,.0f}" if d.amount_range_max else ""
-                } for d in disclosures[:100]])  # Limit to 100 for display
+                disc_df = pd.DataFrame(
+                    [
+                        {
+                            "Date": (
+                                d.transaction_date.strftime("%Y-%m-%d")
+                                if hasattr(d.transaction_date, "strftime")
+                                else str(d.transaction_date)
+                            ),
+                            "Ticker": d.asset_ticker or "—",
+                            "Asset": d.asset_name[:50],
+                            "Type": d.transaction_type,
+                            "Politician": d.politician_bioguide_id,
+                            "Min": f"${d.amount_range_min:,.0f}" if d.amount_range_min else "",
+                            "Max": f"${d.amount_range_max:,.0f}" if d.amount_range_max else "",
+                        }
+                        for d in disclosures[:100]
+                    ]
+                )  # Limit to 100 for display
                 st.dataframe(disc_df, use_container_width=True)
 
         progress_bar.progress(100)
-        status_container.success(f"✅ Scraping completed! Found {len(politicians)} politicians, {len(disclosures)} disclosures")
+        status_container.success(
+            f"✅ Scraping completed! Found {len(politicians)} politicians, {len(disclosures)} disclosures"
+        )
 
         if save_to_db:
             save_politician_trading_to_db(politicians, disclosures)
@@ -754,6 +739,7 @@ def run_senate_watcher_scraper(
     except Exception as e:
         st.error(f"❌ Error: {e}")
         import traceback
+
         st.code(traceback.format_exc())
 
 
@@ -779,9 +765,11 @@ def show_scraper_logs():
     """Display scraper logs"""
     st.subheader("📊 Scraper Logs")
 
-    st.markdown("""
+    st.markdown(
+        """
     View real-time logs from scraping operations and data pull jobs.
-    """)
+    """
+    )
 
     # Get logs from Supabase data_pull_jobs
     try:
@@ -791,7 +779,13 @@ def show_scraper_logs():
 
         if client:
             # Get recent jobs
-            jobs = client.table("data_pull_jobs").select("*").order("created_at", desc=True).limit(50).execute()
+            jobs = (
+                client.table("data_pull_jobs")
+                .select("*")
+                .order("created_at", desc=True)
+                .limit(50)
+                .execute()
+            )
 
             if jobs.data:
                 st.markdown("### Recent Data Pull Jobs")
@@ -799,19 +793,33 @@ def show_scraper_logs():
                 jobs_df = pd.DataFrame(jobs.data)
 
                 # Format dates
-                for col in ['started_at', 'completed_at', 'created_at']:
+                for col in ["started_at", "completed_at", "created_at"]:
                     if col in jobs_df.columns:
-                        jobs_df[col] = pd.to_datetime(jobs_df[col], format='ISO8601', errors='coerce')
+                        jobs_df[col] = pd.to_datetime(
+                            jobs_df[col], format="ISO8601", errors="coerce"
+                        )
 
                 # Display jobs table
-                display_df = jobs_df[[
-                    'created_at', 'job_type', 'status', 'records_found',
-                    'records_new', 'records_updated', 'records_failed'
-                ]].copy()
+                display_df = jobs_df[
+                    [
+                        "created_at",
+                        "job_type",
+                        "status",
+                        "records_found",
+                        "records_new",
+                        "records_updated",
+                        "records_failed",
+                    ]
+                ].copy()
 
                 display_df.columns = [
-                    'Timestamp', 'Job Type', 'Status', 'Found',
-                    'New', 'Updated', 'Failed'
+                    "Timestamp",
+                    "Job Type",
+                    "Status",
+                    "Found",
+                    "New",
+                    "Updated",
+                    "Failed",
                 ]
 
                 st.dataframe(display_df, use_container_width=True)
@@ -821,26 +829,26 @@ def show_scraper_logs():
 
                 selected_job = st.selectbox(
                     "Select Job",
-                    jobs_df['id'].tolist(),
-                    format_func=lambda x: f"{jobs_df[jobs_df['id']==x]['job_type'].values[0]} - {jobs_df[jobs_df['id']==x]['created_at'].values[0]}"
+                    jobs_df["id"].tolist(),
+                    format_func=lambda x: f"{jobs_df[jobs_df['id']==x]['job_type'].values[0]} - {jobs_df[jobs_df['id']==x]['created_at'].values[0]}",
                 )
 
                 if selected_job:
-                    job = jobs_df[jobs_df['id'] == selected_job].iloc[0]
+                    job = jobs_df[jobs_df["id"] == selected_job].iloc[0]
 
                     col1, col2, col3, col4 = st.columns(4)
-                    col1.metric("Status", job['status'])
-                    col2.metric("Records Found", job['records_found'])
-                    col3.metric("New Records", job['records_new'])
-                    col4.metric("Failed", job['records_failed'])
+                    col1.metric("Status", job["status"])
+                    col2.metric("Records Found", job["records_found"])
+                    col3.metric("New Records", job["records_new"])
+                    col4.metric("Failed", job["records_failed"])
 
-                    if job.get('error_message'):
+                    if job.get("error_message"):
                         st.error(f"**Error:** {job['error_message']}")
 
                     # Show config snapshot
-                    if job.get('config_snapshot'):
+                    if job.get("config_snapshot"):
                         with st.expander("Configuration Snapshot"):
-                            st.json(job['config_snapshot'])
+                            st.json(job["config_snapshot"])
 
             else:
                 st.info("No jobs found in database")
@@ -856,33 +864,29 @@ def show_system_logs():
     """Display system logs"""
     st.subheader("📝 System Logs")
 
-    st.markdown("""
+    st.markdown(
+        """
     View application logs, errors, and system events.
-    """)
+    """
+    )
 
     # Log file path
     log_file = Path("/tmp/seed_database.log")
 
     if log_file.exists():
         try:
-            with open(log_file, 'r') as f:
+            with open(log_file, "r") as f:
                 logs = f.readlines()
 
             # Filter options
             col1, col2, col3 = st.columns(3)
 
             with col1:
-                log_level = st.selectbox(
-                    "Log Level",
-                    ["ALL", "ERROR", "WARNING", "INFO", "DEBUG"]
-                )
+                log_level = st.selectbox("Log Level", ["ALL", "ERROR", "WARNING", "INFO", "DEBUG"])
 
             with col2:
                 lines_to_show = st.number_input(
-                    "Lines to Show",
-                    min_value=10,
-                    max_value=1000,
-                    value=100
+                    "Lines to Show", min_value=10, max_value=1000, value=100
                 )
 
             with col3:
@@ -898,51 +902,54 @@ def show_system_logs():
                 filtered_logs = [l for l in filtered_logs if search_term.lower() in l.lower()]
 
             # Display logs
-            st.text_area(
-                "Log Output",
-                "".join(filtered_logs),
-                height=400
-            )
+            st.text_area("Log Output", "".join(filtered_logs), height=400)
 
             # Download button
             st.download_button(
                 "Download Full Logs",
                 "".join(logs),
                 file_name=f"system_logs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt",
-                mime="text/plain"
+                mime="text/plain",
             )
 
         except Exception as e:
             st.error(f"Error reading log file: {e}")
     else:
         st.info("📋 **No logs available yet**")
-        st.markdown("""
+        st.markdown(
+            """
         System logs will appear here automatically after scraping jobs run.
 
         **To generate logs:**
         - Use the "Manual Scrapers" section above to run a data pull
         - Wait for automated jobs to execute
         - Logs will be stored in: `/tmp/seed_database.log`
-        """)
+        """
+        )
 
         # Create example logs display
         st.markdown("### 📝 Example Log Output")
-        st.code("""
+        st.code(
+            """
 2025-10-07 12:00:00 - INFO - Starting data pull job: senate_watcher_seed
 2025-10-07 12:00:05 - INFO - Fetched 8350 Senate transactions
 2025-10-07 12:00:10 - INFO - Upserted 89 politicians (5 new, 84 updated)
 2025-10-07 12:01:30 - INFO - Upserted 8350 disclosures (6353 new, 1893 updated, 104 failed)
 2025-10-07 12:01:31 - INFO - Job completed successfully
-        """, language="log")
+        """,
+            language="log",
+        )
 
 
 def show_job_history():
     """Display job history and statistics"""
     st.subheader("📈 Job History & Statistics")
 
-    st.markdown("""
+    st.markdown(
+        """
     View historical data about scraping jobs, success rates, and trends.
-    """)
+    """
+    )
 
     try:
         from mcli.ml.dashboard.app_integrated import get_supabase_client
@@ -951,15 +958,23 @@ def show_job_history():
 
         if client:
             # Get all jobs
-            jobs = client.table("data_pull_jobs").select("*").order("created_at", desc=True).limit(1000).execute()
+            jobs = (
+                client.table("data_pull_jobs")
+                .select("*")
+                .order("created_at", desc=True)
+                .limit(1000)
+                .execute()
+            )
 
             if jobs.data and len(jobs.data) > 0:
                 jobs_df = pd.DataFrame(jobs.data)
 
                 # Format dates
-                for col in ['started_at', 'completed_at', 'created_at']:
+                for col in ["started_at", "completed_at", "created_at"]:
                     if col in jobs_df.columns:
-                        jobs_df[col] = pd.to_datetime(jobs_df[col], format='ISO8601', errors='coerce')
+                        jobs_df[col] = pd.to_datetime(
+                            jobs_df[col], format="ISO8601", errors="coerce"
+                        )
 
                 # Statistics
                 st.markdown("### Overall Statistics")
@@ -967,8 +982,8 @@ def show_job_history():
                 col1, col2, col3, col4 = st.columns(4)
 
                 total_jobs = len(jobs_df)
-                completed_jobs = len(jobs_df[jobs_df['status'] == 'completed'])
-                failed_jobs = len(jobs_df[jobs_df['status'] == 'failed'])
+                completed_jobs = len(jobs_df[jobs_df["status"] == "completed"])
+                failed_jobs = len(jobs_df[jobs_df["status"] == "failed"])
                 success_rate = (completed_jobs / total_jobs * 100) if total_jobs > 0 else 0
 
                 col1.metric("Total Jobs", total_jobs)
@@ -979,84 +994,94 @@ def show_job_history():
                 # Job type breakdown
                 st.markdown("### Job Type Breakdown")
 
-                job_type_counts = jobs_df['job_type'].value_counts()
+                job_type_counts = jobs_df["job_type"].value_counts()
 
                 fig = px.pie(
-                    values=job_type_counts.values,
-                    names=job_type_counts.index,
-                    title="Jobs by Type"
+                    values=job_type_counts.values, names=job_type_counts.index, title="Jobs by Type"
                 )
                 st.plotly_chart(fig, config={"displayModeBar": True}, use_container_width=True)
 
                 # Status breakdown
                 st.markdown("### Status Breakdown")
 
-                status_counts = jobs_df['status'].value_counts()
+                status_counts = jobs_df["status"].value_counts()
 
                 fig = px.bar(
                     x=status_counts.index,
                     y=status_counts.values,
-                    labels={'x': 'Status', 'y': 'Count'},
-                    title="Jobs by Status"
+                    labels={"x": "Status", "y": "Count"},
+                    title="Jobs by Status",
                 )
                 st.plotly_chart(fig, config={"displayModeBar": True}, use_container_width=True)
 
                 # Timeline
                 st.markdown("### Job Timeline")
 
-                jobs_df['date'] = jobs_df['created_at'].dt.date
+                jobs_df["date"] = jobs_df["created_at"].dt.date
 
-                timeline_df = jobs_df.groupby(['date', 'status']).size().reset_index(name='count')
+                timeline_df = jobs_df.groupby(["date", "status"]).size().reset_index(name="count")
 
                 fig = px.line(
-                    timeline_df,
-                    x='date',
-                    y='count',
-                    color='status',
-                    title="Jobs Over Time"
+                    timeline_df, x="date", y="count", color="status", title="Jobs Over Time"
                 )
                 st.plotly_chart(fig, config={"displayModeBar": True}, use_container_width=True)
 
                 # Records processed
                 st.markdown("### Records Processed")
 
-                records_df = jobs_df[jobs_df['status'] == 'completed'][['created_at', 'records_found', 'records_new', 'records_updated', 'records_failed']].copy()
+                records_df = jobs_df[jobs_df["status"] == "completed"][
+                    [
+                        "created_at",
+                        "records_found",
+                        "records_new",
+                        "records_updated",
+                        "records_failed",
+                    ]
+                ].copy()
 
                 if not records_df.empty:
                     fig = go.Figure()
 
-                    fig.add_trace(go.Scatter(
-                        x=records_df['created_at'],
-                        y=records_df['records_new'],
-                        name='New Records',
-                        mode='lines+markers'
-                    ))
-
-                    fig.add_trace(go.Scatter(
-                        x=records_df['created_at'],
-                        y=records_df['records_updated'],
-                        name='Updated Records',
-                        mode='lines+markers'
-                    ))
-
-                    fig.add_trace(go.Scatter(
-                        x=records_df['created_at'],
-                        y=records_df['records_failed'],
-                        name='Failed Records',
-                        mode='lines+markers'
-                    ))
+                    fig.add_trace(
+                        go.Scatter(
+                            x=records_df["created_at"],
+                            y=records_df["records_new"],
+                            name="New Records",
+                            mode="lines+markers",
+                        )
+                    )
+
+                    fig.add_trace(
+                        go.Scatter(
+                            x=records_df["created_at"],
+                            y=records_df["records_updated"],
+                            name="Updated Records",
+                            mode="lines+markers",
+                        )
+                    )
+
+                    fig.add_trace(
+                        go.Scatter(
+                            x=records_df["created_at"],
+                            y=records_df["records_failed"],
+                            name="Failed Records",
+                            mode="lines+markers",
+                        )
+                    )
 
                     fig.update_layout(
                         title="Records Processed Over Time",
                         xaxis_title="Date",
                         yaxis_title="Count",
-                        hovermode='x unified'
+                        hovermode="x unified",
                     )
 
                     st.plotly_chart(fig, config={"displayModeBar": True}, use_container_width=True)
 
             else:
-                st.info("No job history available yet. Run some scraping jobs to see statistics here.")
+                st.info(
+                    "No job history available yet. Run some scraping jobs to see statistics here."
+                )
 
         else:
             st.warning("Supabase not connected - job history unavailable")
@@ -1064,6 +1089,7 @@ def show_job_history():
     except Exception as e:
         st.error(f"Error loading job history: {e}")
         import traceback
+
         st.code(traceback.format_exc())