ncua-data-analysis 0.1.0__py3-none-any.whl

ncua_data/__init__.py

@@ -0,0 +1,2 @@
+"""Clean, documented NCUA 5300 call report data for credit union analysis."""
+__version__ = "0.1.0"

ncua_data/__main__.py

@@ -0,0 +1,2 @@
+from .cli import main
+raise SystemExit(main())

ncua_data/build.py

@@ -0,0 +1,172 @@
+"""Build the curated tables (dim, fact, metrics, dictionary) from long Parquet."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import duckdb
+
+from .spec import FIELDS
+
+
+def fact_sql(long_glob: str) -> str:
+    cols = []
+    for f in FIELDS:
+        codes = ", ".join(f"'{c}'" for c in f.codes)
+        agg = f"sum(l.value) FILTER (WHERE l.acct IN ({codes}))"
+        # No row in the long table means the credit union reported zero (or left it blank).
+        expr = f"coalesce({agg}, 0)"
+        if f.first_quarter:
+            expr = f"CASE WHEN q.quarter >= '{f.first_quarter}' THEN {expr} END"
+        cols.append(f"{expr} AS {f.name}")
+    return f"""
+        SELECT q.quarter, q.cu_number, {', '.join(cols)}
+        FROM quarter_cus q
+        LEFT JOIN (SELECT * FROM read_parquet('{long_glob}')) l
+          ON l.quarter = q.quarter AND l.cu_number = q.cu_number
+        GROUP BY q.quarter, q.cu_number"""
+
+
+def build(data_dir: Path, out_dir: Path) -> duckdb.DuckDBPyConnection:
+    con = duckdb.connect()
+    con.execute("SET memory_limit='1400MB'; SET threads=2")
+    long_glob = str(data_dir / "long" / "*_long.parquet")
+    foicu_glob = str(data_dir / "long" / "*_foicu.parquet")
+    con.execute(
+        f"""CREATE TABLE quarter_cus AS
+            SELECT quarter, CU_NUMBER::INTEGER AS cu_number
+            FROM read_parquet('{foicu_glob}', union_by_name=true)"""
+    )
+    # One quarter at a time keeps memory low enough for a 2 GB machine.
+    first = True
+    for lf in sorted((data_dir / "long").glob("*_long.parquet")):
+        q = lf.name.split("_")[0]
+        sql = fact_sql(str(lf)).replace("FROM quarter_cus q", f"FROM (SELECT * FROM quarter_cus WHERE quarter = '{q}') q")
+        if first:
+            con.execute(f"CREATE TABLE fact_call_report_curated AS {sql}")
+            first = False
+        else:
+            con.execute(f"INSERT INTO fact_call_report_curated {sql}")
+    return con
+
+
+DIM_SQL = """
+CREATE TABLE dim_credit_union AS
+SELECT quarter,
+       CU_NUMBER::INTEGER AS cu_number,
+       try_cast(RSSD AS BIGINT) AS rssd,
+       trim(CU_NAME) AS name,
+       trim(CITY) AS city,
+       trim(STATE) AS state,
+       trim(CharterState) AS charter_state,
+       trim(ZIP_CODE) AS zip_code,
+       try_cast(COUNTY_CODE AS INTEGER) AS county_code,
+       CASE CU_TYPE WHEN '1' THEN 'federal' WHEN '2' THEN 'state_federally_insured'
+                    WHEN '3' THEN 'state_not_federally_insured' END AS charter_type,
+       CU_TYPE <> '3' AS is_federally_insured,
+       try_cast(Peer_Group AS INTEGER) AS peer_group,
+       CASE try_cast(Peer_Group AS INTEGER)
+            WHEN 1 THEN 'under $2M' WHEN 2 THEN '$2M-$10M' WHEN 3 THEN '$10M-$50M'
+            WHEN 4 THEN '$50M-$100M' WHEN 5 THEN '$100M-$500M' WHEN 6 THEN '$500M+' END AS peer_group_label,
+       lower(IsMDI) IN ('true', '1') AS is_minority_depository,
+       try_cast(LIMITED_INC AS INTEGER) = 1 AS is_low_income,
+       try_cast(YEAR_OPENED AS INTEGER) AS year_opened,
+       trim(TOM_CODE) AS field_of_membership_code,
+       trim(REGION) AS ncua_region
+FROM read_parquet('{foicu}', union_by_name=true)
+"""
+
+# Metrics. Flow fields are year-to-date, so annualise by 4 / quarter number.
+METRICS_SQL = """
+CREATE TABLE metrics AS
+WITH f AS (
+  SELECT f.*, d.is_federally_insured,
+         CAST(substr(f.quarter, 6, 2) AS INTEGER) / 3 AS qn,
+         CAST(substr(f.quarter, 1, 4) AS INTEGER) AS yr
+  FROM fact_call_report_curated f JOIN dim_credit_union d USING (quarter, cu_number)
+), g AS (
+  SELECT f.*, 4.0 / qn AS ann,
+         lag(total_assets, 4) OVER w AS assets_1y_ago,
+         lag(loans_and_leases_total, 4) OVER w AS loans_1y_ago,
+         lag(total_shares_and_deposits, 4) OVER w AS shares_1y_ago,
+         lag(members, 4) OVER w AS members_1y_ago,
+         lag(loans_new_vehicle + loans_used_vehicle, 4) OVER w AS auto_1y_ago,
+         lag(loans_first_lien_residential, 4) OVER w AS first_lien_1y_ago,
+         CASE WHEN count(*) OVER w4 = 4 THEN avg(total_assets) OVER w4 END AS avg_assets_4q,
+         CASE WHEN count(*) OVER w4 = 4 THEN avg(loans_and_leases_total) OVER w4 END AS avg_loans_4q,
+         lag(net_income_ytd, 1) OVER w AS ni_prev, lag(interest_income_ytd, 1) OVER w AS ii_prev,
+         lag(interest_expense_ytd, 1) OVER w AS ie_prev, lag(non_interest_expense_ytd, 1) OVER w AS nie_prev,
+         lag(provision_for_loan_losses_ytd, 1) OVER w AS prov_prev
+  FROM f WINDOW w AS (PARTITION BY cu_number ORDER BY quarter),
+  w4 AS (PARTITION BY cu_number ORDER BY quarter ROWS BETWEEN 3 PRECEDING AND CURRENT ROW)
+)
+SELECT quarter, cu_number,
+  -- size and growth
+  CASE WHEN assets_1y_ago > 0 THEN total_assets / assets_1y_ago - 1 END AS asset_growth_yoy,
+  CASE WHEN loans_1y_ago > 0 THEN loans_and_leases_total / loans_1y_ago - 1 END AS loan_growth_yoy,
+  CASE WHEN shares_1y_ago > 0 THEN total_shares_and_deposits / shares_1y_ago - 1 END AS share_growth_yoy,
+  CASE WHEN members_1y_ago > 0 THEN members / members_1y_ago - 1 END AS member_growth_yoy,
+  CASE WHEN auto_1y_ago > 0 THEN (loans_new_vehicle + loans_used_vehicle) / auto_1y_ago - 1 END AS auto_loan_growth_yoy,
+  CASE WHEN first_lien_1y_ago > 0 THEN loans_first_lien_residential / first_lien_1y_ago - 1 END AS first_lien_growth_yoy,
+  -- balance sheet ratios
+  loans_and_leases_total / nullif(total_shares_and_deposits, 0) AS loan_to_share,
+  loans_and_leases_total / nullif(total_assets, 0) AS loans_to_assets,
+  net_worth / nullif(total_assets, 0) AS net_worth_to_assets,
+  (net_worth - coalesce(cecl_transition_provision, 0)) / nullif(total_assets, 0) AS net_worth_ratio_ex_cecl,
+  allowance_for_credit_losses / nullif(loans_and_leases_total, 0) AS allowance_to_loans,
+  loans_and_leases_total / nullif(members, 0) AS loans_per_member,
+  total_shares_and_deposits / nullif(members, 0) AS shares_per_member,
+  -- loan mix (share of total loans)
+  (loans_new_vehicle + loans_used_vehicle) / nullif(loans_and_leases_total, 0) AS mix_auto,
+  (loans_first_lien_residential + loans_junior_lien_residential + loans_other_real_estate) / nullif(loans_and_leases_total, 0) AS mix_residential_real_estate,
+  loans_credit_card / nullif(loans_and_leases_total, 0) AS mix_credit_card,
+  loans_commercial_total / nullif(loans_and_leases_total, 0) AS mix_commercial,
+  -- share and deposit mix (share of total shares and deposits)
+  shares_share_drafts / nullif(total_shares_and_deposits, 0) AS deposit_mix_share_drafts,
+  shares_regular / nullif(total_shares_and_deposits, 0) AS deposit_mix_regular,
+  shares_money_market / nullif(total_shares_and_deposits, 0) AS deposit_mix_money_market,
+  shares_certificates / nullif(total_shares_and_deposits, 0) AS deposit_mix_certificates,
+  shares_ira_keogh / nullif(total_shares_and_deposits, 0) AS deposit_mix_ira_keogh,
+  deposits_non_member / nullif(total_shares_and_deposits, 0) AS deposit_mix_non_member,
+  total_shares_and_deposits / nullif(accounts_share_drafts + accounts_certificates + accounts_money_market + accounts_ira_keogh + accounts_non_member, 0) AS avg_balance_per_listed_account,
+  -- staffing and branches
+  employees_full_time + employees_part_time / 2.0 AS employees_fte_estimate,
+  members / nullif(employees_full_time + employees_part_time / 2.0, 0) AS members_per_fte,
+  total_assets / nullif(employees_full_time + employees_part_time / 2.0, 0) AS assets_per_fte,
+  employee_compensation_ytd * ann / nullif(employees_full_time + employees_part_time / 2.0, 0) AS compensation_per_fte,
+  non_interest_expense_ytd * ann / nullif(employees_full_time + employees_part_time / 2.0, 0) AS operating_expense_per_fte,
+  total_assets / nullif(branches, 0) AS assets_per_branch,
+  members / nullif(branches, 0) AS members_per_branch,
+  employee_compensation_ytd / nullif(non_interest_expense_ytd, 0) AS compensation_share_of_opex,
+  -- asset quality
+  delinquent_2m_plus / nullif(loans_and_leases_total, 0) AS delinquency_rate,
+  (delinquent_new_vehicle + delinquent_used_vehicle) / nullif(loans_new_vehicle + loans_used_vehicle, 0) AS auto_delinquency_rate,
+  delinquent_credit_card / nullif(loans_credit_card, 0) AS credit_card_delinquency_rate,
+  (chargeoffs_ytd - recoveries_ytd) * ann / nullif(loans_and_leases_total, 0) AS net_chargeoff_rate,
+  (chargeoffs_ytd - recoveries_ytd) * ann / nullif(avg_loans_4q, 0) AS net_chargeoff_rate_avg_loans_4q,
+  provision_for_loan_losses_ytd * ann / nullif(loans_and_leases_total, 0) AS provision_to_loans,
+  -- earnings (annualised from year-to-date)
+  net_income_ytd * ann / nullif(total_assets, 0) AS roa_year_end_assets,
+  net_income_ytd * ann / nullif(avg_assets_4q, 0) AS roa_avg_assets_4q,
+  (interest_income_ytd - interest_expense_ytd) * ann / nullif(total_assets, 0) AS nim_year_end_assets,
+  (interest_income_ytd - interest_expense_ytd) * ann / nullif(avg_assets_4q, 0) AS nim_avg_assets_4q,
+  interest_on_loans_ytd * ann / nullif(loans_and_leases_total, 0) AS loan_yield,
+  interest_expense_ytd * ann / nullif(total_shares_and_deposits, 0) AS cost_of_shares,
+  non_interest_expense_ytd / nullif((interest_income_ytd - interest_expense_ytd) + non_interest_income_ytd, 0) AS efficiency_ratio,
+  non_interest_expense_ytd * ann / nullif(total_assets, 0) AS opex_to_assets,
+  fee_income_ytd / nullif(non_interest_income_ytd, 0) AS fee_share_of_non_interest_income,
+  -- de-cumulated quarterly flows
+  CASE WHEN qn = 1 THEN net_income_ytd ELSE net_income_ytd - ni_prev END AS net_income_quarter,
+  CASE WHEN qn = 1 THEN interest_income_ytd ELSE interest_income_ytd - ii_prev END AS interest_income_quarter,
+  CASE WHEN qn = 1 THEN interest_expense_ytd ELSE interest_expense_ytd - ie_prev END AS interest_expense_quarter,
+  CASE WHEN qn = 1 THEN non_interest_expense_ytd ELSE non_interest_expense_ytd - nie_prev END AS non_interest_expense_quarter,
+  CASE WHEN qn = 1 THEN provision_for_loan_losses_ytd ELSE provision_for_loan_losses_ytd - prov_prev END AS provision_quarter
+FROM g
+"""
+
+
+def build_all(data_dir: Path) -> duckdb.DuckDBPyConnection:
+    con = build(data_dir, data_dir)
+    foicu = str(data_dir / "long" / "*_foicu.parquet")
+    con.execute(DIM_SQL.format(foicu=foicu))
+    con.execute(METRICS_SQL)
+    return con

ncua_data/cli.py

@@ -0,0 +1,50 @@
+"""Command line: download, build, reconcile."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from . import download as dl
+from .build import build_all
+from .dictionary import build_dictionary
+from .ingest import ingest_quarter
+from .reconcile import run as reconcile
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="ncua-data")
+    ap.add_argument("--data", type=Path, default=Path("data"))
+    ap.add_argument("--since", type=int, default=2018, help="first year to download")
+    ap.add_argument("command", choices=["download", "ingest", "build", "reconcile", "all"])
+    a = ap.parse_args(argv)
+    cmds = ["download", "ingest", "build", "reconcile"] if a.command == "all" else [a.command]
+    if "download" in cmds:
+        dl.download(dl.list_quarters(a.since), a.data / "raw")
+    if "ingest" in cmds:
+        for z in sorted((a.data / "raw").glob("*.zip")):
+            q = z.stem.replace("call-report-data-", "")
+            if not (a.data / "long" / f"{q}_dict.parquet").exists():
+                print(ingest_quarter(z, a.data / "work", a.data / "long"))
+    if "build" in cmds or "reconcile" in cmds:
+        con = build_all(a.data)
+        if "build" in cmds:
+            build_dictionary(con)
+            out = a.data / "out"
+            out.mkdir(exist_ok=True)
+            for t in ["dim_credit_union", "fact_call_report_curated", "metrics", "dictionary"]:
+                con.execute(f"COPY {t} TO '{out}/{t}.parquet' (FORMAT PARQUET, COMPRESSION ZSTD)")
+            print("wrote", sorted(p.name for p in out.glob("*.parquet")))
+        if "reconcile" in cmds:
+            res = reconcile(con)
+            (a.data / "out").mkdir(exist_ok=True)
+            (a.data / "out" / "reconciliation.json").write_text(json.dumps(res, indent=2))
+            bad = [r for r in res if not r["ok"]]
+            print(f"{len(res) - len(bad)}/{len(res)} checks within tolerance")
+            for r in bad:
+                print("MISMATCH", r["quarter"], r["check"], r["dataset"], "vs published", r["published"])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ncua_data/dictionary.py

@@ -0,0 +1,93 @@
+"""Build the data dictionary table: one row per column in every published table."""
+from __future__ import annotations
+
+from .spec import FIELDS
+
+METRIC_DOCS = {
+    "asset_growth_yoy": ("Total assets versus the same quarter one year earlier.", "ratio"),
+    "loan_growth_yoy": ("Total loans versus the same quarter one year earlier.", "ratio"),
+    "share_growth_yoy": ("Total shares and deposits versus one year earlier.", "ratio"),
+    "member_growth_yoy": ("Members versus one year earlier.", "ratio"),
+    "auto_loan_growth_yoy": ("New plus used vehicle loans versus one year earlier.", "ratio"),
+    "first_lien_growth_yoy": ("First-lien 1-4 family loans versus one year earlier.", "ratio"),
+    "loan_to_share": ("Loans divided by total shares and deposits. NCUA's headline liquidity ratio.", "ratio"),
+    "loans_to_assets": ("Loans divided by total assets.", "ratio"),
+    "net_worth_to_assets": ("Net worth divided by total assets (1.0 = 100%).", "ratio"),
+    "allowance_to_loans": ("Allowance for credit losses divided by total loans.", "ratio"),
+    "net_worth_ratio_ex_cecl": ("Net worth minus the CECL transition provision, over assets. This is how NCUA publishes its net worth ratio from 2023 on.", "ratio"),
+    "deposit_mix_share_drafts": ("Share draft (checking) accounts as a share of total shares and deposits.", "ratio"),
+    "deposit_mix_regular": ("Regular shares as a share of total shares and deposits.", "ratio"),
+    "deposit_mix_money_market": ("Money market shares as a share of total shares and deposits.", "ratio"),
+    "deposit_mix_certificates": ("Share certificates as a share of total shares and deposits.", "ratio"),
+    "deposit_mix_ira_keogh": ("IRA and Keogh accounts as a share of total shares and deposits.", "ratio"),
+    "deposit_mix_non_member": ("Non-member deposits as a share of total shares and deposits.", "ratio"),
+    "avg_balance_per_listed_account": ("Total shares and deposits over the count of share draft, certificate, money market, IRA and non-member accounts. Rough; regular share account counts are not collected.", "dollars"),
+    "employees_fte_estimate": ("Full-time employees plus half of part-time employees. An estimate, not an NCUA definition.", "count"),
+    "members_per_fte": ("Members per estimated full-time-equivalent employee.", "count"),
+    "assets_per_fte": ("Assets per estimated full-time-equivalent employee.", "dollars"),
+    "compensation_per_fte": ("Annualized employee compensation and benefits per estimated FTE.", "dollars"),
+    "operating_expense_per_fte": ("Annualized non-interest expense per estimated FTE.", "dollars"),
+    "assets_per_branch": ("Assets per branch.", "dollars"),
+    "members_per_branch": ("Members per branch.", "count"),
+    "compensation_share_of_opex": ("Employee compensation and benefits as a share of non-interest expense.", "ratio"),
+    "loans_per_member": ("Average loan dollars per member.", "dollars"),
+    "shares_per_member": ("Average shares and deposits per member.", "dollars"),
+    "mix_auto": ("New plus used vehicle loans as a share of total loans.", "ratio"),
+    "mix_residential_real_estate": ("First lien, junior lien and other real estate loans as a share of total loans.", "ratio"),
+    "mix_credit_card": ("Credit card loans as a share of total loans.", "ratio"),
+    "mix_commercial": ("Commercial loans as a share of total loans.", "ratio"),
+    "delinquency_rate": ("Loans delinquent two or more months divided by total loans. Matches NCUA's published rate.", "ratio"),
+    "auto_delinquency_rate": ("Reportable delinquent vehicle loans divided by vehicle loans.", "ratio"),
+    "credit_card_delinquency_rate": ("Delinquent credit card loans divided by credit card loans.", "ratio"),
+    "net_chargeoff_rate": ("(Charge-offs minus recoveries), annualized from year to date, over year-end loans.", "ratio"),
+    "net_chargeoff_rate_avg_loans_4q": ("Same, over the average of the last four quarter-end loan balances. Closest to NCUA's published ratio but not identical.", "ratio"),
+    "provision_to_loans": ("Provision (credit loss expense) annualized over year-end loans.", "ratio"),
+    "roa_year_end_assets": ("Net income annualized over year-end assets.", "ratio"),
+    "roa_avg_assets_4q": ("Net income annualized over the average of the last four quarter-end assets. NCUA's own denominator is not public, so this can differ by a few basis points.", "ratio"),
+    "nim_year_end_assets": ("(Interest income minus interest expense) annualized over year-end assets.", "ratio"),
+    "nim_avg_assets_4q": ("Same, over four-quarter average assets. NCUA publishes 3.49% for June 2026; this gives about 3.5%.", "ratio"),
+    "loan_yield": ("Interest on loans annualized over year-end loans.", "ratio"),
+    "cost_of_shares": ("Total interest expense annualized over shares and deposits.", "ratio"),
+    "efficiency_ratio": ("Non-interest expense divided by (net interest income plus non-interest income).", "ratio"),
+    "opex_to_assets": ("Non-interest expense annualized over assets.", "ratio"),
+    "fee_share_of_non_interest_income": ("Fee income divided by non-interest income.", "ratio"),
+    "net_income_quarter": ("Net income for the single quarter (year-to-date minus prior quarter).", "dollars"),
+    "interest_income_quarter": ("Interest income for the single quarter.", "dollars"),
+    "interest_expense_quarter": ("Interest expense for the single quarter.", "dollars"),
+    "non_interest_expense_quarter": ("Non-interest expense for the single quarter.", "dollars"),
+    "provision_quarter": ("Provision for loan losses for the single quarter.", "dollars"),
+}
+
+DIM_DOCS = {
+    "quarter": "Report quarter, YYYY-MM of the quarter end (03, 06, 09, 12).",
+    "cu_number": "NCUA charter number. Stable for a credit union over time; disappears when it merges or closes.",
+    "rssd": "Federal Reserve RSSD identifier.",
+    "name": "Credit union name.", "city": "Mailing city.", "state": "Mailing state.",
+    "charter_state": "State of charter (state-chartered only).", "zip_code": "Mailing ZIP.",
+    "county_code": "County code.",
+    "charter_type": "federal, state_federally_insured, or state_not_federally_insured.",
+    "is_federally_insured": "False for the ~85 state-chartered credit unions NCUA does not insure. NCUA's published totals exclude them.",
+    "peer_group": "NCUA asset peer group 1-6.", "peer_group_label": "Peer group asset range.",
+    "is_minority_depository": "Minority depository institution flag.",
+    "is_low_income": "Low-income designation.", "year_opened": "Year the credit union was organized.",
+    "field_of_membership_code": "Type-of-membership code.", "ncua_region": "NCUA region code.",
+}
+
+
+def build_dictionary(con) -> None:
+    rows = []
+    for col, desc in DIM_DOCS.items():
+        rows.append(("dim_credit_union", col, desc, "attribute", None, None, "FOICU.txt", None, None))
+    span = {}
+    for f in FIELDS:
+        r = con.execute(
+            f"SELECT min(quarter), max(quarter) FROM fact_call_report_curated WHERE {f.name} IS NOT NULL AND {f.name} <> 0"
+        ).fetchone()
+        span[f.name] = r
+        rows.append(("fact_call_report_curated", f.name, f.description, f.kind, f.category,
+                     ", ".join(c.replace("ACCT_", "Acct_") for c in f.codes), "FS220* tables", r[0], r[1]))
+    for name, (desc, unit) in METRIC_DOCS.items():
+        rows.append(("metrics", name, desc, unit, "derived", None, "computed from fact_call_report_curated", None, None))
+    con.execute("""CREATE OR REPLACE TABLE dictionary(table_name VARCHAR, column_name VARCHAR, description VARCHAR,
+        kind VARCHAR, category VARCHAR, ncua_account_codes VARCHAR, source VARCHAR, first_quarter_nonzero VARCHAR, last_quarter_nonzero VARCHAR)""")
+    con.executemany("INSERT INTO dictionary VALUES (?,?,?,?,?,?,?,?,?)", rows)

ncua_data/download.py

@@ -0,0 +1,45 @@
+"""Find and download NCUA quarterly call report ZIPs.
+
+Links are scraped from NCUA's quarterly data page rather than built from a
+pattern, because the file naming has changed over the years.
+"""
+from __future__ import annotations
+
+import re
+import sys
+import urllib.request
+from pathlib import Path
+
+BASE = "https://ncua.gov"
+PAGE = BASE + "/analysis/credit-union-corporate-call-report-data/quarterly-data"
+UA = {"User-Agent": "ncua-data-analysis (open source research project)"}
+
+_LINK = re.compile(r'href="(/files/publications/analysis/call-report-data-(\d{4})-(\d{2})\.zip)"', re.I)
+
+
+def _get(url: str) -> bytes:
+    req = urllib.request.Request(url, headers=UA)
+    with urllib.request.urlopen(req, timeout=120) as r:
+        return r.read()
+
+
+def list_quarters(min_year: int = 2018) -> dict[str, str]:
+    """Return {'2026-06': url} for every quarter on the page since min_year."""
+    html = _get(PAGE).decode("utf-8", "replace")
+    out: dict[str, str] = {}
+    for path, y, m in _LINK.findall(html):
+        if int(y) >= min_year:
+            out[f"{y}-{m}"] = BASE + path
+    return dict(sorted(out.items()))
+
+
+def download(quarters: dict[str, str], dest: Path) -> list[Path]:
+    dest.mkdir(parents=True, exist_ok=True)
+    paths = []
+    for q, url in quarters.items():
+        p = dest / f"call-report-data-{q}.zip"
+        if not p.exists() or p.stat().st_size == 0:
+            print(f"downloading {q}", file=sys.stderr)
+            p.write_bytes(_get(url))
+        paths.append(p)
+    return paths

ncua_data/ingest.py

@@ -0,0 +1,88 @@
+"""Load raw NCUA quarterly ZIPs into long-format Parquet files.
+
+Every account value becomes one row (quarter, cu_number, acct, value), keeping
+non-zero values only. That makes the multi-year account churn easy to handle
+and keeps the intermediate store small.
+"""
+from __future__ import annotations
+
+import csv
+import re
+import zipfile
+from pathlib import Path
+
+import duckdb
+
+ACCT_TABLE = re.compile(r"^fs220[a-z]?\.txt$", re.I)
+ID_COLS = ("CU_NUMBER", "CYCLE_DATE", "JOIN_NUMBER", "UPDATE_DATE")
+
+
+def extract(zip_path: Path, dest: Path) -> Path:
+    out = dest / zip_path.stem.replace("call-report-data-", "")
+    if not out.exists():
+        out.mkdir(parents=True)
+        with zipfile.ZipFile(zip_path) as z:
+            z.extractall(out)
+        # NCUA files are Windows-1252 (stray 0x92 apostrophes appear in free text
+        # fields). Re-encode to UTF-8 so every reader agrees.
+        for f in out.iterdir():
+            if f.suffix.lower() == ".txt":
+                f.write_bytes(f.read_bytes().decode("cp1252", errors="replace").encode("utf-8"))
+    return out
+
+
+def _find(folder: Path, name: str) -> Path | None:
+    for p in folder.iterdir():
+        if p.name.lower() == name.lower():
+            return p
+    return None
+
+
+def _unique_header(path: Path) -> list[str]:
+    """Header row with case-insensitive duplicates suffixed (some files repeat a code)."""
+    with open(path, encoding="utf-8", newline="") as fh:
+        header = next(csv.reader(fh))
+    seen: dict[str, int] = {}
+    out = []
+    for h in header:
+        k = h.upper()
+        seen[k] = seen.get(k, 0) + 1
+        out.append(h if seen[k] == 1 else f"{h}__dup{seen[k]}")
+    return out
+
+
+def ingest_quarter(zip_path: Path, work: Path, out_dir: Path) -> dict:
+    """Write long.parquet, foicu.parquet and dict.parquet for one quarter."""
+    quarter = zip_path.stem.replace("call-report-data-", "")
+    folder = extract(zip_path, work)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    con = duckdb.connect()
+    parts = []
+    for p in sorted(folder.iterdir()):
+        if ACCT_TABLE.match(p.name):
+            names = _unique_header(p)
+            parts.append(
+                f"""SELECT CU_NUMBER::INTEGER AS cu_number, upper(acct) AS acct,
+                           try_cast(val AS DOUBLE) AS value, '{p.stem.upper()}' AS source_table
+                    FROM (UNPIVOT (SELECT * FROM read_csv('{p}', all_varchar=true, header=true,
+                                   names={names!r}, ignore_errors=true))
+                          ON COLUMNS('(?i)^acct_')
+                          INTO NAME acct VALUE val)
+                    WHERE try_cast(val AS DOUBLE) <> 0"""
+            )
+    con.execute(
+        f"COPY (SELECT '{quarter}' AS quarter, * FROM ({' UNION ALL '.join(parts)})) "
+        f"TO '{out_dir}/{quarter}_long.parquet' (FORMAT PARQUET, COMPRESSION ZSTD)"
+    )
+    foicu = _find(folder, "FOICU.txt")
+    con.execute(
+        f"COPY (SELECT '{quarter}' AS quarter, * FROM read_csv('{foicu}', all_varchar=true, "
+        f"ignore_errors=true)) TO '{out_dir}/{quarter}_foicu.parquet' (FORMAT PARQUET)"
+    )
+    ad = _find(folder, "AcctDesc.txt")
+    con.execute(
+        f"COPY (SELECT '{quarter}' AS quarter, * FROM read_csv('{ad}', all_varchar=true, "
+        f"ignore_errors=true, strict_mode=false)) TO '{out_dir}/{quarter}_dict.parquet' (FORMAT PARQUET)"
+    )
+    n = con.execute(f"SELECT count(*), count(DISTINCT cu_number), count(DISTINCT acct) FROM '{out_dir}/{quarter}_long.parquet'").fetchone()
+    return {"quarter": quarter, "rows": n[0], "cus": n[1], "accts": n[2]}