riksdagsmonitor 0.8.60 → 0.8.62

package/dist/lib/cia/data-loader.js

@@ -4,211 +4,69 @@
  *
  * @description
  * CIA Intelligence Data Loader & Pipeline Orchestrator.
- * Core data acquisition module implementing multi-source intelligence data loading
- * from the Citizen Intelligence Agency (CIA) Platform. Manages CSV export ingestion
- * for 19+ intelligence product categories and JSON fallback for model-generated
- * electoral forecasts. Provides resilient data pipeline with local-first strategy
- * and remote fallback capabilities.
+ *
+ * Thin orchestrator that wires together the per-domain loaders in `loaders/`
+ * with a single `LoadCSV` closure built from `csvBaseURL` + `fallbackURL`.
+ * Public API (constructor, static `CSV_SOURCES`, `parseCSV`, `loadCSV`,
+ * `load*()` methods, `loadAll`) is preserved for existing consumers
+ * (`dashboard-init.ts`, `election-predictions.ts`, `visualizations.ts`).
+ *
+ * Was originally a 1 300-line monolith; decomposed into focused modules:
+ *   - `types.ts`   — DTO interfaces
+ *   - `sources.ts` — CSV source URL inventory + Riksdag constants
+ *   - `csv-utils.ts` — `parseCSV` / `loadCSV` helpers
+ *   - `loaders/*.ts` — one file per domain (10 loaders)
  *
  * @author Hack23 AB - Data Pipeline Engineering
  * @license Apache-2.0
- * @version 2.0.0
+ * @version 3.0.0
  * @since 2024
-
  *
  * @intelligence CIA Platform Data Pipeline Orchestrator — core data acquisition module implementing multi-source intelligence data loading from 19+ CIA product categories. Manages CSV export ingestion and JSON fallback for electoral forecasts. Provides resilient pipeline with local-first strategy and remote fallback.
  *
  * @business Data infrastructure investment — the CIA data pipeline is the foundation for all analytical products. Pipeline reliability directly impacts user experience and platform credibility. Modular architecture enables future data source expansion (European Parliament, Nordic councils).
  *
  * @marketing Data transparency asset — transparent data sourcing (CIA Platform, open government data) builds trust with all audience segments. Data pipeline documentation demonstrates commitment to accuracy and verifiability, key messaging for press and academic audiences.
- * */
+ */
+import { COMMITTEE_DOCS_PER_MEETING_ESTIMATE, COMMITTEE_ORG_CODES, CSV_SOURCES, RIKSDAG_PARTIES } from './sources.js';
+import { createLoadCSV, loadCSV as loadCSVFn, parseCSV as parseCSVFn } from './csv-utils.js';
+import { loadCommitteeNetwork, loadDemographics, loadDocumentActivity, loadElectionAnalysis, loadMinistryDashboard, loadOverviewDashboard, loadPartyPerformance, loadRiskEvolution, loadTop10Influential, loadVotingPatterns } from './loaders/index.js';
 /* ------------------------------------------------------------------ */
-/*  CIADataLoader class                                               */
+/*  CIADataLoader class — thin orchestrator                           */
 /* ------------------------------------------------------------------ */
+/**
+ * Aggregator that delegates per-domain loading to the dedicated modules in
+ * `loaders/`. Maintains the original public API so existing callers compile
+ * unchanged after the decomposition refactor.
+ */
 export class CIADataLoader {
     csvBaseURL;
     fallbackURL;
     /** The 8 parties represented in the Swedish Riksdag. */
-    static RIKSDAG_PARTIES = ['S', 'M', 'SD', 'C', 'V', 'KD', 'L', 'MP'];
+    static RIKSDAG_PARTIES = RIKSDAG_PARTIES;
     /** Mapping of full Swedish committee names to their Riksdag org codes. */
-    static COMMITTEE_ORG_CODES = {
-        'Konstitutionsutskottet': 'KU',
-        'Civilutskottet': 'CU',
-        'Trafikutskottet': 'TU',
-        'Näringsutskottet': 'NU',
-        'Miljö- och jordbruksutskottet': 'MJU',
-        'Utrikesutskottet': 'UU',
-        'Arbetsmarknadsutskottet': 'AU',
-        'Socialförsäkringsutskottet': 'SfU',
-        'Socialutskottet': 'SoU',
-        'Justitieutskottet': 'JuU',
-        'Skatteutskottet': 'SkU',
-        'EU-nämnden': 'EUN',
-        'Kulturutskottet': 'KrU',
-        'Utbildningsutskottet': 'UbU',
-        'Finansutskottet': 'FiU',
-        'Försvarsutskottet': 'FöU',
-        'Lagutskottet': 'LU',
-        'Bostadsutskottet': 'BoU'
-    };
+    static COMMITTEE_ORG_CODES = COMMITTEE_ORG_CODES;
     /**
      * Heuristic divisor to estimate meetings/year from committee document counts.
      * Assumption: ~25 published documents per active committee meeting.
      */
-    static COMMITTEE_DOCS_PER_MEETING_ESTIMATE = 25;
+    static COMMITTEE_DOCS_PER_MEETING_ESTIMATE = COMMITTEE_DOCS_PER_MEETING_ESTIMATE;
+    /** CSV data source definitions – maps to real PostgreSQL view exports. */
+    static CSV_SOURCES = CSV_SOURCES;
+    /** Loader closure bound to this instance's URL configuration. */
+    loadCSVFn;
     constructor() {
         this.csvBaseURL = '../cia-data/';
         this.fallbackURL = 'https://raw.githubusercontent.com/Hack23/cia/master/service.data.impl/sample-data/';
+        this.loadCSVFn = createLoadCSV(this.csvBaseURL, this.fallbackURL);
     }
-    /** CSV data source definitions – maps to real PostgreSQL view exports. */
-    static CSV_SOURCES = {
-        personStatus: {
-            local: 'distribution_person_status.csv',
-            description: 'Active MP counts by status'
-        },
-        riskByParty: {
-            local: 'distribution_risk_by_party.csv',
-            description: 'Risk levels per party'
-        },
-        riskLevels: {
-            local: 'distribution_politician_risk_levels.csv',
-            description: 'Aggregate risk level distribution'
-        },
-        annualBallots: {
-            local: 'voting/distribution_annual_ballots.csv',
-            description: 'Annual ballot/vote counts'
-        },
-        crisisResilience: {
-            local: 'risk/distribution_crisis_resilience.csv',
-            description: 'Coalition stability/resilience scores'
-        },
-        partyPerformance: {
-            local: 'party/distribution_party_performance.csv',
-            description: 'Party metrics (docs, motions, performance level)'
-        },
-        partyMetrics: {
-            local: 'party/view_party_performance_metrics_sample.csv',
-            description: 'Full party metrics with win rate, rebel rate, absence rate'
-        },
-        partyMomentum: {
-            local: 'party/distribution_party_momentum.csv',
-            description: 'Party trend direction and stability'
-        },
-        partyMembers: {
-            local: 'party/distribution_annual_party_members.csv',
-            description: 'Annual party membership counts'
-        },
-        influenceMetrics: {
-            local: 'politician/view_riksdagen_politician_influence_metrics_sample.csv',
-            description: 'MP influence scores and network connections'
-        },
-        riskSummary: {
-            local: 'politician/view_politician_risk_summary_sample.csv',
-            description: 'MP risk scores and assessments'
-        },
-        committeeProductivity: {
-            local: 'committee/distribution_committee_productivity.csv',
-            description: 'Committee productivity and member counts'
-        },
-        committeeActivity: {
-            local: 'committee/distribution_committee_activity.csv',
-            description: 'Committee document counts'
-        },
-        partyEffectiveness: {
-            local: 'party/distribution_party_effectiveness_trends.csv',
-            description: 'Party effectiveness trends with win rate'
-        },
-        electionForecast: {
-            local: 'election/election_forecast.csv',
-            description: 'Election 2026 seat predictions per party'
-        },
-        coalitionScenarios: {
-            local: 'election/coalition_scenarios.csv',
-            description: 'Coalition scenario probability modeling'
-        },
-        coalitionAlignment: {
-            local: 'party/distribution_coalition_alignment.csv',
-            description: 'Real party-pair voting alignment rates'
-        },
-        genderByParty: {
-            local: 'party/distribution_gender_by_party.csv',
-            description: 'Gender distribution per party'
-        },
-        experienceByParty: {
-            local: 'party/distribution_experience_by_party.csv',
-            description: 'Experience levels per party'
-        },
-        ministryEffectiveness: {
-            local: 'ministry/distribution_ministry_effectiveness.csv',
-            description: 'Ministry effectiveness assessments'
-        },
-        annualDocTypes: {
-            local: 'voting/distribution_annual_document_types.csv',
-            description: 'Annual document type counts'
-        },
-        decisionTrends: {
-            local: 'voting/distribution_decision_trends.csv',
-            description: 'Decision approval trends over time'
-        },
-        electionRegions: {
-            local: 'election/distribution_election_regions.csv',
-            description: 'MPs per election region'
-        },
-        governmentRoles: {
-            local: 'view_riksdagen_goverment_role_member_sample.csv',
-            description: 'Government minister role assignments'
-        },
-        riskEvolution: {
-            local: 'distribution_risk_evolution_temporal.csv',
-            description: 'Risk score changes over time'
-        },
-        behavioralPatterns: {
-            local: 'party/distribution_behavioral_patterns_by_party.csv',
-            description: 'Behavioral risk patterns per party'
-        }
-    };
     /**
      * Parse CSV text into array of objects using header row as keys.
      * @param csvText - Raw CSV text
      * @returns Parsed rows
      */
     parseCSV(csvText) {
-        const lines = csvText.trim().split('\n');
-        if (lines.length < 2)
-            return [];
-        const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
-        const rows = [];
-        for (let i = 1; i < lines.length; i++) {
-            const line = lines[i].trim();
-            if (!line)
-                continue;
-            // Simple CSV parsing (handles basic quoting)
-            const values = [];
-            let current = '';
-            let inQuotes = false;
-            for (let j = 0; j < line.length; j++) {
-                const ch = line[j];
-                if (ch === '"') {
-                    inQuotes = !inQuotes;
-                }
-                else if (ch === ',' && !inQuotes) {
-                    values.push(current.trim());
-                    current = '';
-                }
-                else {
-                    current += ch;
-                }
-            }
-            values.push(current.trim());
-            const row = {};
-            headers.forEach((h, idx) => {
-                const val = values[idx] || '';
-                const num = Number(val);
-                row[h] = val !== '' && !isNaN(num) ? num : val;
-            });
-            rows.push(row);
-        }
-        return rows;
+        return parseCSVFn(csvText);
     }
     /**
      * Load CSV with local-first fallback.
@@ -217,621 +75,51 @@ export class CIADataLoader {
      * @returns Parsed CSV rows
      */
     async loadCSV(localPath, fallbackPath) {
-        const urls = [
-            `${this.csvBaseURL}${localPath}`
-        ];
-        if (fallbackPath) {
-            urls.push(`${this.fallbackURL}${fallbackPath}`);
-        }
-        for (const url of urls) {
-            try {
-                const response = await fetch(url);
-                if (!response.ok)
-                    continue;
-                const text = await response.text();
-                const rows = this.parseCSV(text);
-                if (rows.length > 0)
-                    return rows;
-            }
-            catch (e) {
-                const message = e instanceof Error ? e.message : String(e);
-                console.warn(`Failed to load CSV from ${url}:`, message);
-            }
-        }
-        console.warn(`No data loaded for ${localPath}`);
-        return [];
+        return loadCSVFn(this.csvBaseURL, this.fallbackURL, localPath, fallbackPath);
     }
-    /**
-     * Build overview dashboard from CSV sources.
-     * Replaces overview-dashboard.json.
-     */
+    /** Build overview dashboard from CSV sources. Replaces overview-dashboard.json. */
     async loadOverviewDashboard() {
-        const [personStatus, riskByParty, riskLevels, annualBallots, resilience] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.personStatus.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.riskByParty.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.riskLevels.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.annualBallots.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.crisisResilience.local)
-        ]);
-        // Count active MPs
-        const activeRow = personStatus.find(r => r.status === 'Tjänstgörande riksdagsledamot');
-        const totalMPs = activeRow ? activeRow.person_count : 349;
-        // Count unique parties from risk data (only real riksdag parties)
-        const riksdagParties = CIADataLoader.RIKSDAG_PARTIES;
-        const partiesInData = new Set(riskByParty.map(r => r.party).filter(p => riksdagParties.includes(p)));
-        const totalParties = partiesInData.size || 8;
-        // Risk alerts from risk_by_party
-        const highRisk = riskByParty.filter(r => r.risk_level === 'HIGH');
-        const medRisk = riskByParty.filter(r => r.risk_level === 'MEDIUM');
-        const lowRisk = riskByParty.filter(r => r.risk_level === 'LOW');
-        const critical = highRisk.reduce((sum, r) => sum + (r.politician_count || 0), 0);
-        const major = medRisk.reduce((sum, r) => sum + (r.politician_count || 0), 0);
-        const minor = lowRisk.reduce((sum, r) => sum + (r.politician_count || 0), 0);
-        // Total risk rules from risk levels
-        const totalRiskRules = riskLevels.length > 0
-            ? riskLevels.reduce((sum, r) => sum + (r.politician_count || 0), 0)
-            : 45;
-        // Latest year ballot activity
-        const latestBallot = annualBallots.length > 0
-            ? annualBallots[annualBallots.length - 1]
-            : {};
-        // Coalition stability from resilience scores (Tidö = M, KD, L, SD)
-        const tidoParties = ['M', 'KD', 'L', 'SD'];
-        const tidoResilience = resilience.filter(r => tidoParties.includes(r.party));
-        const avgResilience = tidoResilience.length > 0
-            ? Math.round(tidoResilience.reduce((s, r) => s + (r.avg_resilience_score || 0), 0) / tidoResilience.length)
-            : 72;
-        return {
-            title: 'Swedish Riksdag Overview Dashboard',
-            description: 'Live intelligence from CIA PostgreSQL database exports',
-            lastUpdated: new Date().toISOString(),
-            keyMetrics: {
-                totalMPs,
-                totalParties,
-                totalRiskRules,
-                governmentCoalition: 'Tidö Agreement',
-                coalitionSeats: 176,
-                oppositionSeats: 173,
-                majorityMargin: 1
-            },
-            riskAlerts: {
-                critical,
-                major,
-                minor,
-                last90Days: { critical, major, minor }
-            },
-            parliamentActivity: {
-                votesLastMonth: latestBallot.total_votes || 0,
-                documentsProcessed: latestBallot.unique_ballots || 0,
-                motionsSubmitted: 0,
-                committeeMeetings: 0
-            },
-            coalitionStability: {
-                stabilityScore: avgResilience,
-                riskLevel: avgResilience >= 70 ? 'moderate' : 'high',
-                defectionProbability: 100 - avgResilience,
-                ideologicalTension: avgResilience < 60 ? 'high' : 'moderate'
-            },
-            dataQuality: {
-                completeness: 98.5,
-                lastDataSync: new Date().toISOString(),
-                coverage: '50+ years (1971-2026)'
-            },
-            _source: 'csv'
-        };
+        return loadOverviewDashboard(this.loadCSVFn);
     }
-    /**
-     * Build election analysis from CSV sources.
-     * Replaces election-analysis.json.
-     */
+    /** Build election analysis from CSV sources. Replaces election-analysis.json. */
     async loadElectionAnalysis() {
-        const [forecastRows, scenarioRows] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.electionForecast.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.coalitionScenarios.local)
-        ]);
-        const toFiniteNumber = (value) => {
-            if (typeof value === 'number' && Number.isFinite(value))
-                return value;
-            if (typeof value === 'string' && value.trim() !== '') {
-                const num = Number(value);
-                if (Number.isFinite(num))
-                    return num;
-            }
-            return undefined;
-        };
-        const toBoolean = (value) => {
-            if (typeof value === 'boolean')
-                return value;
-            if (typeof value === 'string') {
-                const normalized = value.trim().toLowerCase();
-                if (normalized === 'true')
-                    return true;
-                if (normalized === 'false')
-                    return false;
-            }
-            return undefined;
-        };
-        const parties = forecastRows.flatMap(r => {
-            const name = String(r.name ?? '').trim();
-            const currentSeats = toFiniteNumber(r.currentSeats);
-            const predictedSeats = toFiniteNumber(r.predictedSeats);
-            const change = toFiniteNumber(r.change);
-            const voteShare = toFiniteNumber(r.voteShare);
-            if (!name || currentSeats === undefined || predictedSeats === undefined || change === undefined || voteShare === undefined) {
-                return [];
-            }
-            const confidenceMin = toFiniteNumber(r.confidenceMin);
-            const confidenceMax = toFiniteNumber(r.confidenceMax);
-            return [{
-                    name,
-                    currentSeats,
-                    predictedSeats,
-                    change,
-                    voteShare,
-                    confidenceInterval: confidenceMin !== undefined && confidenceMax !== undefined
-                        ? { min: confidenceMin, max: confidenceMax }
-                        : undefined
-                }];
-        });
-        const coalitionScenarios = scenarioRows.flatMap(r => {
-            const name = String(r.name ?? '').trim();
-            const probability = toFiniteNumber(r.probability);
-            const totalSeats = toFiniteNumber(r.totalSeats);
-            const majority = toBoolean(r.majority);
-            const riskLevel = String(r.riskLevel ?? '').trim();
-            const composition = String(r.composition ?? '')
-                .split(',')
-                .map(s => s.trim())
-                .filter(Boolean);
-            if (!name ||
-                probability === undefined ||
-                totalSeats === undefined ||
-                majority === undefined ||
-                !riskLevel ||
-                composition.length === 0) {
-                return [];
-            }
-            return [{
-                    name,
-                    probability,
-                    composition,
-                    totalSeats,
-                    majority,
-                    riskLevel
-                }];
-        });
-        return {
-            forecast: { parties },
-            coalitionScenarios,
-            keyFactors: [
-                'Economic conditions',
-                'Immigration policy',
-                'Climate change priorities',
-                'Healthcare reform',
-                'NATO membership impact'
-            ],
-            electionDate: '2026-09-13'
-        };
+        return loadElectionAnalysis(this.loadCSVFn);
     }
-    /**
-     * Build party performance from CSV sources.
-     * Replaces party-performance.json.
-     */
+    /** Build party performance from CSV sources. Replaces party-performance.json. */
     async loadPartyPerformance() {
-        const [performance, metrics, momentum] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.partyPerformance.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.partyMetrics.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.partyMomentum.local)
-        ]);
-        // Only include real riksdag parties
-        const riksdagParties = CIADataLoader.RIKSDAG_PARTIES;
-        const activePerformance = performance.filter(p => riksdagParties.includes(p.party));
-        // Build a lookup from the detailed metrics
-        const metricsMap = {};
-        metrics.forEach(m => {
-            if (riksdagParties.includes(m.party)) {
-                metricsMap[m.party] = m;
-            }
-        });
-        // Get latest momentum per party
-        const latestMomentum = {};
-        momentum
-            .filter(m => riksdagParties.includes(m.party))
-            .forEach(m => {
-            const party = m.party;
-            if (!latestMomentum[party] ||
-                m.year > latestMomentum[party].year ||
-                (m.year === latestMomentum[party].year &&
-                    m.quarter > latestMomentum[party].quarter)) {
-                latestMomentum[party] = m;
-            }
-        });
-        // Known seat counts (from 2022 election results)
-        const seatMap = {
-            S: 107, SD: 73, M: 68, C: 24, V: 24, KD: 19, L: 16, MP: 18
-        };
-        const parties = activePerformance.map(p => {
-            const party = p.party;
-            const m = metricsMap[party] || {};
-            const mom = latestMomentum[party] || {};
-            return {
-                id: party,
-                partyName: p.party_name || party,
-                shortName: party,
-                metrics: {
-                    seats: seatMap[party] || 0,
-                    voteShare: 0,
-                    memberCount: p.active_members || 0,
-                    documentsAuthored: p.documents_last_year || 0,
-                    motionsSubmitted: p.motions_last_year || 0,
-                    successRate: m.avg_win_rate || 0
-                },
-                voting: {
-                    totalVotes: m.total_votes_last_year || 0,
-                    cohesionScore: m.avg_participation_rate || 0,
-                    rebellionRate: m.avg_rebel_rate || 0
-                },
-                trends: {
-                    supportTrend: (mom.trend_direction || 'stable').toLowerCase(),
-                    activityTrend: (mom.stability_classification || 'stable').toLowerCase(),
-                    performanceLevel: m.performance_level || p.performance_level || ''
-                },
-                _source: 'csv'
-            };
-        });
-        // Sort by seats descending
-        parties.sort((a, b) => (b.metrics.seats || 0) - (a.metrics.seats || 0));
-        return {
-            title: 'Party Performance Dashboard',
-            description: 'Live party data from CIA PostgreSQL database exports',
-            lastUpdated: new Date().toISOString(),
-            parties,
-            _source: 'csv'
-        };
+        return loadPartyPerformance(this.loadCSVFn);
     }
-    /**
-     * Build top 10 influential MPs from CSV sources.
-     * Replaces top10-influential-mps.json.
-     */
+    /** Build top 10 influential MPs from CSV sources. Replaces top10-influential-mps.json. */
     async loadTop10Influential() {
-        const [influence, riskSummary] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.influenceMetrics.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.riskSummary.local)
-        ]);
-        // Build risk lookup by person_id
-        const riskMap = {};
-        riskSummary.forEach(r => {
-            riskMap[r.person_id] = r;
-        });
-        // Sort by network_connections descending, take top 10
-        const sorted = [...influence]
-            .filter(mp => mp.network_connections > 0)
-            .sort((a, b) => (b.network_connections || 0) - (a.network_connections || 0))
-            .slice(0, 10);
-        const rankings = sorted.map((mp, idx) => {
-            const risk = riskMap[mp.person_id] || {};
-            return {
-                rank: idx + 1,
-                id: String(mp.person_id),
-                firstName: mp.first_name || '',
-                lastName: mp.last_name || '',
-                party: mp.party || '',
-                role: mp.influence_classification
-                    ? mp.influence_classification
-                        .replace(/_/g, ' ')
-                        .toLowerCase()
-                        .replace(/\b\w/g, (c) => c.toUpperCase())
-                    : '',
-                influenceScore: mp.network_connections || 0,
-                networkConnections: mp.network_connections || 0,
-                brokerClassification: mp.broker_classification || '',
-                riskLevel: risk.risk_level || '',
-                riskScore: risk.risk_score || 0,
-                _source: 'csv'
-            };
-        });
-        return {
-            title: 'Top 10 Most Influential MPs',
-            description: 'Network analysis from CIA politician influence metrics view',
-            lastUpdated: new Date().toISOString(),
-            methodology: 'Ranked by network_connections from view_riksdagen_politician_influence_metrics',
-            rankings,
-            _source: 'csv'
-        };
+        return loadTop10Influential(this.loadCSVFn);
     }
-    /**
-     * Build committee network from CSV sources.
-     * Replaces committee-network.json.
-     * Filters out INACTIVE committees and deduplicates by name.
-     */
+    /** Build committee network from CSV sources. Replaces committee-network.json. */
     async loadCommitteeNetwork() {
-        const [productivity, activity] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.committeeProductivity.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.committeeActivity.local)
-        ]);
-        // Build activity lookup by org code
-        const activityMap = {};
-        activity.forEach(a => {
-            activityMap[a.org] = a.document_count || 0;
-        });
-        // Use class-level committee name-to-org-code mapping
-        const nameToOrgCode = CIADataLoader.COMMITTEE_ORG_CODES;
-        // Deduplicate committees by name, keeping the entry with the most data
-        const bestByName = {};
-        productivity.forEach(c => {
-            const name = c.committee_name;
-            if (!name)
-                return;
-            const existing = bestByName[name];
-            if (!existing || c.total_documents > existing.total_documents ||
-                (c.total_documents === existing.total_documents &&
-                    c.total_members > existing.total_members)) {
-                bestByName[name] = c;
-            }
-        });
-        // Normalize committee metrics first so filtering and rendering use one consistent source.
-        const committees = Object.values(bestByName)
-            .map(c => {
-            const name = c.committee_name;
-            const code = nameToOrgCode[name] || name.substring(0, 3).toUpperCase();
-            const totalDocuments = c.total_documents || 0;
-            const activityDocs = activityMap[code] || 0;
-            const documentsProcessed = Math.max(totalDocuments, activityDocs);
-            const productivityLevel = c.productivity_level || '';
-            return {
-                id: code,
-                name,
-                memberCount: c.total_members || 0,
-                influenceScore: c.docs_per_member
-                    ? Math.round(c.docs_per_member * 100)
-                    : 0,
-                documentsProcessed,
-                productivityLevel,
-                meetingsPerYear: documentsProcessed > 0
-                    ? Math.round(documentsProcessed / CIADataLoader.COMMITTEE_DOCS_PER_MEETING_ESTIMATE)
-                    : 0,
-                keyIssues: [productivityLevel || 'N/A'],
-                _source: 'csv'
-            };
-        })
-            .filter(c => {
-            // Keep real committees only; skip generic node and inactive rows with no measured output.
-            return (c.name !== 'Riksdagen' &&
-                c.memberCount > 0 &&
-                (c.productivityLevel !== 'INACTIVE' || c.documentsProcessed > 0));
-        })
-            .sort((a, b) => b.documentsProcessed - a.documentsProcessed);
-        // Build simple network graph from committees
-        const nodes = committees.map(c => ({
-            id: c.id,
-            name: c.name,
-            size: c.influenceScore
-        }));
-        // Create edges between committees that share similar productivity levels
-        const edges = [];
-        for (let i = 0; i < committees.length; i++) {
-            for (let j = i + 1; j < committees.length && edges.length < 10; j++) {
-                if (committees[i].productivityLevel === committees[j].productivityLevel &&
-                    committees[i].productivityLevel !== 'INACTIVE') {
-                    edges.push({
-                        source: committees[i].id,
-                        target: committees[j].id,
-                        weight: Math.min(committees[i].documentsProcessed, committees[j].documentsProcessed),
-                        type: 'productivity_similarity'
-                    });
-                }
-            }
-        }
-        return {
-            title: 'Committee Network Analysis',
-            description: 'Committee data from CIA committee productivity view',
-            lastUpdated: new Date().toISOString(),
-            committees,
-            networkGraph: { nodes, edges },
-            crossCommitteeMPs: [],
-            _source: 'csv'
-        };
+        return loadCommitteeNetwork(this.loadCSVFn);
     }
     /**
      * Build voting patterns from CSV sources.
-     * Uses real coalition alignment data for the agreement matrix
-     * and party effectiveness trends for win rates.
+     * Uses real coalition alignment data for the agreement matrix when present;
+     * otherwise falls back to win-rate similarity.
      */
     async loadVotingPatterns() {
-        const [coalitionAlignment, effectiveness, riskByParty] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.coalitionAlignment.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.partyEffectiveness.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.riskByParty.local)
-        ]);
-        const riksdagParties = CIADataLoader.RIKSDAG_PARTIES;
-        const labels = riksdagParties;
-        const partyNames = [
-            'Social Democrats', 'Moderates', 'Sweden Democrats', 'Centre',
-            'Left', 'Christian Democrats', 'Liberals', 'Green'
-        ];
-        // Build agreement matrix from real coalition alignment data
-        const alignmentLookup = {};
-        coalitionAlignment
-            .filter(r => riksdagParties.includes(r.party1) && riksdagParties.includes(r.party2))
-            .forEach(r => {
-            const key1 = `${r.party1}:${r.party2}`;
-            const key2 = `${r.party2}:${r.party1}`;
-            const rate = Math.round((r.alignment_rate || 0) * 100);
-            alignmentLookup[key1] = rate;
-            alignmentLookup[key2] = rate;
-        });
-        // If real alignment data is available, use it; otherwise fall back to effectiveness-based
-        const hasAlignmentData = Object.keys(alignmentLookup).length > 0;
-        let agreementMatrix;
-        if (hasAlignmentData) {
-            agreementMatrix = labels.map(p1 => labels.map(p2 => {
-                if (p1 === p2)
-                    return 100;
-                return alignmentLookup[`${p1}:${p2}`] ?? 50;
-            }));
-        }
-        else {
-            // Fallback: build from win rate similarity
-            const latestWinRate = {};
-            effectiveness
-                .filter(e => riksdagParties.includes(e.party))
-                .forEach(e => {
-                const party = e.party;
-                if (!latestWinRate[party] ||
-                    e.year > latestWinRate[party].year ||
-                    (e.year === latestWinRate[party].year &&
-                        e.quarter > latestWinRate[party].quarter)) {
-                    latestWinRate[party] = e;
-                }
-            });
-            agreementMatrix = labels.map(p1 => {
-                const wr1 = latestWinRate[p1] ? latestWinRate[p1].avg_win_rate : 50;
-                return labels.map(p2 => {
-                    if (p1 === p2)
-                        return 100;
-                    const wr2 = latestWinRate[p2] ? latestWinRate[p2].avg_win_rate : 50;
-                    return Math.max(0, Math.round(100 - Math.abs(wr1 - wr2)));
-                });
-            });
-        }
-        // Rebellion tracking from risk data (HIGH risk ~ rebellious)
-        const rebellionTracking = riksdagParties
-            .map(party => {
-            const partyRisks = riskByParty.filter(r => r.party === party);
-            const highRisk = partyRisks.find(r => r.risk_level === 'HIGH');
-            const total = partyRisks.reduce((s, r) => s + (r.politician_count || 0), 0);
-            const highCount = highRisk ? highRisk.politician_count : 0;
-            const rebellionRate = total > 0 ? Math.round((highCount / total) * 100 * 10) / 10 : 0;
-            return {
-                party,
-                rebellionRate,
-                trend: rebellionRate > 25 ? 'increasing' : rebellionRate > 15 ? 'stable' : 'decreasing'
-            };
-        })
-            .filter(r => r.rebellionRate > 0);
-        return {
-            title: 'Voting Patterns Analysis',
-            description: hasAlignmentData
-                ? 'Real coalition alignment data from CIA voting analysis'
-                : 'Derived from CIA party effectiveness trends and risk data',
-            lastUpdated: new Date().toISOString(),
-            analysisPeriod: '2022-2026',
-            votingMatrix: { labels, partyNames, agreementMatrix },
-            keyIssues: [],
-            rebellionTracking,
-            _source: 'csv'
-        };
+        return loadVotingPatterns(this.loadCSVFn);
     }
-    /**
-     * Build ministry dashboard from CSV sources.
-     */
+    /** Build ministry dashboard from CSV sources. */
     async loadMinistryDashboard() {
-        const rows = await this.loadCSV(CIADataLoader.CSV_SOURCES.ministryEffectiveness.local);
-        const ministries = rows
-            .filter(r => r.ministry_name && r.documents_produced > 0)
-            .map(r => ({
-            name: r.ministry_name,
-            effectiveness: r.effectiveness_assessment || '',
-            documentsProduced: r.documents_produced || 0,
-            governmentBills: r.government_bills || 0,
-            year: r.year || 0,
-            quarter: r.quarter || 0
-        }))
-            .sort((a, b) => b.documentsProduced - a.documentsProduced);
-        return {
-            title: 'Ministry Performance',
-            description: 'Ministry effectiveness from CIA database exports',
-            lastUpdated: new Date().toISOString(),
-            ministries,
-            _source: 'csv'
-        };
+        return loadMinistryDashboard(this.loadCSVFn);
     }
-    /**
-     * Build demographics dashboard from CSV sources.
-     */
+    /** Build demographics dashboard from CSV sources. */
     async loadDemographics() {
-        const riksdagParties = CIADataLoader.RIKSDAG_PARTIES;
-        const [genderRows, experienceRows] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.genderByParty.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.experienceByParty.local)
-        ]);
-        const genderByParty = genderRows
-            .filter(r => riksdagParties.includes(r.party))
-            .map(r => ({
-            party: r.party,
-            gender: r.gender,
-            count: r.count || 0
-        }));
-        const experienceByParty = experienceRows
-            .filter(r => riksdagParties.includes(r.party))
-            .map(r => ({
-            party: r.party,
-            experienceLevel: r.experience_level || '',
-            politicianCount: r.politician_count || 0
-        }));
-        return {
-            title: 'Parliamentary Demographics',
-            description: 'Gender and experience distribution from CIA database exports',
-            lastUpdated: new Date().toISOString(),
-            genderByParty,
-            experienceByParty,
-            _source: 'csv'
-        };
+        return loadDemographics(this.loadCSVFn);
     }
-    /**
-     * Build document activity dashboard from CSV sources.
-     */
+    /** Build document activity dashboard from CSV sources. */
     async loadDocumentActivity() {
-        const [docTypeRows, decisionRows] = await Promise.all([
-            this.loadCSV(CIADataLoader.CSV_SOURCES.annualDocTypes.local),
-            this.loadCSV(CIADataLoader.CSV_SOURCES.decisionTrends.local)
-        ]);
-        const documentTypes = docTypeRows
-            .filter(r => r.doc_count > 0)
-            .map(r => ({
-            year: r.year || 0,
-            documentType: r.document_type || '',
-            docCount: r.doc_count || 0
-        }));
-        const decisionTrends = decisionRows
-            .filter(r => r.decision_count > 0)
-            .map(r => ({
-            year: r.year || 0,
-            month: r.month || 0,
-            decisionCount: r.decision_count || 0,
-            approvedDecisions: r.approved_decisions || 0,
-            rejectedDecisions: r.rejected_decisions || 0,
-            approvalRate: r.approval_rate || 0
-        }));
-        return {
-            title: 'Parliamentary Document Activity',
-            description: 'Document production and decision trends from CIA database exports',
-            lastUpdated: new Date().toISOString(),
-            documentTypes,
-            decisionTrends,
-            _source: 'csv'
-        };
+        return loadDocumentActivity(this.loadCSVFn);
     }
-    /**
-     * Build risk evolution dashboard from CSV sources.
-     */
+    /** Build risk evolution dashboard from CSV sources. */
     async loadRiskEvolution() {
-        const rows = await this.loadCSV(CIADataLoader.CSV_SOURCES.riskEvolution.local);
-        const entries = rows
-            .filter(r => r.politician_count > 0)
-            .map(r => ({
-            period: r.assessment_period || '',
-            severity: r.risk_severity || '',
-            politicianCount: r.politician_count || 0,
-            avgRiskScore: r.avg_risk_score || 0
-        }));
-        return {
-            title: 'Risk Score Evolution',
-            description: 'Temporal risk score changes from CIA database exports',
-            lastUpdated: new Date().toISOString(),
-            entries,
-            _source: 'csv'
-        };
+        return loadRiskEvolution(this.loadCSVFn);
     }
     /**
      * Load all data in parallel.