pyconvexity 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

pyconvexity/data/loaders/cache.py

@@ -0,0 +1,212 @@
+"""
+Caching functionality for PyConvexity data operations.
+
+This module handles caching of processed datasets to improve performance.
+"""
+
+import pandas as pd
+import hashlib
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional
+import logging
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+class DataCache:
+    """Manages caching of processed datasets."""
+    
+    def __init__(self, cache_dir: Optional[str] = None):
+        """
+        Initialize the cache manager.
+        
+        Args:
+            cache_dir: Directory to store cache files. Defaults to 'data/cache'
+        """
+        if cache_dir is None:
+            cache_dir = "data/cache"
+        
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Cache metadata file
+        self.metadata_file = self.cache_dir / "cache_metadata.json"
+        self._load_metadata()
+    
+    def _load_metadata(self):
+        """Load cache metadata from file."""
+        if self.metadata_file.exists():
+            try:
+                with open(self.metadata_file, 'r') as f:
+                    self.metadata = json.load(f)
+            except (json.JSONDecodeError, FileNotFoundError):
+                self.metadata = {}
+        else:
+            self.metadata = {}
+    
+    def _save_metadata(self):
+        """Save cache metadata to file."""
+        with open(self.metadata_file, 'w') as f:
+            json.dump(self.metadata, f, indent=2)
+    
+    def _get_cache_key(self, dataset_name: str, filters: Dict[str, Any]) -> str:
+        """Generate a unique cache key for a dataset and filters combination."""
+        # Create a hash of the filters
+        filters_str = json.dumps(filters, sort_keys=True)
+        filters_hash = hashlib.md5(filters_str.encode()).hexdigest()
+        
+        return f"{dataset_name}_{filters_hash}"
+    
+    def _get_cache_file_path(self, cache_key: str) -> Path:
+        """Get the file path for a cache key."""
+        return self.cache_dir / f"{cache_key}.parquet"
+    
+    def get_cached_data(
+        self, 
+        dataset_name: str, 
+        filters: Dict[str, Any]
+    ) -> Optional[pd.DataFrame]:
+        """
+        Retrieve cached data if available and not expired.
+        
+        Args:
+            dataset_name: Name of the dataset
+            filters: Filters applied to the dataset
+            
+        Returns:
+            pandas.DataFrame or None: Cached data if available and valid
+        """
+        cache_key = self._get_cache_key(dataset_name, filters)
+        cache_file = self._get_cache_file_path(cache_key)
+        
+        # Check if cache file exists
+        if not cache_file.exists():
+            return None
+        
+        # Check if cache entry exists in metadata
+        if cache_key not in self.metadata:
+            # Clean up orphaned cache file
+            cache_file.unlink(missing_ok=True)
+            return None
+        
+        # Check if cache is expired (default: 7 days)
+        cache_info = self.metadata[cache_key]
+        created_time = datetime.fromisoformat(cache_info['created'])
+        max_age = timedelta(days=cache_info.get('max_age_days', 7))
+        
+        if datetime.now() - created_time > max_age:
+            logger.info(f"Cache expired for '{dataset_name}', removing...")
+            self._remove_cache_entry(cache_key)
+            return None
+        
+        # Load cached data
+        try:
+            cached_data = pd.read_parquet(cache_file)
+            logger.info(f"Loaded cached data for '{dataset_name}' ({len(cached_data)} rows)")
+            return cached_data
+        except Exception as e:
+            logger.warning(f"Failed to load cached data for '{dataset_name}': {e}")
+            self._remove_cache_entry(cache_key)
+            return None
+    
+    def cache_data(
+        self, 
+        dataset_name: str, 
+        data: pd.DataFrame, 
+        filters: Dict[str, Any],
+        max_age_days: int = 7
+    ):
+        """
+        Cache processed data.
+        
+        Args:
+            dataset_name: Name of the dataset
+            data: Processed pandas DataFrame
+            filters: Filters applied to the dataset
+            max_age_days: Maximum age of cache in days
+        """
+        cache_key = self._get_cache_key(dataset_name, filters)
+        cache_file = self._get_cache_file_path(cache_key)
+        
+        # Save data to parquet file
+        data.to_parquet(cache_file, index=False)
+        
+        # Update metadata
+        self.metadata[cache_key] = {
+            'dataset_name': dataset_name,
+            'filters': filters,
+            'created': datetime.now().isoformat(),
+            'max_age_days': max_age_days,
+            'rows': len(data),
+            'columns': list(data.columns)
+        }
+        
+        self._save_metadata()
+        logger.info(f"Cached data for '{dataset_name}' ({len(data)} rows)")
+    
+    def _remove_cache_entry(self, cache_key: str):
+        """Remove a cache entry and its file."""
+        cache_file = self._get_cache_file_path(cache_key)
+        cache_file.unlink(missing_ok=True)
+        
+        if cache_key in self.metadata:
+            del self.metadata[cache_key]
+            self._save_metadata()
+    
+    def clear_cache(self, dataset_name: Optional[str] = None):
+        """
+        Clear cache entries.
+        
+        Args:
+            dataset_name: If provided, only clear cache for this dataset
+        """
+        keys_to_remove = []
+        
+        for cache_key, info in self.metadata.items():
+            if dataset_name is None or info['dataset_name'] == dataset_name:
+                keys_to_remove.append(cache_key)
+        
+        for key in keys_to_remove:
+            self._remove_cache_entry(key)
+        
+        logger.info(f"Cleared {len(keys_to_remove)} cache entries")
+    
+    def get_cache_info(self) -> Dict[str, Any]:
+        """Get information about the cache."""
+        total_size = 0
+        dataset_counts = {}
+        
+        for cache_key, info in self.metadata.items():
+            dataset_name = info['dataset_name']
+            dataset_counts[dataset_name] = dataset_counts.get(dataset_name, 0) + 1
+            
+            cache_file = self._get_cache_file_path(cache_key)
+            if cache_file.exists():
+                total_size += cache_file.stat().st_size
+        
+        return {
+            'total_entries': len(self.metadata),
+            'total_size_mb': round(total_size / (1024 * 1024), 2),
+            'dataset_counts': dataset_counts,
+            'cache_dir': str(self.cache_dir)
+        }
+    
+    def cleanup_expired_cache(self):
+        """Remove expired cache entries."""
+        expired_keys = []
+        
+        for cache_key, info in self.metadata.items():
+            created_time = datetime.fromisoformat(info['created'])
+            max_age = timedelta(days=info.get('max_age_days', 7))
+            
+            if datetime.now() - created_time > max_age:
+                expired_keys.append(cache_key)
+        
+        for key in expired_keys:
+            self._remove_cache_entry(key)
+        
+        if expired_keys:
+            logger.info(f"Cleaned up {len(expired_keys)} expired cache entries")
+        else:
+            logger.info("No expired cache entries found")

pyconvexity/data/schema/01_core_schema.sql

@@ -34,25 +34,25 @@ CREATE TABLE networks (
 CREATE INDEX idx_networks_name ON networks(name);
 CREATE INDEX idx_networks_created_at ON networks(created_at);
 
--- Network time periods - computed from time axis definition
--- This table is populated automatically based on network time axis
+-- Network time periods - optimized storage using computed timestamps
+-- Instead of storing 75k+ timestamp strings, we compute them from the time axis
+-- This reduces storage from ~3.4MB to ~24 bytes per network
 CREATE TABLE network_time_periods (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
     network_id INTEGER NOT NULL,
-    timestamp DATETIME NOT NULL,
-    period_index INTEGER NOT NULL,  -- 0-based index for array operations
+    period_count INTEGER NOT NULL,      -- Total number of periods (e.g., 8760 for hourly year)
+    start_timestamp INTEGER NOT NULL,   -- Unix timestamp of first period
+    interval_seconds INTEGER NOT NULL,  -- Seconds between periods (3600 for hourly)
+    
+    PRIMARY KEY (network_id),
     
     CONSTRAINT fk_time_periods_network 
         FOREIGN KEY (network_id) REFERENCES networks(id) ON DELETE CASCADE,
-    CONSTRAINT uq_time_period_network_index 
-        UNIQUE (network_id, period_index),
-    CONSTRAINT uq_time_period_network_timestamp 
-        UNIQUE (network_id, timestamp)
+    CONSTRAINT valid_period_count CHECK (period_count > 0),
+    CONSTRAINT valid_interval CHECK (interval_seconds > 0)
 );
 
-CREATE INDEX idx_time_periods_network ON network_time_periods(network_id);
-CREATE INDEX idx_time_periods_timestamp ON network_time_periods(timestamp);
-CREATE INDEX idx_time_periods_index ON network_time_periods(network_id, period_index);
+-- No additional indexes needed - primary key on network_id is sufficient
+-- Timestamps are computed as: start_timestamp + (period_index * interval_seconds)
 
 -- Network locks - prevents concurrent modifications
 CREATE TABLE network_locks (

pyconvexity/data/schema/02_data_metadata.sql

@@ -1,8 +1,8 @@
 -- ============================================================================
--- DATA STORAGE AND METADATA SCHEMA
--- Auxiliary tables for data storage, notes, audit logging, and analysis caching
--- Optimized for atomic operations and data management
--- Version 2.1.0
+-- DATA STORAGE AND METADATA SCHEMA (OPTIMIZED)
+-- Essential tables for data storage and solve results only
+-- Removed unused audit logging and analysis caching for efficiency
+-- Version 2.2.0 - Optimized
 -- ============================================================================
 
 -- ============================================================================
@@ -34,12 +34,9 @@ CREATE TABLE network_data_store (
         CHECK (data_format IN ('json', 'parquet', 'csv', 'binary', 'text', 'yaml', 'toml'))
 );
 
--- Optimized indexes for data retrieval
+-- Minimal indexes for data retrieval
 CREATE INDEX idx_datastore_network ON network_data_store(network_id);
 CREATE INDEX idx_datastore_category ON network_data_store(network_id, category);
-CREATE INDEX idx_datastore_name ON network_data_store(network_id, category, name);
-CREATE INDEX idx_datastore_created_at ON network_data_store(created_at);
-CREATE INDEX idx_datastore_format ON network_data_store(data_format);
 
 -- ============================================================================
 -- DOCUMENTATION AND NOTES
@@ -66,11 +63,8 @@ CREATE TABLE network_notes (
         CHECK (note_type IN ('note', 'todo', 'warning', 'info', 'doc'))
 );
 
+-- Minimal indexes for notes
 CREATE INDEX idx_notes_network ON network_notes(network_id);
-CREATE INDEX idx_notes_title ON network_notes(network_id, title);
-CREATE INDEX idx_notes_type ON network_notes(note_type);
-CREATE INDEX idx_notes_priority ON network_notes(priority);
-CREATE INDEX idx_notes_created_at ON network_notes(created_at);
 
 -- Component-specific notes and documentation
 CREATE TABLE component_notes (
@@ -93,139 +87,13 @@ CREATE TABLE component_notes (
         CHECK (note_type IN ('note', 'todo', 'warning', 'info', 'doc'))
 );
 
+-- Minimal indexes for component notes
 CREATE INDEX idx_component_notes_component ON component_notes(component_id);
-CREATE INDEX idx_component_notes_title ON component_notes(component_id, title);
-CREATE INDEX idx_component_notes_type ON component_notes(note_type);
-CREATE INDEX idx_component_notes_priority ON component_notes(priority);
-CREATE INDEX idx_component_notes_created_at ON component_notes(created_at);
-
--- ============================================================================
--- AUDIT AND CHANGE TRACKING
--- ============================================================================
-
--- Comprehensive audit log for tracking all database changes
-CREATE TABLE audit_log (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    network_id INTEGER,
-    table_name TEXT NOT NULL,
-    record_id INTEGER,
-    operation TEXT NOT NULL CHECK (operation IN ('INSERT', 'UPDATE', 'DELETE')),
-    
-    -- Change data
-    old_values TEXT,                 -- JSON of old values for UPDATE/DELETE
-    new_values TEXT,                 -- JSON of new values for INSERT/UPDATE
-    change_summary TEXT,             -- Human-readable summary of changes
-    affected_fields TEXT,            -- JSON array of changed field names
-    
-    -- Context and metadata
-    user_id TEXT,
-    session_id TEXT,
-    client_info TEXT,                -- Application version, client type, etc.
-    transaction_id TEXT,             -- For grouping related changes
-    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-    
-    -- Performance metrics
-    execution_time_ms INTEGER,       -- Time taken for the operation
-    
-    CONSTRAINT fk_audit_network 
-        FOREIGN KEY (network_id) REFERENCES networks(id) ON DELETE CASCADE
-);
-
--- Optimized indexes for audit queries
-CREATE INDEX idx_audit_network ON audit_log(network_id);
-CREATE INDEX idx_audit_table ON audit_log(table_name);
-CREATE INDEX idx_audit_timestamp ON audit_log(timestamp);
-CREATE INDEX idx_audit_operation ON audit_log(operation);
-CREATE INDEX idx_audit_record ON audit_log(table_name, record_id);
-CREATE INDEX idx_audit_user ON audit_log(user_id);
-CREATE INDEX idx_audit_session ON audit_log(session_id);
-CREATE INDEX idx_audit_transaction ON audit_log(transaction_id);
-
--- Change summary view for recent activity
-CREATE VIEW recent_changes AS
-SELECT 
-    al.id,
-    al.network_id,
-    n.name as network_name,
-    al.table_name,
-    al.record_id,
-    al.operation,
-    al.change_summary,
-    al.user_id,
-    al.timestamp,
-    al.execution_time_ms
-FROM audit_log al
-LEFT JOIN networks n ON al.network_id = n.id
-ORDER BY al.timestamp DESC
-LIMIT 1000;
-
--- ============================================================================
--- NETWORK ANALYSIS AND RESULTS CACHING
--- ============================================================================
-
--- Cache for storing analysis results and computed data
--- This improves performance by avoiding recomputation of expensive operations
-CREATE TABLE network_analysis_cache (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    network_id INTEGER NOT NULL,
-    analysis_type TEXT NOT NULL,     -- 'optimization', 'statistics', 'validation', 'powerflow', etc.
-    analysis_key TEXT NOT NULL,      -- Unique key for this analysis (hash of inputs)
-    analysis_version TEXT,           -- Version of analysis algorithm
-    
-    -- Input tracking
-    input_hash TEXT,                 -- Hash of inputs that generated this result
-    input_summary TEXT,              -- Human-readable summary of inputs
-    dependencies TEXT,               -- JSON array of dependent data (components, attributes, etc.)
-    
-    -- Results storage
-    result_data BLOB NOT NULL,       -- Serialized results (typically JSON or Parquet)
-    result_format TEXT DEFAULT 'json', -- 'json', 'parquet', 'csv', 'binary'
-    result_summary TEXT,             -- Human-readable summary of results
-    result_size_bytes INTEGER,       -- Size of result data for management
-    
-    -- Analysis metadata
-    analysis_time_ms INTEGER,        -- Time taken for analysis
-    status TEXT DEFAULT 'completed', -- 'completed', 'failed', 'in_progress', 'stale'
-    error_message TEXT,              -- If status is 'failed'
-    warnings TEXT,                   -- JSON array of warnings
-    
-    -- Cache management
-    hit_count INTEGER DEFAULT 0,     -- Number of times this cache entry was used
-    last_accessed DATETIME DEFAULT CURRENT_TIMESTAMP,
-    expires_at DATETIME,             -- Optional expiration for cache cleanup
-    
-    -- Metadata
-    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-    created_by TEXT,
-    
-    CONSTRAINT fk_analysis_cache_network 
-        FOREIGN KEY (network_id) REFERENCES networks(id) ON DELETE CASCADE,
-    CONSTRAINT uq_analysis_cache_key 
-        UNIQUE (network_id, analysis_type, analysis_key),
-    CONSTRAINT valid_analysis_status 
-        CHECK (status IN ('completed', 'failed', 'in_progress', 'stale')),
-    CONSTRAINT valid_result_format 
-        CHECK (result_format IN ('json', 'parquet', 'csv', 'binary', 'text'))
-);
-
--- Optimized indexes for cache operations
-CREATE INDEX idx_analysis_cache_network ON network_analysis_cache(network_id);
-CREATE INDEX idx_analysis_cache_type ON network_analysis_cache(network_id, analysis_type);
-CREATE INDEX idx_analysis_cache_key ON network_analysis_cache(analysis_key);
-CREATE INDEX idx_analysis_cache_status ON network_analysis_cache(status);
-CREATE INDEX idx_analysis_cache_expires ON network_analysis_cache(expires_at);
-CREATE INDEX idx_analysis_cache_accessed ON network_analysis_cache(last_accessed);
-CREATE INDEX idx_analysis_cache_created ON network_analysis_cache(created_at);
-CREATE INDEX idx_analysis_cache_size ON network_analysis_cache(result_size_bytes);
 
 -- ============================================================================
 -- SOLVE RESULTS AND STATISTICS
 -- ============================================================================
 
--- Drop the old optimization_results table
-DROP TABLE IF EXISTS optimization_results;
-
 -- Network solve results - stores solver outputs and statistics
 -- This is where PyPSA solve results are stored after successful solves
 CREATE TABLE network_solve_results (
@@ -250,11 +118,9 @@ CREATE TABLE network_solve_results (
     FOREIGN KEY (scenario_id) REFERENCES scenarios(id) ON DELETE CASCADE
 );
 
--- Indexes for performance
+-- Minimal indexes for performance
 CREATE INDEX idx_solve_results_network ON network_solve_results(network_id);
 CREATE INDEX idx_solve_results_scenario ON network_solve_results(scenario_id);
-CREATE INDEX idx_solve_results_status ON network_solve_results(solve_status);
-CREATE INDEX idx_solve_results_solved_at ON network_solve_results(solved_at);
 
 -- ============================================================================
 -- YEAR-BASED SOLVE RESULTS
@@ -284,12 +150,9 @@ CREATE TABLE network_solve_results_by_year (
     CONSTRAINT valid_year CHECK (year >= 1900 AND year <= 2100)
 );
 
--- Indexes for performance
+-- Minimal indexes for performance
 CREATE INDEX idx_solve_results_year_network ON network_solve_results_by_year(network_id);
 CREATE INDEX idx_solve_results_year_scenario ON network_solve_results_by_year(scenario_id);
-CREATE INDEX idx_solve_results_year_year ON network_solve_results_by_year(year);
-CREATE INDEX idx_solve_results_year_network_scenario ON network_solve_results_by_year(network_id, scenario_id);
-CREATE INDEX idx_solve_results_year_created_at ON network_solve_results_by_year(created_at);
 
 -- Optional: Registry of solve type schemas for frontend introspection
 CREATE TABLE solve_type_schemas (
@@ -300,178 +163,11 @@ CREATE TABLE solve_type_schemas (
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 );
 
-/*
-SUGGESTED JSON STRUCTURE FOR results_json:
-
-For PyPSA optimization (solve_type = 'pypsa_optimization'):
-{
-  "core_summary": {
-    "total_generation_mwh": 12500.0,
-    "total_demand_mwh": 12000.0,
-    "total_cost": 1500000.0,
-    "load_factor": 0.96,
-    "unserved_energy_mwh": 0.0
-  },
-  
-  "pypsa_statistics": {
-    "energy_balance": {
-      "gas": 5000.0,
-      "wind": 7500.0,
-      "solar": 0.0
-    },
-    "supply_by_carrier": {
-      "gas": {"total_mwh": 5000.0, "capacity_factor": 0.85},
-      "wind": {"total_mwh": 7500.0, "capacity_factor": 0.42}
-    },
-    "demand_by_carrier": {
-      "electricity": 12000.0
-    },
-    "capacity_factors": {
-      "generator_gas_001": 0.85,
-      "generator_wind_001": 0.42
-    },
-    "curtailment": {
-      "wind": 250.0,
-      "solar": 0.0
-    },
-    "transmission_utilization": {
-      "line_001": 0.75,
-      "line_002": 0.32
-    }
-  },
-  
-  "custom_statistics": {
-    "emissions_by_carrier": {
-      "gas": 2500.0,
-      "wind": 0.0,
-      "solar": 0.0
-    },
-    "total_emissions_tons_co2": 2500.0,
-    "average_price_per_mwh": 125.0,
-    "peak_demand_mw": 2500.0,
-    "renewable_fraction": 0.6
-  },
-  
-  "runtime_info": {
-    "build_time_seconds": 5.2,
-    "solve_time_seconds": 45.1,
-    "result_processing_seconds": 2.3,
-    "component_count": 150,
-    "variable_count": 8760,
-    "constraint_count": 12500,
-    "memory_usage_mb": 256.5
-  },
-  
-  "solver_info": {
-    "solver_name": "highs",
-    "solver_version": "1.6.0",
-    "solver_options": {"presolve": "on", "parallel": "on"},
-    "termination_condition": "optimal",
-    "iterations": 1247,
-    "barrier_iterations": null
-  }
-}
-
-For Monte Carlo sampling (solve_type = 'monte_carlo'):
-{
-  "core_summary": {
-    "scenario_count": 1000,
-    "convergence_achieved": true,
-    "confidence_level": 0.95
-  },
-  
-  "probability_distributions": {
-    "total_cost": {
-      "mean": 1500000.0,
-      "std": 150000.0,
-      "p05": 1250000.0,
-      "p50": 1500000.0,
-      "p95": 1750000.0
-    },
-    "unserved_energy": {
-      "mean": 12.5,
-      "std": 25.2,
-      "p05": 0.0,
-      "p50": 0.0,
-      "p95": 75.0
-    }
-  },
-  
-  "sensitivity_analysis": {
-    "most_influential_parameters": [
-      {"parameter": "wind_capacity", "sensitivity": 0.85},
-      {"parameter": "fuel_price", "sensitivity": 0.72}
-    ]
-  },
-  
-  "runtime_info": {
-    "total_runtime_seconds": 3600.0,
-    "scenarios_per_second": 0.28
-  }
-}
-
-For sensitivity analysis (solve_type = 'sensitivity'):
-{
-  "core_summary": {
-    "parameters_analyzed": 15,
-    "base_case_objective": 1500000.0
-  },
-  
-  "parameter_sensitivities": {
-    "fuel_cost_gas": {
-      "sensitivity_coefficient": 0.85,
-      "objective_range": [1200000.0, 1800000.0],
-      "parameter_range": [50.0, 150.0]
-    },
-    "wind_capacity": {
-      "sensitivity_coefficient": -0.72,
-      "objective_range": [1300000.0, 1700000.0],
-      "parameter_range": [1000.0, 3000.0]
-    }
-  },
-  
-  "tornado_chart_data": [
-    {"parameter": "fuel_cost_gas", "low": -300000.0, "high": 300000.0},
-    {"parameter": "wind_capacity", "low": -200000.0, "high": 200000.0}
-  ]
-}
-*/
-
 -- ============================================================================
--- DATA MANAGEMENT TRIGGERS
+-- UTILITY VIEWS (SIMPLIFIED)
 -- ============================================================================
 
--- Note: Access tracking for analysis cache would need to be handled in application code
--- SQLite doesn't support AFTER SELECT triggers
-
--- Note: Result size calculation should be handled in application code
--- Cannot modify NEW values in SQLite BEFORE INSERT triggers
-
--- Note: Timestamp updates should be handled in application code or with DEFAULT CURRENT_TIMESTAMP
--- SQLite triggers cannot update the same record being modified without recursion issues
-
--- ============================================================================
--- UTILITY VIEWS
--- ============================================================================
-
--- View for network analysis summary
-CREATE VIEW network_analysis_summary AS
-SELECT 
-    n.id as network_id,
-    n.name as network_name,
-    COUNT(DISTINCT nac.analysis_type) as analysis_types_count,
-    COUNT(nac.id) as total_cache_entries,
-    SUM(nac.hit_count) as total_cache_hits,
-    SUM(nac.result_size_bytes) as total_cache_size_bytes,
-    MAX(nac.last_accessed) as last_analysis_accessed,
-    COUNT(or1.id) as optimization_runs_count,
-    MAX(or1.created_at) as last_optimization_run
-FROM networks n
-LEFT JOIN network_analysis_cache nac ON n.id = nac.network_id
-LEFT JOIN optimization_results or1 ON n.id = or1.network_id
-GROUP BY n.id, n.name;
-
--- View for recent network activity
+-- View for recent network activity (simplified without audit/cache tables)
 CREATE VIEW network_activity_summary AS
 SELECT 
     n.id as network_id,
@@ -501,7 +197,7 @@ SELECT
     c.*,
     b.name as bus_name
 FROM components c
-LEFT JOIN components b ON json_extract(c.connectivity, '$.bus_id') = b.id AND b.component_type = 'BUS'
+LEFT JOIN components b ON c.bus_id = b.id AND b.component_type = 'BUS'
 WHERE c.component_type IN ('GENERATOR', 'LOAD', 'STORAGE_UNIT', 'STORE');
 
 -- View for components with dual bus connections (lines, links)
@@ -511,8 +207,8 @@ SELECT
     b0.name as bus0_name,
     b1.name as bus1_name
 FROM components c
-LEFT JOIN components b0 ON json_extract(c.connectivity, '$.bus0_id') = b0.id AND b0.component_type = 'BUS'
-LEFT JOIN components b1 ON json_extract(c.connectivity, '$.bus1_id') = b1.id AND b1.component_type = 'BUS'
+LEFT JOIN components b0 ON c.bus0_id = b0.id AND b0.component_type = 'BUS'
+LEFT JOIN components b1 ON c.bus1_id = b1.id AND b1.component_type = 'BUS'
 WHERE c.component_type IN ('LINE', 'LINK');
 
 -- Unified view for all components with resolved bus connections
@@ -522,17 +218,17 @@ SELECT
     CASE 
         WHEN c.component_type = 'BUS' THEN NULL
         WHEN c.component_type IN ('GENERATOR', 'LOAD', 'STORAGE_UNIT', 'STORE') THEN 
-            (SELECT b.name FROM components b WHERE b.id = json_extract(c.connectivity, '$.bus_id') AND b.component_type = 'BUS')
+            (SELECT b.name FROM components b WHERE b.id = c.bus_id AND b.component_type = 'BUS')
         ELSE NULL
     END as bus_name,
     CASE 
         WHEN c.component_type IN ('LINE', 'LINK') THEN 
-            (SELECT b.name FROM components b WHERE b.id = json_extract(c.connectivity, '$.bus0_id') AND b.component_type = 'BUS')
+            (SELECT b.name FROM components b WHERE b.id = c.bus0_id AND b.component_type = 'BUS')
         ELSE NULL
     END as bus0_name,
     CASE 
         WHEN c.component_type IN ('LINE', 'LINK') THEN 
-            (SELECT b.name FROM components b WHERE b.id = json_extract(c.connectivity, '$.bus1_id') AND b.component_type = 'BUS')
+            (SELECT b.name FROM components b WHERE b.id = c.bus1_id AND b.component_type = 'BUS')
         ELSE NULL
     END as bus1_name
 FROM components c;

pyconvexity/data/sources/__init__.py

@@ -0,0 +1,5 @@
+"""
+Data sources for PyConvexity.
+
+This module contains integrations with external energy data sources.
+"""