pyconvexity 0.1.3__py3-none-any.whl → 0.3.7__py3-none-any.whl

pyconvexity/solvers/pypsa/builder.py

@@ -1,15 +1,16 @@
 """
 Network building functionality for PyPSA solver integration.
 
-Handles loading data from database and constructing PyPSA Network objects.
+Simplified to always use MultiIndex format for consistent multi-period optimization.
 """
 
 import logging
+import json
 import pandas as pd
 from typing import Dict, Any, Optional, Callable
 
 from pyconvexity.models import (
-    list_components_by_type, get_network_time_periods, get_network_config
+    list_components_by_type, get_network_time_periods, get_network_info
 )
 
 logger = logging.getLogger(__name__)
@@ -19,9 +20,8 @@ class NetworkBuilder:
     """
     Builds PyPSA networks from database data.
     
-    This class handles the complex process of loading network components,
-    attributes, and time series data from the database and constructing
-    a properly configured PyPSA Network object.
+    Simplified to always create MultiIndex snapshots for consistent multi-period optimization,
+    even for single-year models.
     """
     
     def __init__(self):
@@ -36,13 +36,8 @@ class NetworkBuilder:
             ) from e
         
         # Import batch loader for efficient data loading
-        try:
-            from pyconvexity.solvers.pypsa.batch_loader import PyPSABatchLoader
-            self.batch_loader = PyPSABatchLoader()
-        except ImportError:
-            # Fallback to individual loading if batch loader not available
-            self.batch_loader = None
-            logger.warning("PyPSABatchLoader not available, using individual component loading")
+        from pyconvexity.solvers.pypsa.batch_loader import PyPSABatchLoader
+        self.batch_loader = PyPSABatchLoader()
     
     def build_network(
         self, 
@@ -87,7 +82,7 @@ class NetworkBuilder:
         if progress_callback:
             progress_callback(20, "Loading components...")
         
-        # Load all components
+        # Load all components using efficient batch loader
         self._load_components(conn, network_id, network, scenario_id, progress_callback)
         
         # NOTE: Snapshot weightings will be set AFTER multi-period optimization setup
@@ -184,35 +179,44 @@ class NetworkBuilder:
         }
     
     def _set_time_index(self, conn, network_id: int, network: 'pypsa.Network'):
-        """Set time index from network time periods."""
+        """Set time index from network time periods - always create MultiIndex for consistency."""
         try:
             time_periods = get_network_time_periods(conn, network_id)
-            if time_periods:
-                # Convert to pandas DatetimeIndex
-                timestamps = [pd.Timestamp(tp.formatted_time) for tp in time_periods]
-                
-                # Set the snapshots to the timestamps
-                network.set_snapshots(timestamps)
+            if not time_periods:
+                logger.error("No time periods found for network")
+                return
                 
-                # Extract years for year-based statistics
-                try:
-                    if hasattr(network.snapshots, 'year'):
-                        years = sorted(network.snapshots.year.unique())
-                        network._available_years = years
-                        logger.info(f"Extracted {len(years)} years from network.snapshots.year: {years}")
-                    else:
-                        # Manually extract years from timestamps
-                        years_from_timestamps = sorted(list(set([ts.year for ts in timestamps])))
-                        network._available_years = years_from_timestamps
-                        logger.info(f"Extracted {len(years_from_timestamps)} years from timestamps: {years_from_timestamps}")
-                        
-                except Exception as year_error:
-                    logger.warning(f"Failed to extract years for year-based statistics: {year_error}")
-                    network._available_years = []
+            # Convert to pandas DatetimeIndex
+            timestamps = [pd.Timestamp(tp.formatted_time) for tp in time_periods]
+            
+            # Extract unique years for investment periods
+            years = sorted(list(set([ts.year for ts in timestamps])))
+            logger.info(f"Found {len(years)} investment periods: {years}")
+            
+            # Always create MultiIndex following PyPSA multi-investment tutorial format
+            # First level: investment periods (years), Second level: timesteps
+            multi_snapshots = []
+            for ts in timestamps:
+                multi_snapshots.append((ts.year, ts))
+            
+            multi_index = pd.MultiIndex.from_tuples(multi_snapshots, names=['period', 'timestep'])
+            
+            # Verify MultiIndex is unique (should always be true now with UTC timestamps)
+            if not multi_index.is_unique:
+                raise ValueError(f"Created MultiIndex is not unique! Check timestamp generation.")
+            
+            logger.info(f"Created MultiIndex with {len(multi_index)} snapshots")
+            network.set_snapshots(multi_index)
+            
+            # Set investment periods for multi-period optimization
+            network.investment_periods = years
+            
+            # Store years for statistics
+            network._available_years = years
+            
+            logger.info(f"Created MultiIndex with {len(multi_index)} snapshots across {len(years)} periods")
+            logger.info(f"Investment periods: {network.investment_periods}")
                     
-            else:
-                logger.warning("No time periods found for network, year-based statistics will not be available")
-                network._available_years = []
         except Exception as e:
             logger.error(f"Failed to set time index: {e}")
             network._available_years = []
@@ -260,15 +264,11 @@ class NetworkBuilder:
         scenario_id: Optional[int],
         progress_callback: Optional[Callable[[int, str], None]] = None
     ):
-        """Load all network components."""
-        # Load component connections if batch loader available
-        if self.batch_loader:
-            connections = self.batch_loader.batch_load_component_connections(conn, network_id)
-            bus_id_to_name = connections['bus_id_to_name']
-            carrier_id_to_name = connections['carrier_id_to_name']
-        else:
-            bus_id_to_name = self._build_bus_id_to_name_map(conn, network_id)
-            carrier_id_to_name = self._build_carrier_id_to_name_map(conn, network_id)
+        """Load all network components using batch loader."""
+        # Load component connections
+        connections = self.batch_loader.batch_load_component_connections(conn, network_id)
+        bus_id_to_name = connections['bus_id_to_name']
+        carrier_id_to_name = connections['carrier_id_to_name']
         
         # Component type mapping for later identification
         component_type_map = {}
@@ -307,22 +307,25 @@ class NetworkBuilder:
         # Store component type mapping on network
         network._component_type_map = component_type_map
     
+    
     def _load_buses(self, conn, network_id: int, network: 'pypsa.Network', scenario_id: Optional[int], component_type_map: Dict[str, str]):
         """Load bus components."""
         buses = list_components_by_type(conn, network_id, 'BUS')
         bus_ids = [bus.id for bus in buses]
         
-        if self.batch_loader:
-            bus_attributes = self.batch_loader.batch_load_component_attributes(conn, bus_ids, scenario_id)
-            bus_timeseries = self.batch_loader.batch_load_component_timeseries(conn, bus_ids, scenario_id)
-        else:
-            bus_attributes = self._load_component_attributes_individually(conn, bus_ids, scenario_id)
-            bus_timeseries = {}
+        bus_attributes = self.batch_loader.batch_load_component_attributes(conn, bus_ids, scenario_id)
+        bus_timeseries = self.batch_loader.batch_load_component_timeseries(conn, bus_ids, scenario_id)
         
         for bus in buses:
             attrs = bus_attributes.get(bus.id, {})
             timeseries = bus_timeseries.get(bus.id, {})
             
+            # Add coordinate data from components table (PyPSA uses 'x' for longitude, 'y' for latitude)
+            if bus.longitude is not None:
+                attrs['x'] = bus.longitude
+            if bus.latitude is not None:
+                attrs['y'] = bus.latitude
+            
             # Merge timeseries into attributes
             attrs.update(timeseries)
             
@@ -338,12 +341,8 @@ class NetworkBuilder:
         
         generator_ids = [gen.id for gen in all_generators]
         
-        if self.batch_loader:
-            generator_attributes = self.batch_loader.batch_load_component_attributes(conn, generator_ids, scenario_id)
-            generator_timeseries = self.batch_loader.batch_load_component_timeseries(conn, generator_ids, scenario_id)
-        else:
-            generator_attributes = self._load_component_attributes_individually(conn, generator_ids, scenario_id)
-            generator_timeseries = {}
+        generator_attributes = self.batch_loader.batch_load_component_attributes(conn, generator_ids, scenario_id)
+        generator_timeseries = self.batch_loader.batch_load_component_timeseries(conn, generator_ids, scenario_id)
         
         for gen in all_generators:
             attrs = generator_attributes.get(gen.id, {})
@@ -373,12 +372,8 @@ class NetworkBuilder:
         loads = list_components_by_type(conn, network_id, 'LOAD')
         load_ids = [load.id for load in loads]
         
-        if self.batch_loader:
-            load_attributes = self.batch_loader.batch_load_component_attributes(conn, load_ids, scenario_id)
-            load_timeseries = self.batch_loader.batch_load_component_timeseries(conn, load_ids, scenario_id)
-        else:
-            load_attributes = self._load_component_attributes_individually(conn, load_ids, scenario_id)
-            load_timeseries = {}
+        load_attributes = self.batch_loader.batch_load_component_attributes(conn, load_ids, scenario_id)
+        load_timeseries = self.batch_loader.batch_load_component_timeseries(conn, load_ids, scenario_id)
         
         for load in loads:
             attrs = load_attributes.get(load.id, {})
@@ -405,12 +400,8 @@ class NetworkBuilder:
         lines = list_components_by_type(conn, network_id, 'LINE')
         line_ids = [line.id for line in lines]
         
-        if self.batch_loader:
-            line_attributes = self.batch_loader.batch_load_component_attributes(conn, line_ids, scenario_id)
-            line_timeseries = self.batch_loader.batch_load_component_timeseries(conn, line_ids, scenario_id)
-        else:
-            line_attributes = self._load_component_attributes_individually(conn, line_ids, scenario_id)
-            line_timeseries = {}
+        line_attributes = self.batch_loader.batch_load_component_attributes(conn, line_ids, scenario_id)
+        line_timeseries = self.batch_loader.batch_load_component_timeseries(conn, line_ids, scenario_id)
         
         for line in lines:
             attrs = line_attributes.get(line.id, {})
@@ -439,12 +430,8 @@ class NetworkBuilder:
         links = list_components_by_type(conn, network_id, 'LINK')
         link_ids = [link.id for link in links]
         
-        if self.batch_loader:
-            link_attributes = self.batch_loader.batch_load_component_attributes(conn, link_ids, scenario_id)
-            link_timeseries = self.batch_loader.batch_load_component_timeseries(conn, link_ids, scenario_id)
-        else:
-            link_attributes = self._load_component_attributes_individually(conn, link_ids, scenario_id)
-            link_timeseries = {}
+        link_attributes = self.batch_loader.batch_load_component_attributes(conn, link_ids, scenario_id)
+        link_timeseries = self.batch_loader.batch_load_component_timeseries(conn, link_ids, scenario_id)
         
         for link in links:
             attrs = link_attributes.get(link.id, {})
@@ -473,12 +460,8 @@ class NetworkBuilder:
         storage_units = list_components_by_type(conn, network_id, 'STORAGE_UNIT')
         storage_ids = [storage.id for storage in storage_units]
         
-        if self.batch_loader:
-            storage_attributes = self.batch_loader.batch_load_component_attributes(conn, storage_ids, scenario_id)
-            storage_timeseries = self.batch_loader.batch_load_component_timeseries(conn, storage_ids, scenario_id)
-        else:
-            storage_attributes = self._load_component_attributes_individually(conn, storage_ids, scenario_id)
-            storage_timeseries = {}
+        storage_attributes = self.batch_loader.batch_load_component_attributes(conn, storage_ids, scenario_id)
+        storage_timeseries = self.batch_loader.batch_load_component_timeseries(conn, storage_ids, scenario_id)
         
         for storage in storage_units:
             attrs = storage_attributes.get(storage.id, {})
@@ -505,12 +488,8 @@ class NetworkBuilder:
         stores = list_components_by_type(conn, network_id, 'STORE')
         store_ids = [store.id for store in stores]
         
-        if self.batch_loader:
-            store_attributes = self.batch_loader.batch_load_component_attributes(conn, store_ids, scenario_id)
-            store_timeseries = self.batch_loader.batch_load_component_timeseries(conn, store_ids, scenario_id)
-        else:
-            store_attributes = self._load_component_attributes_individually(conn, store_ids, scenario_id)
-            store_timeseries = {}
+        store_attributes = self.batch_loader.batch_load_component_attributes(conn, store_ids, scenario_id)
+        store_timeseries = self.batch_loader.batch_load_component_timeseries(conn, store_ids, scenario_id)
         
         for store in stores:
             attrs = store_attributes.get(store.id, {})
@@ -591,66 +570,3 @@ class NetworkBuilder:
         cursor = conn.execute("SELECT id, name FROM carriers WHERE network_id = ?", (network_id,))
         return {row[0]: row[1] for row in cursor.fetchall()}
     
-    def _load_component_attributes_individually(self, conn, component_ids: list, scenario_id: Optional[int]) -> Dict[int, Dict[str, Any]]:
-        """Fallback method to load component attributes individually."""
-        from pyconvexity.models import get_attribute, list_component_attributes
-        from pyconvexity.core.types import AttributeValue
-        from pyconvexity.models.network import get_network_time_periods
-        import pandas as pd
-        
-        # Get network time periods for proper timestamp alignment
-        network_time_periods = None
-        if component_ids:
-            cursor = conn.execute("SELECT network_id FROM components WHERE id = ? LIMIT 1", (component_ids[0],))
-            result = cursor.fetchone()
-            if result:
-                network_id = result[0]
-                try:
-                    network_time_periods = get_network_time_periods(conn, network_id)
-                except Exception as e:
-                    logger.warning(f"Failed to load network time periods: {e}")
-        
-        attributes = {}
-        for comp_id in component_ids:
-            try:
-                attr_names = list_component_attributes(conn, comp_id)
-                comp_attrs = {}
-                for attr_name in attr_names:
-                    try:
-                        attr_value = get_attribute(conn, comp_id, attr_name, scenario_id)
-                        if attr_value is not None:
-                            # Handle different attribute value types
-                            if hasattr(attr_value, 'static_value') and attr_value.static_value is not None:
-                                # Static value
-                                comp_attrs[attr_name] = attr_value.static_value.value()
-                            elif hasattr(attr_value, 'timeseries_value') and attr_value.timeseries_value is not None:
-                                # Timeseries value - convert to pandas Series with proper timestamps
-                                timeseries_points = attr_value.timeseries_value
-                                if timeseries_points:
-                                    # Sort by period_index to ensure correct order
-                                    timeseries_points.sort(key=lambda x: x.period_index)
-                                    values = [point.value for point in timeseries_points]
-                                    
-                                    # Create proper timestamps for PyPSA alignment
-                                    if network_time_periods:
-                                        timestamps = []
-                                        for point in timeseries_points:
-                                            if point.period_index < len(network_time_periods):
-                                                tp = network_time_periods[point.period_index]
-                                                timestamps.append(pd.Timestamp(tp.formatted_time))
-                                            else:
-                                                logger.warning(f"Period index {point.period_index} out of range for network time periods")
-                                                timestamps.append(pd.Timestamp.now())  # Fallback
-                                        comp_attrs[attr_name] = pd.Series(values, index=timestamps)
-                                    else:
-                                        # Fallback: use period_index as index
-                                        period_indices = [point.period_index for point in timeseries_points]
-                                        comp_attrs[attr_name] = pd.Series(values, index=period_indices)
-                    except Exception as e:
-                        logger.debug(f"Failed to load attribute {attr_name} for component {comp_id}: {e}")
-                attributes[comp_id] = comp_attrs
-            except Exception as e:
-                logger.warning(f"Failed to load attributes for component {comp_id}: {e}")
-                attributes[comp_id] = {}
-        
-        return attributes