pyconvexity 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

pyconvexity/solvers/pypsa/batch_loader.py

@@ -19,99 +19,96 @@ class PyPSABatchLoader:
     Simplified batch data loader for PyPSA network construction.
     Always creates MultiIndex timeseries for consistent multi-period optimization.
     """
-    
+
     def __init__(self):
         pass
-    
-    def batch_load_component_attributes(self, conn, component_ids: List[int], scenario_id: Optional[int]) -> Dict[int, Dict[str, Any]]:
-        """Batch load all static attributes for multiple components to avoid N+1 queries"""
+
+    def batch_load_component_attributes(
+        self, conn, component_ids: List[int], scenario_id: Optional[int]
+    ) -> Dict[int, Dict[str, Any]]:
+        """Batch load all static attributes for multiple components to avoid N+1 queries (single network per database)"""
         if not component_ids:
             return {}
-        
+
         # Build a single query to get all attributes for all components
-        placeholders = ','.join(['?' for _ in component_ids])
-        
+        placeholders = ",".join(["?" for _ in component_ids])
+
         # Get all attribute names for all components in one query
-        cursor = conn.execute(f"""
+        cursor = conn.execute(
+            f"""
             SELECT DISTINCT attribute_name 
             FROM component_attributes 
             WHERE component_id IN ({placeholders}) AND storage_type = 'static'
-        """, component_ids)
+        """,
+            component_ids,
+        )
         all_attribute_names = [row[0] for row in cursor.fetchall()]
-        
+
         if not all_attribute_names:
             return {comp_id: {} for comp_id in component_ids}
-        
+
         # Build query to get all attributes for all components
-        attr_placeholders = ','.join(['?' for _ in all_attribute_names])
-        
-        # Resolve scenario IDs for fallback logic
-        scenario_filter_values = []
-        master_id = None
+        attr_placeholders = ",".join(["?" for _ in all_attribute_names])
+
+        # Scenario fallback: scenario_id -> NULL (base network)
+        # Query for both scenario-specific and base network attributes
         if scenario_id is not None:
-            # Get master scenario ID for fallback
-            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = (SELECT network_id FROM components WHERE id = ?) AND is_master = 1", (component_ids[0],))
-            result = cursor.fetchone()
-            if result:
-                master_id = result[0]
-                scenario_filter_values = [scenario_id, master_id]
-            else:
-                scenario_filter_values = [scenario_id]
+            # Get both scenario and base network values (scenario takes precedence)
+            query = f"""
+                SELECT component_id, attribute_name, static_value, data_type, scenario_id
+                FROM component_attributes 
+                WHERE component_id IN ({placeholders}) 
+                AND attribute_name IN ({attr_placeholders})
+                AND (scenario_id = ? OR scenario_id IS NULL)
+                AND storage_type = 'static'
+                ORDER BY component_id, attribute_name, 
+                         CASE WHEN scenario_id = ? THEN 0 ELSE 1 END
+            """
+            query_params = (
+                component_ids + all_attribute_names + [scenario_id, scenario_id]
+            )
         else:
-            # Get master scenario ID
-            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = (SELECT network_id FROM components WHERE id = ?) AND is_master = 1", (component_ids[0],))
-            result = cursor.fetchone()
-            if result:
-                master_id = result[0]
-                scenario_filter_values = [master_id]
-            else:
-                return {comp_id: {} for comp_id in component_ids}
-        
-        scen_placeholders = ','.join(['?' for _ in scenario_filter_values])
-        
-        # Single query to get all attributes
-        # CRITICAL: Order by scenario_id DESC to prioritize current scenario over master
-        query = f"""
-            SELECT component_id, attribute_name, static_value, data_type, scenario_id
-            FROM component_attributes 
-            WHERE component_id IN ({placeholders}) 
-            AND attribute_name IN ({attr_placeholders})
-            AND scenario_id IN ({scen_placeholders})
-            AND storage_type = 'static'
-            ORDER BY component_id, attribute_name, 
-                     CASE WHEN scenario_id = ? THEN 0 ELSE 1 END
-        """
-        
-        # Parameters must match the order of placeholders in the query
-        query_params = component_ids + all_attribute_names + scenario_filter_values + [scenario_id if scenario_id is not None else master_id]
-        
+            # Get only base network attributes (scenario_id IS NULL)
+            query = f"""
+                SELECT component_id, attribute_name, static_value, data_type, scenario_id
+                FROM component_attributes 
+                WHERE component_id IN ({placeholders}) 
+                AND attribute_name IN ({attr_placeholders})
+                AND scenario_id IS NULL
+                AND storage_type = 'static'
+                ORDER BY component_id, attribute_name
+            """
+            query_params = component_ids + all_attribute_names
+
         cursor = conn.execute(query, query_params)
-        
+
         # Group by component_id, preferring current scenario over master
         component_attributes = {}
         for comp_id in component_ids:
             component_attributes[comp_id] = {}
-        
+
         # Process results, preferring current scenario over master
         rows = cursor.fetchall()
-        
+
         for row in rows:
             comp_id, attr_name, static_value_json, data_type, row_scenario_id = row
-            
+
             # Ensure component exists in our dictionary (safety check)
             if comp_id not in component_attributes:
                 continue
-            
+
             # Skip if we already have this attribute from a preferred scenario
             if attr_name in component_attributes[comp_id]:
                 continue
-            
+
             # Parse JSON value
             json_value = json.loads(static_value_json)
-            
+
             # Convert based on data type
             if data_type == "float":
-                value = float(json_value) if isinstance(json_value, (int, float)) else 0.0
+                value = (
+                    float(json_value) if isinstance(json_value, (int, float)) else 0.0
+                )
             elif data_type == "int":
                 value = int(json_value) if isinstance(json_value, (int, float)) else 0
             elif data_type == "boolean":
@@ -120,185 +117,191 @@ class PyPSABatchLoader:
                 value = str(json_value) if isinstance(json_value, str) else ""
             else:
                 value = json_value
-            
+
             component_attributes[comp_id][attr_name] = value
-        
+
         return component_attributes
-    
-    def batch_load_component_connections(self, conn, network_id: int) -> Dict[str, Dict[str, str]]:
-        """Batch load bus and carrier connections to avoid individual lookups"""
+
+    def batch_load_component_connections(self, conn) -> Dict[str, Dict[str, str]]:
+        """Batch load bus and carrier connections to avoid individual lookups (single network per database)"""
         # Get all bus names in one query
-        cursor = conn.execute("""
+        cursor = conn.execute(
+            """
             SELECT id, name FROM components 
-            WHERE network_id = ? AND component_type = 'BUS'
-        """, (network_id,))
+            WHERE component_type = 'BUS'
+        """
+        )
         bus_id_to_name = {row[0]: row[1] for row in cursor.fetchall()}
-        
+
         # Get all carrier names in one query
-        cursor = conn.execute("""
-            SELECT id, name FROM carriers 
-            WHERE network_id = ?
-        """, (network_id,))
+        cursor = conn.execute(
+            """
+            SELECT id, name FROM carriers
+        """
+        )
         carrier_id_to_name = {row[0]: row[1] for row in cursor.fetchall()}
-        
+
         return {
-            'bus_id_to_name': bus_id_to_name,
-            'carrier_id_to_name': carrier_id_to_name
+            "bus_id_to_name": bus_id_to_name,
+            "carrier_id_to_name": carrier_id_to_name,
         }
-    
-    def batch_load_component_timeseries(self, conn, component_ids: List[int], scenario_id: Optional[int]) -> Dict[int, Dict[str, pd.Series]]:
-        """Batch load all timeseries attributes - always create MultiIndex for consistency"""
+
+    def batch_load_component_timeseries(
+        self, conn, component_ids: List[int], scenario_id: Optional[int]
+    ) -> Dict[int, Dict[str, pd.Series]]:
+        """Batch load all timeseries attributes - always create MultiIndex for consistency (single network per database)"""
         if not component_ids:
             return {}
-        
+
         # Get network time periods for proper timestamp alignment
-        cursor = conn.execute("SELECT network_id FROM components WHERE id = ? LIMIT 1", (component_ids[0],))
-        result = cursor.fetchone()
-        if not result:
-            return {comp_id: {} for comp_id in component_ids}
-        
-        network_id = result[0]
-        network_time_periods = get_network_time_periods(conn, network_id)
+        network_time_periods = get_network_time_periods(conn)
         if not network_time_periods:
             logger.warning("No time periods found for network")
             return {comp_id: {} for comp_id in component_ids}
-        
+
         # Convert to timestamps and extract years
         timestamps = [pd.Timestamp(tp.formatted_time) for tp in network_time_periods]
         years = sorted(list(set([ts.year for ts in timestamps])))
-        
+
         # Build a single query to get all timeseries attributes for all components
-        placeholders = ','.join(['?' for _ in component_ids])
-        
+        placeholders = ",".join(["?" for _ in component_ids])
+
         # Get all attribute names for all components in one query
-        cursor = conn.execute(f"""
+        cursor = conn.execute(
+            f"""
             SELECT DISTINCT attribute_name 
             FROM component_attributes 
             WHERE component_id IN ({placeholders}) AND storage_type = 'timeseries'
-        """, component_ids)
+        """,
+            component_ids,
+        )
         all_attribute_names = [row[0] for row in cursor.fetchall()]
-        
+
         if not all_attribute_names:
             return {comp_id: {} for comp_id in component_ids}
-        
+
         # Build query to get all timeseries for all components
-        attr_placeholders = ','.join(['?' for _ in all_attribute_names])
-        
-        # Resolve scenario IDs for fallback logic
-        scenario_filter_values = []
-        master_id = None
+        attr_placeholders = ",".join(["?" for _ in all_attribute_names])
+
+        # Scenario fallback: scenario_id -> NULL (base network)
         if scenario_id is not None:
-            # Get master scenario ID for fallback
-            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = ? AND is_master = 1", (network_id,))
-            result = cursor.fetchone()
-            if result:
-                master_id = result[0]
-                scenario_filter_values = [scenario_id, master_id]
-            else:
-                scenario_filter_values = [scenario_id]
+            # Get both scenario and base network timeseries (scenario takes precedence)
+            query = f"""
+                SELECT component_id, attribute_name, timeseries_data, scenario_id
+                FROM component_attributes 
+                WHERE component_id IN ({placeholders}) 
+                AND attribute_name IN ({attr_placeholders})
+                AND (scenario_id = ? OR scenario_id IS NULL)
+                AND storage_type = 'timeseries'
+                ORDER BY component_id, attribute_name, 
+                         CASE WHEN scenario_id = ? THEN 0 ELSE 1 END
+            """
+            query_params = (
+                component_ids + all_attribute_names + [scenario_id, scenario_id]
+            )
         else:
-            # Get master scenario ID
-            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = ? AND is_master = 1", (network_id,))
-            result = cursor.fetchone()
-            if result:
-                master_id = result[0]
-                scenario_filter_values = [master_id]
-            else:
-                return {comp_id: {} for comp_id in component_ids}
-        
-        scen_placeholders = ','.join(['?' for _ in scenario_filter_values])
-        
-        # Single query to get all timeseries
-        query = f"""
-            SELECT component_id, attribute_name, timeseries_data, scenario_id
-            FROM component_attributes 
-            WHERE component_id IN ({placeholders}) 
-            AND attribute_name IN ({attr_placeholders})
-            AND scenario_id IN ({scen_placeholders})
-            AND storage_type = 'timeseries'
-            ORDER BY component_id, attribute_name, 
-                     CASE WHEN scenario_id = ? THEN 0 ELSE 1 END
-        """
-        
-        # Parameters must match the order of placeholders in the query
-        query_params = component_ids + all_attribute_names + scenario_filter_values + [scenario_id if scenario_id is not None else master_id]
-        
+            # Get only base network timeseries (scenario_id IS NULL)
+            query = f"""
+                SELECT component_id, attribute_name, timeseries_data, scenario_id
+                FROM component_attributes 
+                WHERE component_id IN ({placeholders}) 
+                AND attribute_name IN ({attr_placeholders})
+                AND scenario_id IS NULL
+                AND storage_type = 'timeseries'
+                ORDER BY component_id, attribute_name
+            """
+            query_params = component_ids + all_attribute_names
+
         cursor = conn.execute(query, query_params)
-        
+
         # Group by component_id, preferring current scenario over master
         component_timeseries = {}
         for comp_id in component_ids:
             component_timeseries[comp_id] = {}
-        
+
         # Process results, preferring current scenario over master
         rows = cursor.fetchall()
-        
+
         for row in rows:
             comp_id, attr_name, timeseries_data, row_scenario_id = row
-            
+
             # Ensure component exists in our dictionary (safety check)
             if comp_id not in component_timeseries:
                 continue
-            
+
             # Skip if we already have this attribute from a preferred scenario
             if attr_name in component_timeseries[comp_id]:
                 continue
-            
+
             # Deserialize timeseries data
             try:
                 timeseries = get_timeseries(conn, comp_id, attr_name, row_scenario_id)
                 if timeseries and timeseries.values:
                     values = timeseries.values
-                    
+
                     # Always create MultiIndex following PyPSA multi-investment tutorial format
                     # First level: investment periods (years), Second level: timesteps
                     multi_snapshots = []
-                    for i, ts in enumerate(timestamps[:len(values)]):
+                    for i, ts in enumerate(timestamps[: len(values)]):
                         multi_snapshots.append((ts.year, ts))
-                    
+
                     if multi_snapshots:
-                        multi_index = pd.MultiIndex.from_tuples(multi_snapshots, names=['period', 'timestep'])
-                        component_timeseries[comp_id][attr_name] = pd.Series(values, index=multi_index)
+                        multi_index = pd.MultiIndex.from_tuples(
+                            multi_snapshots, names=["period", "timestep"]
+                        )
+                        component_timeseries[comp_id][attr_name] = pd.Series(
+                            values, index=multi_index
+                        )
                     else:
-                        logger.warning(f"No valid timestamps for timeseries {attr_name}")
-                        
+                        logger.warning(
+                            f"No valid timestamps for timeseries {attr_name}"
+                        )
+
             except Exception as e:
-                logger.warning(f"Failed to load timeseries {attr_name} for component {comp_id}: {e}")
+                logger.warning(
+                    f"Failed to load timeseries {attr_name} for component {comp_id}: {e}"
+                )
                 continue
-        
+
         return component_timeseries
-    
-    def batch_load_all_component_timeseries_by_type(self, conn, network_id: int, component_type: str, scenario_id: Optional[int]) -> Dict[str, pd.DataFrame]:
+
+    def batch_load_all_component_timeseries_by_type(
+        self, conn, component_type: str, scenario_id: Optional[int]
+    ) -> Dict[str, pd.DataFrame]:
         """
-        Load all timeseries attributes for a component type and organize by attribute name.
+        Load all timeseries attributes for a component type and organize by attribute name (single network per database).
         This is a compatibility method for the existing _load_all_component_timeseries interface.
         """
         from pyconvexity.models import list_components_by_type
-        
-        components = list_components_by_type(conn, network_id, component_type)
+
+        components = list_components_by_type(conn, component_type)
         component_ids = [comp.id for comp in components]
-        
+
         # Use batch loading
-        component_timeseries = self.batch_load_component_timeseries(conn, component_ids, scenario_id)
-        
+        component_timeseries = self.batch_load_component_timeseries(
+            conn, component_ids, scenario_id
+        )
+
         # Reorganize by attribute name (matching original interface)
         timeseries_by_attr = {}
-        
+
         for component in components:
             comp_timeseries = component_timeseries.get(component.id, {})
-            
+
             for attr_name, series in comp_timeseries.items():
                 if attr_name not in timeseries_by_attr:
                     timeseries_by_attr[attr_name] = {}
-                
+
                 # Store series in dict first
                 timeseries_by_attr[attr_name][component.name] = series
-        
+
         # Convert to DataFrames all at once to avoid fragmentation
         for attr_name in timeseries_by_attr:
             if timeseries_by_attr[attr_name]:
-                timeseries_by_attr[attr_name] = pd.DataFrame(timeseries_by_attr[attr_name])
+                timeseries_by_attr[attr_name] = pd.DataFrame(
+                    timeseries_by_attr[attr_name]
+                )
             else:
                 timeseries_by_attr[attr_name] = pd.DataFrame()
-        
+
         return timeseries_by_attr