pyconvexity 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

pyconvexity/models/attributes.py

@@ -8,14 +8,14 @@ with support for both static values and timeseries data.
 import sqlite3
 import json
 import logging
-from typing import Dict, Any, Optional, List
+from typing import Dict, Any, Optional, List, Union
 import pandas as pd
 from io import BytesIO
 import pyarrow as pa
 import pyarrow.parquet as pq
 
 from pyconvexity.core.types import (
-    StaticValue, TimeseriesPoint, AttributeValue, TimePeriod
+    StaticValue, Timeseries, TimeseriesMetadata, AttributeValue, TimePeriod
 )
 from pyconvexity.core.errors import (
     ComponentNotFound, AttributeNotFound, ValidationError, TimeseriesError
@@ -86,7 +86,7 @@ def set_timeseries_attribute(
     conn: sqlite3.Connection,
     component_id: int,
     attribute_name: str,
-    timeseries: List[TimeseriesPoint],
+    timeseries: Union[Timeseries, List[float]],
     scenario_id: Optional[int] = None
 ) -> None:
     """
@@ -96,7 +96,7 @@ def set_timeseries_attribute(
         conn: Database connection
         component_id: Component ID
         attribute_name: Name of the attribute
-        timeseries: List of timeseries points
+        timeseries: Timeseries object or list of float values
         scenario_id: Scenario ID (uses master scenario if None)
         
     Raises:
@@ -116,25 +116,34 @@ def set_timeseries_attribute(
     if not rule.allows_timeseries:
         raise ValidationError(f"Attribute '{attribute_name}' for {component_type} does not allow timeseries values")
     
-    # 4. Serialize timeseries to Parquet
-    parquet_data = serialize_timeseries_to_parquet(timeseries)
+    # 4. Convert input to values array
+    if isinstance(timeseries, Timeseries):
+        values = timeseries.values
+    elif isinstance(timeseries, list) and all(isinstance(v, (int, float)) for v in timeseries):
+        # Direct values array
+        values = [float(v) for v in timeseries]
+    else:
+        raise ValueError("timeseries must be Timeseries or List[float]")
     
-    # 5. Resolve scenario ID (get master scenario if None)
+    # 5. Serialize to binary format (ultra-fast, matches Rust exactly)
+    binary_data = serialize_values_to_binary(values)
+    
+    # 6. Resolve scenario ID (get master scenario if None)
     resolved_scenario_id = resolve_scenario_id(conn, component_id, scenario_id)
     
-    # 6. Remove any existing attribute for this scenario
+    # 7. Remove any existing attribute for this scenario
     cursor = conn.cursor()
     cursor.execute(
         "DELETE FROM component_attributes WHERE component_id = ? AND attribute_name = ? AND scenario_id = ?",
         (component_id, attribute_name, resolved_scenario_id)
     )
     
-    # 7. Insert new timeseries attribute
+    # 8. Insert new timeseries attribute
     cursor.execute(
         """INSERT INTO component_attributes 
            (component_id, attribute_name, scenario_id, storage_type, timeseries_data, data_type, unit, is_input) 
            VALUES (?, ?, ?, 'timeseries', ?, ?, ?, ?)""",
-        (component_id, attribute_name, resolved_scenario_id, parquet_data,
+        (component_id, attribute_name, resolved_scenario_id, binary_data,
          rule.data_type, rule.unit, rule.is_input)
     )
 
@@ -239,22 +248,19 @@ def get_attribute(
         if not timeseries_data:
             raise ValidationError("Timeseries attribute missing data")
         
-        # Get network_id from component to load time periods
-        cursor = conn.execute("SELECT network_id FROM components WHERE id = ?", (component_id,))
-        network_row = cursor.fetchone()
+        # Deserialize from binary format to new efficient Timeseries format
+        values = deserialize_values_from_binary(timeseries_data)
         
-        network_time_periods = None
-        if network_row:
-            network_id = network_row[0]
-            try:
-                from pyconvexity.models.network import get_network_time_periods
-                network_time_periods = get_network_time_periods(conn, network_id)
-            except Exception as e:
-                logger.warning(f"Failed to load network time periods for timestamp computation: {e}")
+        timeseries = Timeseries(
+            values=values,
+            length=len(values),
+            start_index=0,
+            data_type=data_type,
+            unit=unit,
+            is_input=True  # Default, could be enhanced with actual is_input from DB
+        )
         
-        # Deserialize from Parquet with proper timestamp computation
-        timeseries_points = deserialize_timeseries_from_parquet(timeseries_data, network_time_periods)
-        return AttributeValue.timeseries(timeseries_points)
+        return AttributeValue.timeseries(timeseries)
     
     else:
         raise ValidationError(f"Unknown storage type: {storage_type}")
@@ -322,64 +328,195 @@ def get_master_scenario_id(conn: sqlite3.Connection, network_id: int) -> int:
     return result[0]
 
 
-# Timeseries serialization functions
+# ============================================================================
+# EFFICIENT TIMESERIES SERIALIZATION - MATCHES RUST IMPLEMENTATION EXACTLY
+# ============================================================================
 
-def serialize_timeseries_to_parquet(timeseries: List[TimeseriesPoint]) -> bytes:
-    """Serialize timeseries to Parquet format - EXACT MATCH WITH RUST SCHEMA."""
-    # Define the exact schema to match Rust expectations
-    schema = pa.schema([
-        ('period_index', pa.int32()),
-        ('value', pa.float64())
-        ])
+def serialize_values_to_binary(values: List[float]) -> bytes:
+    """
+    Serialize f32 values to binary format - EXACT MATCH WITH RUST.
     
-    if not timeseries:
-        # Return empty parquet file with correct schema
-        empty_period_array = pa.array([], type=pa.int32())
-        empty_value_array = pa.array([], type=pa.float64())
-        table = pa.table([empty_period_array, empty_value_array], schema=schema)
-    else:
-        # Create PyArrow table with EXPLICIT schema to ensure data types match Rust
-        period_indices = [p.period_index for p in timeseries]
-        values = [p.value for p in timeseries]
-        
-        # Create arrays with explicit types to ensure Int32 for period_index
-        period_array = pa.array(period_indices, type=pa.int32())
-        value_array = pa.array(values, type=pa.float64())
-        
-        table = pa.table([period_array, value_array], schema=schema)
+    Ultra-fast binary format: just raw Float32 array, little-endian.
+    """
+    if not values:
+        return b''
+    
+    import struct
+    buffer = bytearray(len(values) * 4)  # 4 bytes per Float32
     
-    # Serialize to Parquet bytes with SNAPPY compression (match Rust)
-    buffer = BytesIO()
-    pq.write_table(table, buffer, compression='snappy')
-    return buffer.getvalue()
+    for i, value in enumerate(values):
+        # Pack as little-endian Float32 to match Rust exactly
+        struct.pack_into('<f', buffer, i * 4, float(value))
+    
+    return bytes(buffer)
 
 
-def deserialize_timeseries_from_parquet(data: bytes, network_time_periods: Optional[List[TimePeriod]] = None) -> List[TimeseriesPoint]:
-    """Deserialize timeseries from Parquet format - EXACT MATCH WITH RUST."""
+def deserialize_values_from_binary(data: bytes) -> List[float]:
+    """
+    Deserialize f32 values from binary format - EXACT MATCH WITH RUST.
+    
+    Ultra-fast deserialization: read raw Float32 values only.
+    """
     if not data:
         return []
 
-    buffer = BytesIO(data)
-    table = pq.read_table(buffer)
+    # Ensure data length is multiple of 4 (Float32 size)
+    if len(data) % 4 != 0:
+        raise ValueError("Invalid binary data length - must be multiple of 4 bytes")
+
+    import struct
+    values = []
+    
+    # Ultra-fast deserialization: read raw Float32 values
+    for i in range(0, len(data), 4):
+        value = struct.unpack('<f', data[i:i+4])[0]  # Little-endian Float32
+        values.append(value)
+
+    return values
+
+
+def get_timeseries_length_from_binary(data: bytes) -> int:
+    """Get the length of a timeseries without deserializing the full data."""
+    if not data:
+        return 0
+    
+    # Ultra-fast: just divide by 4 bytes per Float32
+    if len(data) % 4 != 0:
+        raise ValueError("Invalid binary data length - must be multiple of 4 bytes")
+    
+    return len(data) // 4
+
 
-    # Convert to pandas for easier handling
-    df = table.to_pandas()
+# ============================================================================
+# UNIFIED TIMESERIES FUNCTIONS - MATCH RUST API
+# ============================================================================
 
-    points = []
-    for _, row in df.iterrows():
-        period_index = int(row['period_index'])
+def get_timeseries(
+    conn: sqlite3.Connection,
+    component_id: int,
+    attribute_name: str,
+    scenario_id: Optional[int] = None,
+    start_index: Optional[int] = None,
+    end_index: Optional[int] = None,
+    max_points: Optional[int] = None
+) -> Timeseries:
+    """
+    Get timeseries data with unified interface matching Rust implementation.
+    
+    Args:
+        conn: Database connection
+        component_id: Component ID
+        attribute_name: Name of the attribute
+        scenario_id: Scenario ID (uses master scenario if None)
+        start_index: Start index for range queries
+        end_index: End index for range queries
+        max_points: Maximum number of points (for sampling)
         
-        # Compute timestamp from period_index using network time periods if available
-        if network_time_periods and 0 <= period_index < len(network_time_periods):
-            timestamp = network_time_periods[period_index].timestamp
-        else:
-            # Fallback: use period_index as timestamp (matching previous behavior for compatibility)
-            timestamp = period_index
+    Returns:
+        Timeseries object with efficient array-based data
         
-        points.append(TimeseriesPoint(
-            timestamp=timestamp,
-            value=float(row['value']),
-            period_index=period_index
-        ))
+    Raises:
+        ComponentNotFound: If component doesn't exist
+        AttributeNotFound: If attribute doesn't exist
+    """
+    # Get the attribute value
+    attr_value = get_attribute(conn, component_id, attribute_name, scenario_id)
+    
+    if not attr_value.is_timeseries():
+        raise ValueError(f"Attribute '{attribute_name}' is not a timeseries")
+    
+    timeseries = attr_value.as_timeseries()
+    if not timeseries:
+        raise ValueError("Failed to get timeseries data")
+    
+    # Apply range filtering if requested
+    if start_index is not None and end_index is not None:
+        timeseries = timeseries.slice(start_index, end_index)
+    
+    # Apply sampling if requested
+    if max_points is not None:
+        timeseries = timeseries.sample(max_points)
+    
+    return timeseries
+
 
-    return points
+def get_timeseries_metadata(
+    conn: sqlite3.Connection,
+    component_id: int,
+    attribute_name: str,
+    scenario_id: Optional[int] = None
+) -> TimeseriesMetadata:
+    """
+    Get timeseries metadata without loading the full data.
+    
+    Args:
+        conn: Database connection
+        component_id: Component ID
+        attribute_name: Name of the attribute
+        scenario_id: Scenario ID (uses master scenario if None)
+        
+    Returns:
+        TimeseriesMetadata with length and type information
+    """
+    # Get basic attribute info without loading full data
+    cursor = conn.cursor()
+    
+    # Get network_id from component
+    cursor.execute("SELECT network_id FROM components WHERE id = ?", (component_id,))
+    result = cursor.fetchone()
+    if not result:
+        raise ComponentNotFound(component_id)
+    
+    network_id = result[0]
+    
+    # Get master scenario ID
+    master_scenario_id = get_master_scenario_id(conn, network_id)
+    current_scenario_id = scenario_id if scenario_id is not None else master_scenario_id
+    
+    # Get timeseries metadata
+    cursor.execute(
+        """SELECT timeseries_data, data_type, unit, is_input
+           FROM component_attributes 
+           WHERE component_id = ? AND attribute_name = ? AND storage_type = 'timeseries' AND scenario_id = ?""",
+        (component_id, attribute_name, current_scenario_id)
+    )
+    result = cursor.fetchone()
+    
+    # Try fallback to master scenario if not found
+    if not result and current_scenario_id != master_scenario_id:
+        cursor.execute(
+            """SELECT timeseries_data, data_type, unit, is_input
+               FROM component_attributes 
+               WHERE component_id = ? AND attribute_name = ? AND storage_type = 'timeseries' AND scenario_id = ?""",
+            (component_id, attribute_name, master_scenario_id)
+        )
+        result = cursor.fetchone()
+    
+    if not result:
+        raise AttributeNotFound(component_id, attribute_name)
+    
+    timeseries_data, data_type, unit, is_input = result
+    
+    # Get length without full deserialization
+    length = get_timeseries_length_from_binary(timeseries_data)
+    
+    # Get time range from network time periods
+    try:
+        from pyconvexity.models.network import get_network_time_periods
+        time_periods = get_network_time_periods(conn, network_id)
+        start_time = time_periods[0].timestamp if time_periods else 0
+        end_time = time_periods[-1].timestamp if time_periods else 0
+    except Exception:
+        start_time = 0
+        end_time = length - 1
+    
+    return TimeseriesMetadata(
+        length=length,
+        start_time=start_time,
+        end_time=end_time,
+        start_index=0,
+        end_index=length,
+        data_type=data_type,
+        unit=unit,
+        is_input=is_input
+    )

pyconvexity/models/network.py

@@ -9,7 +9,7 @@ import sqlite3
 import json
 import logging
 from typing import Dict, Any, Optional, List
-from datetime import datetime
+from datetime import datetime, timezone
 
 from pyconvexity.core.types import (
     CreateNetworkRequest, TimePeriod, Network
@@ -105,7 +105,7 @@ def get_network_time_periods(
     network_id: int
 ) -> List[TimePeriod]:
     """
-    Get network time periods.
+    Get network time periods using optimized storage.
     
     Args:
         conn: Database connection
@@ -115,28 +115,30 @@ def get_network_time_periods(
         List of TimePeriod objects ordered by period_index
     """
     cursor = conn.execute("""
-        SELECT timestamp, period_index 
+        SELECT period_count, start_timestamp, interval_seconds 
         FROM network_time_periods 
-        WHERE network_id = ? 
-        ORDER BY period_index
+        WHERE network_id = ?
     """, (network_id,))
     
+    row = cursor.fetchone()
+    if not row:
+        return []  # No time periods defined
+    
+    period_count, start_timestamp, interval_seconds = row
+    
+    # Generate all time periods computationally
     periods = []
-    for row in cursor.fetchall():
-        timestamp_str, period_index = row
+    for period_index in range(period_count):
+        timestamp = start_timestamp + (period_index * interval_seconds)
         
-        # Convert datetime string to Unix timestamp
-        try:
-            dt = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
-            timestamp = int(dt.timestamp())
-        except ValueError:
-            # Fallback: use period_index as timestamp
-            timestamp = period_index
+        # Format timestamp as string for compatibility - ALWAYS use UTC to avoid DST duplicates
+        dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+        formatted_time = dt.strftime("%Y-%m-%d %H:%M:%S")
         
         periods.append(TimePeriod(
             timestamp=timestamp,
             period_index=period_index,
-            formatted_time=timestamp_str
+            formatted_time=formatted_time
         ))
     
     return periods

pyconvexity/solvers/pypsa/api.py

@@ -109,7 +109,30 @@ def solve_network(
         )
         
         if progress_callback:
-            progress_callback(100, "Solve completed successfully")
+            progress_callback(95, "Solve completed successfully")
+        
+        # Optimize database after successful solve (if solve was successful)
+        if solve_result.get("success", False):
+            try:
+                if progress_callback:
+                    progress_callback(98, "Optimizing database...")
+                
+                from pyconvexity.core.database import should_optimize_database, optimize_database
+                
+                # Only optimize if there's significant free space (>5% threshold for post-solve)
+                if should_optimize_database(conn, free_space_threshold_percent=5.0):
+                    logger.info("Running database optimization after successful solve")
+                    optimization_result = optimize_database(conn)
+                    logger.info(f"Database optimization completed: {optimization_result['space_reclaimed']:,} bytes reclaimed")
+                else:
+                    logger.debug("Skipping database optimization - insufficient free space")
+                    
+            except Exception as e:
+                # Don't fail the solve if optimization fails
+                logger.warning(f"Database optimization failed (non-critical): {e}")
+        
+        if progress_callback:
+            progress_callback(100, "Complete")
         
         # Return simple status if requested (for sidecar/async usage)
         # Results are now stored in database regardless of this flag

pyconvexity/solvers/pypsa/batch_loader.py

@@ -1,7 +1,6 @@
 """
 PyPSA Batch Data Loader
-Optimized batch loading functions for PyPSA network construction.
-Eliminates N+1 query patterns for improved performance.
+Simplified to always create MultiIndex timeseries for consistent multi-period optimization.
 """
 
 import logging
@@ -9,16 +8,16 @@ import pandas as pd
 import json
 from typing import Dict, Any, List, Optional
 
-# Import functions directly from pyconvexity
-from pyconvexity.models.attributes import deserialize_timeseries_from_parquet
+from pyconvexity.models.attributes import get_timeseries
+from pyconvexity.models import get_network_time_periods
 
 logger = logging.getLogger(__name__)
 
 
 class PyPSABatchLoader:
     """
-    Optimized batch data loader for PyPSA network construction.
-    Eliminates N+1 query patterns by loading data in batches.
+    Simplified batch data loader for PyPSA network construction.
+    Always creates MultiIndex timeseries for consistent multi-period optimization.
     """
     
     def __init__(self):
@@ -148,23 +147,25 @@ class PyPSABatchLoader:
         }
     
     def batch_load_component_timeseries(self, conn, component_ids: List[int], scenario_id: Optional[int]) -> Dict[int, Dict[str, pd.Series]]:
-        """Batch load all timeseries attributes for multiple components to avoid N+1 queries"""
+        """Batch load all timeseries attributes - always create MultiIndex for consistency"""
         if not component_ids:
             return {}
         
         # Get network time periods for proper timestamp alignment
-        network_time_periods = None
-        if component_ids:
-            cursor = conn.execute("SELECT network_id FROM components WHERE id = ? LIMIT 1", (component_ids[0],))
-            result = cursor.fetchone()
-            if result:
-                network_id = result[0]
-                try:
-                    from pyconvexity.models.network import get_network_time_periods
-                    network_time_periods = get_network_time_periods(conn, network_id)
-                    logger.debug(f"Loaded {len(network_time_periods)} time periods for timeseries alignment")
-                except Exception as e:
-                    logger.warning(f"Failed to load network time periods: {e}")
+        cursor = conn.execute("SELECT network_id FROM components WHERE id = ? LIMIT 1", (component_ids[0],))
+        result = cursor.fetchone()
+        if not result:
+            return {comp_id: {} for comp_id in component_ids}
+        
+        network_id = result[0]
+        network_time_periods = get_network_time_periods(conn, network_id)
+        if not network_time_periods:
+            logger.warning("No time periods found for network")
+            return {comp_id: {} for comp_id in component_ids}
+        
+        # Convert to timestamps and extract years
+        timestamps = [pd.Timestamp(tp.formatted_time) for tp in network_time_periods]
+        years = sorted(list(set([ts.year for ts in timestamps])))
         
         # Build a single query to get all timeseries attributes for all components
         placeholders = ','.join(['?' for _ in component_ids])
@@ -188,7 +189,7 @@ class PyPSABatchLoader:
         master_id = None
         if scenario_id is not None:
             # Get master scenario ID for fallback
-            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = (SELECT network_id FROM components WHERE id = ?) AND is_master = 1", (component_ids[0],))
+            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = ? AND is_master = 1", (network_id,))
             result = cursor.fetchone()
             if result:
                 master_id = result[0]
@@ -197,7 +198,7 @@ class PyPSABatchLoader:
                 scenario_filter_values = [scenario_id]
         else:
             # Get master scenario ID
-            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = (SELECT network_id FROM components WHERE id = ?) AND is_master = 1", (component_ids[0],))
+            cursor = conn.execute("SELECT id FROM scenarios WHERE network_id = ? AND is_master = 1", (network_id,))
             result = cursor.fetchone()
             if result:
                 master_id = result[0]
@@ -208,7 +209,6 @@ class PyPSABatchLoader:
         scen_placeholders = ','.join(['?' for _ in scenario_filter_values])
         
         # Single query to get all timeseries
-        # CRITICAL: Order by scenario_id to prioritize current scenario over master
         query = f"""
             SELECT component_id, attribute_name, timeseries_data, scenario_id
             FROM component_attributes 
@@ -246,31 +246,24 @@ class PyPSABatchLoader:
             
             # Deserialize timeseries data
             try:
-                timeseries_points = deserialize_timeseries_from_parquet(timeseries_data, network_time_periods)
-                if timeseries_points:
-                    # Create pandas Series from timeseries points
-                    # Sort by period_index to ensure correct order
-                    timeseries_points.sort(key=lambda x: x.period_index)
-                    values = [point.value for point in timeseries_points]
+                timeseries = get_timeseries(conn, comp_id, attr_name, row_scenario_id)
+                if timeseries and timeseries.values:
+                    values = timeseries.values
+                    
+                    # Always create MultiIndex following PyPSA multi-investment tutorial format
+                    # First level: investment periods (years), Second level: timesteps
+                    multi_snapshots = []
+                    for i, ts in enumerate(timestamps[:len(values)]):
+                        multi_snapshots.append((ts.year, ts))
                     
-                    # Create proper timestamps for PyPSA alignment
-                    if network_time_periods:
-                        # Use formatted timestamps from network time periods
-                        timestamps = []
-                        for point in timeseries_points:
-                            if point.period_index < len(network_time_periods):
-                                tp = network_time_periods[point.period_index]
-                                timestamps.append(pd.Timestamp(tp.formatted_time))
-                            else:
-                                logger.warning(f"Period index {point.period_index} out of range for network time periods")
-                                timestamps.append(pd.Timestamp.now())  # Fallback
-                        component_timeseries[comp_id][attr_name] = pd.Series(values, index=timestamps)
+                    if multi_snapshots:
+                        multi_index = pd.MultiIndex.from_tuples(multi_snapshots, names=['period', 'timestep'])
+                        component_timeseries[comp_id][attr_name] = pd.Series(values, index=multi_index)
                     else:
-                        # Fallback: use period_index as index
-                        period_indices = [point.period_index for point in timeseries_points]
-                        component_timeseries[comp_id][attr_name] = pd.Series(values, index=period_indices)
+                        logger.warning(f"No valid timestamps for timeseries {attr_name}")
+                        
             except Exception as e:
-                logger.warning(f"Failed to deserialize timeseries {attr_name} for component {comp_id}: {e}")
+                logger.warning(f"Failed to load timeseries {attr_name} for component {comp_id}: {e}")
                 continue
         
         return component_timeseries