pyconvexity 0.1.3__py3-none-any.whl → 0.3.7__py3-none-any.whl

pyconvexity/io/excel_importer.py

@@ -13,7 +13,7 @@ import json
 
 # Import functions directly from pyconvexity
 from pyconvexity.core.database import open_connection
-from pyconvexity.core.types import StaticValue, TimeseriesPoint, CreateNetworkRequest
+from pyconvexity.core.types import StaticValue, CreateNetworkRequest
 from pyconvexity.core.errors import AttributeNotFound, ValidationError
 from pyconvexity.models import (
     list_components_by_type, create_component, update_component, create_network,
@@ -22,6 +22,8 @@ from pyconvexity.models import (
     get_network_info, delete_attribute
 )
 from pyconvexity.validation import get_validation_rule
+from pyconvexity.timeseries import set_timeseries
+from pyconvexity.models.attributes import set_timeseries_attribute as set_timeseries_conn
 
 logger = logging.getLogger(__name__)
 
@@ -302,14 +304,20 @@ class ExcelModelImporter:
             
             self.logger.info(f"Generating {len(timestamps)} time periods from {start_time} to {end_time} at {time_resolution} resolution")
             
-            # Insert time periods into database
-            for period_index, timestamp in enumerate(timestamps):
-                timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M:%S')
-                
-                conn.execute("""
-                    INSERT INTO network_time_periods (network_id, timestamp, period_index)
-                    VALUES (?, ?, ?)
-                """, (network_id, timestamp_str, period_index))
+            # Insert optimized time periods metadata
+            period_count = len(timestamps)
+            start_timestamp = int(timestamps[0].timestamp())
+            
+            # Calculate interval in seconds
+            if len(timestamps) > 1:
+                interval_seconds = int((timestamps[1] - timestamps[0]).total_seconds())
+            else:
+                interval_seconds = 3600  # Default to hourly
+            
+            conn.execute("""
+                INSERT INTO network_time_periods (network_id, period_count, start_timestamp, interval_seconds)
+                VALUES (?, ?, ?, ?)
+            """, (network_id, period_count, start_timestamp, interval_seconds))
             
             self.logger.info(f"Successfully created {len(timestamps)} time periods for network {network_id}")
             
@@ -765,76 +773,65 @@ class ExcelModelImporter:
                             component = next((c for c in components if c.name == component_name), None)
                         
                         if component:
-                            # Create timeseries data
-                            timeseries_data = []
-                            matched_timestamps = 0
-                            fallback_timestamps = 0
-                            skipped_timestamps = 0
+                            # Create timeseries data using efficient array format
+                            timeseries_values = []
                             filled_missing_values = 0
                             
-                            for i, (timestamp, value) in enumerate(zip(timestamps, df[column])):
-                                # Convert timestamp string to match network time periods
-                                timestamp_str = str(timestamp).strip()
+                            # Debug: Show first few timestamps for comparison
+                            if len(timestamps) > 0 and len(network_time_periods) > 0:
+                                excel_first = str(timestamps.iloc[0]).strip()
+                                excel_last = str(timestamps.iloc[-1]).strip() if len(timestamps) > 1 else excel_first
+                                network_first = network_time_periods[0].formatted_time
+                                network_last = network_time_periods[-1].formatted_time if len(network_time_periods) > 1 else network_first
+                                
+                                self.logger.info(f"TIMESERIES DEBUG: Timestamp comparison for '{component_name}.{attr_name}':")
+                                self.logger.info(f"  Excel range: '{excel_first}' to '{excel_last}' ({len(timestamps)} periods)")
+                                self.logger.info(f"  Network range: '{network_first}' to '{network_last}' ({len(network_time_periods)} periods)")
+                            
+                            # Take the first N values from Excel where N = expected network periods
+                            # This puts responsibility on user to format Excel correctly
+                            max_periods = min(len(timestamps), len(network_time_periods), len(df[column]))
+                            
+                            for i in range(max_periods):
+                                value = df[column].iloc[i]
                                 
                                 # Handle missing values - use 0.0 as default
                                 if pd.isna(value):
                                     actual_value = 0.0
                                     filled_missing_values += 1
-                                    self.logger.debug(f"Using default value 0.0 for missing data at timestamp {timestamp_str} in {component_name}.{attr_name}")
                                 else:
                                     try:
                                         actual_value = float(value)
                                     except (ValueError, TypeError):
                                         actual_value = 0.0
                                         filled_missing_values += 1
-                                        self.logger.debug(f"Using default value 0.0 for invalid data '{value}' at timestamp {timestamp_str} in {component_name}.{attr_name}")
-                                
-                                # Try to find matching time period
-                                if timestamp_str in time_period_map:
-                                    period = time_period_map[timestamp_str]
-                                    timeseries_point = TimeseriesPoint(
-                                        timestamp=period.timestamp,
-                                        value=actual_value,
-                                        period_index=period.period_index
-                                    )
-                                    matched_timestamps += 1
-                                else:
-                                    # Fallback: try to parse timestamp and use index
-                                    try:
-                                        parsed_timestamp = pd.to_datetime(timestamp_str)
-                                        timeseries_point = TimeseriesPoint(
-                                            timestamp=int(parsed_timestamp.timestamp()),
-                                            value=actual_value,
-                                            period_index=i
-                                        )
-                                        fallback_timestamps += 1
-                                    except Exception as e:
-                                        skipped_timestamps += 1
-                                        self.logger.warning(f"Failed to parse timestamp '{timestamp_str}' in {component_name}.{attr_name}: {e}")
-                                        continue
                                 
-                                timeseries_data.append(timeseries_point)
+                                timeseries_values.append(actual_value)
                             
-                            final_ts_length = len(timeseries_data)
+                            final_ts_length = len(timeseries_values)
                             self.logger.info(f"TIMESERIES DEBUG: Component '{component_name}.{attr_name}': "
                                            f"Excel rows={excel_ts_length}, "
-                                           f"Matched={matched_timestamps}, "
-                                           f"Fallback={fallback_timestamps}, "
-                                           f"Skipped={skipped_timestamps}, "
+                                           f"Network periods={expected_length}, "
+                                           f"Used={max_periods}, "
                                            f"Filled missing={filled_missing_values}, "
                                            f"Final length={final_ts_length}")
                             
                             if filled_missing_values > 0:
                                 self.logger.warning(f"TIMESERIES DEBUG: Filled {filled_missing_values} missing/invalid values with 0.0 for '{component_name}.{attr_name}'")
                             
+                            if excel_ts_length != expected_length:
+                                self.logger.warning(f"TIMESERIES DEBUG: LENGTH MISMATCH for '{component_name}.{attr_name}': "
+                                                  f"Excel has {excel_ts_length} rows, network expects {expected_length} periods")
+                            
                             if final_ts_length != expected_length:
                                 self.logger.warning(f"TIMESERIES DEBUG: FINAL LENGTH MISMATCH for '{component_name}.{attr_name}': "
                                                   f"Expected {expected_length}, got {final_ts_length} (difference: {final_ts_length - expected_length})")
                             
-                            if timeseries_data:
+                            if timeseries_values:
                                 try:
-                                    set_timeseries_attribute(
-                                            conn, component.id, attr_name, timeseries_data, scenario_id
+                                    # Use new efficient timeseries API
+                                    set_timeseries_conn(
+                                        conn, component.id, attr_name, timeseries_values, scenario_id
                                     )
                                     imported['attributes'] += 1
                                     self.logger.info(f"TIMESERIES DEBUG: Successfully imported {final_ts_length} points for '{component_name}.{attr_name}'")