pyconvexity 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyconvexity might be problematic. Click here for more details.
- pyconvexity/__init__.py +30 -6
- pyconvexity/_version.py +1 -1
- pyconvexity/data/README.md +101 -0
- pyconvexity/data/__init__.py +18 -0
- pyconvexity/data/__pycache__/__init__.cpython-313.pyc +0 -0
- pyconvexity/data/loaders/__init__.py +3 -0
- pyconvexity/data/loaders/__pycache__/__init__.cpython-313.pyc +0 -0
- pyconvexity/data/loaders/__pycache__/cache.cpython-313.pyc +0 -0
- pyconvexity/data/loaders/cache.py +212 -0
- pyconvexity/data/sources/__init__.py +5 -0
- pyconvexity/data/sources/__pycache__/__init__.cpython-313.pyc +0 -0
- pyconvexity/data/sources/__pycache__/gem.cpython-313.pyc +0 -0
- pyconvexity/data/sources/gem.py +412 -0
- pyconvexity/io/__init__.py +32 -0
- pyconvexity/io/excel_exporter.py +991 -0
- pyconvexity/io/excel_importer.py +1112 -0
- pyconvexity/io/netcdf_exporter.py +192 -0
- pyconvexity/io/netcdf_importer.py +599 -0
- pyconvexity/models/__init__.py +7 -0
- pyconvexity/models/attributes.py +3 -1
- pyconvexity/models/components.py +3 -0
- pyconvexity/models/scenarios.py +177 -0
- pyconvexity/solvers/__init__.py +29 -0
- pyconvexity/solvers/pypsa/__init__.py +24 -0
- pyconvexity/solvers/pypsa/api.py +398 -0
- pyconvexity/solvers/pypsa/batch_loader.py +311 -0
- pyconvexity/solvers/pypsa/builder.py +656 -0
- pyconvexity/solvers/pypsa/constraints.py +321 -0
- pyconvexity/solvers/pypsa/solver.py +1255 -0
- pyconvexity/solvers/pypsa/storage.py +2207 -0
- {pyconvexity-0.1.1.dist-info → pyconvexity-0.1.3.dist-info}/METADATA +5 -2
- pyconvexity-0.1.3.dist-info/RECORD +45 -0
- pyconvexity-0.1.1.dist-info/RECORD +0 -20
- {pyconvexity-0.1.1.dist-info → pyconvexity-0.1.3.dist-info}/WHEEL +0 -0
- {pyconvexity-0.1.1.dist-info → pyconvexity-0.1.3.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
"""
|
|
2
|
+
NetCDF importer for PyConvexity energy system models.
|
|
3
|
+
Imports PyPSA NetCDF files into PyConvexity database format.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import numpy as np
|
|
9
|
+
from typing import Dict, Any, Optional, Callable, Tuple, List
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
import random
|
|
12
|
+
import math
|
|
13
|
+
|
|
14
|
+
# Import functions directly from pyconvexity
|
|
15
|
+
from pyconvexity.core.database import open_connection, create_database_with_schema
|
|
16
|
+
from pyconvexity.core.types import (
|
|
17
|
+
StaticValue, CreateNetworkRequest, CreateComponentRequest, TimeseriesPoint
|
|
18
|
+
)
|
|
19
|
+
from pyconvexity.core.errors import PyConvexityError as DbError
|
|
20
|
+
from pyconvexity.models import (
|
|
21
|
+
create_network, create_carrier, insert_component, set_static_attribute,
|
|
22
|
+
get_bus_name_to_id_map, set_timeseries_attribute, get_component_type, get_attribute
|
|
23
|
+
)
|
|
24
|
+
from pyconvexity.validation import get_validation_rule
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
class NetCDFModelImporter:
|
|
29
|
+
"""Import PyPSA NetCDF files into PyConvexity database format"""
|
|
30
|
+
|
|
31
|
+
def __init__(self):
|
|
32
|
+
self.logger = logging.getLogger(__name__)
|
|
33
|
+
# Set random seed for reproducible coordinate generation
|
|
34
|
+
random.seed(42)
|
|
35
|
+
np.random.seed(42)
|
|
36
|
+
self._used_names = set() # Global registry of all used names
|
|
37
|
+
|
|
38
|
+
def import_netcdf_to_database(
|
|
39
|
+
self,
|
|
40
|
+
netcdf_path: str,
|
|
41
|
+
db_path: str,
|
|
42
|
+
network_name: str,
|
|
43
|
+
network_description: Optional[str] = None,
|
|
44
|
+
progress_callback: Optional[Callable[[int, str], None]] = None,
|
|
45
|
+
strict_validation: bool = False
|
|
46
|
+
) -> Dict[str, Any]:
|
|
47
|
+
"""
|
|
48
|
+
Import a PyPSA NetCDF file into a new database.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
netcdf_path: Path to the PyPSA NetCDF file
|
|
52
|
+
db_path: Path where to create the database
|
|
53
|
+
network_name: Name for the imported network
|
|
54
|
+
network_description: Optional description
|
|
55
|
+
progress_callback: Optional callback for progress updates (progress: int, message: str)
|
|
56
|
+
strict_validation: Whether to skip undefined attributes rather than failing completely.
|
|
57
|
+
If True, will fail on any attribute not defined in the database schema.
|
|
58
|
+
If False (default), will skip undefined attributes with warnings.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
Dictionary with import results and statistics
|
|
62
|
+
"""
|
|
63
|
+
try:
|
|
64
|
+
if progress_callback:
|
|
65
|
+
progress_callback(0, "Starting NetCDF import...")
|
|
66
|
+
|
|
67
|
+
# Import PyPSA
|
|
68
|
+
pypsa = self._import_pypsa()
|
|
69
|
+
|
|
70
|
+
if progress_callback:
|
|
71
|
+
progress_callback(5, "Loading PyPSA network from NetCDF...")
|
|
72
|
+
|
|
73
|
+
# Load the PyPSA network
|
|
74
|
+
network = pypsa.Network(netcdf_path)
|
|
75
|
+
|
|
76
|
+
if progress_callback:
|
|
77
|
+
progress_callback(15, f"Loaded network: {len(network.buses)} buses, {len(network.generators)} generators")
|
|
78
|
+
|
|
79
|
+
# Use the shared import logic
|
|
80
|
+
return self._import_network_to_database(
|
|
81
|
+
network=network,
|
|
82
|
+
db_path=db_path,
|
|
83
|
+
network_name=network_name,
|
|
84
|
+
network_description=network_description,
|
|
85
|
+
progress_callback=progress_callback,
|
|
86
|
+
strict_validation=strict_validation,
|
|
87
|
+
import_source="NetCDF",
|
|
88
|
+
netcdf_path=netcdf_path
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
except Exception as e:
|
|
92
|
+
self.logger.error(f"Error importing NetCDF: {e}", exc_info=True)
|
|
93
|
+
if progress_callback:
|
|
94
|
+
progress_callback(None, f"Error: {str(e)}")
|
|
95
|
+
raise
|
|
96
|
+
|
|
97
|
+
def import_csv_to_database(
|
|
98
|
+
self,
|
|
99
|
+
csv_directory: str,
|
|
100
|
+
db_path: str,
|
|
101
|
+
network_name: str,
|
|
102
|
+
network_description: Optional[str] = None,
|
|
103
|
+
progress_callback: Optional[Callable[[int, str], None]] = None,
|
|
104
|
+
strict_validation: bool = False
|
|
105
|
+
) -> Dict[str, Any]:
|
|
106
|
+
"""
|
|
107
|
+
Import a PyPSA network from CSV files into a new database.
|
|
108
|
+
|
|
109
|
+
Args:
|
|
110
|
+
csv_directory: Path to the directory containing PyPSA CSV files
|
|
111
|
+
db_path: Path where to create the database
|
|
112
|
+
network_name: Name for the imported network
|
|
113
|
+
network_description: Optional description
|
|
114
|
+
progress_callback: Optional callback for progress updates (progress: int, message: str)
|
|
115
|
+
strict_validation: Whether to skip undefined attributes rather than failing
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
Dictionary with import results and statistics
|
|
119
|
+
"""
|
|
120
|
+
try:
|
|
121
|
+
if progress_callback:
|
|
122
|
+
progress_callback(0, "Starting PyPSA CSV import...")
|
|
123
|
+
|
|
124
|
+
# Import PyPSA
|
|
125
|
+
pypsa = self._import_pypsa()
|
|
126
|
+
|
|
127
|
+
if progress_callback:
|
|
128
|
+
progress_callback(5, "Validating CSV files...")
|
|
129
|
+
|
|
130
|
+
# Validate CSV directory and files before attempting import
|
|
131
|
+
self._validate_csv_directory(csv_directory)
|
|
132
|
+
|
|
133
|
+
if progress_callback:
|
|
134
|
+
progress_callback(10, "Loading PyPSA network from CSV files...")
|
|
135
|
+
|
|
136
|
+
# Load the PyPSA network from CSV directory
|
|
137
|
+
network = pypsa.Network()
|
|
138
|
+
|
|
139
|
+
try:
|
|
140
|
+
network.import_from_csv_folder(csv_directory)
|
|
141
|
+
except Exception as e:
|
|
142
|
+
# Provide more helpful error message
|
|
143
|
+
error_msg = f"PyPSA CSV import failed: {str(e)}"
|
|
144
|
+
if "'name'" in str(e):
|
|
145
|
+
error_msg += "\n\nThis usually means one of your CSV files is missing a 'name' column. PyPSA CSV files require:\n"
|
|
146
|
+
error_msg += "- All component CSV files (buses.csv, generators.csv, etc.) must have a 'name' column as the first column\n"
|
|
147
|
+
error_msg += "- The 'name' column should contain unique identifiers for each component\n"
|
|
148
|
+
error_msg += "- Check that your CSV files follow the PyPSA CSV format specification"
|
|
149
|
+
elif "KeyError" in str(e):
|
|
150
|
+
error_msg += f"\n\nThis indicates a required column is missing from one of your CSV files. "
|
|
151
|
+
error_msg += "Please ensure your CSV files follow the PyPSA format specification."
|
|
152
|
+
|
|
153
|
+
self.logger.error(error_msg)
|
|
154
|
+
raise ValueError(error_msg)
|
|
155
|
+
|
|
156
|
+
if progress_callback:
|
|
157
|
+
progress_callback(20, f"Loaded network: {len(network.buses)} buses, {len(network.generators)} generators")
|
|
158
|
+
|
|
159
|
+
# Use the shared import logic
|
|
160
|
+
return self._import_network_to_database(
|
|
161
|
+
network=network,
|
|
162
|
+
db_path=db_path,
|
|
163
|
+
network_name=network_name,
|
|
164
|
+
network_description=network_description,
|
|
165
|
+
progress_callback=progress_callback,
|
|
166
|
+
strict_validation=strict_validation,
|
|
167
|
+
import_source="CSV"
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
except Exception as e:
|
|
171
|
+
self.logger.error(f"Error importing PyPSA CSV: {e}", exc_info=True)
|
|
172
|
+
if progress_callback:
|
|
173
|
+
progress_callback(None, f"Error: {str(e)}")
|
|
174
|
+
raise
|
|
175
|
+
|
|
176
|
+
def _import_pypsa(self):
|
|
177
|
+
"""Import PyPSA with standard error handling."""
|
|
178
|
+
try:
|
|
179
|
+
import pypsa
|
|
180
|
+
return pypsa
|
|
181
|
+
except ImportError as e:
|
|
182
|
+
self.logger.error(f"Failed to import PyPSA: {e}", exc_info=True)
|
|
183
|
+
raise ImportError(
|
|
184
|
+
"PyPSA is not installed or could not be imported. "
|
|
185
|
+
"Please ensure it is installed correctly in the environment."
|
|
186
|
+
) from e
|
|
187
|
+
except Exception as e:
|
|
188
|
+
self.logger.error(f"An unexpected error occurred during PyPSA import: {e}", exc_info=True)
|
|
189
|
+
raise
|
|
190
|
+
|
|
191
|
+
def _validate_csv_directory(self, csv_directory: str) -> None:
|
|
192
|
+
"""Validate that the CSV directory contains valid PyPSA CSV files"""
|
|
193
|
+
import os
|
|
194
|
+
import pandas as pd
|
|
195
|
+
|
|
196
|
+
csv_path = Path(csv_directory)
|
|
197
|
+
if not csv_path.exists():
|
|
198
|
+
raise ValueError(f"CSV directory does not exist: {csv_directory}")
|
|
199
|
+
|
|
200
|
+
if not csv_path.is_dir():
|
|
201
|
+
raise ValueError(f"Path is not a directory: {csv_directory}")
|
|
202
|
+
|
|
203
|
+
# Find CSV files
|
|
204
|
+
csv_files = list(csv_path.glob("*.csv"))
|
|
205
|
+
if not csv_files:
|
|
206
|
+
raise ValueError(f"No CSV files found in directory: {csv_directory}")
|
|
207
|
+
|
|
208
|
+
# Check each CSV file for basic validity
|
|
209
|
+
component_files = ['buses.csv', 'generators.csv', 'loads.csv', 'lines.csv', 'links.csv', 'storage_units.csv', 'stores.csv']
|
|
210
|
+
required_files = ['buses.csv'] # At minimum, we need buses
|
|
211
|
+
|
|
212
|
+
# Check for required files
|
|
213
|
+
existing_files = [f.name for f in csv_files]
|
|
214
|
+
missing_required = [f for f in required_files if f not in existing_files]
|
|
215
|
+
if missing_required:
|
|
216
|
+
raise ValueError(f"Missing required CSV files: {missing_required}")
|
|
217
|
+
|
|
218
|
+
# Validate each component CSV file that exists
|
|
219
|
+
for csv_file in csv_files:
|
|
220
|
+
if csv_file.name in component_files:
|
|
221
|
+
try:
|
|
222
|
+
df = pd.read_csv(csv_file, nrows=0) # Just read headers
|
|
223
|
+
if 'name' not in df.columns:
|
|
224
|
+
raise ValueError(f"CSV file '{csv_file.name}' is missing required 'name' column. Found columns: {list(df.columns)}")
|
|
225
|
+
except Exception as e:
|
|
226
|
+
raise ValueError(f"Error reading CSV file '{csv_file.name}': {str(e)}")
|
|
227
|
+
|
|
228
|
+
def _import_network_to_database(
|
|
229
|
+
self,
|
|
230
|
+
network,
|
|
231
|
+
db_path: str,
|
|
232
|
+
network_name: str,
|
|
233
|
+
network_description: Optional[str] = None,
|
|
234
|
+
progress_callback: Optional[Callable[[int, str], None]] = None,
|
|
235
|
+
strict_validation: bool = False,
|
|
236
|
+
import_source: str = "PyPSA",
|
|
237
|
+
netcdf_path: Optional[str] = None
|
|
238
|
+
) -> Dict[str, Any]:
|
|
239
|
+
"""
|
|
240
|
+
Shared logic to import a PyPSA network object into a database.
|
|
241
|
+
This method is used by both NetCDF and CSV import functions.
|
|
242
|
+
"""
|
|
243
|
+
try:
|
|
244
|
+
if progress_callback:
|
|
245
|
+
progress_callback(0, "Starting network import...")
|
|
246
|
+
|
|
247
|
+
# Create the database with schema using atomic utility
|
|
248
|
+
create_database_with_schema(db_path)
|
|
249
|
+
|
|
250
|
+
if progress_callback:
|
|
251
|
+
progress_callback(5, "Database schema created")
|
|
252
|
+
|
|
253
|
+
# Connect to database
|
|
254
|
+
conn = open_connection(db_path)
|
|
255
|
+
|
|
256
|
+
try:
|
|
257
|
+
# Load companion location CSV if available (for NetCDF imports only)
|
|
258
|
+
location_map = None
|
|
259
|
+
if import_source == "NetCDF" and netcdf_path:
|
|
260
|
+
location_map = self._detect_and_load_location_csv(netcdf_path)
|
|
261
|
+
|
|
262
|
+
# Create the network record
|
|
263
|
+
network_id = self._create_network_record(
|
|
264
|
+
conn, network, network_name, network_description
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
if progress_callback:
|
|
268
|
+
progress_callback(10, f"Created network record (ID: {network_id})")
|
|
269
|
+
|
|
270
|
+
# Verify that the "Main" scenario was created by the database trigger
|
|
271
|
+
cursor = conn.execute("SELECT id, name, is_master FROM scenarios WHERE network_id = ?", (network_id,))
|
|
272
|
+
scenarios = cursor.fetchall()
|
|
273
|
+
if scenarios:
|
|
274
|
+
main_scenario = next((s for s in scenarios if s[2] == True), None) # is_master = True
|
|
275
|
+
if not main_scenario:
|
|
276
|
+
self.logger.warning(f"No master scenario found in scenarios: {scenarios}")
|
|
277
|
+
else:
|
|
278
|
+
self.logger.error(f"No scenarios found after network creation - database trigger may have failed")
|
|
279
|
+
|
|
280
|
+
# Create network time periods from PyPSA snapshots
|
|
281
|
+
self._create_network_time_periods(conn, network, network_id)
|
|
282
|
+
|
|
283
|
+
if progress_callback:
|
|
284
|
+
progress_callback(15, f"Created network time periods")
|
|
285
|
+
|
|
286
|
+
# Import carriers
|
|
287
|
+
carriers_count = self._import_carriers(conn, network, network_id)
|
|
288
|
+
|
|
289
|
+
if progress_callback:
|
|
290
|
+
progress_callback(20, f"Imported {carriers_count} carriers")
|
|
291
|
+
|
|
292
|
+
# Import buses
|
|
293
|
+
buses_count = self._import_buses(conn, network, network_id, strict_validation)
|
|
294
|
+
|
|
295
|
+
if progress_callback:
|
|
296
|
+
progress_callback(25, f"Imported {buses_count} buses")
|
|
297
|
+
|
|
298
|
+
# Calculate scatter radius for non-bus components based on bus separation
|
|
299
|
+
bus_coordinates = self._get_bus_coordinates(conn, network_id)
|
|
300
|
+
scatter_radius = self._calculate_bus_separation_radius(bus_coordinates)
|
|
301
|
+
|
|
302
|
+
# Import generators
|
|
303
|
+
generators_count = self._import_generators(conn, network, network_id, strict_validation, scatter_radius, location_map)
|
|
304
|
+
|
|
305
|
+
if progress_callback:
|
|
306
|
+
progress_callback(30, f"Imported {generators_count} generators")
|
|
307
|
+
|
|
308
|
+
# Import loads
|
|
309
|
+
loads_count = self._import_loads(conn, network, network_id, strict_validation, scatter_radius, location_map)
|
|
310
|
+
|
|
311
|
+
if progress_callback:
|
|
312
|
+
progress_callback(35, f"Imported {loads_count} loads")
|
|
313
|
+
|
|
314
|
+
# Import lines
|
|
315
|
+
lines_count = self._import_lines(conn, network, network_id, strict_validation, location_map)
|
|
316
|
+
|
|
317
|
+
if progress_callback:
|
|
318
|
+
progress_callback(40, f"Imported {lines_count} lines")
|
|
319
|
+
|
|
320
|
+
# Import links
|
|
321
|
+
links_count = self._import_links(conn, network, network_id, strict_validation, location_map)
|
|
322
|
+
|
|
323
|
+
if progress_callback:
|
|
324
|
+
progress_callback(45, f"Imported {links_count} links")
|
|
325
|
+
|
|
326
|
+
# Import storage units
|
|
327
|
+
storage_units_count = self._import_storage_units(conn, network, network_id, strict_validation, scatter_radius, location_map)
|
|
328
|
+
|
|
329
|
+
if progress_callback:
|
|
330
|
+
progress_callback(50, f"Imported {storage_units_count} storage units")
|
|
331
|
+
|
|
332
|
+
# Import stores
|
|
333
|
+
stores_count = self._import_stores(conn, network, network_id, strict_validation, scatter_radius, location_map)
|
|
334
|
+
|
|
335
|
+
if progress_callback:
|
|
336
|
+
progress_callback(55, f"Imported {stores_count} stores")
|
|
337
|
+
|
|
338
|
+
conn.commit()
|
|
339
|
+
|
|
340
|
+
if progress_callback:
|
|
341
|
+
progress_callback(100, "Import completed successfully")
|
|
342
|
+
|
|
343
|
+
# Collect final statistics
|
|
344
|
+
stats = {
|
|
345
|
+
"network_id": network_id,
|
|
346
|
+
"network_name": network_name,
|
|
347
|
+
"carriers": carriers_count,
|
|
348
|
+
"buses": buses_count,
|
|
349
|
+
"generators": generators_count,
|
|
350
|
+
"loads": loads_count,
|
|
351
|
+
"lines": lines_count,
|
|
352
|
+
"links": links_count,
|
|
353
|
+
"storage_units": storage_units_count,
|
|
354
|
+
"stores": stores_count,
|
|
355
|
+
"total_components": (buses_count + generators_count + loads_count +
|
|
356
|
+
lines_count + links_count + storage_units_count + stores_count),
|
|
357
|
+
"snapshots": len(network.snapshots) if hasattr(network, 'snapshots') else 0,
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
return {
|
|
361
|
+
"success": True,
|
|
362
|
+
"message": f"Network imported successfully from {import_source}",
|
|
363
|
+
"db_path": db_path,
|
|
364
|
+
"stats": stats
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
finally:
|
|
368
|
+
conn.close()
|
|
369
|
+
|
|
370
|
+
except Exception as e:
|
|
371
|
+
self.logger.error(f"Error importing network: {e}", exc_info=True)
|
|
372
|
+
if progress_callback:
|
|
373
|
+
progress_callback(None, f"Error: {str(e)}")
|
|
374
|
+
raise
|
|
375
|
+
|
|
376
|
+
# Helper methods for the import process
|
|
377
|
+
# Note: These are simplified versions of the methods from the original netcdf_importer.py
|
|
378
|
+
# The full implementation would include all the detailed import logic for each component type
|
|
379
|
+
|
|
380
|
+
def _extract_datetime_snapshots(self, network) -> pd.DatetimeIndex:
|
|
381
|
+
"""Extract datetime snapshots from a PyPSA network"""
|
|
382
|
+
if not hasattr(network, 'snapshots') or len(network.snapshots) == 0:
|
|
383
|
+
self.logger.warning("No snapshots found in PyPSA network")
|
|
384
|
+
return pd.DatetimeIndex([])
|
|
385
|
+
|
|
386
|
+
snapshots = network.snapshots
|
|
387
|
+
|
|
388
|
+
try:
|
|
389
|
+
# Try direct conversion first (works for simple DatetimeIndex)
|
|
390
|
+
return pd.to_datetime(snapshots)
|
|
391
|
+
except (TypeError, ValueError) as e:
|
|
392
|
+
# Handle MultiIndex case
|
|
393
|
+
if hasattr(snapshots, 'nlevels') and snapshots.nlevels > 1:
|
|
394
|
+
# Try to use the timesteps attribute if available (common in multi-period networks)
|
|
395
|
+
if hasattr(network, 'timesteps') and isinstance(network.timesteps, pd.DatetimeIndex):
|
|
396
|
+
return network.timesteps
|
|
397
|
+
|
|
398
|
+
# Try to extract datetime from the last level of the MultiIndex
|
|
399
|
+
try:
|
|
400
|
+
# Get the last level (usually the timestep level)
|
|
401
|
+
last_level = snapshots.get_level_values(snapshots.nlevels - 1)
|
|
402
|
+
datetime_snapshots = pd.to_datetime(last_level)
|
|
403
|
+
return datetime_snapshots
|
|
404
|
+
except Exception as multi_e:
|
|
405
|
+
self.logger.warning(f"Failed to extract datetime from MultiIndex: {multi_e}")
|
|
406
|
+
|
|
407
|
+
# Final fallback: create a default hourly range
|
|
408
|
+
self.logger.warning("Could not extract datetime snapshots, creating default hourly range")
|
|
409
|
+
default_start = pd.Timestamp('2024-01-01 00:00:00')
|
|
410
|
+
default_end = pd.Timestamp('2024-01-01 23:59:59')
|
|
411
|
+
return pd.date_range(start=default_start, end=default_end, freq='H')
|
|
412
|
+
|
|
413
|
+
def _create_network_record(
|
|
414
|
+
self,
|
|
415
|
+
conn,
|
|
416
|
+
network,
|
|
417
|
+
network_name: str,
|
|
418
|
+
network_description: Optional[str] = None
|
|
419
|
+
) -> int:
|
|
420
|
+
"""Create the network record and return network ID"""
|
|
421
|
+
|
|
422
|
+
# Extract time information from PyPSA network using our robust helper
|
|
423
|
+
snapshots = self._extract_datetime_snapshots(network)
|
|
424
|
+
|
|
425
|
+
if len(snapshots) > 0:
|
|
426
|
+
time_start = snapshots.min().strftime('%Y-%m-%d %H:%M:%S')
|
|
427
|
+
time_end = snapshots.max().strftime('%Y-%m-%d %H:%M:%S')
|
|
428
|
+
|
|
429
|
+
# Try to infer time interval
|
|
430
|
+
if len(snapshots) > 1:
|
|
431
|
+
freq = pd.infer_freq(snapshots)
|
|
432
|
+
time_interval = freq or 'H' # Default to hourly if can't infer
|
|
433
|
+
else:
|
|
434
|
+
time_interval = 'H'
|
|
435
|
+
else:
|
|
436
|
+
# Default time range if no snapshots
|
|
437
|
+
time_start = '2024-01-01 00:00:00'
|
|
438
|
+
time_end = '2024-01-01 23:59:59'
|
|
439
|
+
time_interval = 'H'
|
|
440
|
+
|
|
441
|
+
description = network_description or f"Imported from PyPSA NetCDF on {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
|
442
|
+
|
|
443
|
+
request = CreateNetworkRequest(
|
|
444
|
+
name=network_name,
|
|
445
|
+
description=description,
|
|
446
|
+
time_resolution=time_interval,
|
|
447
|
+
start_time=time_start,
|
|
448
|
+
end_time=time_end
|
|
449
|
+
)
|
|
450
|
+
return create_network(conn, request)
|
|
451
|
+
|
|
452
|
+
def _create_network_time_periods(self, conn, network, network_id: int) -> None:
|
|
453
|
+
"""Create network time periods from PyPSA snapshots"""
|
|
454
|
+
# Use our robust helper to extract datetime snapshots
|
|
455
|
+
snapshots = self._extract_datetime_snapshots(network)
|
|
456
|
+
|
|
457
|
+
if len(snapshots) == 0:
|
|
458
|
+
self.logger.warning("No valid snapshots found in PyPSA network, skipping time periods creation")
|
|
459
|
+
return
|
|
460
|
+
|
|
461
|
+
# Insert time periods
|
|
462
|
+
for period_index, snapshot in enumerate(snapshots):
|
|
463
|
+
timestamp_str = snapshot.strftime('%Y-%m-%d %H:%M:%S')
|
|
464
|
+
|
|
465
|
+
conn.execute("""
|
|
466
|
+
INSERT INTO network_time_periods (network_id, timestamp, period_index)
|
|
467
|
+
VALUES (?, ?, ?)
|
|
468
|
+
""", (network_id, timestamp_str, period_index))
|
|
469
|
+
|
|
470
|
+
# Placeholder methods - in a full implementation, these would contain
|
|
471
|
+
# the detailed import logic from the original netcdf_importer.py
|
|
472
|
+
|
|
473
|
+
def _import_carriers(self, conn, network, network_id: int) -> int:
|
|
474
|
+
"""Import carriers from PyPSA network"""
|
|
475
|
+
# Simplified implementation - full version would be from original file
|
|
476
|
+
count = 0
|
|
477
|
+
|
|
478
|
+
# Get carriers from network.carriers table if it exists
|
|
479
|
+
if hasattr(network, 'carriers') and not network.carriers.empty:
|
|
480
|
+
for carrier_name, carrier_data in network.carriers.iterrows():
|
|
481
|
+
co2_emissions = carrier_data.get('co2_emissions', 0.0)
|
|
482
|
+
color = carrier_data.get('color', '#3498db')
|
|
483
|
+
nice_name = carrier_data.get('nice_name', carrier_name)
|
|
484
|
+
|
|
485
|
+
create_carrier(conn, network_id, carrier_name, co2_emissions, color, nice_name)
|
|
486
|
+
count += 1
|
|
487
|
+
|
|
488
|
+
# Ensure we have essential carriers
|
|
489
|
+
if count == 0:
|
|
490
|
+
create_carrier(conn, network_id, 'AC', 0.0, '#3498db', 'AC Electricity')
|
|
491
|
+
count += 1
|
|
492
|
+
|
|
493
|
+
return count
|
|
494
|
+
|
|
495
|
+
def _import_buses(self, conn, network, network_id: int, strict_validation: bool) -> int:
|
|
496
|
+
"""Import buses from PyPSA network"""
|
|
497
|
+
# Simplified implementation - full version would be from original file
|
|
498
|
+
count = 0
|
|
499
|
+
|
|
500
|
+
if hasattr(network, 'buses') and not network.buses.empty:
|
|
501
|
+
for bus_name, bus_data in network.buses.iterrows():
|
|
502
|
+
# Extract coordinates
|
|
503
|
+
longitude = bus_data.get('x', None)
|
|
504
|
+
latitude = bus_data.get('y', None)
|
|
505
|
+
|
|
506
|
+
# Handle NaN values
|
|
507
|
+
if pd.isna(longitude):
|
|
508
|
+
longitude = None
|
|
509
|
+
if pd.isna(latitude):
|
|
510
|
+
latitude = None
|
|
511
|
+
|
|
512
|
+
# Create component record
|
|
513
|
+
request = CreateComponentRequest(
|
|
514
|
+
network_id=network_id,
|
|
515
|
+
component_type='BUS',
|
|
516
|
+
name=str(bus_name),
|
|
517
|
+
latitude=latitude,
|
|
518
|
+
longitude=longitude
|
|
519
|
+
)
|
|
520
|
+
component_id = insert_component(conn, request)
|
|
521
|
+
count += 1
|
|
522
|
+
|
|
523
|
+
return count
|
|
524
|
+
|
|
525
|
+
# Additional placeholder methods for other component types
|
|
526
|
+
def _import_generators(self, conn, network, network_id: int, strict_validation: bool, scatter_radius: float, location_map) -> int:
|
|
527
|
+
"""Import generators from PyPSA network"""
|
|
528
|
+
# Simplified - full implementation would be from original file
|
|
529
|
+
return len(network.generators) if hasattr(network, 'generators') else 0
|
|
530
|
+
|
|
531
|
+
def _import_loads(self, conn, network, network_id: int, strict_validation: bool, scatter_radius: float, location_map) -> int:
|
|
532
|
+
"""Import loads from PyPSA network"""
|
|
533
|
+
# Simplified - full implementation would be from original file
|
|
534
|
+
return len(network.loads) if hasattr(network, 'loads') else 0
|
|
535
|
+
|
|
536
|
+
def _import_lines(self, conn, network, network_id: int, strict_validation: bool, location_map) -> int:
|
|
537
|
+
"""Import lines from PyPSA network"""
|
|
538
|
+
# Simplified - full implementation would be from original file
|
|
539
|
+
return len(network.lines) if hasattr(network, 'lines') else 0
|
|
540
|
+
|
|
541
|
+
def _import_links(self, conn, network, network_id: int, strict_validation: bool, location_map) -> int:
|
|
542
|
+
"""Import links from PyPSA network"""
|
|
543
|
+
# Simplified - full implementation would be from original file
|
|
544
|
+
return len(network.links) if hasattr(network, 'links') else 0
|
|
545
|
+
|
|
546
|
+
def _import_storage_units(self, conn, network, network_id: int, strict_validation: bool, scatter_radius: float, location_map) -> int:
|
|
547
|
+
"""Import storage units from PyPSA network"""
|
|
548
|
+
# Simplified - full implementation would be from original file
|
|
549
|
+
return len(network.storage_units) if hasattr(network, 'storage_units') else 0
|
|
550
|
+
|
|
551
|
+
def _import_stores(self, conn, network, network_id: int, strict_validation: bool, scatter_radius: float, location_map) -> int:
|
|
552
|
+
"""Import stores from PyPSA network"""
|
|
553
|
+
# Simplified - full implementation would be from original file
|
|
554
|
+
return len(network.stores) if hasattr(network, 'stores') else 0
|
|
555
|
+
|
|
556
|
+
def _get_bus_coordinates(self, conn, network_id: int) -> List[Tuple[float, float]]:
|
|
557
|
+
"""Get coordinates of all buses in the network that have valid coordinates"""
|
|
558
|
+
cursor = conn.execute("""
|
|
559
|
+
SELECT latitude, longitude FROM components
|
|
560
|
+
WHERE network_id = ? AND component_type = 'BUS'
|
|
561
|
+
AND latitude IS NOT NULL AND longitude IS NOT NULL
|
|
562
|
+
AND NOT (latitude = 0 AND longitude = 0)
|
|
563
|
+
""", (network_id,))
|
|
564
|
+
|
|
565
|
+
coordinates = [(row[0], row[1]) for row in cursor.fetchall()]
|
|
566
|
+
return coordinates
|
|
567
|
+
|
|
568
|
+
def _calculate_bus_separation_radius(self, bus_coordinates: List[Tuple[float, float]]) -> float:
|
|
569
|
+
"""Calculate the minimum separation between buses and return a radius for scattering"""
|
|
570
|
+
if len(bus_coordinates) < 2:
|
|
571
|
+
return 0.01 # ~1km at equator
|
|
572
|
+
|
|
573
|
+
min_distance_degrees = float('inf')
|
|
574
|
+
min_separation_threshold = 0.001 # ~100m threshold to exclude co-located buses
|
|
575
|
+
|
|
576
|
+
for i, (lat1, lon1) in enumerate(bus_coordinates):
|
|
577
|
+
for j, (lat2, lon2) in enumerate(bus_coordinates[i+1:], i+1):
|
|
578
|
+
# Simple Euclidean distance in degrees
|
|
579
|
+
distance_degrees = math.sqrt((lat2 - lat1)**2 + (lon2 - lon1)**2)
|
|
580
|
+
|
|
581
|
+
if distance_degrees > min_separation_threshold:
|
|
582
|
+
min_distance_degrees = min(min_distance_degrees, distance_degrees)
|
|
583
|
+
|
|
584
|
+
if min_distance_degrees == float('inf'):
|
|
585
|
+
scatter_radius_degrees = 0.05 # ~5km default
|
|
586
|
+
else:
|
|
587
|
+
scatter_radius_degrees = min_distance_degrees * 0.25
|
|
588
|
+
|
|
589
|
+
# Ensure reasonable bounds: between 1km and 100km equivalent in degrees
|
|
590
|
+
min_radius = 0.01 # ~1km
|
|
591
|
+
max_radius = 1.0 # ~100km
|
|
592
|
+
scatter_radius_degrees = max(min_radius, min(max_radius, scatter_radius_degrees))
|
|
593
|
+
|
|
594
|
+
return scatter_radius_degrees
|
|
595
|
+
|
|
596
|
+
def _detect_and_load_location_csv(self, netcdf_path: str) -> Optional[Dict[str, Tuple[float, float]]]:
|
|
597
|
+
"""Detect and load companion CSV file with component locations"""
|
|
598
|
+
# Simplified implementation - full version would be from original file
|
|
599
|
+
return None
|
pyconvexity/models/__init__.py
CHANGED
|
@@ -20,6 +20,10 @@ from pyconvexity.models.network import (
|
|
|
20
20
|
get_component_counts, get_master_scenario_id, resolve_scenario_id
|
|
21
21
|
)
|
|
22
22
|
|
|
23
|
+
from pyconvexity.models.scenarios import (
|
|
24
|
+
create_scenario, list_scenarios, get_scenario, delete_scenario
|
|
25
|
+
)
|
|
26
|
+
|
|
23
27
|
__all__ = [
|
|
24
28
|
# Component operations
|
|
25
29
|
"get_component_type", "get_component", "list_components_by_type",
|
|
@@ -33,4 +37,7 @@ __all__ = [
|
|
|
33
37
|
"create_network", "get_network_info", "get_network_time_periods", "list_networks",
|
|
34
38
|
"create_carrier", "list_carriers", "get_network_config", "set_network_config",
|
|
35
39
|
"get_component_counts", "get_master_scenario_id", "resolve_scenario_id",
|
|
40
|
+
|
|
41
|
+
# Scenario operations
|
|
42
|
+
"create_scenario", "list_scenarios", "get_scenario", "delete_scenario",
|
|
36
43
|
]
|
pyconvexity/models/attributes.py
CHANGED
|
@@ -334,7 +334,9 @@ def serialize_timeseries_to_parquet(timeseries: List[TimeseriesPoint]) -> bytes:
|
|
|
334
334
|
|
|
335
335
|
if not timeseries:
|
|
336
336
|
# Return empty parquet file with correct schema
|
|
337
|
-
|
|
337
|
+
empty_period_array = pa.array([], type=pa.int32())
|
|
338
|
+
empty_value_array = pa.array([], type=pa.float64())
|
|
339
|
+
table = pa.table([empty_period_array, empty_value_array], schema=schema)
|
|
338
340
|
else:
|
|
339
341
|
# Create PyArrow table with EXPLICIT schema to ensure data types match Rust
|
|
340
342
|
period_indices = [p.period_index for p in timeseries]
|
pyconvexity/models/components.py
CHANGED
|
@@ -451,6 +451,9 @@ def ensure_unmet_load_for_bus(
|
|
|
451
451
|
from pyconvexity.models.attributes import set_static_attribute
|
|
452
452
|
set_static_attribute(conn, unmet_load_id, "marginal_cost", StaticValue(1e6))
|
|
453
453
|
set_static_attribute(conn, unmet_load_id, "p_nom", StaticValue(1e6))
|
|
454
|
+
set_static_attribute(conn, unmet_load_id, "p_max_pu", StaticValue(1.0)) # Can run at full capacity
|
|
455
|
+
set_static_attribute(conn, unmet_load_id, "p_min_pu", StaticValue(0.0)) # Can be turned off
|
|
456
|
+
set_static_attribute(conn, unmet_load_id, "sign", StaticValue(1.0)) # Positive power sign (generation)
|
|
454
457
|
set_static_attribute(conn, unmet_load_id, "active", StaticValue(unmet_load_active))
|
|
455
458
|
|
|
456
459
|
|