pyconvexity 0.1.1__tar.gz → 0.1.3__tar.gz

{pyconvexity-0.1.1 → pyconvexity-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyconvexity
-Version: 0.1.1
+Version: 0.1.3
 Summary: Python library for energy system modeling and optimization with PyPSA
 Author-email: Convexity Team <info@convexity.com>
 License: MIT
@@ -32,6 +32,9 @@ Requires-Dist: xlsxwriter>=3.0.0; extra == "excel"
 Provides-Extra: netcdf
 Requires-Dist: netcdf4>=1.6.0; extra == "netcdf"
 Requires-Dist: xarray>=2022.3.0; extra == "netcdf"
+Provides-Extra: data
+Requires-Dist: country-converter>=1.0.0; extra == "data"
+Requires-Dist: pyyaml>=6.0.0; extra == "data"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
@@ -40,7 +43,7 @@ Requires-Dist: isort>=5.10.0; extra == "dev"
 Requires-Dist: mypy>=1.0.0; extra == "dev"
 Requires-Dist: pre-commit>=2.20.0; extra == "dev"
 Provides-Extra: all
-Requires-Dist: pyconvexity[excel,netcdf,pypsa]; extra == "all"
+Requires-Dist: pyconvexity[data,excel,netcdf,pypsa]; extra == "all"
 
 # PyConvexity
 

{pyconvexity-0.1.1 → pyconvexity-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyconvexity"
-version = "0.1.1"
+version = "0.1.3"
 description = "Python library for energy system modeling and optimization with PyPSA"
 readme = "README.md"
 license = {text = "MIT"}
@@ -44,6 +44,10 @@ netcdf = [
     "netcdf4>=1.6.0",
     "xarray>=2022.3.0",
 ]
+data = [
+    "country-converter>=1.0.0",
+    "pyyaml>=6.0.0",
+]
 dev = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0",
@@ -53,7 +57,7 @@ dev = [
     "pre-commit>=2.20.0",
 ]
 all = [
-    "pyconvexity[pypsa,excel,netcdf]",
+    "pyconvexity[pypsa,excel,netcdf,data]",
 ]
 
 [project.urls]

{pyconvexity-0.1.1 → pyconvexity-0.1.3}/src/pyconvexity/__init__.py

@@ -48,6 +48,9 @@ from pyconvexity.models import (
     create_network, get_network_info, get_network_time_periods, list_networks,
     create_carrier, list_carriers, get_network_config, set_network_config,
     get_master_scenario_id, resolve_scenario_id,
+    
+    # Scenario operations
+    create_scenario, list_scenarios, get_scenario, delete_scenario,
 )
 
 from pyconvexity.validation import (
@@ -93,28 +96,49 @@ __all__ = [
     "create_carrier", "list_carriers", "get_network_config", "set_network_config",
     "get_master_scenario_id", "resolve_scenario_id",
     
+    # Scenario operations
+    "create_scenario", "list_scenarios", "get_scenario", "delete_scenario",
+    
     # Validation
     "get_validation_rule", "list_validation_rules", "validate_timeseries_alignment",
 ]
 
+# Data module imports
+try:
+    from pyconvexity import data
+    __all__.append("data")
+except ImportError:
+    # Data dependencies not available
+    pass
+
 # Optional imports with graceful fallbacks
 try:
-    from pyconvexity.solvers.pypsa import PyPSASolver
-    __all__.append("PyPSASolver")
+    from pyconvexity.solvers.pypsa import (
+        solve_network, build_pypsa_network, solve_pypsa_network,
+        load_network_components, apply_constraints, store_solve_results
+    )
+    __all__.extend([
+        "solve_network", "build_pypsa_network", "solve_pypsa_network",
+        "load_network_components", "apply_constraints", "store_solve_results"
+    ])
 except ImportError:
     # PyPSA not available
     pass
 
+# Excel I/O functionality
 try:
-    from pyconvexity.io.excel import ExcelImporter, ExcelExporter
-    __all__.extend(["ExcelImporter", "ExcelExporter"])
+    from pyconvexity.io import ExcelModelExporter, ExcelModelImporter
+    __all__.extend([
+        "ExcelModelExporter", "ExcelModelImporter"
+    ])
 except ImportError:
     # Excel dependencies not available
     pass
 
+
 try:
-    from pyconvexity.io.netcdf import NetCDFImporter, NetCDFExporter
-    __all__.extend(["NetCDFImporter", "NetCDFExporter"])
+    from pyconvexity.io import NetCDFModelExporter, NetCDFModelImporter
+    __all__.extend(["NetCDFModelExporter", "NetCDFModelImporter"])
 except ImportError:
     # NetCDF dependencies not available
     pass

{pyconvexity-0.1.1 → pyconvexity-0.1.3}/src/pyconvexity/_version.py

@@ -1,2 +1,2 @@
 # This file is automatically updated by GitHub Actions during release
-__version__ = "0.1.1"  # Default version for local development
+__version__ = "0.1.2"  # Default version for local development

pyconvexity-0.1.3/src/pyconvexity/data/README.md

@@ -0,0 +1,101 @@
+# PyConvexity Data Module
+
+The `pyconvexity.data` module provides functions for loading external energy data and integrating it with PyConvexity models. This is a simple, expert-friendly toolbox for working with real-world energy data.
+
+## Installation
+
+Install PyConvexity with data dependencies:
+
+```bash
+pip install pyconvexity[data]
+```
+
+## Current Data Sources
+
+### Global Energy Monitor (GEM)
+
+Load power plant data from GEM's Global Integrated Power dataset.
+
+**Setup:**
+1. Download the GEM Excel file: `Global-Integrated-Power-August-2025.xlsx`
+2. Place it in a `data/raw/global-energy-monitor/` directory, or set the path manually
+
+**Usage:**
+
+```python
+import pyconvexity as px
+
+# Load generators for a specific country
+generators = px.data.get_generators_from_gem(
+    country="USA",  # ISO 3-letter country code
+    technology_types=["solar", "wind", "nuclear"],  # Optional filter
+    min_capacity_mw=100.0  # Optional minimum capacity
+)
+
+# Create a network and add generators
+px.create_database_with_schema("my_model.db")
+
+with px.database_context("my_model.db") as conn:
+    network_id = px.create_network(conn, network_req)
+    
+    # Create carriers
+    carriers = {}
+    for carrier_name in generators['carrier'].unique():
+        carriers[carrier_name] = px.create_carrier(conn, network_id, carrier_name)
+    
+    # Add generators to network
+    generator_ids = px.data.add_gem_generators_to_network(
+        conn, network_id, generators, carrier_mapping=carriers
+    )
+```
+
+## Data Output Format
+
+The `get_generators_from_gem()` function returns a pandas DataFrame with these columns:
+
+- `plant_name`: Name of the power plant
+- `country_iso_3`: ISO 3-letter country code
+- `category`: Energy category (nuclear, thermal, renewables, storage, etc.)
+- `carrier`: Energy carrier (coal, gas, solar, wind, nuclear, etc.)
+- `type`: Technology type (subcritical, combined-cycle, photovoltaic, etc.)
+- `capacity_mw`: Capacity in megawatts
+- `start_year`: Year the plant started operation
+- `latitude`: Latitude coordinate
+- `longitude`: Longitude coordinate
+
+## Technology Mapping
+
+GEM technologies are automatically mapped to a standardized schema:
+
+- **Nuclear**: pressurized-water-reactor, boiling-water-reactor, small-modular-reactor
+- **Thermal**: subcritical, supercritical, combined-cycle, gas-turbine
+- **Renewables**: photovoltaic, thermal (solar), onshore/offshore (wind), run-of-river (hydro)
+- **Storage**: lithium-ion (battery), pumped-hydro
+- **Bioenergy**: biomass, biogas
+
+## Caching
+
+Data is automatically cached for 7 days to improve performance. You can:
+
+```python
+# Disable caching
+generators = px.data.get_generators_from_gem(country="USA", use_cache=False)
+
+# Clear cache
+cache = px.data.DataCache()
+cache.clear_cache('gem_generators')
+```
+
+## Examples
+
+See `examples/gem_data_example.py` for a complete working example.
+
+## Future Data Sources
+
+The framework is designed to be extensible. Planned additions include:
+
+- IRENA Global Energy Atlas (renewable resource data)
+- World Bank energy statistics
+- IEA World Energy Outlook data
+- OpenStreetMap transmission infrastructure
+- NASA weather data for renewable profiles

pyconvexity-0.1.3/src/pyconvexity/data/__init__.py

@@ -0,0 +1,18 @@
+"""
+PyConvexity Data Module
+
+Provides functions for loading external energy data and integrating it with PyConvexity models.
+This module offers a simple, expert-friendly toolbox for working with real-world energy data.
+"""
+
+from .sources.gem import get_generators_from_gem, add_gem_generators_to_network
+from .loaders.cache import DataCache
+
+__all__ = [
+    # GEM (Global Energy Monitor) functions
+    "get_generators_from_gem",
+    "add_gem_generators_to_network",
+    
+    # Caching utilities
+    "DataCache",
+]

pyconvexity-0.1.3/src/pyconvexity/data/loaders/__init__.py

@@ -0,0 +1,3 @@
+"""
+Data loaders and caching utilities for PyConvexity.
+"""

pyconvexity-0.1.3/src/pyconvexity/data/loaders/cache.py

@@ -0,0 +1,212 @@
+"""
+Caching functionality for PyConvexity data operations.
+
+This module handles caching of processed datasets to improve performance.
+"""
+
+import pandas as pd
+import hashlib
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional
+import logging
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+class DataCache:
+    """Manages caching of processed datasets."""
+    
+    def __init__(self, cache_dir: Optional[str] = None):
+        """
+        Initialize the cache manager.
+        
+        Args:
+            cache_dir: Directory to store cache files. Defaults to 'data/cache'
+        """
+        if cache_dir is None:
+            cache_dir = "data/cache"
+        
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Cache metadata file
+        self.metadata_file = self.cache_dir / "cache_metadata.json"
+        self._load_metadata()
+    
+    def _load_metadata(self):
+        """Load cache metadata from file."""
+        if self.metadata_file.exists():
+            try:
+                with open(self.metadata_file, 'r') as f:
+                    self.metadata = json.load(f)
+            except (json.JSONDecodeError, FileNotFoundError):
+                self.metadata = {}
+        else:
+            self.metadata = {}
+    
+    def _save_metadata(self):
+        """Save cache metadata to file."""
+        with open(self.metadata_file, 'w') as f:
+            json.dump(self.metadata, f, indent=2)
+    
+    def _get_cache_key(self, dataset_name: str, filters: Dict[str, Any]) -> str:
+        """Generate a unique cache key for a dataset and filters combination."""
+        # Create a hash of the filters
+        filters_str = json.dumps(filters, sort_keys=True)
+        filters_hash = hashlib.md5(filters_str.encode()).hexdigest()
+        
+        return f"{dataset_name}_{filters_hash}"
+    
+    def _get_cache_file_path(self, cache_key: str) -> Path:
+        """Get the file path for a cache key."""
+        return self.cache_dir / f"{cache_key}.parquet"
+    
+    def get_cached_data(
+        self, 
+        dataset_name: str, 
+        filters: Dict[str, Any]
+    ) -> Optional[pd.DataFrame]:
+        """
+        Retrieve cached data if available and not expired.
+        
+        Args:
+            dataset_name: Name of the dataset
+            filters: Filters applied to the dataset
+            
+        Returns:
+            pandas.DataFrame or None: Cached data if available and valid
+        """
+        cache_key = self._get_cache_key(dataset_name, filters)
+        cache_file = self._get_cache_file_path(cache_key)
+        
+        # Check if cache file exists
+        if not cache_file.exists():
+            return None
+        
+        # Check if cache entry exists in metadata
+        if cache_key not in self.metadata:
+            # Clean up orphaned cache file
+            cache_file.unlink(missing_ok=True)
+            return None
+        
+        # Check if cache is expired (default: 7 days)
+        cache_info = self.metadata[cache_key]
+        created_time = datetime.fromisoformat(cache_info['created'])
+        max_age = timedelta(days=cache_info.get('max_age_days', 7))
+        
+        if datetime.now() - created_time > max_age:
+            logger.info(f"Cache expired for '{dataset_name}', removing...")
+            self._remove_cache_entry(cache_key)
+            return None
+        
+        # Load cached data
+        try:
+            cached_data = pd.read_parquet(cache_file)
+            logger.info(f"Loaded cached data for '{dataset_name}' ({len(cached_data)} rows)")
+            return cached_data
+        except Exception as e:
+            logger.warning(f"Failed to load cached data for '{dataset_name}': {e}")
+            self._remove_cache_entry(cache_key)
+            return None
+    
+    def cache_data(
+        self, 
+        dataset_name: str, 
+        data: pd.DataFrame, 
+        filters: Dict[str, Any],
+        max_age_days: int = 7
+    ):
+        """
+        Cache processed data.
+        
+        Args:
+            dataset_name: Name of the dataset
+            data: Processed pandas DataFrame
+            filters: Filters applied to the dataset
+            max_age_days: Maximum age of cache in days
+        """
+        cache_key = self._get_cache_key(dataset_name, filters)
+        cache_file = self._get_cache_file_path(cache_key)
+        
+        # Save data to parquet file
+        data.to_parquet(cache_file, index=False)
+        
+        # Update metadata
+        self.metadata[cache_key] = {
+            'dataset_name': dataset_name,
+            'filters': filters,
+            'created': datetime.now().isoformat(),
+            'max_age_days': max_age_days,
+            'rows': len(data),
+            'columns': list(data.columns)
+        }
+        
+        self._save_metadata()
+        logger.info(f"Cached data for '{dataset_name}' ({len(data)} rows)")
+    
+    def _remove_cache_entry(self, cache_key: str):
+        """Remove a cache entry and its file."""
+        cache_file = self._get_cache_file_path(cache_key)
+        cache_file.unlink(missing_ok=True)
+        
+        if cache_key in self.metadata:
+            del self.metadata[cache_key]
+            self._save_metadata()
+    
+    def clear_cache(self, dataset_name: Optional[str] = None):
+        """
+        Clear cache entries.
+        
+        Args:
+            dataset_name: If provided, only clear cache for this dataset
+        """
+        keys_to_remove = []
+        
+        for cache_key, info in self.metadata.items():
+            if dataset_name is None or info['dataset_name'] == dataset_name:
+                keys_to_remove.append(cache_key)
+        
+        for key in keys_to_remove:
+            self._remove_cache_entry(key)
+        
+        logger.info(f"Cleared {len(keys_to_remove)} cache entries")
+    
+    def get_cache_info(self) -> Dict[str, Any]:
+        """Get information about the cache."""
+        total_size = 0
+        dataset_counts = {}
+        
+        for cache_key, info in self.metadata.items():
+            dataset_name = info['dataset_name']
+            dataset_counts[dataset_name] = dataset_counts.get(dataset_name, 0) + 1
+            
+            cache_file = self._get_cache_file_path(cache_key)
+            if cache_file.exists():
+                total_size += cache_file.stat().st_size
+        
+        return {
+            'total_entries': len(self.metadata),
+            'total_size_mb': round(total_size / (1024 * 1024), 2),
+            'dataset_counts': dataset_counts,
+            'cache_dir': str(self.cache_dir)
+        }
+    
+    def cleanup_expired_cache(self):
+        """Remove expired cache entries."""
+        expired_keys = []
+        
+        for cache_key, info in self.metadata.items():
+            created_time = datetime.fromisoformat(info['created'])
+            max_age = timedelta(days=info.get('max_age_days', 7))
+            
+            if datetime.now() - created_time > max_age:
+                expired_keys.append(cache_key)
+        
+        for key in expired_keys:
+            self._remove_cache_entry(key)
+        
+        if expired_keys:
+            logger.info(f"Cleaned up {len(expired_keys)} expired cache entries")
+        else:
+            logger.info("No expired cache entries found")

pyconvexity-0.1.3/src/pyconvexity/data/sources/__init__.py

@@ -0,0 +1,5 @@
+"""
+Data sources for PyConvexity.
+
+This module contains integrations with external energy data sources.
+"""