pyconvexity 0.1.2__tar.gz → 0.1.4__tar.gz

{pyconvexity-0.1.2 → pyconvexity-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyconvexity
-Version: 0.1.2
+Version: 0.1.4
 Summary: Python library for energy system modeling and optimization with PyPSA
 Author-email: Convexity Team <info@convexity.com>
 License: MIT
@@ -32,6 +32,9 @@ Requires-Dist: xlsxwriter>=3.0.0; extra == "excel"
 Provides-Extra: netcdf
 Requires-Dist: netcdf4>=1.6.0; extra == "netcdf"
 Requires-Dist: xarray>=2022.3.0; extra == "netcdf"
+Provides-Extra: data
+Requires-Dist: country-converter>=1.0.0; extra == "data"
+Requires-Dist: pyyaml>=6.0.0; extra == "data"
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
@@ -40,7 +43,7 @@ Requires-Dist: isort>=5.10.0; extra == "dev"
 Requires-Dist: mypy>=1.0.0; extra == "dev"
 Requires-Dist: pre-commit>=2.20.0; extra == "dev"
 Provides-Extra: all
-Requires-Dist: pyconvexity[excel,netcdf,pypsa]; extra == "all"
+Requires-Dist: pyconvexity[data,excel,netcdf,pypsa]; extra == "all"
 
 # PyConvexity
 

{pyconvexity-0.1.2 → pyconvexity-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyconvexity"
-version = "0.1.2"
+version = "0.1.4"
 description = "Python library for energy system modeling and optimization with PyPSA"
 readme = "README.md"
 license = {text = "MIT"}
@@ -44,6 +44,10 @@ netcdf = [
     "netcdf4>=1.6.0",
     "xarray>=2022.3.0",
 ]
+data = [
+    "country-converter>=1.0.0",
+    "pyyaml>=6.0.0",
+]
 dev = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0",
@@ -53,7 +57,7 @@ dev = [
     "pre-commit>=2.20.0",
 ]
 all = [
-    "pyconvexity[pypsa,excel,netcdf]",
+    "pyconvexity[pypsa,excel,netcdf,data]",
 ]
 
 [project.urls]
@@ -77,7 +81,7 @@ profile = "black"
 line_length = 100
 
 [tool.mypy]
-python_version = "3.9"
+python_version = "0.1.4"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{pyconvexity-0.1.2 → pyconvexity-0.1.4}/src/pyconvexity/__init__.py

@@ -21,7 +21,8 @@ from pyconvexity.core.errors import (
 
 from pyconvexity.core.types import (
     StaticValue,
-    TimeseriesPoint,
+    Timeseries,
+    TimeseriesMetadata,
     Component,
     Network,
     CreateNetworkRequest,
@@ -33,6 +34,12 @@ from pyconvexity.core.database import (
     database_context,
     open_connection,
     validate_database,
+    # Database maintenance functions
+    vacuum_database,
+    analyze_database,
+    optimize_database,
+    get_database_size_info,
+    should_optimize_database,
 )
 
 # Import main API functions
@@ -48,12 +55,21 @@ from pyconvexity.models import (
     create_network, get_network_info, get_network_time_periods, list_networks,
     create_carrier, list_carriers, get_network_config, set_network_config,
     get_master_scenario_id, resolve_scenario_id,
+    
+    # Scenario operations
+    create_scenario, list_scenarios, get_scenario, delete_scenario,
 )
 
 from pyconvexity.validation import (
     get_validation_rule, list_validation_rules, validate_timeseries_alignment
 )
 
+# High-level timeseries API - recommended for new code
+from pyconvexity.timeseries import (
+    get_timeseries, set_timeseries, get_timeseries_metadata,
+    get_multiple_timeseries, timeseries_to_numpy, numpy_to_timeseries
+)
+
 # High-level API functions
 __all__ = [
     # Version info
@@ -62,7 +78,8 @@ __all__ = [
     
     # Core types
     "StaticValue",
-    "TimeseriesPoint", 
+    "Timeseries",
+    "TimeseriesMetadata", 
     "Component",
     "Network",
     "CreateNetworkRequest",
@@ -74,6 +91,13 @@ __all__ = [
     "open_connection",
     "validate_database",
     
+    # Database maintenance
+    "vacuum_database",
+    "analyze_database", 
+    "optimize_database",
+    "get_database_size_info",
+    "should_optimize_database",
+    
     # Exceptions
     "PyConvexityError",
     "DatabaseError",
@@ -93,28 +117,53 @@ __all__ = [
     "create_carrier", "list_carriers", "get_network_config", "set_network_config",
     "get_master_scenario_id", "resolve_scenario_id",
     
+    # Scenario operations
+    "create_scenario", "list_scenarios", "get_scenario", "delete_scenario",
+    
     # Validation
     "get_validation_rule", "list_validation_rules", "validate_timeseries_alignment",
+    
+    # High-level timeseries API
+    "get_timeseries", "set_timeseries", "get_timeseries_metadata",
+    "get_multiple_timeseries", "timeseries_to_numpy", "numpy_to_timeseries",
 ]
 
+# Data module imports
+try:
+    from pyconvexity import data
+    __all__.append("data")
+except ImportError:
+    # Data dependencies not available
+    pass
+
 # Optional imports with graceful fallbacks
 try:
-    from pyconvexity.solvers.pypsa import PyPSASolver
-    __all__.append("PyPSASolver")
+    from pyconvexity.solvers.pypsa import (
+        solve_network, build_pypsa_network, solve_pypsa_network,
+        load_network_components, apply_constraints, store_solve_results
+    )
+    __all__.extend([
+        "solve_network", "build_pypsa_network", "solve_pypsa_network",
+        "load_network_components", "apply_constraints", "store_solve_results"
+    ])
 except ImportError:
     # PyPSA not available
     pass
 
+# Excel I/O functionality
 try:
-    from pyconvexity.io.excel import ExcelImporter, ExcelExporter
-    __all__.extend(["ExcelImporter", "ExcelExporter"])
+    from pyconvexity.io import ExcelModelExporter, ExcelModelImporter
+    __all__.extend([
+        "ExcelModelExporter", "ExcelModelImporter"
+    ])
 except ImportError:
     # Excel dependencies not available
     pass
 
+
 try:
-    from pyconvexity.io.netcdf import NetCDFImporter, NetCDFExporter
-    __all__.extend(["NetCDFImporter", "NetCDFExporter"])
+    from pyconvexity.io import NetCDFModelExporter, NetCDFModelImporter
+    __all__.extend(["NetCDFModelExporter", "NetCDFModelImporter"])
 except ImportError:
     # NetCDF dependencies not available
     pass

pyconvexity-0.1.4/src/pyconvexity/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.4"

{pyconvexity-0.1.2 → pyconvexity-0.1.4}/src/pyconvexity/core/__init__.py

@@ -16,7 +16,6 @@ from pyconvexity.core.errors import (
 
 from pyconvexity.core.types import (
     StaticValue,
-    TimeseriesPoint,
     AttributeValue,
     ValidationRule,
     Component,
@@ -46,7 +45,6 @@ __all__ = [
     
     # Types
     "StaticValue",
-    "TimeseriesPoint",
     "AttributeValue", 
     "ValidationRule",
     "Component",

{pyconvexity-0.1.2 → pyconvexity-0.1.4}/src/pyconvexity/core/database.py

@@ -90,6 +90,13 @@ def open_connection(db_path: str, read_only: bool = False) -> sqlite3.Connection
         conn.row_factory = sqlite3.Row  # Enable column access by name
         conn.execute("PRAGMA foreign_keys = ON")  # Enable foreign key constraints
         
+        # Configure for concurrent access (WAL mode for better concurrency)
+        if not read_only:
+            conn.execute("PRAGMA journal_mode = WAL")  # Write-Ahead Logging for concurrency
+            conn.execute("PRAGMA synchronous = NORMAL")  # Faster than FULL, still safe
+            conn.execute("PRAGMA wal_autocheckpoint = 1000")  # Less frequent checkpoints
+            conn.execute("PRAGMA temp_store = MEMORY")  # Faster temporary operations
+        
         # Set reasonable timeouts
         conn.execute("PRAGMA busy_timeout = 30000")  # 30 second timeout
         
@@ -183,6 +190,13 @@ def create_database_with_schema(db_path: str) -> None:
         # Enable foreign key constraints
         conn.execute("PRAGMA foreign_keys = ON")
         
+        # Configure for concurrent access
+        conn.execute("PRAGMA journal_mode = WAL")
+        conn.execute("PRAGMA synchronous = NORMAL")
+        conn.execute("PRAGMA wal_autocheckpoint = 1000")
+        conn.execute("PRAGMA temp_store = MEMORY")
+        conn.execute("PRAGMA busy_timeout = 30000")
+        
         # Execute schemas in order
         for filename in schema_files:
             schema_file = schema_dir / filename
@@ -317,3 +331,147 @@ def check_database_compatibility(conn: sqlite3.Connection) -> dict:
         result["warnings"].append("No version information found in database")
     
     return result
+
+
+# ============================================================================
+# DATABASE MAINTENANCE FUNCTIONS
+# ============================================================================
+
+def vacuum_database(conn: sqlite3.Connection) -> None:
+    """
+    Run VACUUM to reclaim database space and defragment.
+    
+    VACUUM rebuilds the database file, repacking it into a minimal amount of disk space.
+    This is useful after deleting large amounts of data or after many INSERT/UPDATE/DELETE operations.
+    
+    Args:
+        conn: Database connection
+        
+    Note:
+        VACUUM can take a significant amount of time on large databases and requires
+        temporary disk space up to twice the size of the original database.
+    """
+    import logging
+    logger = logging.getLogger(__name__)
+    
+    logger.info("Running VACUUM to reclaim database space and defragment")
+    conn.execute("VACUUM")
+    logger.info("VACUUM completed successfully")
+
+
+def analyze_database(conn: sqlite3.Connection) -> None:
+    """
+    Run ANALYZE to update query planner statistics.
+    
+    ANALYZE gathers statistics about the contents of tables and indices.
+    These statistics are used by the query planner to help make better choices about how to perform queries.
+    
+    Args:
+        conn: Database connection
+    """
+    import logging
+    logger = logging.getLogger(__name__)
+    
+    logger.info("Running ANALYZE to update query planner statistics")
+    conn.execute("ANALYZE")
+    logger.info("ANALYZE completed successfully")
+
+
+def optimize_database(conn: sqlite3.Connection) -> dict:
+    """
+    Run complete database optimization (VACUUM + ANALYZE).
+    
+    This performs both VACUUM and ANALYZE operations in the correct order:
+    1. VACUUM first to reclaim space and defragment
+    2. ANALYZE to update statistics with the new layout
+    
+    Args:
+        conn: Database connection
+        
+    Returns:
+        Dictionary with optimization results including before/after size information
+    """
+    import logging
+    import time
+    logger = logging.getLogger(__name__)
+    
+    logger.info("Running database optimization (VACUUM + ANALYZE)")
+    start_time = time.time()
+    
+    # Get size before optimization
+    size_before = get_database_size_info(conn)
+    
+    # VACUUM first to reclaim space and defragment
+    vacuum_database(conn)
+    
+    # Then ANALYZE to update statistics with the new layout
+    analyze_database(conn)
+    
+    # Get size after optimization
+    size_after = get_database_size_info(conn)
+    
+    optimization_time = time.time() - start_time
+    
+    result = {
+        "success": True,
+        "optimization_time": optimization_time,
+        "size_before": size_before,
+        "size_after": size_after,
+        "space_reclaimed": size_before["total_size"] - size_after["total_size"],
+        "free_pages_reclaimed": size_before["free_pages"] - size_after["free_pages"]
+    }
+    
+    logger.info(f"Database optimization completed in {optimization_time:.2f} seconds")
+    logger.info(f"Space reclaimed: {result['space_reclaimed']:,} bytes ({result['space_reclaimed']/1024/1024:.1f} MB)")
+    
+    return result
+
+
+def get_database_size_info(conn: sqlite3.Connection) -> dict:
+    """
+    Get detailed information about database size and space usage.
+    
+    Args:
+        conn: Database connection
+        
+    Returns:
+        Dictionary with size information including total, used, and free space
+    """
+    # Get page count, page size, and freelist count
+    page_count = conn.execute("PRAGMA page_count").fetchone()[0]
+    page_size = conn.execute("PRAGMA page_size").fetchone()[0]
+    freelist_count = conn.execute("PRAGMA freelist_count").fetchone()[0]
+    
+    total_size = page_count * page_size
+    free_size = freelist_count * page_size
+    used_size = total_size - free_size
+    
+    return {
+        "total_size": total_size,
+        "used_size": used_size,
+        "free_size": free_size,
+        "page_count": page_count,
+        "page_size": page_size,
+        "free_pages": freelist_count,
+        "utilization_percent": (used_size / total_size * 100) if total_size > 0 else 0
+    }
+
+
+def should_optimize_database(conn: sqlite3.Connection, free_space_threshold_percent: float = 10.0) -> bool:
+    """
+    Check if database would benefit from optimization based on free space.
+    
+    Args:
+        conn: Database connection
+        free_space_threshold_percent: Threshold percentage of free space to trigger optimization
+        
+    Returns:
+        True if optimization is recommended, False otherwise
+    """
+    size_info = get_database_size_info(conn)
+    
+    if size_info["total_size"] == 0:
+        return False
+    
+    free_space_percent = (size_info["free_size"] / size_info["total_size"]) * 100
+    return free_space_percent >= free_space_threshold_percent

{pyconvexity-0.1.2 → pyconvexity-0.1.4}/src/pyconvexity/core/types.py

@@ -38,8 +38,14 @@ class StaticValue:
         Rust stores: 123.45, 42, true, "hello"
         Not: {"Float": 123.45}, {"Integer": 42}, etc.
         """
+        import math
+        
         if "Float" in self.data:
-            return json.dumps(self.data["Float"])
+            float_val = self.data["Float"]
+            # Ensure finite values only
+            if not math.isfinite(float_val):
+                raise ValueError(f"Cannot serialize non-finite float value: {float_val}")
+            return json.dumps(float_val)
         elif "Integer" in self.data:
             return json.dumps(self.data["Integer"])
         elif "Boolean" in self.data:
@@ -100,21 +106,94 @@ class StaticValue:
 
 
 @dataclass
-class TimeseriesPoint:
+class Timeseries:
     """
-    A single point in a time series.
+    Efficient timeseries data structure matching the new Rust implementation.
     
-    Mirrors Rust TimeseriesPoint with exact field matching.
+    Stores values as a flat array for maximum performance, matching the 
+    unified Rust Timeseries struct.
     """
-    timestamp: int
-    value: float
-    period_index: int
+    values: List[float]
+    length: int
+    start_index: int
+    data_type: str
+    unit: Optional[str]
+    is_input: bool
     
     def __post_init__(self):
-        # Ensure types are correct
-        self.timestamp = int(self.timestamp)
-        self.value = float(self.value)
-        self.period_index = int(self.period_index)
+        # Ensure length matches values array
+        self.length = len(self.values)
+        # Ensure all values are float32-compatible
+        self.values = [float(v) for v in self.values]
+    
+    def get_value(self, index: int) -> Optional[float]:
+        """Get value at specific index."""
+        if 0 <= index < len(self.values):
+            return self.values[index]
+        return None
+    
+    def get_range(self, start: int, end: int) -> List[float]:
+        """Get a range of values efficiently."""
+        end = min(end, len(self.values))
+        start = min(start, end)
+        return self.values[start:end]
+    
+    def sample(self, max_points: int) -> 'Timeseries':
+        """Apply sampling if the timeseries is too large."""
+        if len(self.values) <= max_points:
+            return self
+        
+        step = len(self.values) // max_points
+        sampled_values = []
+        
+        for i in range(0, len(self.values), max(1, step)):
+            sampled_values.append(self.values[i])
+        
+        # Always include the last point if not already included
+        if self.values and sampled_values[-1] != self.values[-1]:
+            sampled_values.append(self.values[-1])
+        
+        return Timeseries(
+            values=sampled_values,
+            length=len(sampled_values),
+            start_index=self.start_index,
+            data_type=self.data_type,
+            unit=self.unit,
+            is_input=self.is_input
+        )
+    
+    def slice(self, start_index: int, end_index: int) -> 'Timeseries':
+        """Apply range filtering."""
+        start = max(0, start_index - self.start_index)
+        end = max(0, end_index - self.start_index)
+        end = min(end, len(self.values))
+        start = min(start, end)
+        
+        return Timeseries(
+            values=self.values[start:end],
+            length=end - start,
+            start_index=self.start_index + start,
+            data_type=self.data_type,
+            unit=self.unit,
+            is_input=self.is_input
+        )
+
+
+@dataclass
+class TimeseriesMetadata:
+    """
+    Metadata about a timeseries without loading the full data.
+    
+    Mirrors Rust TimeseriesMetadata struct.
+    """
+    length: int
+    start_time: int
+    end_time: int
+    start_index: int
+    end_index: int
+    data_type: str
+    unit: Optional[str]
+    is_input: bool
 
 
 @dataclass
@@ -168,21 +247,22 @@ class AttributeValue:
     """
     Represents either a static value or timeseries data for a component attribute.
     
+    Uses efficient Timeseries format for optimal performance.
     Mirrors Rust AttributeValue enum.
     """
     
-    def __init__(self, value: Union[StaticValue, List[TimeseriesPoint]]):
+    def __init__(self, value: Union[StaticValue, Timeseries]):
         if isinstance(value, StaticValue):
             self.variant = "Static"
             self.static_value = value
             self.timeseries_value = None
-        elif isinstance(value, list) and all(isinstance(p, TimeseriesPoint) for p in value):
+        elif isinstance(value, Timeseries):
             self.variant = "Timeseries"
             self.static_value = None
             self.timeseries_value = value
         else:
             raise ValueError(
-                f"AttributeValue must be StaticValue or List[TimeseriesPoint], got {type(value)}"
+                f"AttributeValue must be StaticValue or Timeseries, got {type(value)}"
             )
     
     @classmethod
@@ -191,9 +271,10 @@ class AttributeValue:
         return cls(value)
     
     @classmethod
-    def timeseries(cls, points: List[TimeseriesPoint]) -> 'AttributeValue':
-        """Create a timeseries attribute value"""
-        return cls(points)
+    def timeseries(cls, timeseries: Timeseries) -> 'AttributeValue':
+        """Create a timeseries attribute value (new format)"""
+        return cls(timeseries)
+    
     
     def is_static(self) -> bool:
         """Check if this is a static value"""
@@ -203,11 +284,17 @@ class AttributeValue:
         """Check if this is a timeseries value"""
         return self.variant == "Timeseries"
     
+    def as_timeseries(self) -> Optional[Timeseries]:
+        """Get the timeseries data in new format"""
+        return self.timeseries_value if self.is_timeseries() else None
+    
+    
     def __repr__(self) -> str:
         if self.is_static():
             return f"AttributeValue.static({self.static_value})"
         else:
-            return f"AttributeValue.timeseries({len(self.timeseries_value)} points)"
+            length = len(self.timeseries_value.values) if self.timeseries_value else 0
+            return f"AttributeValue.timeseries({length} points)"
 
 
 @dataclass

pyconvexity-0.1.4/src/pyconvexity/data/README.md

@@ -0,0 +1,101 @@
+# PyConvexity Data Module
+
+The `pyconvexity.data` module provides functions for loading external energy data and integrating it with PyConvexity models. This is a simple, expert-friendly toolbox for working with real-world energy data.
+
+## Installation
+
+Install PyConvexity with data dependencies:
+
+```bash
+pip install pyconvexity[data]
+```
+
+## Current Data Sources
+
+### Global Energy Monitor (GEM)
+
+Load power plant data from GEM's Global Integrated Power dataset.
+
+**Setup:**
+1. Download the GEM Excel file: `Global-Integrated-Power-August-2025.xlsx`
+2. Place it in a `data/raw/global-energy-monitor/` directory, or set the path manually
+
+**Usage:**
+
+```python
+import pyconvexity as px
+
+# Load generators for a specific country
+generators = px.data.get_generators_from_gem(
+    country="USA",  # ISO 3-letter country code
+    technology_types=["solar", "wind", "nuclear"],  # Optional filter
+    min_capacity_mw=100.0  # Optional minimum capacity
+)
+
+# Create a network and add generators
+px.create_database_with_schema("my_model.db")
+
+with px.database_context("my_model.db") as conn:
+    network_id = px.create_network(conn, network_req)
+    
+    # Create carriers
+    carriers = {}
+    for carrier_name in generators['carrier'].unique():
+        carriers[carrier_name] = px.create_carrier(conn, network_id, carrier_name)
+    
+    # Add generators to network
+    generator_ids = px.data.add_gem_generators_to_network(
+        conn, network_id, generators, carrier_mapping=carriers
+    )
+```
+
+## Data Output Format
+
+The `get_generators_from_gem()` function returns a pandas DataFrame with these columns:
+
+- `plant_name`: Name of the power plant
+- `country_iso_3`: ISO 3-letter country code
+- `category`: Energy category (nuclear, thermal, renewables, storage, etc.)
+- `carrier`: Energy carrier (coal, gas, solar, wind, nuclear, etc.)
+- `type`: Technology type (subcritical, combined-cycle, photovoltaic, etc.)
+- `capacity_mw`: Capacity in megawatts
+- `start_year`: Year the plant started operation
+- `latitude`: Latitude coordinate
+- `longitude`: Longitude coordinate
+
+## Technology Mapping
+
+GEM technologies are automatically mapped to a standardized schema:
+
+- **Nuclear**: pressurized-water-reactor, boiling-water-reactor, small-modular-reactor
+- **Thermal**: subcritical, supercritical, combined-cycle, gas-turbine
+- **Renewables**: photovoltaic, thermal (solar), onshore/offshore (wind), run-of-river (hydro)
+- **Storage**: lithium-ion (battery), pumped-hydro
+- **Bioenergy**: biomass, biogas
+
+## Caching
+
+Data is automatically cached for 7 days to improve performance. You can:
+
+```python
+# Disable caching
+generators = px.data.get_generators_from_gem(country="USA", use_cache=False)
+
+# Clear cache
+cache = px.data.DataCache()
+cache.clear_cache('gem_generators')
+```
+
+## Examples
+
+See `examples/gem_data_example.py` for a complete working example.
+
+## Future Data Sources
+
+The framework is designed to be extensible. Planned additions include:
+
+- IRENA Global Energy Atlas (renewable resource data)
+- World Bank energy statistics
+- IEA World Energy Outlook data
+- OpenStreetMap transmission infrastructure
+- NASA weather data for renewable profiles

pyconvexity-0.1.4/src/pyconvexity/data/__init__.py

@@ -0,0 +1,18 @@
+"""
+PyConvexity Data Module
+
+Provides functions for loading external energy data and integrating it with PyConvexity models.
+This module offers a simple, expert-friendly toolbox for working with real-world energy data.
+"""
+
+from .sources.gem import get_generators_from_gem, add_gem_generators_to_network
+from .loaders.cache import DataCache
+
+__all__ = [
+    # GEM (Global Energy Monitor) functions
+    "get_generators_from_gem",
+    "add_gem_generators_to_network",
+    
+    # Caching utilities
+    "DataCache",
+]