fram-core 0.0.0__py3-none-any.whl → 0.1.0__py3-none-any.whl

framcore/utils/__init__.py

@@ -0,0 +1,35 @@
+# framcore/lib/__init__.py
+
+from framcore.utils.get_supported_components import get_supported_components
+from framcore.utils.node_flow_utils import (
+    FlowInfo,
+    get_component_to_nodes,
+    get_flow_infos,
+    get_node_to_commodity,
+    get_transports_by_commodity,
+    is_transport_by_commodity,
+)
+from framcore.utils.global_energy_equivalent import get_hydro_downstream_energy_equivalent, set_global_energy_equivalent
+from framcore.utils.storage_subsystems import get_one_commodity_storage_subsystems
+from framcore.utils.isolate_subnodes import isolate_subnodes
+from framcore.utils.get_regional_volumes import get_regional_volumes, RegionalVolumes
+from framcore.utils.loaders import add_loaders_if, add_loaders, replace_loader_path
+
+__all__ = [
+    "FlowInfo",
+    "RegionalVolumes",
+    "add_loaders",
+    "add_loaders_if",
+    "get_component_to_nodes",
+    "get_flow_infos",
+    "get_hydro_downstream_energy_equivalent",
+    "get_node_to_commodity",
+    "get_one_commodity_storage_subsystems",
+    "get_regional_volumes",
+    "get_supported_components",
+    "get_transports_by_commodity",
+    "is_transport_by_commodity",
+    "isolate_subnodes",
+    "replace_loader_path",
+    "set_global_energy_equivalent",
+]

framcore/utils/get_regional_volumes.py

@@ -0,0 +1,387 @@
+from __future__ import annotations
+
+import contextlib
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+import numpy as np
+from numpy.typing import NDArray
+
+from framcore.attributes import FlowVolume
+from framcore.components import Component, Flow, Node
+from framcore.events import send_warning_event
+from framcore.expressions import get_unit_conversion_factor
+from framcore.expressions._utils import _load_model_and_create_model_db
+from framcore.metadata import Member
+from framcore.querydbs import QueryDB
+from framcore.timeindexes import FixedFrequencyTimeIndex, SinglePeriodTimeIndex
+from framcore.utils import FlowInfo, get_flow_infos, get_node_to_commodity, get_supported_components
+
+if TYPE_CHECKING:
+    from framcore import Model
+
+
+class RegionalVolumes:
+    """
+    Container for regional energy volumes.
+
+    Stores production, consumption, import, and export vectors for each node and category.
+    Provides methods to access these aggregated results.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the RegionalVolumes instance with empty dictionaries for production, consumption, import, and export."""
+        self._production: dict[str, dict[str, NDArray]] = dict()
+        self._consumption: dict[str, dict[str, NDArray]] = dict()
+        self._export: dict[str, dict[str, NDArray]] = dict()
+        self._import: dict[str, dict[str, NDArray]] = dict()
+
+    def get_production(self) -> dict[str, dict[str, NDArray]]:
+        """Return dict with production vector by category for each node."""
+        return self._production
+
+    def get_consumption(self) -> dict[str, dict[str, NDArray]]:
+        """Return dict with consumption vector by category for each node."""
+        return self._consumption
+
+    def get_export(self) -> dict[str, dict[str, NDArray]]:
+        """Return nested dict with export vector for each trade partner to an exporting node."""
+        return self._export
+
+    def get_import(self) -> dict[str, dict[str, NDArray]]:
+        """Return nested dict with import vector for each trade partner to an importing node."""
+        return self._import
+
+
+def _get_meta_value(key: str, v: Node | Flow, category_level: str) -> str:
+    """Get member meta value from component."""
+    meta = v.get_meta(category_level)
+    if not isinstance(meta, Member):
+        message = f"Expected for key {key} metadata of type Member, got {meta}"
+        raise ValueError(message)
+    return meta.get_value()
+
+
+def _get_vector(
+    flow: Flow,
+    is_ingoing: bool,
+    commodity: str,
+    node_to_commodity: dict[str, str],
+    db: QueryDB,
+    data_period: SinglePeriodTimeIndex,
+    scenario_period: FixedFrequencyTimeIndex,
+    unit: str,
+    is_float32: bool,
+) -> FlowVolume:
+    arrows = flow.get_arrows()
+    if len(arrows) == 1:
+        volume = flow.get_volume()
+        return volume.get_scenario_vector(
+            db=db,
+            scenario_horizon=scenario_period,
+            level_period=data_period,
+            unit=unit,
+            is_float32=is_float32,
+        )
+
+    arrows = [a for a in flow.get_arrows() if a.is_ingoing() == is_ingoing and node_to_commodity[a.get_node()] == commodity]
+    if len(arrows) != 1:
+        message = f"Expected one arrow, got {arrows}"
+        raise ValueError(message)
+    arrow = arrows[0]
+
+    arrow_volumes = flow.get_arrow_volumes()
+
+    if arrow in arrow_volumes:
+        volume = arrow_volumes[arrow]
+        return volume.get_scenario_vector(
+            db=db,
+            scenario_horizon=scenario_period,
+            level_period=data_period,
+            unit=unit,
+            is_float32=is_float32,
+        )
+
+    # we have to calculate using volume and conversion
+    volume = flow.get_volume()
+
+    main_node = flow.get_main_node()
+    main_arrows = [a for a in flow.get_arrows() if a.get_node() == main_node]
+    if len(main_arrows) != 1:
+        message = f"Expected exactly one arrow connected to main node of flow. Got {main_arrows}"
+        raise ValueError(message)
+    main_arrow = main_arrows[0]
+
+    if arrow == main_arrow:
+        return volume.get_scenario_vector(
+            db=db,
+            scenario_horizon=scenario_period,
+            level_period=data_period,
+            unit=unit,
+            is_float32=is_float32,
+        )
+
+    main_units = main_arrow.get_conversion_unit_set(db)
+    if not main_units:
+        return volume.get_scenario_vector(
+            db=db,
+            scenario_horizon=scenario_period,
+            level_period=data_period,
+            unit=unit,
+            is_float32=is_float32,
+        )
+
+    # we must convert to correct unit
+    arrow_units = arrow.get_conversion_unit_set(db)
+
+    a_main_unit = next(iter(main_units))
+    a_arrow_unit = next(iter(arrow_units))
+
+    unit_conversion_factor = get_unit_conversion_factor(
+        from_unit=f"(({a_arrow_unit}) / ({a_main_unit}))",
+        to_unit=unit,
+    )
+
+    vector = volume.get_scenario_vector(
+        db=db,
+        scenario_horizon=scenario_period,
+        level_period=data_period,
+        unit=a_main_unit,
+        is_float32=is_float32,
+    )
+
+    if arrow.has_profile():
+        conversion_vector = arrow.get_scenario_vector(
+            db=db,
+            scenario_horizon=scenario_period,
+            level_period=data_period,
+            unit=a_arrow_unit,
+            is_float32=is_float32,
+        )
+        np.multiply(vector, conversion_vector, out=vector)
+        np.multiply(vector, unit_conversion_factor, out=vector)
+        return vector
+
+    conversion_value = arrow.get_data_value(
+        db=db,
+        scenario_horizon=scenario_period,
+        level_period=data_period,
+        unit=a_arrow_unit,
+    )
+    np.multiply(vector, conversion_value * unit_conversion_factor, out=vector)
+
+    return vector
+
+
+# TODO: More options: node_category, consumption_category, production_category, with_trade_partners
+
+
+def _check_category(category: str, flow_id: str, flow_info: FlowInfo) -> None:
+    pass
+
+
+def get_regional_volumes(  # noqa C901
+    db: Model | QueryDB,
+    commodity: str,
+    node_category: str,
+    production_category: str,
+    consumption_category: str,
+    data_period: SinglePeriodTimeIndex,
+    scenario_period: FixedFrequencyTimeIndex,
+    unit: str,
+    is_float32: bool = True,
+) -> RegionalVolumes:
+    """
+    Calculate aggregated production, consumption, import and export for member in node_category.
+
+    Decompose the model components into nodes and flows. Analyze the flows to determine their contribution to production, consumption, import, and export if
+    they are associated with the specified commodity. Group these contributions based on the provided node_category, production_category, and
+    consumption_category metadata.
+
+    Args:
+        db (Model | QueryDB): Model or QueryDB to use
+        commodity (str): Commodity to consider
+        node_category (str): Meta key for node category to group the results by
+        production_category (str): Meta key for production category to group the results by
+        consumption_category (str): Meta key for consumption category to group the results by
+        data_period (SinglePeriodTimeIndex): Consider results for this data period
+        scenario_period (FixedFrequencyTimeIndex): Consider results for this scenario period
+        unit (str): Unit to use for the results
+        is_float32 (bool): Use float32 for calculations and results if True
+
+    """
+    db = _load_model_and_create_model_db(db)
+
+    if not isinstance(is_float32, bool):
+        message = f"Expected bool for is_float32, got {is_float32}"
+        raise ValueError(message)
+
+    domain_components = {k: v for k, v in db.get_data().items() if isinstance(v, Component)}
+
+    graph: dict[str, Node | Flow] = get_supported_components(
+        components=domain_components,
+        supported_types=(Node, Flow),
+        forbidden_types=tuple(),
+    )
+
+    flows: dict[str, Flow] = {k: v for k, v in graph.items() if isinstance(v, Flow)}
+    nodes: dict[str, Node] = {k: v for k, v in graph.items() if isinstance(v, Node)}
+
+    node_to_commodity = get_node_to_commodity(graph)
+
+    # only nodes of prefered commodity
+    nodes_of_commodity: dict[str, Node] = {k: v for k, v in nodes.items() if v.get_commodity() == commodity}
+
+    # Mapping of node to category of prefered node level
+    node_to_category: dict[str, str] = {k: _get_meta_value(k, v, node_category) for k, v in nodes_of_commodity.items()}
+
+    category_to_nodes: dict[str, set[str]] = defaultdict(set)
+    visited = set()
+    for node_id, category in node_to_category.items():
+        assert node_id not in visited, f"{node_id} is duplicated"
+        category_to_nodes[category].add(node_id)
+        visited.add(node_id)
+
+    direct_production: dict[str, dict[str, list[Flow]]] = dict()
+    direct_consumption: dict[str, dict[str, list[Flow]]] = dict()
+    converted_production: dict[str, dict[str, list[Flow]]] = dict()
+    converted_consumption: dict[str, dict[str, list[Flow]]] = dict()
+    import_: dict[str, dict[str, list[Flow]]] = dict()
+    export: dict[str, dict[str, list[Flow]]] = dict()
+
+    for flow_id, flow in flows.items():
+        flow_infos = get_flow_infos(flow, node_to_commodity)
+
+        prod_category = None
+        cons_category = None
+        with contextlib.suppress(Exception):
+            prod_category = _get_meta_value(flow_id, flow, production_category)
+        with contextlib.suppress(Exception):
+            cons_category = _get_meta_value(flow_id, flow, consumption_category)
+
+        for flow_info in flow_infos:
+            flow_info: FlowInfo
+            if flow_info.category == "direct_in" and flow_info.commodity_in == commodity:
+                _check_category(prod_category, flow_id, flow_info)
+                node_category = node_to_category[flow_info.node_in]
+                if node_category not in direct_production:
+                    direct_production[node_category] = defaultdict(list)
+                direct_production[node_category][prod_category].append(flow)
+
+            elif flow_info.category == "conversion" and flow_info.commodity_in == commodity:
+                _check_category(prod_category, flow_id, flow_info)
+                node_category = node_to_category[flow_info.node_in]
+                if node_category not in converted_production:
+                    converted_production[node_category] = defaultdict(list)
+                converted_production[node_category][prod_category].append(flow)
+
+            elif flow_info.category == "direct_out" and flow_info.commodity_out == commodity:
+                _check_category(cons_category, flow_id, flow_info)
+                node_category = node_to_category[flow_info.node_out]
+                if node_category not in direct_consumption:
+                    direct_consumption[node_category] = defaultdict(list)
+                direct_consumption[node_category][cons_category].append(flow)
+
+            elif flow_info.category == "conversion" and flow_info.commodity_out == commodity:
+                _check_category(cons_category, flow_id, flow_info)
+                node_category = node_to_category[flow_info.node_out]
+                if node_category not in converted_consumption:
+                    converted_consumption[node_category] = defaultdict(list)
+                converted_consumption[node_category][cons_category].append(flow)
+
+            elif flow_info.category == "transport":
+                if node_to_commodity[flow_info.node_in] != commodity:
+                    continue
+                category_in = node_to_category[flow_info.node_in]
+                category_out = node_to_category[flow_info.node_out]
+                if category_in == category_out:
+                    continue
+
+                if category_in not in import_:
+                    import_[category_in] = defaultdict(list)
+                import_[category_in][category_out].append(flow)
+
+                if category_out not in export:
+                    export[category_out] = defaultdict(list)
+                export[category_out][category_in].append(flow)
+
+    num_periods = scenario_period.get_num_periods()
+    dtype = np.float32 if is_float32 else np.float64
+
+    out = RegionalVolumes()
+
+    # direct
+    for flow_dict, out_dict, is_ingoing in [(direct_production, out.get_production(), True), (direct_consumption, out.get_consumption(), False)]:
+        for node_category, flow_categories in flow_dict.items():
+            if node_category not in out_dict:
+                out_dict[node_category] = dict()
+            for flow_category, flows in flow_categories.items():
+                x = np.zeros(num_periods, dtype=dtype)
+                for flow in set(flows):
+                    try:
+                        vector = _get_vector(
+                            flow=flow,
+                            is_ingoing=is_ingoing,
+                            commodity=commodity,
+                            node_to_commodity=node_to_commodity,
+                            db=db,
+                            scenario_period=scenario_period,
+                            data_period=data_period,
+                            unit=unit,
+                            is_float32=is_float32,
+                        )
+                        np.add(x, vector, out=x)
+                    except Exception as e:
+                        send_warning_event(flow, f"Could not get direct production or consumption for flow {flow}: {e}")
+                out_dict[node_category][flow_category] = x
+
+    # converted
+    for flow_dict, out_dict, is_ingoing in [(converted_production, out.get_production(), True), (converted_consumption, out.get_consumption(), False)]:
+        for node_category, flow_categories in flow_dict.items():
+            if node_category not in out_dict:
+                out_dict[node_category] = dict()
+            for flow_category, flows in flow_categories.items():
+                x = out_dict[node_category][flow_category] if flow_category in out_dict[node_category] else np.zeros(num_periods, dtype=dtype)
+                for flow in set(flows):
+                    try:
+                        vector = _get_vector(
+                            flow=flow,
+                            is_ingoing=is_ingoing,
+                            commodity=commodity,
+                            node_to_commodity=node_to_commodity,
+                            db=db,
+                            scenario_period=scenario_period,
+                            data_period=data_period,
+                            unit=unit,
+                            is_float32=is_float32,
+                        )
+                        np.add(x, vector, out=x)
+                    except Exception as e:
+                        send_warning_event(flow, f"Could not get indirect production or consumption for flow {flow}: {e}")
+                out_dict[node_category][flow_category] = x
+
+    # trade
+    for flow_dict, out_dict, is_ingoing in [(import_, out.get_import(), True), (export, out.get_export(), False)]:
+        for category, trade_partners in flow_dict.items():
+            out_dict[category] = dict()
+            for trade_partner, flows in trade_partners.items():
+                x = np.zeros(num_periods, dtype=dtype)
+                for flow in set(flows):
+                    try:
+                        vector = _get_vector(
+                            flow=flow,
+                            is_ingoing=is_ingoing,
+                            commodity=commodity,
+                            node_to_commodity=node_to_commodity,
+                            db=db,
+                            scenario_period=scenario_period,
+                            data_period=data_period,
+                            unit=unit,
+                            is_float32=is_float32,
+                        )
+                        np.add(x, vector, out=x)
+                    except Exception as e:
+                        send_warning_event(flow, f"Could not get trade for flow {flow}: {e}")
+                out_dict[category][trade_partner] = x
+
+    return out

framcore/utils/get_supported_components.py

@@ -0,0 +1,60 @@
+from framcore.components import Component
+
+
+def get_supported_components(
+    components: dict[str, Component],
+    supported_types: tuple[type[Component]],
+    forbidden_types: tuple[type[Component]],
+) -> dict[str, Component]:
+    """Return simplified version of components in compliance with specified component types.See description in Component."""
+    output: dict[str, Component] = {}
+    errors: list[str] = []
+
+    _simplify_until_supported(
+        output,
+        errors,
+        components,
+        supported_types,
+        forbidden_types,
+    )
+
+    if errors:
+        message = "\n".join(errors)
+        raise ValueError(message)
+
+    return output
+
+
+def _simplify_until_supported(
+    output: dict[str, Component],
+    errors: list[str],
+    candidates: dict[str, Component],
+    supported_types: tuple[type[Component]],
+    forbidden_types: tuple[type[Component]],
+) -> None:
+    for name, component in candidates.items():
+        if isinstance(component, forbidden_types):
+            message = f"{component.get_top_parent()} has forbidden component {component}"
+            errors.append(message)
+
+        elif isinstance(component, supported_types):
+            output[name] = component
+
+        else:
+            simpler_components = component.get_simpler_components(name)
+
+            if not simpler_components:
+                message = (
+                    f"Failed to support component. Reached bottom level component {component} with top level "
+                    f"parent {component.get_top_parent()}. No component in the hierarchy was supported."
+                )
+                errors.append(message)
+
+            else:
+                _simplify_until_supported(
+                    output,
+                    errors,
+                    simpler_components,
+                    supported_types,
+                    forbidden_types,
+                )

framcore/utils/global_energy_equivalent.py

@@ -0,0 +1,63 @@
+from framcore.components import Component, HydroModule
+from framcore.curves import Curve
+from framcore.expressions import Expr
+from framcore.metadata import LevelExprMeta
+from framcore.timevectors import ConstantTimeVector, TimeVector
+
+
+def get_hydro_downstream_energy_equivalent(
+    data: dict[str, Component | TimeVector | Curve | Expr],
+    module_name: str,
+    power_node: str | None = None,
+) -> Expr:
+    """
+    Get the expression for the sum downstream energy equivalent for a hydro module.
+
+    - If power node is given, only count downstream energy equivalents that are connected to the power node.
+    - Energy equivalents are collected from hydro generators downstream, and the main topology follows the release_to attribute.
+    - Transport pumps are included in the downstream topology, but counted as negative energy equivalents.
+
+    Args:
+        data (dict[str, Component | TimeVector | Curve | Expr]): The dict containing the components.
+        module_name (str): The name of the hydro module to start from.
+        power_node (str): Optional power node to filter energy equivalents.
+
+    """
+    if data[module_name].get_pump() and data[module_name].get_pump().get_from_module() == module_name:  # transport pump
+        pump_power_node = data[module_name].get_pump().get_power_node()
+        pump_to = data[module_name].get_pump().get_to_module()
+        energy_equivalent = get_hydro_downstream_energy_equivalent(data, pump_to, power_node)  # continue downstream of pump_to module
+        if power_node in (pump_power_node, None):
+            return energy_equivalent - data[module_name].get_pump().get_energy_equivalent().get_level()  # pumps has negative energy equivalents
+        return energy_equivalent
+
+    energy_equivalent = 0
+    if data[module_name].get_generator():  # hydro generator
+        module_power_node = data[module_name].get_generator().get_power_node()
+        if power_node in (module_power_node, None):
+            energy_equivalent += data[module_name].get_generator().get_energy_equivalent().get_level()
+    if data[module_name].get_release_to():  # continue from release_to module
+        release_to = data[module_name].get_release_to()
+        energy_equivalent += get_hydro_downstream_energy_equivalent(data, release_to, power_node)
+    return energy_equivalent
+
+
+def set_global_energy_equivalent(data: dict[str, Component | TimeVector | Curve | Expr], metakey_energy_eq_downstream: str) -> None:
+    """
+    Loop through data dict and set the downstream energy equivalent for all HydroModules.
+
+    Send a warning event if a HydroModule has no downstream energy equivalents.
+
+    Args:
+        data (dict[str, Component | TimeVector | Curve | Expr]): The dict containing the components.
+        metakey_energy_eq_downstream (str): The meta key to use for storing the downstream energy equivalent.
+
+    """
+    for module_name, module in data.items():
+        if isinstance(module, HydroModule) and module.get_reservoir():
+            energy_equivalent = get_hydro_downstream_energy_equivalent(data, module_name)
+            if energy_equivalent == 0:
+                message = f"HydroModule {module_name} has no downstream energy equivalents."
+                module.send_warning_event(message)
+                energy_equivalent = ConstantTimeVector(scalar=0.0, unit="kWh/m3", is_max_level=False)
+            module.add_meta(metakey_energy_eq_downstream, LevelExprMeta(energy_equivalent))