dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

dsgrid/units/constants.py

@@ -0,0 +1,113 @@
+KILO = 1_000
+MEGA = 1_000_000
+GIGA = 1_000_000_000
+TERA = 1_000_000_000_000
+
+KILO_TO_MEGA = KILO / MEGA
+KILO_TO_GIGA = KILO / GIGA
+KILO_TO_TERA = KILO / TERA
+
+MEGA_TO_KILO = MEGA / KILO
+MEGA_TO_GIGA = MEGA / GIGA
+MEGA_TO_TERA = MEGA / TERA
+
+GIGA_TO_KILO = GIGA / KILO
+GIGA_TO_MEGA = GIGA / MEGA
+GIGA_TO_TERA = GIGA / TERA
+
+TERA_TO_KILO = TERA / KILO
+TERA_TO_MEGA = TERA / MEGA
+TERA_TO_GIGA = TERA / GIGA
+
+KWH = "kWh"
+MWH = "MWh"
+GWH = "GWh"
+TWH = "TWh"
+THERM = "therm"
+MBTU = "MBtu"
+
+KW = "kW"
+MW = "MW"
+GW = "GW"
+TW = "TW"
+
+THERM_TO_KWH = 29.307107017222222
+THERM_TO_MWH = THERM_TO_KWH * KILO_TO_MEGA
+THERM_TO_GWH = THERM_TO_KWH * KILO_TO_GIGA
+THERM_TO_TWH = THERM_TO_KWH * KILO_TO_TERA
+
+KWH_TO_THERM = 1 / THERM_TO_KWH
+MWH_TO_THERM = 1 / THERM_TO_MWH
+GWH_TO_THERM = 1 / THERM_TO_GWH
+TWH_TO_THERM = 1 / THERM_TO_TWH
+
+# BTU conversion is based on EIA. This website says 1 kWh = 3,412 BTU.
+# https://www.eia.gov/energyexplained/units-and-calculators/energy-conversion-calculators.php
+# The more precise number below comes from ResStock at
+# https://github.com/NREL/resstock/blob/2e0a82a7bfad0f17ff75a3c66c91a5d72265a847/resources/hpxml-measures/HPXMLtoOpenStudio/resources/unit_conversions.rb
+MBTU_TO_KWH = 293.0710701722222
+MBTU_TO_MWH = MBTU_TO_KWH * KILO_TO_MEGA
+MBTU_TO_GWH = MBTU_TO_KWH * KILO_TO_GIGA
+MBTU_TO_TWH = MBTU_TO_KWH * KILO_TO_TERA
+
+KWH_TO_MBTU = 1 / MBTU_TO_KWH
+MWH_TO_MBTU = 1 / MBTU_TO_MWH
+GWH_TO_MBTU = 1 / MBTU_TO_GWH
+TWH_TO_MBTU = 1 / MBTU_TO_TWH
+
+MBTU_TO_THERM = 10.0
+THERM_TO_MBTU = 1 / MBTU_TO_THERM
+
+ENERGY_UNITS = (KWH, MWH, GWH, TWH, THERM, MBTU)
+POWER_UNITS = (KW, MW, GW, TW)
+
+
+# Constants for unit conversions
+__all__ = (
+    "ENERGY_UNITS",
+    "POWER_UNITS",
+    "KILO",
+    "MEGA",
+    "GIGA",
+    "TERA",
+    "KILO_TO_MEGA",
+    "KILO_TO_GIGA",
+    "KILO_TO_TERA",
+    "MEGA_TO_KILO",
+    "MEGA_TO_GIGA",
+    "MEGA_TO_TERA",
+    "GIGA_TO_KILO",
+    "GIGA_TO_MEGA",
+    "GIGA_TO_TERA",
+    "TERA_TO_KILO",
+    "TERA_TO_MEGA",
+    "TERA_TO_GIGA",
+    "KWH",
+    "MWH",
+    "GWH",
+    "TWH",
+    "THERM",
+    "MBTU",
+    "KW",
+    "MW",
+    "GW",
+    "TW",
+    "THERM_TO_KWH",
+    "THERM_TO_MWH",
+    "THERM_TO_GWH",
+    "THERM_TO_TWH",
+    "KWH_TO_THERM",
+    "MWH_TO_THERM",
+    "GWH_TO_THERM",
+    "TWH_TO_THERM",
+    "MBTU_TO_KWH",
+    "MBTU_TO_MWH",
+    "MBTU_TO_GWH",
+    "MBTU_TO_TWH",
+    "KWH_TO_MBTU",
+    "MWH_TO_MBTU",
+    "GWH_TO_MBTU",
+    "TWH_TO_MBTU",
+    "MBTU_TO_THERM",
+    "THERM_TO_MBTU",
+)

dsgrid/units/convert.py

@@ -0,0 +1,71 @@
+import logging
+
+import dsgrid.units.energy as energy
+import dsgrid.units.power as power
+from dsgrid.common import VALUE_COLUMN
+from dsgrid.spark.functions import except_all, is_dataframe_empty, join
+from dsgrid.spark.types import DataFrame, F
+from dsgrid.units.constants import ENERGY_UNITS, POWER_UNITS
+from dsgrid.utils.spark import get_unique_values
+
+
+logger = logging.getLogger(__name__)
+
+
+def convert_units_unpivoted(
+    df: DataFrame,
+    metric_column: str,
+    from_records: DataFrame,
+    from_to_records: DataFrame | None,
+    to_unit_records: DataFrame,
+) -> DataFrame:
+    """Convert the value column of the dataframe to the target units.
+
+    Parameters
+    ----------
+    df : DataFrame
+        Load data table
+    metric_column : str
+        Column in dataframe with metric record IDs
+    from_records : DataFrame
+        Metric dimension records for the columns being converted
+    from_to_records : DataFrame | None
+        Records that map the dimension IDs in columns to the target IDs
+        If None, mapping is not required and from_records contain the units.
+    to_unit_records : DataFrame
+        Metric dimension records for the target IDs
+    """
+    unit_col = "unit"  # must match EnergyEndUse.unit
+    tmp1 = from_records.select("id", unit_col).withColumnRenamed(unit_col, "from_unit")
+    if from_to_records is None:
+        unit_df = tmp1.select("id", "from_unit")
+    else:
+        tmp2 = from_to_records.select("from_id", "to_id")
+        unit_df = (
+            join(tmp1, tmp2, "id", "from_id")
+            .select(F.col("to_id").alias("id"), "from_unit")
+            .distinct()
+        )
+    if is_dataframe_empty(
+        except_all(unit_df, to_unit_records.select("id", F.col("unit").alias("from_unit")))
+    ):
+        logger.debug("Return early because the units match.")
+        return df
+
+    df = join(df, unit_df, metric_column, "id").drop("id")
+    tmp3 = to_unit_records.select("id", "unit").withColumnRenamed(unit_col, "to_unit")
+    df = join(df, tmp3, metric_column, "id").drop("id")
+    logger.debug("Converting units from column %s", metric_column)
+
+    units = get_unique_values(to_unit_records, unit_col)
+    if units.issubset(ENERGY_UNITS):
+        func = energy.from_any_to_any
+    elif units.issubset(POWER_UNITS):
+        func = power.from_any_to_any
+    else:
+        msg = f"Unsupported unit conversion: {units}"
+        raise ValueError(msg)
+
+    return df.withColumn(VALUE_COLUMN, func("from_unit", "to_unit", VALUE_COLUMN)).drop(
+        "from_unit", "to_unit"
+    )

dsgrid/units/energy.py

@@ -0,0 +1,145 @@
+"""Contains functions to perform unit conversion of energy."""
+
+import logging
+
+from dsgrid.spark.types import DataFrame, F
+from dsgrid.units.constants import (
+    GIGA_TO_KILO,
+    GIGA_TO_MEGA,
+    GIGA_TO_TERA,
+    GWH,
+    GWH_TO_MBTU,
+    GWH_TO_THERM,
+    KILO_TO_GIGA,
+    KILO_TO_MEGA,
+    KILO_TO_TERA,
+    KWH,
+    KWH_TO_MBTU,
+    KWH_TO_THERM,
+    MBTU,
+    MBTU_TO_GWH,
+    MBTU_TO_KWH,
+    MBTU_TO_MWH,
+    MBTU_TO_THERM,
+    MBTU_TO_TWH,
+    MEGA_TO_GIGA,
+    MEGA_TO_KILO,
+    MEGA_TO_TERA,
+    MWH,
+    MWH_TO_MBTU,
+    MWH_TO_THERM,
+    TERA_TO_GIGA,
+    TERA_TO_KILO,
+    TERA_TO_MEGA,
+    THERM,
+    THERM_TO_GWH,
+    THERM_TO_KWH,
+    THERM_TO_MBTU,
+    THERM_TO_MWH,
+    THERM_TO_TWH,
+    TWH,
+    TWH_TO_MBTU,
+    TWH_TO_THERM,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def to_kwh(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to kWh."""
+    return (
+        F.when(F.col(unit_col) == KWH, F.col(value_col))
+        .when(F.col(unit_col) == MWH, (F.col(value_col) * MEGA_TO_KILO))
+        .when(F.col(unit_col) == GWH, (F.col(value_col) * GIGA_TO_KILO))
+        .when(F.col(unit_col) == TWH, (F.col(value_col) * TERA_TO_KILO))
+        .when(F.col(unit_col) == THERM, (F.col(value_col) * THERM_TO_KWH))
+        .when(F.col(unit_col) == MBTU, (F.col(value_col) * MBTU_TO_KWH))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_mwh(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to mWh."""
+    return (
+        F.when(F.col(unit_col) == KWH, (F.col(value_col) * KILO_TO_MEGA))
+        .when(F.col(unit_col) == MWH, F.col(value_col))
+        .when(F.col(unit_col) == GWH, (F.col(value_col) * GIGA_TO_MEGA))
+        .when(F.col(unit_col) == TWH, (F.col(value_col) * TERA_TO_MEGA))
+        .when(F.col(unit_col) == THERM, (F.col(value_col) * THERM_TO_MWH))
+        .when(F.col(unit_col) == MBTU, (F.col(value_col) * MBTU_TO_MWH))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_gwh(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to gWh."""
+    return (
+        F.when(F.col(unit_col) == KWH, (F.col(value_col) * KILO_TO_GIGA))
+        .when(F.col(unit_col) == MWH, (F.col(value_col) * MEGA_TO_GIGA))
+        .when(F.col(unit_col) == GWH, F.col(value_col))
+        .when(F.col(unit_col) == TWH, (F.col(value_col) * TERA_TO_GIGA))
+        .when(F.col(unit_col) == THERM, (F.col(value_col) * THERM_TO_GWH))
+        .when(F.col(unit_col) == MBTU, (F.col(value_col) * MBTU_TO_GWH))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_twh(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to tWh."""
+    return (
+        F.when(F.col(unit_col) == KWH, (F.col(value_col) * KILO_TO_TERA))
+        .when(F.col(unit_col) == MWH, (F.col(value_col) * MEGA_TO_TERA))
+        .when(F.col(unit_col) == GWH, (F.col(value_col) * GIGA_TO_TERA))
+        .when(F.col(unit_col) == TWH, F.col(value_col))
+        .when(F.col(unit_col) == THERM, (F.col(value_col) * THERM_TO_TWH))
+        .when(F.col(unit_col) == MBTU, (F.col(value_col) * MBTU_TO_TWH))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_therm(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to therm."""
+    return (
+        F.when(F.col(unit_col) == KWH, (F.col(value_col) * KWH_TO_THERM))
+        .when(F.col(unit_col) == MWH, (F.col(value_col) * MWH_TO_THERM))
+        .when(F.col(unit_col) == GWH, (F.col(value_col) * GWH_TO_THERM))
+        .when(F.col(unit_col) == TWH, (F.col(value_col) * TWH_TO_THERM))
+        .when(F.col(unit_col) == THERM, F.col(value_col))
+        .when(F.col(unit_col) == MBTU, (F.col(value_col) * MBTU_TO_THERM))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_mbtu(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to MBtu."""
+    return (
+        F.when(F.col(unit_col) == KWH, (F.col(value_col) * KWH_TO_MBTU))
+        .when(F.col(unit_col) == MWH, (F.col(value_col) * MWH_TO_MBTU))
+        .when(F.col(unit_col) == GWH, (F.col(value_col) * GWH_TO_MBTU))
+        .when(F.col(unit_col) == TWH, (F.col(value_col) * TWH_TO_MBTU))
+        .when(F.col(unit_col) == THERM, (F.col(value_col) * THERM_TO_MBTU))
+        .when(F.col(unit_col) == MBTU, F.col(value_col))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def from_any_to_any(from_unit_col: str, to_unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column of energy based on from/to columns."""
+    return (
+        F.when(F.col(from_unit_col) == F.col(to_unit_col), F.col(value_col))
+        .when(F.col(from_unit_col) == "", F.col(value_col))
+        .when(F.col(to_unit_col) == KWH, to_kwh(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == MWH, to_mwh(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == GWH, to_gwh(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == TWH, to_twh(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == THERM, to_therm(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == MBTU, to_mbtu(from_unit_col, value_col))
+        .otherwise(None)
+    )

dsgrid/units/power.py

@@ -0,0 +1,87 @@
+"""Contains functions to perform unit conversion of power."""
+
+import logging
+
+from dsgrid.spark.types import DataFrame, F
+from dsgrid.units.constants import (
+    GIGA_TO_KILO,
+    GIGA_TO_MEGA,
+    GIGA_TO_TERA,
+    GW,
+    KILO_TO_GIGA,
+    KILO_TO_MEGA,
+    KILO_TO_TERA,
+    KW,
+    MEGA_TO_GIGA,
+    MEGA_TO_KILO,
+    MEGA_TO_TERA,
+    MW,
+    TERA_TO_GIGA,
+    TERA_TO_KILO,
+    TERA_TO_MEGA,
+    TW,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def to_kw(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to kW."""
+    return (
+        F.when(F.col(unit_col) == KW, F.col(value_col))
+        .when(F.col(unit_col) == MW, (F.col(value_col) * MEGA_TO_KILO))
+        .when(F.col(unit_col) == GW, (F.col(value_col) * GIGA_TO_KILO))
+        .when(F.col(unit_col) == TW, (F.col(value_col) * TERA_TO_KILO))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_mw(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to MW."""
+    return (
+        F.when(F.col(unit_col) == KW, (F.col(value_col) * KILO_TO_MEGA))
+        .when(F.col(unit_col) == MW, F.col(value_col))
+        .when(F.col(unit_col) == GW, (F.col(value_col) * GIGA_TO_MEGA))
+        .when(F.col(unit_col) == TW, (F.col(value_col) * TERA_TO_MEGA))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_gw(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to GW."""
+    return (
+        F.when(F.col(unit_col) == KW, (F.col(value_col) * KILO_TO_GIGA))
+        .when(F.col(unit_col) == MW, (F.col(value_col) * MEGA_TO_GIGA))
+        .when(F.col(unit_col) == GW, F.col(value_col))
+        .when(F.col(unit_col) == TW, (F.col(value_col) * TERA_TO_GIGA))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def to_tw(unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column to TW."""
+    return (
+        F.when(F.col(unit_col) == KW, (F.col(value_col) * KILO_TO_TERA))
+        .when(F.col(unit_col) == MW, (F.col(value_col) * MEGA_TO_TERA))
+        .when(F.col(unit_col) == GW, (F.col(value_col) * GIGA_TO_TERA))
+        .when(F.col(unit_col) == TW, F.col(value_col))
+        .when(F.col(unit_col) == "", F.col(value_col))
+        .otherwise(None)
+    )
+
+
+def from_any_to_any(from_unit_col: str, to_unit_col: str, value_col: str) -> DataFrame:
+    """Convert a column of power based on from/to columns."""
+    return (
+        F.when(F.col(from_unit_col) == F.col(to_unit_col), F.col(value_col))
+        .when(F.col(from_unit_col) == "", F.col(value_col))
+        .when(F.col(to_unit_col) == KW, to_kw(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == MW, to_mw(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == GW, to_gw(from_unit_col, value_col))
+        .when(F.col(to_unit_col) == TW, to_tw(from_unit_col, value_col))
+        .otherwise(None)
+    )