domainiac 1.0.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

domainiac/__init__.py

@@ -1 +1,8 @@
-from .managers import MeteringManager, PlantManager, ResourceManager, UnitManager
+from .managers import (
+    MeteringManager,
+    NWPManager,
+    PlantManager,
+    ResourceManager,
+    UnitManager,
+)
+from .modeling import Coordinate, Group, Neighborhood, NWPParameter, NWPProvider, Plant

domainiac/managers/__init__.py

@@ -1,5 +1,6 @@
 from .masterdata_manager import MasterdataManager
 from .metering_manager import MeteringManager
+from .nwp_manager import NWPManager
 from .plant_manager import PlantManager
 from .resource_manager import ResourceManager
 from .unit_manager import UnitManager

domainiac/managers/masterdata_manager.py

@@ -59,7 +59,10 @@ class MasterdataManager:
 
         # Apply the filters
         for column, value in filters.items():
-            df = df[df[column] == value].reset_index()
+            if isinstance(value, list):
+                df = df[df[column].isin(value)].reset_index()
+            else:
+                df = df[df[column] == value].reset_index()
 
         for column in columns:
             if column not in df.columns:

domainiac/managers/nwp_manager.py

@@ -0,0 +1,298 @@
+import datamazing.pandas as pdz
+import numpy as np
+import pandas as pd
+from sklearn.neighbors import KDTree
+
+from ..modeling.nwp import Coordinate, Neighborhood, NWPParameter, NWPProvider
+
+
+class NWPManager:
+    """
+    Contains all logic concerning NWP data.
+    This includes:
+    - Getting NWP data
+    - Getting average NWP data
+    - Finding closest NWP coordinates
+    """
+
+    NWP_INTERPOLATION_METHODS = {
+        # Wind and temperature are estimates
+        # of the instantaneous value at time t.
+        # We use linear interpolation.
+        NWPParameter.WIND: ("linear", "instantanous"),
+        NWPParameter.TEMPERATURE: ("linear", "instantanous"),
+        # solar is an estimate of the average value
+        # in the interval [t-1, t].
+        # We use PCHIP interpolation to ensure
+        # a smooth shape and non-negative values
+        # (https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.PchipInterpolator.html)
+        NWPParameter.SOLAR: ("pchip", "interval_average"),
+    }
+
+    NWP_RESOLUTIONS = {
+        (NWPProvider.ECMWF, NWPParameter.WIND): pd.Timedelta("PT3H"),
+        (NWPProvider.ECMWF, NWPParameter.TEMPERATURE): pd.Timedelta("PT3H"),
+        (NWPProvider.ECMWF, NWPParameter.SOLAR): pd.Timedelta("PT1H"),
+        (NWPProvider.DMI, NWPParameter.WIND): pd.Timedelta("PT1H"),
+        (NWPProvider.DMI, NWPParameter.TEMPERATURE): pd.Timedelta("PT1H"),
+        (NWPProvider.DMI, NWPParameter.SOLAR): pd.Timedelta("PT1H"),
+        (NWPProvider.CONWX, NWPParameter.WIND): pd.Timedelta("PT1H"),
+        (NWPProvider.CONWX, NWPParameter.TEMPERATURE): pd.Timedelta("PT1H"),
+        (NWPProvider.CONWX, NWPParameter.SOLAR): pd.Timedelta("PT1H"),
+    }
+
+    def __init__(
+        self,
+        db: pdz.Database,
+        time_interval: pdz.TimeInterval,
+        resolution: pd.Timedelta,
+    ):
+        self.db = db
+        self.time_interval = time_interval
+        self.resolution = resolution
+
+        self._nwp_coordinates_kd_tree = dict()
+        self._nwp_table_name_prefix = "forecastNwp"
+
+    @classmethod
+    def get_nwp_resolution(
+        cls, provider: NWPProvider, parameter: NWPParameter
+    ) -> pd.Timedelta:
+        """
+        Get the resolution of the NWP parameter.
+        """
+        return cls.NWP_RESOLUTIONS[(provider, parameter)]
+
+    @classmethod
+    def get_nwp_interpolation_method(cls, parameter: NWPParameter) -> tuple[str, str]:
+        """
+        Get the interpolation method and temporal aggregation for the NWP parameter.
+        """
+        return cls.NWP_INTERPOLATION_METHODS[parameter]
+
+    @staticmethod
+    def calculate_wind_speed_from_vectors(u: pd.Series, v: pd.Series) -> pd.Series:
+        return np.sqrt(u**2 + v**2)
+
+    def _get_table_name(
+        self,
+        provider: NWPProvider,
+        parameter: NWPParameter,
+    ) -> str:
+        """
+        Get the table name for a given NWP parameter.
+        """
+        return self._nwp_table_name_prefix + provider.value + parameter.value
+
+    def _get_kd_tree(self, provider: NWPProvider, parameter: NWPParameter) -> KDTree:
+        """
+        Create a KDTree from the coordinates from the nwp parameter table.
+        This is done lazily, so that the KDTree is only created once.
+        """
+        key = (provider, parameter)
+        if key not in self._nwp_coordinates_kd_tree:
+            # Base KDTree on one hour of data
+            query_time_interval = pdz.TimeInterval(
+                self.time_interval.left, self.time_interval.left + pd.Timedelta("PT1H")
+            )
+
+            df = self.db.query(
+                table_name=self._get_table_name(provider, parameter),
+                time_interval=query_time_interval,
+            )
+
+            self._nwp_coordinates_kd_tree[key] = dict()
+
+            # Make KDTree for altitude
+            self._nwp_coordinates_kd_tree[key]["altitude"] = KDTree(
+                df[["altitude_m"]].drop_duplicates()
+            )
+
+            # Make KDTree for latitude and longitude
+            self._nwp_coordinates_kd_tree[key]["plane"] = KDTree(
+                df[["latitude", "longitude"]].drop_duplicates()
+            )
+
+        return self._nwp_coordinates_kd_tree[key]
+
+    def get_nwp_neighbors(
+        self,
+        provider: NWPProvider,
+        parameter: NWPParameter,
+        neighborhood: Neighborhood,
+    ) -> list[Coordinate]:
+        """
+        Find n closest coordinates based on the NWP parameter table.
+
+        Note: This assumes that coordinates does not change over time.
+        """
+        kd_tree = self._get_kd_tree(provider, parameter)
+
+        altitude_indices = kd_tree["altitude"].query(
+            np.c_[neighborhood.coordinate.altitude],
+            k=1,
+            return_distance=False,
+        )
+        # TODO: Allow this to handle multiple altitudes
+        nearest_altitude = kd_tree["altitude"].data.base[altitude_indices][0][0][0]
+
+        plane_indices = kd_tree["plane"].query(
+            np.c_[neighborhood.coordinate.latitude, neighborhood.coordinate.longitude],
+            k=neighborhood.num_neighbors,
+            return_distance=False,
+        )
+        nearest_planes = kd_tree["plane"].data.base[plane_indices][0]
+
+        coordinates = [
+            Coordinate(latitude=plane[0], longitude=plane[1], altitude=nearest_altitude)
+            for plane in nearest_planes
+        ]
+
+        return coordinates
+
+    def _get_nwp_parameter(
+        self,
+        provider: NWPProvider,
+        parameter: NWPParameter,
+        coordinate: Coordinate,
+    ) -> pd.DataFrame:
+        """
+        Get NWP data for a given parameter, coordinate and altitude.
+        """
+        table = self._get_table_name(provider, parameter)
+
+        filters = {
+            "latitude": coordinate.latitude,
+            "longitude": coordinate.longitude,
+            "altitude_m": coordinate.altitude,
+        }
+
+        # Pad time interval with one resolution
+        # This is needed for the interpolation to work as we shift back in time
+        nwp_resolution = self.get_nwp_resolution(provider, parameter)
+
+        # Pad time interval to make sure interpolation is possible
+        padded_time_interval = pdz.TimeInterval(
+            self.time_interval.left - (nwp_resolution / 2),
+            self.time_interval.right + (nwp_resolution * 1.5),
+        )
+
+        df = self.db.query(table, padded_time_interval, filters=filters)
+
+        df = df.drop(
+            columns=[
+                "created_time_utc",
+                "altitude_m",
+                "longitude",
+                "latitude",
+            ]
+        )
+
+        return df
+
+    def _get_interpolated_nwp_parameter(
+        self,
+        provider: NWPProvider,
+        parameter: NWPParameter,
+        coordinate: Coordinate,
+    ):
+        """
+        Get NWP data for a given parameter, coordinate and altitude,
+        using the proper interpolation technique.
+        """
+        df = self._get_nwp_parameter(provider, parameter, coordinate)
+
+        nwp_resolution = self.get_nwp_resolution(provider, parameter)
+        interpolation_method, temporal_aggregation = self.get_nwp_interpolation_method(
+            parameter
+        )
+
+        match temporal_aggregation:
+            # if the parameter is an interval average,
+            # it makes best sense to interpolate from
+            # midpoint to midpoint. Otherwise, we just
+            # interpolate regularly.
+            case "interval_average":
+                shift = -1 / 2
+            case "instantanous":
+                shift = 0
+
+        df_shifted = pdz.shift_time(df, on="time_utc", period=shift * nwp_resolution)
+
+        # Interpolate
+        df_resampled = pdz.resample(
+            df_shifted, on="time_utc", resolution=self.resolution / 2
+        ).interpolate(interpolation_method)
+
+        # Shift back time, so that the output time series again represents
+        # the value by the endpoint.
+        df_re_shifted = pdz.shift_time(
+            df_resampled, on="time_utc", period=-shift * self.resolution
+        )
+
+        # Make sure times is divisible by the resolution
+        df_resolution = df_re_shifted[
+            df_re_shifted["time_utc"].dt.round(self.resolution)
+            == df_re_shifted["time_utc"]
+        ]
+
+        return df_resolution
+
+    def get_nwp_parameter(
+        self,
+        provider: NWPProvider,
+        parameter: NWPParameter,
+        coordinate: Coordinate,
+    ):
+        df_nwp = self._get_interpolated_nwp_parameter(provider, parameter, coordinate)
+
+        # Ensure data is within time_interval defined by user
+        df_nwp = df_nwp.query(
+            f"time_utc >= '{self.time_interval.left}' "
+            f"and time_utc <= '{self.time_interval.right}'",
+        )
+
+        return df_nwp
+
+    def get_nwp_parameter_in_neighborhood(
+        self,
+        provider: NWPProvider,
+        parameter: NWPParameter,
+        neighborhood: Neighborhood,
+    ) -> pd.DataFrame:
+        """
+        Get average NWP data for a given parameter, coordinates and altitude.
+        """
+        dfs = []
+
+        neighbors = self.get_nwp_neighbors(provider, parameter, neighborhood)
+        for coordinate in neighbors:
+            df_coordinate = self.get_nwp_parameter(provider, parameter, coordinate)
+            dfs.append(df_coordinate)
+
+        df = pd.concat(dfs)
+        df = pdz.group(df, "time_utc").agg("mean")
+
+        return df
+
+    def get_nwp(
+        self,
+        provider: NWPProvider,
+        parameters: list[NWPParameter],
+        neighborhood: Neighborhood,
+    ) -> pd.DataFrame:
+        """
+        Get weather features for a given coordinate.
+        """
+        dfs = []
+        for parameter in parameters:
+            df_param = self.get_nwp_parameter_in_neighborhood(
+                provider,
+                parameter,
+                neighborhood,
+            )
+            dfs.append(df_param)
+
+        df = pdz.merge_many(dfs=dfs, on=["time_utc"])
+
+        return df

domainiac/managers/plant_manager.py

@@ -24,7 +24,13 @@ class PlantManager(MasterdataManager):
     def get_plants(
         self,
         filters: dict = {},
-        columns: list = [
+        columns: list | None = None,
+    ) -> pd.DataFrame:
+        """Gets the plants for a given plant type.
+        Filters for plants valid at the end of time interval.
+        Filters by default for plants in operation.
+        """
+        default_columns = [
             "plant_id",
             "masterdata_gsrn",
             "datahub_gsrn_e18",
@@ -33,12 +39,11 @@ class PlantManager(MasterdataManager):
             "is_tso_connected",
             "valid_from_date_utc",
             "valid_to_date_utc",
-        ],
-    ) -> pd.DataFrame:
-        """Gets the plants for a given plant type.
-        Filters for plants valid at the end of time interval.
-        Filters by default for plants in operation.
-        """
+        ]
+        if not columns:
+            columns = default_columns
+        else:
+            columns = list(set(default_columns + columns))
         return self.get_data("masterdataPlant", filters=filters, columns=columns)
 
     def get_installed_power_timeseries(self, gsrn: str) -> pd.DataFrame:
@@ -62,3 +67,37 @@ class PlantManager(MasterdataManager):
         return df_plant.filter(["time_utc", "installed_power_MW"]).reset_index(
             drop=True
         )
+
+    def _get_corrected_installed_power(
+        self, gsrn: str, df_invalid_periods: pd.DataFrame
+    ):
+        df_times = self.time_interval.to_range(self.resolution).to_frame(
+            index=False, name="time_utc"
+        )
+        df = self.get_installed_power_timeseries(gsrn=gsrn)
+
+        # explode invalid periods to time series
+        df_invalid_periods = df_invalid_periods.query(f"masterdata_gsrn == '{gsrn}'")
+        df_invalid_periods = pdz.merge(
+            df_times,
+            df_invalid_periods,
+            left_time="time_utc",
+            right_period=("start_date_utc", "end_date_utc"),
+        )
+
+        df = pdz.merge(
+            df,
+            df_invalid_periods,
+            on="time_utc",
+            how="left",
+        )
+
+        # correct installed power for invalid periods
+        df["installed_power_MW"] = df["installed_power_MW"].where(
+            df["corrected_installed_power_MW"].isnull(),
+            df["corrected_installed_power_MW"],
+        )
+
+        df = df[["time_utc", "installed_power_MW"]]
+
+        return df

domainiac/managers/unit_manager.py

@@ -24,18 +24,19 @@ class UnitManager(MasterdataManager):
     def get_units(
         self,
         filters: dict = {},
-        columns: list = [
-            "masterdata_gsrn",
-            "installed_power_MW",
-            "plant_id",
-            "price_area",
-            "valid_from_date_utc",
-            "valid_to_date_utc",
-            "power_system_resource_type",
-        ],
+        columns: list | None = None,
     ) -> pd.DataFrame:
         """Gets the units for a given unit type.
         Filters for units valid at the end of time interval.
         Filters by default for units in operation.
         """
+        default_columns = [
+            "masterdata_gsrn",
+            "plant_id",
+            "power_system_resource_type",
+        ]
+        if not columns:
+            columns = default_columns
+        else:
+            columns = list(set(default_columns + columns))
         return self.get_data("masterdataUnit", filters=filters, columns=columns)

domainiac/modeling/__init__.py

@@ -0,0 +1,2 @@
+from .nwp import Coordinate, Neighborhood, NWPParameter, NWPProvider
+from .plant import Group, Plant

domainiac/modeling/nwp.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+class NWPParameter(Enum):
+    SOLAR = "Solar"
+    WIND = "Wind"
+    TEMPERATURE = "Temperature"
+
+
+class NWPProvider(Enum):
+    ECMWF = "Ecmwf"
+    CONWX = "Conwx"
+    DMI = "Dmi"
+
+
+@dataclass(frozen=True)
+class Coordinate:
+    latitude: float
+    longitude: float
+    altitude: float
+
+
+@dataclass(frozen=True)
+class Neighborhood:
+    coordinate: Coordinate
+    num_neighbors: int

domainiac/modeling/plant.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass
+
+import pandas as pd
+
+from .nwp import Coordinate
+
+
+@dataclass(frozen=True)
+class Plant:
+    plant_gsrn: str
+    plant_name: str
+    datahub_gsrn_e18: str
+    price_area: str
+    coordinate: Coordinate
+    installed_power_MW: float
+
+    @classmethod
+    def plants_from_df(cls, df: pd.DataFrame) -> list["Plant"]:
+        plants = []
+        for _, row in df.iterrows():
+            plant = cls(
+                plant_gsrn=row["plant_gsrn"],
+                plant_name=row["plant_name"],
+                datahub_gsrn_e18=row["datahub_gsrn_e18"],
+                price_area=row["price_area"],
+                coordinate=Coordinate(
+                    latitude=row["latitude"],
+                    longitude=row["longitude"],
+                    altitude=0,
+                ),
+                installed_power_MW=row["installed_power_MW"],
+            )
+            plants.append(plant)
+        return plants
+
+
+@dataclass(frozen=True)
+class Group:
+    coordinate: Coordinate
+    installed_power_MW: float
+    identifiers: dict[str, str]
+
+    @classmethod
+    def groups_from_df(cls, df: pd.DataFrame, identifiers: list[str]) -> list["Group"]:
+        groups = []
+        for _, row in df.iterrows():
+            identifiers = {identifier: row[identifier] for identifier in identifiers}
+            group = cls(
+                identifiers=identifiers,
+                installed_power_MW=row["installed_power_MW"],
+                coordinate=Coordinate(
+                    latitude=row["latitude"],
+                    longitude=row["longitude"],
+                    altitude=0,
+                ),
+            )
+            groups.append(group)
+        return groups

{domainiac-1.0.0.dist-info → domainiac-2.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: domainiac
-Version: 1.0.0
+Version: 2.0.0
 Summary: Package for working with Energinet data, but with specialized functions used for Enigma.
 Author: Team Enigma
 Author-email: gridop-enigma@energinet.dk
@@ -9,4 +9,5 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Requires-Dist: datamazing (>=5.0.2,<6.0.0)
 Requires-Dist: pandas (>=2.2.0,<3.0.0)
+Requires-Dist: scikit-learn (>=1.3.0,<2.0.0)
 Requires-Dist: typeguard (>=4.2.1,<5.0.0)

domainiac-2.0.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+domainiac/__init__.py,sha256=Wuix4jscZD4sy7YB0RAedAo_eiRW5AcPlef58lKvg6s,207
+domainiac/managers/__init__.py,sha256=x7NZxFAuJig2TZGJ71niwMyPyDvxjaiE647BJQU0qD8,256
+domainiac/managers/masterdata_manager.py,sha256=H-Dk-dvZZ3bqgV66Xf1V4GQnQWprzLqSKdfD1MPLQgo,2223
+domainiac/managers/metering_manager.py,sha256=NXGHuNjWiAUE08fGIPAfvwbzlKESbKaO5uyxLNyZKC0,995
+domainiac/managers/nwp_manager.py,sha256=qGr4c1ZbUnVRmhX_LmjaobBKN6BBXUn5bbYVKa_5lo4,9921
+domainiac/managers/plant_manager.py,sha256=o0HkOs1nf1uri7OMMH-VgPTAONWjuEE80SyD9RDf0n8,3186
+domainiac/managers/resource_manager.py,sha256=QMkNcrgyXLLlkCR9kg4-nFjTaYHtlsldex-b_8lAmO0,3730
+domainiac/managers/unit_manager.py,sha256=DYF-AiBHzUGZJE5XlTLcszvgIQkSzZs_s9o35l4bkVk,1219
+domainiac/modeling/__init__.py,sha256=Ac5feYZ9gVLayU1Cb1ACqOyDrmvcxoOvZuUkOeXVKf8,101
+domainiac/modeling/nwp.py,sha256=PolrBdQn8W-e1M0_pefvmLn2mr4HT0NqlYlkyCV0dds,438
+domainiac/modeling/plant.py,sha256=48KEcfiMk90fOAzOeI5qtf1FcGvpyBNIRmN2zvx9CLc,1699
+domainiac/wrappers/__init__.py,sha256=vZOw9maXgVoAvudZqioD-GTPkgPI6fm7_CUELQcR_-g,43
+domainiac/wrappers/cache_wrapper.py,sha256=jDg-Lt_y-YzItyP-74tGULOxb07s_CcutmOHP1Ie00Q,355
+domainiac-2.0.0.dist-info/METADATA,sha256=O0KM7oPIYCtDKihwlHr8leFJ7XErIsjb8WMobGibAic,510
+domainiac-2.0.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+domainiac-2.0.0.dist-info/RECORD,,

{domainiac-1.0.0.dist-info → domainiac-2.0.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 2.0.1
 Root-Is-Purelib: true
 Tag: py3-none-any

domainiac-1.0.0.dist-info/RECORD

@@ -1,12 +0,0 @@
-domainiac/__init__.py,sha256=7-KoUot9OZgzMoOLhuHInlvyt9EfL5QjuZagTbAJksU,82
-domainiac/managers/__init__.py,sha256=QzkZNv8_3G0TPcfQia45RWBnASYR6WIuY6jlFBeley0,220
-domainiac/managers/masterdata_manager.py,sha256=tl37saXdH1aeek3Oo58nTxl-eL2txKnmpwVHkyij7n0,2099
-domainiac/managers/metering_manager.py,sha256=NXGHuNjWiAUE08fGIPAfvwbzlKESbKaO5uyxLNyZKC0,995
-domainiac/managers/plant_manager.py,sha256=sEVc3_RaJaG-beT7kv3rgvCtaWLCuXuhxy21mk360Og,1944
-domainiac/managers/resource_manager.py,sha256=QMkNcrgyXLLlkCR9kg4-nFjTaYHtlsldex-b_8lAmO0,3730
-domainiac/managers/unit_manager.py,sha256=8B0hRu_tlZfaZYKDnM0NAQoJ6a1s_8DicBM9v6GEkYE,1174
-domainiac/wrappers/__init__.py,sha256=vZOw9maXgVoAvudZqioD-GTPkgPI6fm7_CUELQcR_-g,43
-domainiac/wrappers/cache_wrapper.py,sha256=jDg-Lt_y-YzItyP-74tGULOxb07s_CcutmOHP1Ie00Q,355
-domainiac-1.0.0.dist-info/METADATA,sha256=LWQrY1F34IUdCbZbsXqQ8Di6DOMVidN7bq1deCbzUFU,465
-domainiac-1.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-domainiac-1.0.0.dist-info/RECORD,,