skypro 1.2.0__py3-none-any.whl

skypro/commands/pull_elexon_imbalance/README.md

@@ -0,0 +1,4 @@
+# pull-elexon-imbalance
+
+This is a script to pulls a months worth of half-hourly imbalance price and volume data from Elexon and saves it to disk.
+The data is saved in monthly CSV files in the directory defined by the `MARKET_DATA_DIR` variable in the environment configuration.

skypro/commands/pull_elexon_imbalance/main.py

@@ -0,0 +1,104 @@
+import functools
+import json
+import logging
+
+import requests
+from datetime import date, datetime, timedelta
+from io import StringIO
+from typing import Callable
+
+import pandas as pd
+
+from skypro.common.cli_utils.cli_utils import get_user_ack_of_warning_or_exit, read_yaml_file
+from skypro.common.data.utility import prepare_data_dir
+
+from skypro.common.timeutils.month_str import get_first_and_last_date
+from skypro.commands.pull_elexon_imbalance.utils import daterange, with_retries
+
+ELEXON_API_MAX_RETRIES = 5
+ELEXON_API_RETRY_DELAY = timedelta(seconds=1)
+
+
+def pull_elexon_imbalance(month_str: str, env_file_path: str):
+    """
+    Pulls a months worth of half-hourly imbalance price and volume data from Elexon and saves it to disk.
+    The data is saved in monthly CSV files in the directory defined by the MARKET_DATA_DIR in the environment configuration.
+    """
+
+    start_date, end_date = get_first_and_last_date(month_str)
+
+    today = datetime.now().date()
+    if end_date > today:
+        end_date = today
+        get_user_ack_of_warning_or_exit(f"The month has not ended yet, so data will be incomplete after {end_date}")
+
+    env_config = read_yaml_file(env_file_path)
+    data_dir = env_config["vars"]["MARKET_DATA_DIR"]
+
+    df = _fetch_multiple_days(
+        start=start_date,
+        end=end_date,
+        fetch_func=_fetch_day,
+    )
+
+    df["price"] = df["price"] / 10  # £/MW to p/kW
+
+    prices_file_path = prepare_data_dir(data_dir, "elexon", "imbalance_price", start_date)
+    volume_file_path = prepare_data_dir(data_dir, "elexon", "imbalance_volume",  start_date)
+
+    logging.info(f"Saving pricing data to '{prices_file_path}'")
+    df[["spUTCTime", "spClockTime", "price"]].to_csv(prices_file_path, index=False)
+    logging.info(f"Saving volume data to '{volume_file_path}'")
+    df[["spUTCTime", "spClockTime", "volume"]].to_csv(volume_file_path, index=False)
+
+
+def _fetch_multiple_days(start: date, end: date, fetch_func: Callable) -> pd.DataFrame:
+    """
+    The elexon API pulls for a single day, so this function calls the elexon API repeatedly for each day and stacks up
+    the results.
+    """
+    df = pd.DataFrame()
+    for day in daterange(start, end):
+
+        logging.info(f"Fetching imbalance data for '{str(day)}'...")
+        day_df = with_retries(  # The Elexon API can be busy/unreliable at times so use retries to get past temporary failures
+            functools.partial(fetch_func, day),
+            ELEXON_API_MAX_RETRIES,
+            ELEXON_API_RETRY_DELAY
+        )
+
+        df = pd.concat([df, day_df])
+
+    # The values come through in the opposite order to what you'd expect
+    df = df.sort_values(by=["spUTCTime"], ignore_index=True)
+
+    return df
+
+
+def _fetch_day(day: date) -> pd.DataFrame:
+    """
+    Pulls a single days worth of imbalance data from Elexon.
+    """
+
+    day_str = day.isoformat()
+    # Send a GET request to the API
+    response = requests.get(
+        url=f"https://data.elexon.co.uk/bmrs/api/v1/balancing/settlement/system-prices/{day_str}",
+        params={"format": "json"}
+    )
+    response.raise_for_status()
+
+    json_data = json.load(StringIO(response.text))
+    day_df = pd.DataFrame.from_dict(json_data["data"])
+
+    day_df["startTime"] = pd.to_datetime(day_df["startTime"], utc=True)
+    day_df = day_df[["startTime", "systemSellPrice", "netImbalanceVolume"]]
+    day_df = day_df.rename(columns={
+        "startTime": "spUTCTime",
+        "systemSellPrice": "price",
+        "netImbalanceVolume": "volume"
+    })
+
+    day_df.insert(1, "spClockTime", day_df["spUTCTime"].dt.tz_convert("Europe/London"))
+
+    return day_df

skypro/commands/pull_elexon_imbalance/utils.py

@@ -0,0 +1,33 @@
+import logging
+from datetime import date, timedelta
+from time import sleep
+from typing import Callable, Any
+
+
+def with_retries(func: Callable, max_attempts: int, delay: timedelta) -> Any:
+    """
+    This will call the given `func`, and retry up to `max_attempt` times if an exception is encountered.
+    """
+    attempts = 0
+    while True:
+        try:
+            attempts += 1
+            result = func()
+            return result
+        except Exception as e:
+            logging.warning(f"Call to function failed: {e}")
+            if attempts < max_attempts:
+                logging.info(f"Retrying {attempts}/{max_attempts}")
+                sleep(delay.total_seconds())
+                continue
+            else:
+                logging.error("Max attempts reached.")
+                raise e
+
+
+def daterange(start: date, end: date):
+    """
+    Yields every date between start and end.
+    """
+    for n in range(int((end - start).days + 1)):
+        yield start + timedelta(days=n)

skypro/commands/report/README.md

@@ -0,0 +1,24 @@
+# Skypro reporting
+
+The reporting CLI tool collates historical metering actuals and rate/pricing information about a microgrid, and then analyses the data to produce a report which gives:
+- Supplier invoice estimates, i.e. the costs associated with imports and revenues associated with exports.
+- Summaries of energy flows in the microgrid, and their associated prices and costs.
+- Solar statistics, including self-use.
+- BESS statistics, including cycling and roundtrip efficiency.
+
+The tool also reports on data inconsistencies by presenting 'Notices' to the user, which are graded according to how serious the issue is.
+Some data inconsistencies are always going to be present because, for example, meters are not 100% accurate.
+However, other data inconsistencies may be transient, for example, some metering data may be temporarily missing and so the reporting tool may need to make approximations.  
+
+
+## Usage
+
+To run a microgrid report and plot the results: `skypro report -c <report-config-file> -m 2025-04 --plot`
+
+See `skypro report -h` for help with command line options.
+
+## Configuration
+
+See the integration tests for an example configuration YAML file with inline comments: `src/tests/integration/fixtures/reporting/config.yaml`
+
+Also, see the `skypro/commands/report/config/config.py` file for the configuration definition in code with comments.

skypro/commands/report/config/config.py

@@ -0,0 +1,140 @@
+from typing import Dict, Optional
+from uuid import UUID
+
+import yaml
+from marshmallow_dataclass import dataclass
+from skypro.common.config.bill_match import BillMatchLineItem
+from skypro.common.config.path_field import PathField, PathType
+from skypro.common.config.rates_dataclasses import Rates
+from skypro.common.config.utility import field_with_opts
+from skypro.common.config.data_source import MeterReadingDataSource, PlotMeterReadingDataSource, BessReadingDataSource
+
+"""
+This file contains configuration schema that is used for reporting actuals on microgrid costs/revenues and battery performance.
+The higher-level configuration structures are defined towards the end of the file, and the lower-level structures towards the top.
+"""
+
+
+@dataclass
+class MicrogridMeter:
+    """
+    Configures the data source for a microgrid-level meter (e.g. an Acuvim meter)
+    """
+    data_source: MeterReadingDataSource = field_with_opts(key="dataSource")
+
+
+@dataclass
+class MicrogridFeederMeter:
+    """
+    Configures the data source for a microgrid-level feeder meter (e.g. an Acuvim meter), alongside the
+    ID that the feeder is assigned in the Flows database.
+    """
+    data_source: MeterReadingDataSource = field_with_opts(key="dataSource")
+    feeder_flows_id: UUID = field_with_opts(key="feederId")
+
+
+@dataclass
+class MicrogridMeters:
+    """
+    Holds configuration for all the microgrid-level meters on a site
+    """
+    bess_inverter: MicrogridMeter = field_with_opts(key="bessInverter")
+    main_incomer: MicrogridMeter = field_with_opts(key="mainIncomer")
+    ev_charger: MicrogridMeter = field_with_opts(key="evCharger")
+    feeder_1: MicrogridFeederMeter = field_with_opts(key="feeder1")
+    feeder_2: MicrogridFeederMeter = field_with_opts(key="feeder2")
+
+
+@dataclass
+class PlotMeters:
+    """
+    Configures the data source for plot-level meters (e.g. Emlite meters)
+    """
+    data_source: PlotMeterReadingDataSource = field_with_opts(key="dataSource")
+
+
+@dataclass
+class Metering:
+    """
+    Holds configuration for both plot-level and microgrid-level meters
+    """
+    plot_meters: Optional[PlotMeters] = field_with_opts(key="plotMeters")
+    microgrid_meters: MicrogridMeters = field_with_opts(key="microgridMeters")
+
+
+@dataclass
+class Bess:
+    """
+    Configures the size of a BESS in a microgrid, as well as the data source for BESS readings (which
+    hold things like state of energy kWh etc)
+    """
+    energy_capacity: float = field_with_opts(key="energyCapacity")
+    data_source: BessReadingDataSource = field_with_opts(key="dataSource")
+
+
+@dataclass
+class GridConnection:
+    """
+    Configures the size of the sites grid connection in kVA (this is used for calculating things like
+    the £/kVA/day costs).
+    """
+    import_capacity: float = field_with_opts(key="importCapacity")
+    export_capacity: float = field_with_opts(key="exportCapacity")
+
+
+@dataclass
+class BillMatchImportOrExport:
+    """
+    Configures a set of 'line items' that appear on a Suppliers invoice. Suppliers each have their own
+    way of formatting their invoices, and this allows us to express our costs/revenues in the same way as the
+    Suppliers, so we can easily compare. For example, some may have a line for "non commodity", and "duos", etc.
+    """
+    line_items: Dict[str, BillMatchLineItem] = field_with_opts(key="lineItems")
+
+
+@dataclass
+class BillMatch:
+    """
+    Optionally configures the import and export Supplier invoice formats, see above.
+    """
+    import_direction: Optional[BillMatchImportOrExport] = field_with_opts(key="import")
+    export_direction: Optional[BillMatchImportOrExport] = field_with_opts(key="export")
+
+
+@dataclass
+class Reporting:
+    """
+    Holds all configuration required to configure reporting of actual costs/revenues of a microgrid.
+    """
+    metering: Metering
+    bess: Bess
+    grid_connection: GridConnection = field_with_opts(key="gridConnection")
+    bill_match: BillMatch = field_with_opts(key="billMatch")
+    profiles_save_dir: PathType = field_with_opts(key="profilesSaveDir")  # Optionally save out the load and solar profiles to disk (useful for running simulations with actual solar and load profiles).
+    rates: Rates
+
+
+@dataclass
+class Config:
+    reporting: Reporting
+
+
+def parse_config(file_path: str, env_vars: dict) -> Config:
+    """
+    Reads in the given file, and returns the parsed Config instance.
+    Variables may be specified to substitute into any file paths that are specified in the config - e.g. some paths
+    appear like "$SKYPRO_DIR/my_sims/blah.yaml" and the $SKYPRO_DIR will be substituted if there is a corresponding
+    entry in `env_vars`.
+    """
+
+    # Read in the main config file
+    with open(file_path) as config_file:
+        # Here we parse the config file as YAML, which is a superset of JSON so allows us to parse JSON files as well
+        config_dict = yaml.safe_load(config_file)
+
+        # Set up the variables that are substituted into file paths
+        PathField.vars_for_substitution = env_vars
+
+        config = Config.Schema().load(config_dict)
+
+    return config