shrecc 0.0.5.dev2__tar.gz → 0.1.0.dev2__tar.gz

shrecc-0.1.0.dev2/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include shrecc/data *.csv *.json

{shrecc-0.0.5.dev2/shrecc.egg-info → shrecc-0.1.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shrecc
-Version: 0.0.5.dev2
+Version: 0.1.0.dev2
 Summary: SHRECC: Smooth Hourly Resolution Electricity Consumption Calculation
 Author-email: Sabina Bednářová <sabina.bednarova@list.lu>
 Maintainer-email: Sabina Bednářová <sabina.bednarova@list.lu>
@@ -27,12 +27,14 @@ Requires-Dist: pandas<3,>=2
 Requires-Dist: numpy<3
 Requires-Dist: numpy>2; python_version >= "3.13"
 Requires-Dist: requests>=2.32
-Requires-Dist: fs
 Requires-Dist: scipy>=1.14
 Requires-Dist: appdirs
+Requires-Dist: packaging
+Requires-Dist: tqdm
+Requires-Dist: xarray
+Requires-Dist: python-calamine>=0.6.2
 Provides-Extra: testing
-Requires-Dist: shrecc; extra == "testing"
-Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest>=8; extra == "testing"
 Requires-Dist: pytest-cov; extra == "testing"
 Requires-Dist: python-coveralls; extra == "testing"
 Provides-Extra: dev
@@ -45,6 +47,12 @@ Requires-Dist: pytest-randomly; extra == "dev"
 Requires-Dist: setuptools; extra == "dev"
 Requires-Dist: ruff>=0.11.12; extra == "dev"
 Requires-Dist: black>=25.1.0; extra == "dev"
+Provides-Extra: premise
+Requires-Dist: premise; extra == "premise"
+Provides-Extra: notebooks
+Requires-Dist: jupyterlab; extra == "notebooks"
+Requires-Dist: jupyter-black>=0.4.0; extra == "notebooks"
+Requires-Dist: matplotlib>=3.10.9; extra == "notebooks"
 Provides-Extra: docs
 Requires-Dist: furo==2024.1.29; extra == "docs"
 Requires-Dist: myst_parser==2.0.0; extra == "docs"
@@ -56,6 +64,7 @@ Requires-Dist: myst-parser>=2.0.0; extra == "docs"
 Requires-Dist: sphinx-design>=0.6.1; extra == "docs"
 Requires-Dist: sphinx-copybutton>=0.5.2; extra == "docs"
 Requires-Dist: sphinx-autoapi>=3.5.0; extra == "docs"
+Requires-Dist: astroid<4; extra == "docs"
 Requires-Dist: sphinx-notfound-page>=1.1.0; extra == "docs"
 Requires-Dist: myst-nb>=1.2.0; extra == "docs"
 Dynamic: license-file
@@ -66,22 +75,29 @@ Simple Hourly Resolution Electricity Consumption Calculation
 
 ## Description
 
-SHRECC package is a python package directly compatible with Brightway2 or Brightway2.5 to create time-aware electricity databases. For any given year and countries (check availability on https://api.energy-charts.info/), download and prepare data for low-voltage electricity consumption.
+SHRECC creates time-resolved electricity-consumption databases for Brightway.
+It uses measured Energy Charts data for historical years and ENTSO-E TYNDP
+scenario data for prospective years, while exposing the same workflow for both.
 
 ## Features
 
-- **High-resolution electricity mixes** – Generates electricity life cycle inventories (LCIs) with **hourly** resolution, enhancing accuracy for life cycle assessment (LCA).
-- **Brightway2/2.5 compatibility** – Seamlessly integrates with Brightway, allowing direct use in existing LCA models.
-- **Dynamic temporal representation** – Users can select electricity mixes by **hour, month, or season**, addressing fluctuations in renewable energy generation and consumption.
-- **Automated data retrieval** – Pulls electricity production, trade, and consumption data from the **Energy Charts API**, ensuring up-to-date datasets.
-- **Ecoinvent matching** – Aligns with **ecoinvent classifications**, converting from ENTSO-E datasets.
-- **User-controlled updates** – Enables **one-time or recurring** updates, allowing continuous tracking of electricity mix evolution over time.
-- **Optimized impact assessments** – Helps reduce uncertainty and improve **decision-making for electricity-intensive technologies** by considering real-time electricity mix variations.
+- Hourly production, trade, consumption-volume, and consumption-mix results.
+- One reduced country-network solver shared by Energy Charts and TYNDP data.
+- Historical ecoinvent allocation and prospective premise/IAM mapping.
+- Hourly, daily, weekly, monthly, and explicit timestamp selections.
+- Inspectable intermediate results before any Brightway database is changed.
+- Resumable source acquisition and compressed, time-chunked result caches.
+- Brightway 2 and 2.5 database writing through `NewDatabase`.
+- Multi-year runs with a separate, year-labelled foreground database for each
+  requested year.
 
 ## Documentation
 
 The full documentation is hosted at [Read the Docs page for shrecc](https://shrecc.readthedocs.io/en/latest/)
 
+The internal data flow and module responsibilities are summarized in
+[docs/architecture.md](https://git.list.lu/shrecc_project/SHRECC/-/blob/main/docs/architecture.md).
+
 ## Installation
 
 `shrecc` can be installed from pypi or from source.
@@ -95,6 +111,12 @@ You can install it with pip (or any other pypi compatible util like `uv` or `poe
 pip install shrecc
 ```
 
+Install optional premise geography support or the notebook environment with:
+
+```
+pip install "shrecc[premise,notebooks]"
+```
+
 ### From source 
 
 To install shrecc from source, clone the code and then install the package and if necessary the dependencies manually.
@@ -102,8 +124,49 @@ To install shrecc from source, clone the code and then install the package and i
 
 ## Usage
 
-You can find usage examples in the Jupyter notebook in this repo: [notebooks/example.ipynb](notebooks/example.ipynb)
-_and_ in the documentation at [read the docs](https://shrecc.readthedocs.io/en/latest/content/notebooks/notebooks.html).
+The repository contains a minimal
+[getting-started notebook](https://git.list.lu/shrecc_project/shrecc/-/blob/develop/notebooks/1_shrecc_get_started.ipynb)
+and a more detailed
+[analysis notebook](https://git.list.lu/shrecc_project/SHRECC/-/blob/develop/notebooks/2_shrecc_analysis.ipynb).
+Both are also available in the documentation (only for main releases) at
+[read the docs](https://shrecc.readthedocs.io/en/latest/content/notebooks/).
+
+The harmonized historical/prospective workflow is available through
+`NewDatabase`:
+
+```python
+from shrecc import NewDatabase
+
+electricity = NewDatabase(
+    scenario="DE",
+    years=[2035, 2040, 2050],
+    climate_year=2009,
+    bg_db_name={
+        2035: "premise-remind-eu-2035",
+        2040: "premise-remind-eu-2040",
+        2050: "premise-remind-eu-2050",
+    },
+    my_db_name="shrecc_tyndp_DE_june_noon",
+    countries=["ES", "FR", "DE", "IT", "PT", "BE", "NL", "LU", "AT", "CH"],
+    time_range=["2040-06-01 00:00:00", "2040-06-30 23:00:00"],
+    hour_range=[10, 14],
+    project_name="SHRECCei311",
+    source="auto",
+)
+
+electricity.create()
+electricity.write()
+```
+
+For a multi-year run, the month/day/time selection is reused for each year and
+`write()` creates one foreground database per year. A year suffix is added to
+`my_db_name` automatically unless explicit year-specific names are supplied.
+Foreground activity names and metadata also record their model year.
+
+Intermediate canonical results and mapped tables remain available on the class
+instance through `results(year)` and `table(year)`. Use
+`mapping_report(year)` to inspect any historical technologies assigned to a
+country-specific high-voltage fallback activity.
 
 
 ## Contributing

shrecc-0.1.0.dev2/README.md

@@ -0,0 +1,113 @@
+# SHRECC
+
+Simple Hourly Resolution Electricity Consumption Calculation
+
+## Description
+
+SHRECC creates time-resolved electricity-consumption databases for Brightway.
+It uses measured Energy Charts data for historical years and ENTSO-E TYNDP
+scenario data for prospective years, while exposing the same workflow for both.
+
+## Features
+
+- Hourly production, trade, consumption-volume, and consumption-mix results.
+- One reduced country-network solver shared by Energy Charts and TYNDP data.
+- Historical ecoinvent allocation and prospective premise/IAM mapping.
+- Hourly, daily, weekly, monthly, and explicit timestamp selections.
+- Inspectable intermediate results before any Brightway database is changed.
+- Resumable source acquisition and compressed, time-chunked result caches.
+- Brightway 2 and 2.5 database writing through `NewDatabase`.
+- Multi-year runs with a separate, year-labelled foreground database for each
+  requested year.
+
+## Documentation
+
+The full documentation is hosted at [Read the Docs page for shrecc](https://shrecc.readthedocs.io/en/latest/)
+
+The internal data flow and module responsibilities are summarized in
+[docs/architecture.md](https://git.list.lu/shrecc_project/SHRECC/-/blob/main/docs/architecture.md).
+
+## Installation
+
+`shrecc` can be installed from pypi or from source.
+
+### From pypi
+
+The package is published at [pypi.org/projects/shrecc](https://pypi.org/project/shrecc).
+You can install it with pip (or any other pypi compatible util like `uv` or `poetry` as follows:
+
+```
+pip install shrecc
+```
+
+Install optional premise geography support or the notebook environment with:
+
+```
+pip install "shrecc[premise,notebooks]"
+```
+
+### From source 
+
+To install shrecc from source, clone the code and then install the package and if necessary the dependencies manually.
+
+
+## Usage
+
+The repository contains a minimal
+[getting-started notebook](https://git.list.lu/shrecc_project/shrecc/-/blob/develop/notebooks/1_shrecc_get_started.ipynb)
+and a more detailed
+[analysis notebook](https://git.list.lu/shrecc_project/SHRECC/-/blob/develop/notebooks/2_shrecc_analysis.ipynb).
+Both are also available in the documentation (only for main releases) at
+[read the docs](https://shrecc.readthedocs.io/en/latest/content/notebooks/).
+
+The harmonized historical/prospective workflow is available through
+`NewDatabase`:
+
+```python
+from shrecc import NewDatabase
+
+electricity = NewDatabase(
+    scenario="DE",
+    years=[2035, 2040, 2050],
+    climate_year=2009,
+    bg_db_name={
+        2035: "premise-remind-eu-2035",
+        2040: "premise-remind-eu-2040",
+        2050: "premise-remind-eu-2050",
+    },
+    my_db_name="shrecc_tyndp_DE_june_noon",
+    countries=["ES", "FR", "DE", "IT", "PT", "BE", "NL", "LU", "AT", "CH"],
+    time_range=["2040-06-01 00:00:00", "2040-06-30 23:00:00"],
+    hour_range=[10, 14],
+    project_name="SHRECCei311",
+    source="auto",
+)
+
+electricity.create()
+electricity.write()
+```
+
+For a multi-year run, the month/day/time selection is reused for each year and
+`write()` creates one foreground database per year. A year suffix is added to
+`my_db_name` automatically unless explicit year-specific names are supplied.
+Foreground activity names and metadata also record their model year.
+
+Intermediate canonical results and mapped tables remain available on the class
+instance through `results(year)` and `table(year)`. Use
+`mapping_report(year)` to inspect any historical technologies assigned to a
+country-specific high-voltage fallback activity.
+
+
+## Contributing
+
+Please take a look at the [DEVELOPPING.md](https://git.list.lu/shrecc_project/shrecc/-/blob/main/DEVELOPPING.md) file for details on how to contribute code to the repository.
+
+## License
+
+Copyright © 2025 Luxembourg Institute of Science and Technology
+Licensed under the MIT License.
+
+## Authors
+
+* Sabina Bednářová (<sabina.bednarova@list.lu>)
+* Thomas Gibon (<thomas.gibon@list.lu>)

{shrecc-0.0.5.dev2 → shrecc-0.1.0.dev2}/pyproject.toml

@@ -37,9 +37,12 @@ dependencies = [
    "numpy<3",
    'numpy>2;python_version>="3.13"',
    "requests>=2.32",
-   "fs",
    "scipy>=1.14",
    "appdirs",
+   "packaging",
+   "tqdm",
+   "xarray",
+   "python-calamine>=0.6.2",
 ]
 license="MIT"
 license-files = ["LICENSE"]
@@ -50,11 +53,8 @@ homepage = "https://git.list.lu/shrecc_project/SHRECC"
 tracker = "https://git.list.lu/shrecc_project/SHRECC/issues"
 
 [project.optional-dependencies]
-# Getting recursive dependencies to work is a pain, this
-# seems to work, at least for now
 testing = [
-    "shrecc",
-    "pytest",
+    "pytest>=8",
     "pytest-cov",
     "python-coveralls"
 ]
@@ -69,6 +69,14 @@ dev = [
     "ruff>=0.11.12",
     "black>=25.1.0",
 ]
+premise = [
+    "premise",
+]
+notebooks = [
+    "jupyterlab",
+    "jupyter-black>=0.4.0",
+    "matplotlib>=3.10.9",
+]
 docs = [
     "furo==2024.1.29",
     "myst_parser==2.0.0",
@@ -80,13 +88,18 @@ docs = [
     "sphinx-design>=0.6.1",
     "sphinx-copybutton>=0.5.2",
     "sphinx-autoapi>=3.5.0",
+    "astroid<4",
     "sphinx-notfound-page>=1.1.0",
     "myst-nb>=1.2.0",
  ]
 
 [tool.setuptools]
 include-package-data = true
-packages = ["shrecc"]
+packages = ["shrecc", "shrecc.data", "shrecc.data.tyndp"]
+
+[tool.setuptools.package-data]
+"shrecc.data" = ["*.csv", "*.json"]
+"shrecc.data.tyndp" = ["*.csv", "*.json"]
 
 [tool.setuptools.dynamic]
 version = {attr = "shrecc.__version__"}

{shrecc-0.0.5.dev2 → shrecc-0.1.0.dev2}/shrecc/__init__.py

@@ -1,4 +1,4 @@
-# Copyright © 2024 Luxembourg Institute of Science and Technology
+# Copyright © 2024-2026 Luxembourg Institute of Science and Technology
 # Licensed under the MIT License (see LICENSE file for details).
 # Authors: [Sabina Bednářová, Thomas Gibon]
 
@@ -7,13 +7,14 @@
 
 __all__ = (
     "__version__",
+    "NewDatabase",
     "create_database",
     "filt_cutoff",
     "get_data",
     "data_processing",
 )
 from .database import create_database, filt_cutoff
-from .download import get_data
-from .treatment import data_processing
+from .energy_charts import data_processing, get_data
+from .pipeline import NewDatabase
 
-__version__ = "0.0.5.dev2"
+__version__ = "0.1.0.dev2"

shrecc-0.1.0.dev2/shrecc/_legacy_database.py

@@ -0,0 +1,156 @@
+"""Historical file-based database-table preparation retained for compatibility."""
+
+from datetime import datetime
+from importlib.resources import files
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from shrecc.mapping import load_ecoinvent_mapping as load_mapping_data
+from shrecc.result_store import (
+    load_pickle as load_from_pickle,
+    save_pickle as save_to_pickle,
+)
+
+
+def load_time_series_data(path_to_data, year):
+    """Load a legacy sparse consumption matrix and restore its labels."""
+    path_to_data = Path(path_to_data)
+    filename = path_to_data / f"{year}" / f"indices_{year}.pkl"
+    indices = load_from_pickle(filename)
+    Z_cons_sp = load_from_pickle(path_to_data / f"{year}" / f"Z_cons_{year}.pkl")
+    Z_cons = pd.DataFrame(
+        np.float32(Z_cons_sp.todense()),
+        index=indices["index"],
+        columns=indices["columns"],
+    )
+    return Z_cons.reorder_levels(["time", "source", "country"], axis=1).sort_index(
+        axis=1
+    )
+
+
+def prepare_consumption_data(Z_cons):
+    """Remove legacy trade rows and arrange rows for activity mapping."""
+    Z_cons = Z_cons.sort_index()
+
+    if "trade" in Z_cons.index.get_level_values("source"):
+        Z_cons_to_multiply = Z_cons.drop("trade", axis=0).copy()
+    else:
+        Z_cons_to_multiply = Z_cons.copy()
+    Z_cons_to_multiply.index.names = ["source", "geography_mix"]
+
+    return Z_cons_to_multiply.swaplevel()
+
+
+def apply_mapping(Z_cons_to_multiply, el_map_all_norm):
+    """Apply the historical matrix-form technology mapping."""
+    el_map_to_multiply = el_map_all_norm.reindex(
+        Z_cons_to_multiply.index, axis=1
+    ).astype("float32")
+    el_map_to_multiply["NIE"] = el_map_all_norm["GB"][
+        el_map_to_multiply["NIE"].columns
+    ].astype("float32")
+    el_map_to_multiply["UK"] = el_map_all_norm["GB"][
+        el_map_to_multiply["UK"].columns
+    ].astype("float32")
+
+    el_map_to_multiply = el_map_to_multiply.fillna(0)
+    LCI_cons = el_map_to_multiply.dot(Z_cons_to_multiply)
+    return LCI_cons.fillna(0) if LCI_cons.isna().sum().sum() else LCI_cons
+
+
+def tech_mapping(year, path_to_data, path_to_mapping=None):
+    """Map a legacy consumption matrix and normalize each mix to one kWh."""
+    now = datetime.now()
+    print(f"{now} Mapping technologies...")
+    if path_to_mapping:
+        el_map_all_norm = load_mapping_data(path_to_mapping)
+    else:
+        el_map_all_norm = load_mapping_data(files("shrecc.data"))
+    Z_cons = load_time_series_data(path_to_data, year)
+    Z_cons_to_multiply = prepare_consumption_data(Z_cons)
+    filename = Path(path_to_data / f"{year}" / f"LCI_cons_scaled_{year}.pkl")
+    if filename.exists():
+        LCI_cons_scaled = load_from_pickle(filename)
+    else:
+        LCI_cons = apply_mapping(Z_cons_to_multiply, el_map_all_norm)
+        mapped_total = LCI_cons.sum()
+        filename = Path(path_to_data / f"{year}" / f"Z_load_{year}.pkl")
+        load = load_from_pickle(filename)
+        load_difference_row = (
+            "RER",
+            "electricity, high voltage, European attribute mix",
+            "electricity, high voltage",
+            "kWh",
+        )
+
+        load_stacked = load.stack()
+        load_stacked.index.names = ["country", "time"]
+        load_stacked = load_stacked[load_stacked != 0]
+        merged = load_stacked.reset_index(name="load_value").merge(
+            mapped_total.reset_index(), how="inner", on=["time", "country"]
+        )
+        merged.rename(columns={0: "sum"}, inplace=True)
+        merged["difference"] = merged["load_value"] - merged["sum"]
+        merged = merged[merged["difference"] > 0]
+        merged.set_index(["time", "source", "country"], inplace=True)
+
+        LCI_cons.sort_index(inplace=True)
+        LCI_cons.sort_index(axis=1, inplace=True)
+        LCI_cons.loc[load_difference_row] = merged["difference"]
+        LCI_cons.loc[load_difference_row] = LCI_cons.loc[
+            load_difference_row
+        ].fillna(0)
+        LCI_cons_scaled = LCI_cons / LCI_cons.sum()
+        save_to_pickle(
+            LCI_cons_scaled,
+            Path(path_to_data / f"{year}" / f"LCI_cons_scaled_{year}.pkl"),
+        )
+    now = datetime.now()
+    print(f"{now} Technologies mapped.")
+    return LCI_cons_scaled
+
+
+def filter_by_countries(dataframe, countries):
+    """Filter a legacy activity table to selected consumer countries."""
+    if "country" not in dataframe.columns.names:
+        print("Couldnt find country to filter")
+        return None
+
+    return dataframe.loc[
+        :, dataframe.columns.get_level_values("country").isin(countries)
+    ]
+
+
+def filter_by_times(dataframe, times):
+    """Filter a legacy activity table to specific timestamps."""
+    available_times = pd.to_datetime(dataframe.columns.get_level_values("time"))
+    requested_times = pd.to_datetime(times)
+    return dataframe.loc[:, available_times.isin(requested_times)]
+
+
+def filter_by_range(dataframe, general_range, refined_range, freq):
+    """Filter and aggregate a legacy activity table over a time range."""
+    df_filt = dataframe.loc[:, general_range[0] : general_range[1]]
+    if refined_range and len(refined_range) > 1:
+        timestamp = pd.date_range(
+            start=general_range[0], end=general_range[1], freq=freq
+        )
+        timestamps_range = timestamp[
+            (timestamp.hour >= refined_range[0]) & (timestamp.hour <= refined_range[1])
+        ]
+        df_filt = df_filt.loc[
+            :,
+            pd.to_datetime(df_filt.columns.get_level_values("time")).isin(
+                timestamps_range
+            ),
+        ]
+    elif refined_range and len(refined_range) == 1:
+        timestamp = pd.date_range(
+            start=general_range[0], end=general_range[1], freq=freq
+        )
+        df_filt = df_filt.loc[
+            :, pd.to_datetime(df_filt.columns.get_level_values("time")).isin(timestamp)
+        ]
+    return df_filt.T.groupby(level="country").mean().T

shrecc-0.0.5.dev2/shrecc/treatment.py → shrecc-0.1.0.dev2/shrecc/_legacy_treatment.py

@@ -2,7 +2,8 @@
 # Licensed under the MIT License (see LICENSE file for details).
 # Authors: [Sabina Bednářová, Thomas Gibon]
 
-import pickle
+"""Historical full-matrix treatment retained for validation."""
+
 from datetime import datetime
 from importlib.resources import files
 from pathlib import Path
@@ -14,8 +15,13 @@ from pypardiso import spsolve
 from scipy.linalg import block_diag
 from scipy.sparse import coo_array, csc_matrix, csr_matrix, identity
 
+from shrecc.result_store import (
+    load_pickle as load_from_pickle,
+    save_pickle as save_to_pickle,
+)
+
 
-def data_processing(data_df, year, path_to_data=None):
+def _legacy_data_processing(data_df, year, path_to_data=None):
     """
     Processes data, adds missing countries, and correctly divides them between consumption and demand.
 
@@ -141,35 +147,6 @@ def data_processing(data_df, year, path_to_data=None):
     print(f"{now} ..all done!")
 
 
-def save_to_pickle(obj, filename):
-    """
-    Saves an object to a pickle file.
-
-    Args:
-        obj: The object to be saved.
-        filename (Path): Path to the filename where the object will be saved.
-
-    Returns:
-        None
-    """
-    with open(filename, "wb") as f:
-        pickle.dump(obj, f)
-
-
-def load_from_pickle(filename):
-    """
-    Loads an object from a pickle file.
-
-    Args:
-        filename (Path): Path to the filename where the object will be saved.
-
-    Returns:
-        object: The object loaded from the pickle file.
-    """
-    with open(filename, "rb") as f:
-        return pickle.load(f)
-
-
 def add_missing_elements(df, existing_elements, all_elements, axis=0, fill_value=0):
     """
     Adds missing elements (rows or columns) to a dataframe.