premise 2.1.0.dev3__tar.gz → 2.1.0.dev5__tar.gz

{premise-2.1.0.dev3/premise.egg-info → premise-2.1.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: premise
-Version: 2.1.0.dev3
+Version: 2.1.0.dev5
 Summary: Coupling IAM output to ecoinvent LCA database ecoinvent for prospective LCA
 Author-email: Romain Sacchi <romain.sacchi@psi.ch>, Alois Dirnaichner <dirnaichner@pik-potsdam.de>,  Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Romain Sacchi <romain.sacchi@psi.ch>

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/__init__.py

@@ -1,5 +1,5 @@
 __all__ = ("NewDatabase", "clear_cache", "get_regions_definition")
-__version__ = (2, 1, 0, "dev3")
+__version__ = (2, 1, 0, "dev5")
 
 
 from premise.new_database import NewDatabase

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/export.py

@@ -27,9 +27,8 @@ from scipy import sparse as nsp
 from . import __version__
 from .data_collection import get_delimiter
 from .filesystem_constants import DATA_DIR
-from .geomap import Geomap
 from .inventory_imports import get_correspondence_bio_flows
-from .utils import reset_all_codes
+from .utils import dump_database, load_database, reset_all_codes
 from .validation import BaseDatasetValidator
 
 FILEPATH_SIMAPRO_UNITS = DATA_DIR / "utils" / "export" / "simapro_units.yml"
@@ -976,6 +975,7 @@ def prepare_db_for_export(
 
 
 def _prepare_database(scenario, db_name, original_database, keep_uncertainty_data):
+
     scenario["database"] = prepare_db_for_export(
         scenario,
         name=db_name,

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/filesystem_constants.py

@@ -19,5 +19,8 @@ USER_DATA_BASE_DIR.mkdir(parents=True, exist_ok=True)
 DIR_CACHED_DB = USER_DATA_BASE_DIR / "cache"
 DIR_CACHED_DB.mkdir(parents=True, exist_ok=True)
 
+DIR_CACHED_FILES = USER_DATA_BASE_DIR / "cached_files"
+DIR_CACHED_FILES.mkdir(parents=True, exist_ok=True)
+
 USER_LOGS_DIR = platformdirs.user_log_path(appname="premise", appauthor="pylca")
 USER_LOGS_DIR.mkdir(parents=True, exist_ok=True)

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/logger.py

@@ -3,12 +3,9 @@ Module to create a logger with the given handler.
 """
 
 import logging.config
-from logging.handlers import QueueHandler
 from multiprocessing import Queue
 from pathlib import Path
 
-import yaml
-
 from .filesystem_constants import DATA_DIR
 
 LOG_CONFIG = DATA_DIR / "utils" / "logging" / "logconfig.yaml"
@@ -58,4 +55,12 @@ def empty_log_files():
     for file in DIR_LOG_REPORT.iterdir():
         # if suffix is ".log"
         if file.suffix == ".log":
-            file.unlink()
+            try:
+                file.unlink()
+            except PermissionError:
+                try:
+                    # instead, let's empty the file
+                    with open(file, "w") as f:
+                        f.write("")
+                except PermissionError:
+                    pass

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/new_database.py

@@ -40,17 +40,19 @@ from .filesystem_constants import DIR_CACHED_DB, IAM_OUTPUT_DIR, INVENTORY_DIR
 from .fuels import _update_fuels
 from .heat import _update_heat
 from .inventory_imports import AdditionalInventory, DefaultInventory
-from .logger import empty_log_files
 from .report import generate_change_report, generate_summary_report
 from .steel import _update_steel
 from .transport import _update_vehicles
 from .utils import (
     clear_existing_cache,
     create_scenario_list,
+    delete_all_pickles,
+    dump_database,
     eidb_label,
     hide_messages,
     info_on_utils_functions,
     load_constants,
+    load_database,
     print_version,
     warning_about_biogenic_co2,
 )
@@ -580,8 +582,6 @@ class NewDatabase:
             data = self.__import_additional_inventories(self.additional_inventories)
             self.database.extend(data)
 
-        data = pickle.dumps(self.database, -1)
-
         print("- Fetching IAM data")
         # use multiprocessing to speed up the process
         if self.multiprocessing:
@@ -591,10 +591,6 @@ class NewDatabase:
             for scenario in self.scenarios:
                 _fetch_iam_data(scenario)
 
-        # add database to scenarios
-        for scenario in self.scenarios:
-            scenario["database"] = pickle.loads(data)
-
         print("Done!")
 
     def __find_cached_db(self, db_name: str) -> List[dict]:
@@ -898,57 +894,35 @@ class NewDatabase:
             [item for item in sectors if item not in sector_update_methods]
         )
 
-        # Outer tqdm progress bar for sectors
-        with tqdm(total=len(sectors), desc="Updating sectors", ncols=70) as pbar_outer:
-            for sector in sectors:
-                pbar_outer.set_description(f"Updating: {sector}")
-
-                # skip "external" if no external scenarios are provided
-                if sector == "external" and not any(
-                    "external scenarios" in scenario for scenario in self.scenarios
-                ):
-                    continue
-
-                # Prepare the function and arguments
-                update_func = sector_update_methods[sector]["func"]
-                fixed_args = sector_update_methods[sector]["args"]
-
-                if self.multiprocessing:
-                    # Process scenarios in parallel for the current sector
-                    with ProcessPool(processes=multiprocessing.cpu_count()) as pool:
-                        # Prepare the tasks with all necessary arguments
-                        tasks = [
-                            (scenario,) + fixed_args for scenario in self.scenarios
-                        ]
-
-                        # we cannot pickle the datapackage contained under "external scenarios" in each scenario
-                        # so we replace it by the file path of the datapackage before processing
-
-                        for task in tasks:
-                            if "external scenarios" in task[0]:
-                                for external_scenario in task[0]["external scenarios"]:
-                                    if isinstance(
-                                        external_scenario["data"],
-                                        datapackage.DataPackage,
-                                    ):
-                                        external_scenario["data"] = external_scenario[
-                                            "data"
-                                        ].base_path
-
-                        # Use starmap for preserving the order of tasks
-                        results = pool.starmap(update_func, tasks)
-
-                        # Update self.scenarios based on the ordered results
-                        for s, updated_scenario in enumerate(results):
-                            self.scenarios[s] = updated_scenario
-
-                else:
-                    # Process scenarios in sequence for the current sector
-                    for s, scenario in enumerate(self.scenarios):
-                        self.scenarios[s] = update_func(scenario, *fixed_args)
+        # unlink all files in the cache directory
+        delete_all_pickles()
 
+        with tqdm(
+            total=len(self.scenarios), desc="Processing scenarios", ncols=70
+        ) as pbar_outer:
+            for scenario in self.scenarios:
+                # add database to scenarios
+                scenario["database"] = pickle.loads(pickle.dumps(self.database, -1))
+                for sector in sectors:
+                    if sector in scenario.get("applied functions", []):
+                        print(
+                            f"Function to update {sector} already applied to scenario."
+                        )
+                        continue
+
+                    # Prepare the function and arguments
+                    update_func = sector_update_methods[sector]["func"]
+                    fixed_args = sector_update_methods[sector]["args"]
+                    scenario = update_func(scenario, *fixed_args)
+
+                    if "applied functions" not in scenario:
+                        scenario["applied functions"] = []
+                    scenario["applied functions"].append(sector)
+
+                # dump database
+                dump_database(scenario)
                 # Manually update the outer progress bar after each sector is completed
-                pbar_outer.update(1)
+                pbar_outer.update()
         print("Done!\n")
 
     def write_superstructure_db_to_brightway(
@@ -973,6 +947,7 @@ class NewDatabase:
             )
 
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name=name,
@@ -1002,6 +977,9 @@ class NewDatabase:
         # generate change report from logs
         self.generate_change_report()
 
+        for scenario in self.scenarios:
+            del scenario["database"]
+
     def write_db_to_brightway(self, name: [str, List[str]] = None):
         """
         Register the new database into an open brightway project.
@@ -1039,18 +1017,19 @@ class NewDatabase:
         print("Write new database(s) to Brightway.")
 
         for s, scenario in enumerate(self.scenarios):
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name=name[s],
                 original_database=self.database,
                 keep_uncertainty_data=self.keep_uncertainty_data,
             )
-
-        for scen, scenario in enumerate(self.scenarios):
             write_brightway_database(
                 scenario["database"],
-                name[scen],
+                name[s],
             )
+            # delete the database from the scenario
+            del scenario["database"]
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1073,10 +1052,18 @@ class NewDatabase:
 
         """
 
+        def scenario_name(scenario):
+            name = scenario["pathway"]
+
+            if "external scenarios" in scenario:
+                for external in scenario["external scenarios"]:
+                    name += f"-{external['scenario']}"
+            return name
+
         if filepath is not None:
             if isinstance(filepath, str):
                 filepath = [
-                    (Path(filepath) / s["model"] / s["pathway"] / str(s["year"]))
+                    (Path(filepath) / s["model"] / scenario_name(s) / str(s["year"]))
                     for s in self.scenarios
                 ]
             elif isinstance(filepath, list):
@@ -1094,36 +1081,15 @@ class NewDatabase:
 
         print("Write new database(s) to matrix.")
 
-        # use multiprocessing to speed up the process
-
-        if self.multiprocessing:
-            with ProcessPool(processes=multiprocessing.cpu_count()) as pool:
-                args = [
-                    (scenario, "database", self.database, self.keep_uncertainty_data)
-                    for scenario in self.scenarios
-                ]
-                results = pool.starmap(_prepare_database, args)
-
-            for s, scenario in enumerate(self.scenarios):
-                self.scenarios[s] = results[s]
-
-            with ProcessPool(processes=multiprocessing.cpu_count()) as pool:
-                args = [
-                    Export(scenario, filepath[scen], self.version)
-                    for scen, scenario in enumerate(self.scenarios)
-                ]
-                pool.map(_export_to_matrices, args)
-        else:
-            for scenario in self.scenarios:
-                _prepare_database(
-                    scenario=scenario,
-                    db_name="database",
-                    original_database=self.database,
-                    keep_uncertainty_data=self.keep_uncertainty_data,
-                )
-
-            for scen, scenario in enumerate(self.scenarios):
-                Export(scenario, filepath[scen], self.version).export_db_to_matrices()
+        for s, scenario in enumerate(self.scenarios):
+            scenario = load_database(scenario)
+            _prepare_database(
+                scenario=scenario,
+                db_name="database",
+                original_database=self.database,
+                keep_uncertainty_data=self.keep_uncertainty_data,
+            )
+            Export(scenario, filepath[s], self.version).export_db_to_matrices()
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1149,15 +1115,15 @@ class NewDatabase:
         # use multiprocessing to speed up the process
 
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name="database",
                 original_database=self.database,
                 keep_uncertainty_data=self.keep_uncertainty_data,
             )
-
-        for scenario in self.scenarios:
             Export(scenario, filepath, self.version).export_db_to_simapro()
+            del scenario["database"]
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1183,17 +1149,17 @@ class NewDatabase:
         # use multiprocessing to speed up the process
 
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name="database",
                 original_database=self.database,
                 keep_uncertainty_data=self.keep_uncertainty_data,
             )
-
-        for scenario in self.scenarios:
             Export(scenario, filepath, self.version).export_db_to_simapro(
                 olca_compartments=True
             )
+            del scenario["database"]
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1215,6 +1181,7 @@ class NewDatabase:
 
         # use multiprocessing to speed up the process
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name=name,
@@ -1232,6 +1199,9 @@ class NewDatabase:
             scenario_list=list_scenarios,
         )
 
+        for scenario in self.scenarios:
+            del scenario["database"]
+
         cached_inventories.extend(extra_inventories)
 
         list_scenarios = ["original"] + list_scenarios

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/transformation.py

@@ -310,23 +310,6 @@ def find_fuel_efficiency(
             print(
                 f"Warning: {dataset['name'], dataset['location']} has no energy input"
             )
-            for exc in dataset["exchanges"]:
-                print(
-                    exc["type"],
-                    exc["name"],
-                    exc["amount"],
-                    exc.get("product"),
-                    exc["unit"],
-                )
-            from pprint import pprint
-
-            print()
-            pprint(fuel_filters)
-            print()
-            pprint(fuel_specs)
-            print()
-            pprint(fuel_map_reverse)
-            print()
 
     if energy_input != 0 and float(energy_out) != 0:
         current_efficiency = float(energy_out) / energy_input

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/transport.py

@@ -517,26 +517,6 @@ class Transport(BaseTransformation):
         Create vehicle market (fleet average) datasets.
         """
 
-        # # create and regionalize vehicle datasets
-        # vehicle_datasets = list(ws.get_many(
-        #     self.database,
-        #     ws.either(*[
-        #         ws.contains("name", size)
-        #         for size in self.mapping[self.vehicle_type]["sizes"]
-        #     ]),
-        #     ws.contains("name", self.vehicle_type),
-        #     ws.equals("unit", "unit")
-        # ))
-        #
-        # for ds in vehicle_datasets:
-        #     new_vehicle_datasets = self.fetch_proxies(
-        #         subset = vehicle_datasets,
-        #         name=ds["name"],
-        #         ref_prod=ds["reference product"],
-        #     )
-        #
-        #     self.database.extend(new_vehicle_datasets.values())
-
         # create and regionalize transport datasets
         filters = [
             ws.either(

{premise-2.1.0.dev3 → premise-2.1.0.dev5}/premise/utils.py

@@ -3,6 +3,7 @@ Various utils functions.
 """
 
 import os
+import pickle
 import sys
 import uuid
 from datetime import datetime
@@ -21,7 +22,12 @@ from wurst.searching import biosphere, equals, get_many, technosphere
 
 from . import __version__
 from .data_collection import get_delimiter
-from .filesystem_constants import DATA_DIR, DIR_CACHED_DB, VARIABLES_DIR
+from .filesystem_constants import (
+    DATA_DIR,
+    DIR_CACHED_DB,
+    DIR_CACHED_FILES,
+    VARIABLES_DIR,
+)
 from .geomap import Geomap
 
 FUELS_PROPERTIES = VARIABLES_DIR / "fuels_variables.yaml"
@@ -343,3 +349,53 @@ def create_scenario_list(scenarios: list) -> list:
         list_scenarios.append(name)
 
     return list_scenarios
+
+
+def dump_database(scenario):
+    """
+    Dump database to a pickle file.
+    :param scenario: scenario dictionary
+    """
+
+    if "database filepath" in scenario:
+        return scenario
+
+    # generate random name
+    name = f"{uuid.uuid4().hex}.pickle"
+    # dump as pickle
+    with open(DIR_CACHED_FILES / name, "wb") as f:
+        pickle.dump(scenario["database"], f)
+    scenario["database filepath"] = DIR_CACHED_FILES / name
+    del scenario["database"]
+
+    return scenario
+
+
+def load_database(scenario):
+    """
+    Load database from a pickle file.
+    :param scenario: scenario dictionary
+    """
+
+    if "database" in scenario:
+        return scenario
+
+    filepath = scenario["database filepath"]
+
+    # load pickle
+    with open(filepath, "rb") as f:
+        scenario["database"] = pickle.load(f)
+    del scenario["database filepath"]
+    # delete the file
+    filepath.unlink()
+
+    return scenario
+
+
+def delete_all_pickles():
+    """
+    Delete all pickle files in the cache folder.
+    """
+
+    for file in DIR_CACHED_FILES.glob("*.pickle"):
+        file.unlink()

{premise-2.1.0.dev3 → premise-2.1.0.dev5/premise.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: premise
-Version: 2.1.0.dev3
+Version: 2.1.0.dev5
 Summary: Coupling IAM output to ecoinvent LCA database ecoinvent for prospective LCA
 Author-email: Romain Sacchi <romain.sacchi@psi.ch>, Alois Dirnaichner <dirnaichner@pik-potsdam.de>,  Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Romain Sacchi <romain.sacchi@psi.ch>