premise 2.1.0.dev3__tar.gz → 2.1.0.dev4__tar.gz

{premise-2.1.0.dev3/premise.egg-info → premise-2.1.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: premise
-Version: 2.1.0.dev3
+Version: 2.1.0.dev4
 Summary: Coupling IAM output to ecoinvent LCA database ecoinvent for prospective LCA
 Author-email: Romain Sacchi <romain.sacchi@psi.ch>, Alois Dirnaichner <dirnaichner@pik-potsdam.de>,  Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Romain Sacchi <romain.sacchi@psi.ch>

{premise-2.1.0.dev3 → premise-2.1.0.dev4}/premise/__init__.py

@@ -1,5 +1,5 @@
 __all__ = ("NewDatabase", "clear_cache", "get_regions_definition")
-__version__ = (2, 1, 0, "dev3")
+__version__ = (2, 1, 0, "dev4")
 
 
 from premise.new_database import NewDatabase

{premise-2.1.0.dev3 → premise-2.1.0.dev4}/premise/export.py

@@ -27,9 +27,8 @@ from scipy import sparse as nsp
 from . import __version__
 from .data_collection import get_delimiter
 from .filesystem_constants import DATA_DIR
-from .geomap import Geomap
 from .inventory_imports import get_correspondence_bio_flows
-from .utils import reset_all_codes
+from .utils import dump_database, load_database, reset_all_codes
 from .validation import BaseDatasetValidator
 
 FILEPATH_SIMAPRO_UNITS = DATA_DIR / "utils" / "export" / "simapro_units.yml"
@@ -976,6 +975,7 @@ def prepare_db_for_export(
 
 
 def _prepare_database(scenario, db_name, original_database, keep_uncertainty_data):
+
     scenario["database"] = prepare_db_for_export(
         scenario,
         name=db_name,

{premise-2.1.0.dev3 → premise-2.1.0.dev4}/premise/filesystem_constants.py

@@ -19,5 +19,8 @@ USER_DATA_BASE_DIR.mkdir(parents=True, exist_ok=True)
 DIR_CACHED_DB = USER_DATA_BASE_DIR / "cache"
 DIR_CACHED_DB.mkdir(parents=True, exist_ok=True)
 
+DIR_CACHED_FILES = USER_DATA_BASE_DIR / "cached_files"
+DIR_CACHED_FILES.mkdir(parents=True, exist_ok=True)
+
 USER_LOGS_DIR = platformdirs.user_log_path(appname="premise", appauthor="pylca")
 USER_LOGS_DIR.mkdir(parents=True, exist_ok=True)

{premise-2.1.0.dev3 → premise-2.1.0.dev4}/premise/new_database.py

@@ -40,17 +40,19 @@ from .filesystem_constants import DIR_CACHED_DB, IAM_OUTPUT_DIR, INVENTORY_DIR
 from .fuels import _update_fuels
 from .heat import _update_heat
 from .inventory_imports import AdditionalInventory, DefaultInventory
-from .logger import empty_log_files
 from .report import generate_change_report, generate_summary_report
 from .steel import _update_steel
 from .transport import _update_vehicles
 from .utils import (
     clear_existing_cache,
     create_scenario_list,
+    delete_all_pickles,
+    dump_database,
     eidb_label,
     hide_messages,
     info_on_utils_functions,
     load_constants,
+    load_database,
     print_version,
     warning_about_biogenic_co2,
 )
@@ -580,8 +582,6 @@ class NewDatabase:
             data = self.__import_additional_inventories(self.additional_inventories)
             self.database.extend(data)
 
-        data = pickle.dumps(self.database, -1)
-
         print("- Fetching IAM data")
         # use multiprocessing to speed up the process
         if self.multiprocessing:
@@ -591,10 +591,6 @@ class NewDatabase:
             for scenario in self.scenarios:
                 _fetch_iam_data(scenario)
 
-        # add database to scenarios
-        for scenario in self.scenarios:
-            scenario["database"] = pickle.loads(data)
-
         print("Done!")
 
     def __find_cached_db(self, db_name: str) -> List[dict]:
@@ -898,57 +894,25 @@ class NewDatabase:
             [item for item in sectors if item not in sector_update_methods]
         )
 
-        # Outer tqdm progress bar for sectors
-        with tqdm(total=len(sectors), desc="Updating sectors", ncols=70) as pbar_outer:
-            for sector in sectors:
-                pbar_outer.set_description(f"Updating: {sector}")
-
-                # skip "external" if no external scenarios are provided
-                if sector == "external" and not any(
-                    "external scenarios" in scenario for scenario in self.scenarios
-                ):
-                    continue
-
-                # Prepare the function and arguments
-                update_func = sector_update_methods[sector]["func"]
-                fixed_args = sector_update_methods[sector]["args"]
-
-                if self.multiprocessing:
-                    # Process scenarios in parallel for the current sector
-                    with ProcessPool(processes=multiprocessing.cpu_count()) as pool:
-                        # Prepare the tasks with all necessary arguments
-                        tasks = [
-                            (scenario,) + fixed_args for scenario in self.scenarios
-                        ]
-
-                        # we cannot pickle the datapackage contained under "external scenarios" in each scenario
-                        # so we replace it by the file path of the datapackage before processing
-
-                        for task in tasks:
-                            if "external scenarios" in task[0]:
-                                for external_scenario in task[0]["external scenarios"]:
-                                    if isinstance(
-                                        external_scenario["data"],
-                                        datapackage.DataPackage,
-                                    ):
-                                        external_scenario["data"] = external_scenario[
-                                            "data"
-                                        ].base_path
-
-                        # Use starmap for preserving the order of tasks
-                        results = pool.starmap(update_func, tasks)
-
-                        # Update self.scenarios based on the ordered results
-                        for s, updated_scenario in enumerate(results):
-                            self.scenarios[s] = updated_scenario
-
-                else:
-                    # Process scenarios in sequence for the current sector
-                    for s, scenario in enumerate(self.scenarios):
-                        self.scenarios[s] = update_func(scenario, *fixed_args)
+        # unlink all files in the cache directory
+        delete_all_pickles()
 
+        with tqdm(
+            total=len(self.scenarios), desc="Processing scenarios", ncols=70
+        ) as pbar_outer:
+            for scenario in self.scenarios:
+                # add database to scenarios
+                scenario["database"] = pickle.loads(pickle.dumps(self.database, -1))
+                for sector in sectors:
+                    # Prepare the function and arguments
+                    update_func = sector_update_methods[sector]["func"]
+                    fixed_args = sector_update_methods[sector]["args"]
+                    scenario = update_func(scenario, *fixed_args)
+
+                # dump database
+                dump_database(scenario)
                 # Manually update the outer progress bar after each sector is completed
-                pbar_outer.update(1)
+                pbar_outer.update()
         print("Done!\n")
 
     def write_superstructure_db_to_brightway(
@@ -973,6 +937,7 @@ class NewDatabase:
             )
 
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name=name,
@@ -1002,6 +967,9 @@ class NewDatabase:
         # generate change report from logs
         self.generate_change_report()
 
+        for scenario in self.scenarios:
+            del scenario["database"]
+
     def write_db_to_brightway(self, name: [str, List[str]] = None):
         """
         Register the new database into an open brightway project.
@@ -1039,18 +1007,19 @@ class NewDatabase:
         print("Write new database(s) to Brightway.")
 
         for s, scenario in enumerate(self.scenarios):
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name=name[s],
                 original_database=self.database,
                 keep_uncertainty_data=self.keep_uncertainty_data,
             )
-
-        for scen, scenario in enumerate(self.scenarios):
             write_brightway_database(
                 scenario["database"],
-                name[scen],
+                name[s],
             )
+            # delete the database from the scenario
+            del scenario["database"]
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1094,36 +1063,16 @@ class NewDatabase:
 
         print("Write new database(s) to matrix.")
 
-        # use multiprocessing to speed up the process
-
-        if self.multiprocessing:
-            with ProcessPool(processes=multiprocessing.cpu_count()) as pool:
-                args = [
-                    (scenario, "database", self.database, self.keep_uncertainty_data)
-                    for scenario in self.scenarios
-                ]
-                results = pool.starmap(_prepare_database, args)
-
-            for s, scenario in enumerate(self.scenarios):
-                self.scenarios[s] = results[s]
-
-            with ProcessPool(processes=multiprocessing.cpu_count()) as pool:
-                args = [
-                    Export(scenario, filepath[scen], self.version)
-                    for scen, scenario in enumerate(self.scenarios)
-                ]
-                pool.map(_export_to_matrices, args)
-        else:
-            for scenario in self.scenarios:
-                _prepare_database(
-                    scenario=scenario,
-                    db_name="database",
-                    original_database=self.database,
-                    keep_uncertainty_data=self.keep_uncertainty_data,
-                )
-
-            for scen, scenario in enumerate(self.scenarios):
-                Export(scenario, filepath[scen], self.version).export_db_to_matrices()
+        for s, scenario in enumerate(self.scenarios):
+            scenario = load_database(scenario)
+            _prepare_database(
+                scenario=scenario,
+                db_name="database",
+                original_database=self.database,
+                keep_uncertainty_data=self.keep_uncertainty_data,
+            )
+            Export(scenario, filepath[s], self.version).export_db_to_matrices()
+            del scenario["database"]
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1149,15 +1098,15 @@ class NewDatabase:
         # use multiprocessing to speed up the process
 
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name="database",
                 original_database=self.database,
                 keep_uncertainty_data=self.keep_uncertainty_data,
             )
-
-        for scenario in self.scenarios:
             Export(scenario, filepath, self.version).export_db_to_simapro()
+            del scenario["database"]
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1183,17 +1132,17 @@ class NewDatabase:
         # use multiprocessing to speed up the process
 
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name="database",
                 original_database=self.database,
                 keep_uncertainty_data=self.keep_uncertainty_data,
             )
-
-        for scenario in self.scenarios:
             Export(scenario, filepath, self.version).export_db_to_simapro(
                 olca_compartments=True
             )
+            del scenario["database"]
 
         # generate scenario report
         self.generate_scenario_report()
@@ -1215,6 +1164,7 @@ class NewDatabase:
 
         # use multiprocessing to speed up the process
         for scenario in self.scenarios:
+            scenario = load_database(scenario)
             _prepare_database(
                 scenario=scenario,
                 db_name=name,
@@ -1232,6 +1182,9 @@ class NewDatabase:
             scenario_list=list_scenarios,
         )
 
+        for scenario in self.scenarios:
+            del scenario["database"]
+
         cached_inventories.extend(extra_inventories)
 
         list_scenarios = ["original"] + list_scenarios

{premise-2.1.0.dev3 → premise-2.1.0.dev4}/premise/utils.py

@@ -3,6 +3,7 @@ Various utils functions.
 """
 
 import os
+import pickle
 import sys
 import uuid
 from datetime import datetime
@@ -21,7 +22,12 @@ from wurst.searching import biosphere, equals, get_many, technosphere
 
 from . import __version__
 from .data_collection import get_delimiter
-from .filesystem_constants import DATA_DIR, DIR_CACHED_DB, VARIABLES_DIR
+from .filesystem_constants import (
+    DATA_DIR,
+    DIR_CACHED_DB,
+    DIR_CACHED_FILES,
+    VARIABLES_DIR,
+)
 from .geomap import Geomap
 
 FUELS_PROPERTIES = VARIABLES_DIR / "fuels_variables.yaml"
@@ -343,3 +349,47 @@ def create_scenario_list(scenarios: list) -> list:
         list_scenarios.append(name)
 
     return list_scenarios
+
+
+def dump_database(scenario):
+    """
+    Dump database to a pickle file.
+    :param scenario: scenario dictionary
+    """
+
+    # generate random name
+    name = f"{uuid.uuid4().hex}.pickle"
+    # dump as pickle
+    with open(DIR_CACHED_FILES / name, "wb") as f:
+        pickle.dump(scenario["database"], f)
+    scenario["database filepath"] = DIR_CACHED_FILES / name
+    del scenario["database"]
+
+    return scenario
+
+
+def load_database(scenario):
+    """
+    Load database from a pickle file.
+    :param scenario: scenario dictionary
+    """
+
+    filepath = scenario["database filepath"]
+
+    # load pickle
+    with open(filepath, "rb") as f:
+        scenario["database"] = pickle.load(f)
+    del scenario["database filepath"]
+    # delete the file
+    filepath.unlink()
+
+    return scenario
+
+
+def delete_all_pickles():
+    """
+    Delete all pickle files in the cache folder.
+    """
+
+    for file in DIR_CACHED_FILES.glob("*.pickle"):
+        file.unlink()

{premise-2.1.0.dev3 → premise-2.1.0.dev4/premise.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: premise
-Version: 2.1.0.dev3
+Version: 2.1.0.dev4
 Summary: Coupling IAM output to ecoinvent LCA database ecoinvent for prospective LCA
 Author-email: Romain Sacchi <romain.sacchi@psi.ch>, Alois Dirnaichner <dirnaichner@pik-potsdam.de>,  Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Romain Sacchi <romain.sacchi@psi.ch>