hestia-earth-aggregation 0.21.14__py3-none-any.whl → 0.21.16__py3-none-any.whl

hestia_earth/aggregation/__init__.py

@@ -88,12 +88,14 @@ def aggregate(
         filter_by_country=filter_by_country,
         include_covariance=include_covariance,
     )
-    logger.info("time=%s, unit=ms", current_time_ms() - now)
+    logger.info("Aggregations finished. time=%s, unit=ms", current_time_ms() - now)
+    now = current_time_ms()
     aggregations = (
         [recalculate(agg, product) for agg in aggregations]
         if should_recalculate(product)
         else aggregations
     )
+    logger.info("Recalculations finished. time=%s, unit=ms", current_time_ms() - now)
     aggregations = [
         calculate_score(cycle=agg, countries=countries) for agg in aggregations
     ]

hestia_earth/aggregation/aggregate_cycles.py

@@ -157,7 +157,10 @@ def aggregate_country(
     functional_unit = cycles_aggregated[0].get("functionalUnit")
     include_matrix = aggregate_with_matrix(product)
     cycles_aggregated = non_empty_list(
-        [format_terms_results(cycle, product, country) for cycle in cycles_aggregated]
+        [
+            format_terms_results(cycle, product, country, start_year, end_year)
+            for cycle in cycles_aggregated
+        ]
     )
     cycles_aggregated = non_empty_list(
         map(

hestia_earth/aggregation/config/Cycle/processedFood.json

@@ -29,17 +29,29 @@
     },
     {
       "key": "emissions",
-      "model": "ecoinventV3AndEmberClimate",
+      "model": "worldSteel",
       "value": "cycle",
       "runStrategy": "always",
       "mergeStrategy": "list",
       "mergeArgs": {
         "replaceThreshold": ["value", 0.01]
-      }
+      },
+      "stage": 2
     },
     {
       "key": "emissions",
-      "model": "ecoinventV3",
+      "model": "bafu2025",
+      "value": "cycle",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "mergeArgs": {
+        "replaceThreshold": ["value", 0.01]
+      },
+      "stage": 2
+    },
+    {
+      "key": "emissions",
+      "model": "ecoinventV3AndEmberClimate",
       "value": "cycle",
       "runStrategy": "always",
       "mergeStrategy": "list",

hestia_earth/aggregation/utils/aggregate_country_nodes.py

@@ -61,7 +61,11 @@ from .site import (
     format_site_description,
     site_parent_region_id,
 )
-from .emission import get_method_tier, get_method_model
+from .emission import (
+    get_method_tier,
+    get_method_model,
+    has_value_without_transformation,
+)
 from .property import aggregate_properties
 from .distribution import generate_blank_node_distribution
 from .covariance import add_covariance_cycles, generate_covariance_cycles
@@ -110,6 +114,8 @@ class BlankNodeFormatted(TypedDict, total=False):
     completeness_key: Optional[str]
     start_date: Optional[str]
     end_date: str
+    inputs: List[dict]
+    transformation: List[dict]
 
 
 # --- Cycle
@@ -243,6 +249,8 @@ def _group_cycle_blank_node(cycle: dict, product: dict, product_value: float):
                 "economicValueShare": non_empty_list([_blank_node_evs(blank_nodes)]),
                 # needed for background emissions
                 "inputs": _map_values(blank_nodes, "inputs"),
+                # needed to exclude emissions with only transformation value
+                "transformation": _map_values(blank_nodes, "transformation"),
                 "properties": _map_values(blank_nodes, "properties"),
                 "methodTier": _map_values(blank_nodes, "methodTier"),
                 "methodModel": _map_values(blank_nodes, "methodModel"),
@@ -293,20 +301,16 @@ def _group_cycle_blank_nodes(
     end_year: int = None,
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, Dict[str, BlankNodeFormatted]]:
-        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(cycle, list_key, start_year, end_year)
         values = reduce(
             _group_cycle_blank_node(cycle, product, product_value), blank_nodes, {}
         )
         # after combining all values, need to compute the final statistical values
         group[list_key] = {
-            k: v | _compute_blank_node_stats(v) for k, v in values.items()
+            k: v | _compute_blank_node_stats(v)
+            for k, v in values.items()
+            if list_key != "emissions" or has_value_without_transformation(v)
         }
-        logger.debug(
-            "function=_group_cycle_blank_nodes, list_key=%s, time=%s",
-            list_key,
-            current_time_ms() - now,
-        )
         return group
 
     return grouper
@@ -323,6 +327,7 @@ def _format_cycle(
     Returns the list of formatted Cycles, and the list of Site ids to be downloaded.
     Note: if a Site does not have an `@id`, it means it's nested within the Cycle.
     """
+    now = current_time_ms()
     product_value = _cycle_product_value(cycle, product)
     data: CycleFormatted = (
         (
@@ -355,8 +360,9 @@ def _format_cycle(
     )
     if data:
         logger.debug(
-            "id=%s, yield=%s, organic=%s, irrigated=%s",
+            "id=%s, time=%sms, yield=%s, organic=%s, irrigated=%s",
             cycle.get("@id"),
+            current_time_ms() - now,
             product_value,
             data["organic"],
             data["irrigated"],
@@ -412,7 +418,6 @@ def _combine_cycle_blank_nodes(
     cycles: List[CycleFormatted], completeness: Dict[str, int], cycle_count: int
 ):
     def combine(group: dict, list_key: str):
-        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(cycle.get(list_key, {}).keys()) for cycle in cycles]))
         group[list_key] = reduce(
@@ -420,11 +425,6 @@ def _combine_cycle_blank_nodes(
             keys,
             {},
         )
-        logger.debug(
-            "function=_combine_cycle_blank_nodes, list_key=%s, time=%s",
-            list_key,
-            current_time_ms() - now,
-        )
         return group
 
     return combine
@@ -524,14 +524,8 @@ def _group_site_blank_nodes(
     site: SiteJSONLD, start_year: int = None, end_year: int = None
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, BlankNodeFormatted]:
-        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(site, list_key, start_year, end_year)
         group[list_key] = reduce(_group_site_blank_node(), blank_nodes, {})
-        logger.debug(
-            "function=_group_site_blank_nodes, list_key=%s, time=%s",
-            list_key,
-            current_time_ms() - now,
-        )
         return group
 
     return grouper
@@ -578,15 +572,9 @@ def _combine_site_blank_node(sites: List[SiteFormatted], list_key: str):
 
 def _combine_site_blank_nodes(sites: List[SiteFormatted]):
     def combine(group: dict, list_key: str):
-        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(site.get(list_key, {}).keys()) for site in sites]))
         group[list_key] = reduce(_combine_site_blank_node(sites, list_key), keys, {})
-        logger.debug(
-            "function=_combine_site_blank_nodes, list_key=%s, time=%s",
-            list_key,
-            current_time_ms() - now,
-        )
         return group
 
     return combine
@@ -672,15 +660,9 @@ def _aggregate_formatted(
     data: Union[CycleFormatted, SiteFormatted], aggregated_keys: List[str]
 ):
     def aggregate(key: str):
-        now = current_time_ms()
         values = data.get(key, {}).values()
         logger.debug(f"Aggregating {len(values)} {key}...")
         values = list(map(_aggregate_blank_node, values))
-        logger.debug(
-            "function=_aggregate_formatted, key=%s, time=%s",
-            key,
-            current_time_ms() - now,
-        )
         return values
 
     return reduce(lambda group, key: group | {key: aggregate(key)}, aggregated_keys, {})

hestia_earth/aggregation/utils/aggregate_weighted.py

@@ -6,11 +6,9 @@ from hestia_earth.utils.tools import (
     list_sum,
     list_average,
     is_boolean,
-    current_time_ms,
 )
 from hestia_earth.utils.blank_node import get_node_value
 
-from hestia_earth.aggregation.log import logger
 from . import weighted_average, _min, _max, _sd, format_evs, pick
 from .emission import get_method_tier, get_method_model
 from .property import aggregate_properties
@@ -190,7 +188,6 @@ def _aggregate_nodes(
     aggregate_keys: List[str], data: dict, weights: dict, missing_weights_node_id_func
 ):
     def aggregate_single(key: str):
-        now = current_time_ms()
         aggregates_map: dict = data.get(key)
         terms = aggregates_map.keys()
         values = non_empty_list(
@@ -199,9 +196,6 @@ def _aggregate_nodes(
                 terms,
             )
         )
-        logger.debug(
-            "function=_aggregate_nodes, key=%s, time=%s", key, current_time_ms() - now
-        )
         return values
 
     return reduce(

hestia_earth/aggregation/utils/cycle.py

@@ -41,15 +41,18 @@ from .practice import new_practice, organic_practice
 from .product import new_product
 from .weights import format_weights
 
-_MEAN_DATE_DESCRIPTION = (
-    "Additional notes: the mean endDate of all aggregated Cycles is "
-)
-
 
 def _timestamp():
     return datetime.now().strftime("%Y%m%d")
 
 
+def _mean_date_description(start: int, end: int, average: int):
+    return (
+        f"While the aggregation is for the period {start} - {end}, "
+        + f"the average end date of all aggregated Cycles is {average}."
+    )
+
+
 def _mean_date(dates: list):
     years = [int(d.split("-")[0]) for d in dates]
     return str(round(mean(years))) if years else ""
@@ -58,13 +61,17 @@ def _mean_date(dates: list):
 def _combine_mean_dates(cycles: list):
     # parse the mean date from the description of sub-country nodes
     descriptions = non_empty_list(map(lambda c: c.get("description"), cycles))
-    dates = [v.split(_MEAN_DATE_DESCRIPTION)[-1] for v in descriptions]
+    dates = [v[-5:-1] for v in descriptions]
     return _mean_date(dates)
 
 
 def _aggregated_weights(weights: dict):
-    description = format_weights(weights.values())
-    return ("Sub-aggregation weighting: " + description) if description else ""
+    description = format_weights(weights.values(), include_default_combinations=True)
+    return (
+        f"This aggregation uses the following weights on sub-systems: ({description})"
+        if description
+        else ""
+    )
 
 
 def is_organic(cycle: dict):
@@ -183,7 +190,9 @@ def _format_results(
     return {}
 
 
-def format_terms_results(data: dict, product: dict, country: dict):
+def format_terms_results(
+    data: dict, product: dict, country: dict, start_year: int, end_year: int
+):
     inputs = data.get("inputs", [])
     practices = data.get("practices", [])
     products = data.get("products", [])
@@ -193,8 +202,11 @@ def format_terms_results(data: dict, product: dict, country: dict):
         cycle = _format_results(
             cycle=_create_cycle(
                 {
-                    "description": _MEAN_DATE_DESCRIPTION
-                    + _mean_date(data.get("endDate"))
+                    "description": _mean_date_description(
+                        start=start_year,
+                        end=end_year,
+                        average=_mean_date(data.get("endDate")),
+                    )
                 }
             ),
             product=product,
@@ -259,7 +271,11 @@ def format_country_results(
                     non_empty_list(
                         [
                             _aggregated_weights(weights),
-                            _MEAN_DATE_DESCRIPTION + _combine_mean_dates(cycles),
+                            _mean_date_description(
+                                start=cycle.get("startDate"),
+                                end=cycle.get("endDate"),
+                                average=_combine_mean_dates(cycles),
+                            ),
                         ]
                     )
                 ),

hestia_earth/aggregation/utils/emission.py

@@ -69,3 +69,18 @@ def get_method_model(emissions: list):
     values = non_empty_list(flatten([e.get("methodModel", []) for e in emissions]))
     values = list({v["@id"]: v for v in values}.values())
     return values[0] if len(values) == 1 else None
+
+
+def has_value_without_transformation(blank_node: dict):
+    values = blank_node.get("value", [])
+    transformations = blank_node.get("transformation", [])
+    return (
+        not transformations
+        or len(transformations) != len(values)
+        or any(
+            [
+                value is not None and transformations[index] is None
+                for index, value in enumerate(values)
+            ]
+        )
+    )

hestia_earth/aggregation/utils/quality_score.py

@@ -175,16 +175,19 @@ def _calculate_score_emissions_system_boundary(cycle: dict, *args):
     # get all emissions in the system boundary
     all_emissions_ids = cycle_emissions_in_system_boundary(cycle)
     # ignore emissions that should be skipped in aggregation
-    all_emissions_ids = [
+    skipped_emission_ids = [
         id
         for id in all_emissions_ids
-        if should_aggregate_blank_node(
+        if not should_aggregate_blank_node(
             {
                 "@type": SchemaType.EMISSION.value,
                 "term": {"@id": id, "termType": TermTermType.EMISSION.value},
             }
         )
     ]
+    all_emissions_ids = [
+        id for id in all_emissions_ids if id not in skipped_emission_ids
+    ]
 
     # get all emissions in the Cycle that are in the system boundary
     emissions_ids = list(
@@ -207,6 +210,7 @@ def _calculate_score_emissions_system_boundary(cycle: dict, *args):
         included_emissions=len(emissions_ids),
         all_included=all_included,
         missing_emissions=";".join(missing_emissions),
+        excluded_emissions=";".join(skipped_emission_ids),
     )
 
     return all_included

hestia_earth/aggregation/utils/weights.py

@@ -16,18 +16,40 @@ from .term import (
 )
 
 
+def _weight_to_key(weights: list):
+    return "_".join(non_empty_list(weights))
+
+
 def _format_weight_key(key: str):
     return ", ".join(key.split("_")).capitalize()
 
 
-def format_weights(weights: list):
+_WEIGHTS_KEYS = [["organic", "conventional"], ["irrigated", "non-irrigated"]]
+_WEIGHTS_COMBINATIONS = [
+    _weight_to_key([organic_key, irrigated_key])
+    for organic_key in _WEIGHTS_KEYS[0]
+    for irrigated_key in _WEIGHTS_KEYS[1]
+]
+
+
+def format_weights(weights: list, include_default_combinations: bool = False):
     total = list_sum(non_empty_list([w.get("weight") for w in weights]))
-    return ";".join(
+    included_keys = non_empty_list([weight.get("key") for weight in weights])
+    return "; ".join(
         [
-            f"{_format_weight_key(weight.get('key'))}: {round(weight.get('weight') * 100 / total, 2)}"
+            f"{_format_weight_key(weight.get('key'))}: {round(weight.get('weight') * 100 / total, 2)}%"
             for weight in weights
             if weight.get("key")
         ]
+        + (
+            [
+                f"{_format_weight_key(key)}: 0%"
+                for key in _WEIGHTS_COMBINATIONS
+                if key not in included_keys
+            ]
+            if include_default_combinations and included_keys
+            else []
+        )
     )
 
 
@@ -119,13 +141,11 @@ def _country_weights(
     weight = (organic_weight if node.get("organic", False) else 1 - organic_weight) * (
         irrigated_weight if node.get("irrigated", False) else 1 - irrigated_weight
     )
-    key = "_".join(
-        non_empty_list(
-            [
-                "organic" if node.get("organic", False) else "conventional",
-                "irrigated" if node.get("irrigated", False) else "non-irrigated",
-            ]
-        )
+    key = _weight_to_key(
+        [
+            "organic" if node.get("organic", False) else "conventional",
+            "irrigated" if node.get("irrigated", False) else "non-irrigated",
+        ]
     )
     return {node_id: {"weight": weight, "completeness": completeness, "key": key}}
 

hestia_earth/aggregation/version.py

@@ -1 +1 @@
-VERSION = "0.21.14"
+VERSION = "0.21.16"

{hestia_earth_aggregation-0.21.14.dist-info → hestia_earth_aggregation-0.21.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_aggregation
-Version: 0.21.14
+Version: 0.21.16
 Summary: HESTIA's aggregation engine.
 Home-page: https://gitlab.com/hestia-earth/hestia-aggregation-engine
 Author: HESTIA Team

{hestia_earth_aggregation-0.21.14.dist-info → hestia_earth_aggregation-0.21.16.dist-info}/RECORD

@@ -1,19 +1,19 @@
-hestia_earth/aggregation/__init__.py,sha256=5r0icTYP5F4DFIX-Xm1Ms3inpAUfpXQwOmO2y_DV3zs,3202
-hestia_earth/aggregation/aggregate_cycles.py,sha256=9BRX41d0uLhKLUPBlALxAoRMfPriDj3Dhy0sTgmTS5Y,7812
+hestia_earth/aggregation/__init__.py,sha256=kSK6RYcK0YnRGx6TknaukhuhbnxRHofLI_8PA0-k8Yg,3339
+hestia_earth/aggregation/aggregate_cycles.py,sha256=MR-SseQRMGFUOKiBJcLVysjdjw5J-QfJ258VxGhasFw,7868
 hestia_earth/aggregation/log.py,sha256=0CibXDtKqFfE-R86qziM2uH7ATUZ6I9p_bVpRSn_e_o,2351
 hestia_earth/aggregation/recalculate_cycles.py,sha256=xCO_tj5BAC407vZ36EVuxeOBmX2y82EXRrTx5W6g4CY,484
-hestia_earth/aggregation/version.py,sha256=zqCwaEmEyCPuIxRLzAP6Ds1U0koeoYfXnJ0e800-MWo,20
-hestia_earth/aggregation/config/Cycle/processedFood.json,sha256=JWelgL-ZLnurS9R8Y-HOXjiWZ5k3kAuUQHDoMcukEq8,1127
+hestia_earth/aggregation/version.py,sha256=Mlni8p7cuSE5CwxrazB-rArGWlKBFeBsDeVs2L8bT_E,20
+hestia_earth/aggregation/config/Cycle/processedFood.json,sha256=OBYXxleifxg9EJfZsqR1dgCxlEe2lIrCBqMe_Jo5XE4,1387
 hestia_earth/aggregation/utils/__init__.py,sha256=nVVLgVFZ4415CgQAqgv4jDamA7gDmzOWk5SY4vshAQs,6583
-hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=c2rosOtI7B8vwb2CbDye-YPx4xyPc-o4kKKPsyAVoyU,28456
-hestia_earth/aggregation/utils/aggregate_weighted.py,sha256=CUTqR8z3_MRnGhzAOiFswy68Twe1GaNCyrSd5x7qf_Q,7037
+hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=FkfiOcWIE9a8duowZgOtFiyr4m-UwnawXRiZ-geRrTY,27924
+hestia_earth/aggregation/utils/aggregate_weighted.py,sha256=OzrYhbL7g-X_1x7BLEwkxqLlDOLQ7CdFV8UuQTzQEz4,6817
 hestia_earth/aggregation/utils/blank_node.py,sha256=f8JBcd5_SLqW8juzFAfbZysAESaZXj8JX6C7rDkLEoc,16812
 hestia_earth/aggregation/utils/combine.py,sha256=EmXUZksdkrtQ6o1gMOG5WMivUhL28NKSM9D4cPohJiE,4904
 hestia_earth/aggregation/utils/completeness.py,sha256=eZ759PAdOSshhys-YKKPEpAYxR3sTrEjI--WJTEwmus,4633
 hestia_earth/aggregation/utils/covariance.py,sha256=2VaNGPvyD4ub3rR-OBn8VIEIeAz4JOefl6ZlAtXKj4U,5884
-hestia_earth/aggregation/utils/cycle.py,sha256=gzU2KofOVGBZvOPTymJGpsbbouABjSzUM35OJ42sFl8,15954
+hestia_earth/aggregation/utils/cycle.py,sha256=d65RvgqJiq4sr12SShLKjM50MO95V07I88LU7p__Ixc,16496
 hestia_earth/aggregation/utils/distribution.py,sha256=2XQKXVu-1XUy1zEHgThERupaj4RizXKO5F-VY-QQlMo,6935
-hestia_earth/aggregation/utils/emission.py,sha256=6MpDdf_izru0PGmkr2y9gOf9x_zfafyvSJSWALsseos,2554
+hestia_earth/aggregation/utils/emission.py,sha256=2roXjGmDuS7a3muPJoEOqg7E-TZyQsnoavhU6b3B-Z8,2983
 hestia_earth/aggregation/utils/group.py,sha256=qU9sAuJu0RG80yLs90nsyoOcU57Pb6M1WKwT1TlJEkM,4765
 hestia_earth/aggregation/utils/input.py,sha256=pZ1bcdHR3y4S-3b0JZllbEUWdEyVZbltxv-07j7SucI,1092
 hestia_earth/aggregation/utils/lookup.py,sha256=osJiXTRns5KeBMz3mRsanmFQTATrwcMVBzB07xrb71M,2046
@@ -22,14 +22,14 @@ hestia_earth/aggregation/utils/measurement.py,sha256=tdQwjo1hPb3iAaboBQiua832q27
 hestia_earth/aggregation/utils/practice.py,sha256=MXZsQoeSXxe2vmdjvyS58HZN9jejPZNbVnczTc3HxxU,2252
 hestia_earth/aggregation/utils/product.py,sha256=dEUCy0eakyKyD2vu1RRnF4bdAI-jpS-qePNF8ZGhHFE,886
 hestia_earth/aggregation/utils/property.py,sha256=n_aHZ4qASv_wcDJ_HrA1T2kFX8R2zUEo91LCPL5xWkw,917
-hestia_earth/aggregation/utils/quality_score.py,sha256=Jfxbdj9_UgZUB0w9RbiMCX0soVqqe7okrHw1f2vwLUw,8743
+hestia_earth/aggregation/utils/quality_score.py,sha256=PPNRW1wEkVqP7QExyu5dcr8R_Z_DWkmT3zmQG9u470Y,8914
 hestia_earth/aggregation/utils/queries.py,sha256=kzob6CHL1a60CRJdGICiWS98LKJRyq3uaToyuQyl9RI,17564
 hestia_earth/aggregation/utils/site.py,sha256=nc1N4IRiTki69IbCX4G6lagQ5U7IpsOsqiPuHwbl1HM,5455
 hestia_earth/aggregation/utils/source.py,sha256=SOiE-jB5WNtbRxreUWQ8c04ywzrwICy1aGyRL3-L0RY,686
 hestia_earth/aggregation/utils/term.py,sha256=uVAfBYnxNkcTNfzuwwHur6JGBahNrPwR2pN-__sj9zk,2758
-hestia_earth/aggregation/utils/weights.py,sha256=3WO2_26bj1RB2h524Gknu0btvTFJucqMtn-tag339aY,6843
-hestia_earth_aggregation-0.21.14.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
-hestia_earth_aggregation-0.21.14.dist-info/METADATA,sha256=DId0r59jDje3k-jfnNbBa1G9mcBRvi2rAbEjyvZQGnU,2481
-hestia_earth_aggregation-0.21.14.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-hestia_earth_aggregation-0.21.14.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
-hestia_earth_aggregation-0.21.14.dist-info/RECORD,,
+hestia_earth/aggregation/utils/weights.py,sha256=Qk5hCr_K24DzkeE4Sa241pGb6m00iaWHIXFQAJ-cDGc,7518
+hestia_earth_aggregation-0.21.16.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
+hestia_earth_aggregation-0.21.16.dist-info/METADATA,sha256=38XSLUkdQu3ePpK38m0b_ZjiLPjWqANxTFmEk9dshCs,2481
+hestia_earth_aggregation-0.21.16.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+hestia_earth_aggregation-0.21.16.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
+hestia_earth_aggregation-0.21.16.dist-info/RECORD,,