hestia-earth-aggregation 0.21.16__py3-none-any.whl → 0.21.18__py3-none-any.whl

hestia_earth/aggregation/config/Cycle/crop.json

@@ -0,0 +1,63 @@
+{
+  "models": [
+    {
+      "key": "inputs",
+      "model": "cycle",
+      "value": "input.hestiaAggregatedData",
+      "runStrategy": "always",
+      "mergeStrategy": "list"
+    },
+    {
+      "key": "emissions",
+      "model": "linkedImpactAssessment",
+      "value": "emissions",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "mergeArgs": {
+        "replaceThreshold": ["value", 0.01]
+      }
+    },
+    {
+      "key": "emissions",
+      "model": "ecoalimV9",
+      "value": "cycle",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "mergeArgs": {
+        "replaceThreshold": ["value", 0.01]
+      }
+    },
+    {
+      "key": "emissions",
+      "model": "worldSteel",
+      "value": "cycle",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "mergeArgs": {
+        "replaceThreshold": ["value", 0.01]
+      },
+      "stage": 2
+    },
+    {
+      "key": "emissions",
+      "model": "bafu2025",
+      "value": "cycle",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "mergeArgs": {
+        "replaceThreshold": ["value", 0.01]
+      },
+      "stage": 2
+    },
+    {
+      "key": "emissions",
+      "model": "ecoinventV3AndEmberClimate",
+      "value": "cycle",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "mergeArgs": {
+        "replaceThreshold": ["value", 0.01]
+      }
+    }
+  ]
+}

hestia_earth/aggregation/recalculate_cycles.py

@@ -4,10 +4,11 @@ from hestia_earth.orchestrator import run
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 CONFIG_PATH = os.path.join(CURRENT_DIR, "config", "Cycle")
+_TERM_TYPES = os.listdir(CONFIG_PATH)
 
 
 def should_recalculate(product: dict):
-    return f"{product.get('termType')}.json" in os.listdir(CONFIG_PATH)
+    return f"{product.get('termType')}.json" in _TERM_TYPES
 
 
 def recalculate(cycle: dict, product: dict):

hestia_earth/aggregation/utils/aggregate_country_nodes.py

@@ -65,6 +65,7 @@ from .emission import (
     get_method_tier,
     get_method_model,
     has_value_without_transformation,
+    include_emission,
 )
 from .property import aggregate_properties
 from .distribution import generate_blank_node_distribution
@@ -205,6 +206,10 @@ def _group_cycle_blank_node(cycle: dict, product: dict, product_value: float):
         term_id = term.get("@id")
         group_key = blank_node_group_key(blank_node)
 
+        # skip background emissions when recalculated
+        if node_type == "Emission" and not include_emission(blank_node, product):
+            return group
+
         is_aggregated_product = node_type == "Product" and term_id == product.get("@id")
 
         site_type = cycle.get("site", {}).get("siteType")
@@ -223,9 +228,9 @@ def _group_cycle_blank_node(cycle: dict, product: dict, product_value: float):
             is_aggregated_product=is_aggregated_product,
         )
 
-        # only primary data can be incomplete, otherwise skip the blank node
         if all(
             [
+                # only primary data can be incomplete, otherwise skip the blank node
                 complete is False,
                 not blank_node_data.get("primary"),
                 blank_node_data.get("primaryPercent", 0) <= 0,

hestia_earth/aggregation/utils/cycle.py

@@ -68,7 +68,7 @@ def _combine_mean_dates(cycles: list):
 def _aggregated_weights(weights: dict):
     description = format_weights(weights.values(), include_default_combinations=True)
     return (
-        f"This aggregation uses the following weights on sub-systems: ({description})"
+        f"This aggregation uses the following weights on sub-systems: {description}"
         if description
         else ""
     )
@@ -177,9 +177,7 @@ def _format_results(
         "products": non_empty_list(map(_format_aggregate(new_product), products)),
     }
     # aggregate emissions after as it needs inputs and products
-    cycle["emissions"] = non_empty_list(
-        map(_format_aggregate(new_emission(product)), emissions)
-    )
+    cycle["emissions"] = non_empty_list(map(_format_aggregate(new_emission), emissions))
 
     # set the primary product
     primary_product = find_term_match(cycle.get("products", []), product.get("@id"))

hestia_earth/aggregation/utils/emission.py

@@ -2,62 +2,58 @@ from hestia_earth.schema import (
     EmissionMethodTier,
     SchemaType,
     EmissionStatsDefinition,
-    TermTermType,
 )
 from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.lookup_utils import is_in_system_boundary
 from hestia_earth.utils.tools import flatten, non_empty_list
 
+from hestia_earth.aggregation.recalculate_cycles import should_recalculate
 from . import _unique_nodes, _set_dict_single
 from .term import METHOD_MODEL
 
 _DEFAULT_TIER = EmissionMethodTier.TIER_1.value
-_SKIP_BACKGROUND_EMISSIONS = [TermTermType.PROCESSEDFOOD.value]
 
 
-def _include_emission(emission: dict, product: dict):
+def include_emission(emission: dict, product: dict):
     return any(
         [
             emission.get("methodTier") != EmissionMethodTier.BACKGROUND.value,
-            product.get("termType") not in _SKIP_BACKGROUND_EMISSIONS,
+            not should_recalculate(product),
         ]
     )
 
 
-def new_emission(product: dict):
-    def emission(data: dict):
-        term = data.get("term", {})
-        # only add emissions included in the System Boundary
-        if is_in_system_boundary(term.get("@id")):
-            node = {"@type": SchemaType.EMISSION.value}
-            node["term"] = linked_node(term)
-            value = data.get("value")
-            if value is not None:
-                node["value"] = [value]
-                node["statsDefinition"] = EmissionStatsDefinition.CYCLES.value
-
-            node["methodTier"] = data.get("methodTier") or _DEFAULT_TIER
-            node["methodModel"] = data.get("methodModel") or METHOD_MODEL
-
-            inputs = data.get("inputs", [])
-            # compute list of unique inputs, required for `background` emissions
-            if inputs:
-                _set_dict_single(
-                    node,
-                    "inputs",
-                    list(map(linked_node, _unique_nodes(inputs))),
-                    strict=True,
-                )
-
-            if node.get("methodTier") != EmissionMethodTier.NOT_RELEVANT.value:
-                _set_dict_single(
-                    node, "distribution", data.get("distribution"), strict=True
-                )
-
-            _set_dict_single(node, "description", data.get("description"), strict=True)
-            return node if _include_emission(node, product) else None
-
-    return emission
+def new_emission(data: dict):
+    term = data.get("term", {})
+    # only add emissions included in the System Boundary
+    if is_in_system_boundary(term.get("@id")):
+        node = {"@type": SchemaType.EMISSION.value}
+        node["term"] = linked_node(term)
+        value = data.get("value")
+        if value is not None:
+            node["value"] = [value]
+            node["statsDefinition"] = EmissionStatsDefinition.CYCLES.value
+
+        node["methodTier"] = data.get("methodTier") or _DEFAULT_TIER
+        node["methodModel"] = data.get("methodModel") or METHOD_MODEL
+
+        inputs = data.get("inputs", [])
+        # compute list of unique inputs, required for `background` emissions
+        if inputs:
+            _set_dict_single(
+                node,
+                "inputs",
+                list(map(linked_node, _unique_nodes(inputs))),
+                strict=True,
+            )
+
+        if node.get("methodTier") != EmissionMethodTier.NOT_RELEVANT.value:
+            _set_dict_single(
+                node, "distribution", data.get("distribution"), strict=True
+            )
+
+        _set_dict_single(node, "description", data.get("description"), strict=True)
+        return node
 
 
 def get_method_tier(emissions: list):

hestia_earth/aggregation/utils/management.py

@@ -1,6 +1,7 @@
 from hestia_earth.schema import ManagementJSONLD
 from hestia_earth.utils.model import linked_node
-from hestia_earth.utils.tools import safe_parse_date
+from hestia_earth.utils.tools import safe_parse_date, flatten
+from hestia_earth.utils.date import gapfill_datestr
 
 from . import match_dates, _set_dict_single
 from .queries import TIME_PERIOD, _get_time_ranges, _current_date
@@ -45,6 +46,34 @@ def aggregated_dates(blank_node: dict):
     )
 
 
+def split_by_dates(blank_nodes: list, start_year: int):
+    # if the blank_node overlaps with the start period of the aggregation, we need to split it in 2 parts
+    def split_data(blank_node: dict):
+        start_date = safe_parse_date(blank_node.get("startDate"), default=None)
+        end_date = safe_parse_date(blank_node.get("endDate"), default=None)
+        is_overlapping = all([start_date is not None, end_date is not None]) and all(
+            [start_date.year < start_year, start_year < end_date.year]
+        )
+        return (
+            [
+                blank_node
+                | {
+                    "startDate": blank_node.get("startDate"),
+                    "endDate": gapfill_datestr(start_year - 1, "end"),
+                },
+                blank_node
+                | {
+                    "startDate": gapfill_datestr(start_year, "start"),
+                    "endDate": blank_node.get("endDate"),
+                },
+            ]
+            if is_overlapping
+            else blank_node
+        )
+
+    return flatten(map(split_data, blank_nodes))
+
+
 def filter_management(blank_nodes: list, start_year: int = None, end_year: int = None):
     def update_dates(blank_node: dict):
         start_date = safe_parse_date(blank_node.get("startDate"), default=None)
@@ -72,7 +101,9 @@ def filter_management(blank_nodes: list, start_year: int = None, end_year: int =
     """
     return [
         update_dates(v)
-        for v in blank_nodes
+        for v in (
+            split_by_dates(blank_nodes, start_year) if start_year else blank_nodes
+        )
         if not start_year
         or not end_year
         or match_dates(v, start_year - TIME_PERIOD, end_year)

hestia_earth/aggregation/utils/queries.py

@@ -15,7 +15,7 @@ from hestia_earth.aggregation.log import logger
 from . import HestiaError, _save_json, pick
 from .term import DEFAULT_COUNTRY_NAME, _fetch_countries, is_global as is_global_country
 
-SEARCH_LIMIT = 10000
+SEARCH_LIMIT = int(os.getenv("SEARCH_LIMIT", "10000"))
 _MATCH_AGGREGATED_QUERY = {"match": {"aggregated": "true"}}
 _MATCH_AGGREGATED_VALIDATED_QUERY = {"match": {"aggregatedDataValidated": "true"}}
 # enable matching any product

hestia_earth/aggregation/version.py

@@ -1 +1 @@
-VERSION = "0.21.16"
+VERSION = "0.21.18"

{hestia_earth_aggregation-0.21.16.dist-info → hestia_earth_aggregation-0.21.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_aggregation
-Version: 0.21.16
+Version: 0.21.18
 Summary: HESTIA's aggregation engine.
 Home-page: https://gitlab.com/hestia-earth/hestia-aggregation-engine
 Author: HESTIA Team

{hestia_earth_aggregation-0.21.16.dist-info → hestia_earth_aggregation-0.21.18.dist-info}/RECORD

@@ -1,35 +1,36 @@
 hestia_earth/aggregation/__init__.py,sha256=kSK6RYcK0YnRGx6TknaukhuhbnxRHofLI_8PA0-k8Yg,3339
 hestia_earth/aggregation/aggregate_cycles.py,sha256=MR-SseQRMGFUOKiBJcLVysjdjw5J-QfJ258VxGhasFw,7868
 hestia_earth/aggregation/log.py,sha256=0CibXDtKqFfE-R86qziM2uH7ATUZ6I9p_bVpRSn_e_o,2351
-hestia_earth/aggregation/recalculate_cycles.py,sha256=xCO_tj5BAC407vZ36EVuxeOBmX2y82EXRrTx5W6g4CY,484
-hestia_earth/aggregation/version.py,sha256=Mlni8p7cuSE5CwxrazB-rArGWlKBFeBsDeVs2L8bT_E,20
+hestia_earth/aggregation/recalculate_cycles.py,sha256=igMxKd7_K0MFlAt3Q_OynvHxDo3ldTixWbXjaC4M71c,510
+hestia_earth/aggregation/version.py,sha256=l0WM7ge0CXaUMWDGZjVN-Bv5P4kpSeFzwkayyG9kADw,20
+hestia_earth/aggregation/config/Cycle/crop.json,sha256=OBYXxleifxg9EJfZsqR1dgCxlEe2lIrCBqMe_Jo5XE4,1387
 hestia_earth/aggregation/config/Cycle/processedFood.json,sha256=OBYXxleifxg9EJfZsqR1dgCxlEe2lIrCBqMe_Jo5XE4,1387
 hestia_earth/aggregation/utils/__init__.py,sha256=nVVLgVFZ4415CgQAqgv4jDamA7gDmzOWk5SY4vshAQs,6583
-hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=FkfiOcWIE9a8duowZgOtFiyr4m-UwnawXRiZ-geRrTY,27924
+hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=RIJYaWNNdnlCif6EgF-mG_HsfwuchSOHEgvE8eYl-NU,28116
 hestia_earth/aggregation/utils/aggregate_weighted.py,sha256=OzrYhbL7g-X_1x7BLEwkxqLlDOLQ7CdFV8UuQTzQEz4,6817
 hestia_earth/aggregation/utils/blank_node.py,sha256=f8JBcd5_SLqW8juzFAfbZysAESaZXj8JX6C7rDkLEoc,16812
 hestia_earth/aggregation/utils/combine.py,sha256=EmXUZksdkrtQ6o1gMOG5WMivUhL28NKSM9D4cPohJiE,4904
 hestia_earth/aggregation/utils/completeness.py,sha256=eZ759PAdOSshhys-YKKPEpAYxR3sTrEjI--WJTEwmus,4633
 hestia_earth/aggregation/utils/covariance.py,sha256=2VaNGPvyD4ub3rR-OBn8VIEIeAz4JOefl6ZlAtXKj4U,5884
-hestia_earth/aggregation/utils/cycle.py,sha256=d65RvgqJiq4sr12SShLKjM50MO95V07I88LU7p__Ixc,16496
+hestia_earth/aggregation/utils/cycle.py,sha256=ohFc2rtzi7cJZatBll8JFoAFK0PB7CzVtccnyvzkfsU,16471
 hestia_earth/aggregation/utils/distribution.py,sha256=2XQKXVu-1XUy1zEHgThERupaj4RizXKO5F-VY-QQlMo,6935
-hestia_earth/aggregation/utils/emission.py,sha256=2roXjGmDuS7a3muPJoEOqg7E-TZyQsnoavhU6b3B-Z8,2983
+hestia_earth/aggregation/utils/emission.py,sha256=fmzi2GHhG1E5Ioi00D1vVIPZwqaDB7P4-NxGwX64KyI,2745
 hestia_earth/aggregation/utils/group.py,sha256=qU9sAuJu0RG80yLs90nsyoOcU57Pb6M1WKwT1TlJEkM,4765
 hestia_earth/aggregation/utils/input.py,sha256=pZ1bcdHR3y4S-3b0JZllbEUWdEyVZbltxv-07j7SucI,1092
 hestia_earth/aggregation/utils/lookup.py,sha256=osJiXTRns5KeBMz3mRsanmFQTATrwcMVBzB07xrb71M,2046
-hestia_earth/aggregation/utils/management.py,sha256=wrgNHhWuDTWmpRdKNi9BKg7GQRL8CJVAjcW1aeRaiyM,2891
+hestia_earth/aggregation/utils/management.py,sha256=tr46zD8HrmRjEQnhMJ5s9eYapd8CKs4Gr99aZGLo1kY,4117
 hestia_earth/aggregation/utils/measurement.py,sha256=tdQwjo1hPb3iAaboBQiua832q27xZb_a9KqIp3iQkuE,1463
 hestia_earth/aggregation/utils/practice.py,sha256=MXZsQoeSXxe2vmdjvyS58HZN9jejPZNbVnczTc3HxxU,2252
 hestia_earth/aggregation/utils/product.py,sha256=dEUCy0eakyKyD2vu1RRnF4bdAI-jpS-qePNF8ZGhHFE,886
 hestia_earth/aggregation/utils/property.py,sha256=n_aHZ4qASv_wcDJ_HrA1T2kFX8R2zUEo91LCPL5xWkw,917
 hestia_earth/aggregation/utils/quality_score.py,sha256=PPNRW1wEkVqP7QExyu5dcr8R_Z_DWkmT3zmQG9u470Y,8914
-hestia_earth/aggregation/utils/queries.py,sha256=kzob6CHL1a60CRJdGICiWS98LKJRyq3uaToyuQyl9RI,17564
+hestia_earth/aggregation/utils/queries.py,sha256=S9JZ3km_7EkbPDV0piQxfXMDF0qa2eXKdN0mz7oIkf8,17598
 hestia_earth/aggregation/utils/site.py,sha256=nc1N4IRiTki69IbCX4G6lagQ5U7IpsOsqiPuHwbl1HM,5455
 hestia_earth/aggregation/utils/source.py,sha256=SOiE-jB5WNtbRxreUWQ8c04ywzrwICy1aGyRL3-L0RY,686
 hestia_earth/aggregation/utils/term.py,sha256=uVAfBYnxNkcTNfzuwwHur6JGBahNrPwR2pN-__sj9zk,2758
 hestia_earth/aggregation/utils/weights.py,sha256=Qk5hCr_K24DzkeE4Sa241pGb6m00iaWHIXFQAJ-cDGc,7518
-hestia_earth_aggregation-0.21.16.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
-hestia_earth_aggregation-0.21.16.dist-info/METADATA,sha256=38XSLUkdQu3ePpK38m0b_ZjiLPjWqANxTFmEk9dshCs,2481
-hestia_earth_aggregation-0.21.16.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-hestia_earth_aggregation-0.21.16.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
-hestia_earth_aggregation-0.21.16.dist-info/RECORD,,
+hestia_earth_aggregation-0.21.18.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
+hestia_earth_aggregation-0.21.18.dist-info/METADATA,sha256=1WmgTNGYc4EQFS6ltVH_UQkThgZAm1gM2z84cuCiDLk,2481
+hestia_earth_aggregation-0.21.18.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+hestia_earth_aggregation-0.21.18.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
+hestia_earth_aggregation-0.21.18.dist-info/RECORD,,