hestia-earth-aggregation 0.21.13__py3-none-any.whl → 0.21.15__py3-none-any.whl

hestia_earth/aggregation/utils/aggregate_country_nodes.py

@@ -10,6 +10,7 @@ from hestia_earth.utils.tools import (
     flatten,
     is_number,
     is_boolean,
+    current_time_ms,
 )
 from hestia_earth.utils.blank_node import (
     ArrayTreatment,
@@ -60,7 +61,11 @@ from .site import (
     format_site_description,
     site_parent_region_id,
 )
-from .emission import get_method_tier, get_method_model
+from .emission import (
+    get_method_tier,
+    get_method_model,
+    has_value_without_transformation,
+)
 from .property import aggregate_properties
 from .distribution import generate_blank_node_distribution
 from .covariance import add_covariance_cycles, generate_covariance_cycles
@@ -109,6 +114,8 @@ class BlankNodeFormatted(TypedDict, total=False):
     completeness_key: Optional[str]
     start_date: Optional[str]
     end_date: str
+    inputs: List[dict]
+    transformation: List[dict]
 
 
 # --- Cycle
@@ -242,6 +249,8 @@ def _group_cycle_blank_node(cycle: dict, product: dict, product_value: float):
                 "economicValueShare": non_empty_list([_blank_node_evs(blank_nodes)]),
                 # needed for background emissions
                 "inputs": _map_values(blank_nodes, "inputs"),
+                # needed to exclude emissions with only transformation value
+                "transformation": _map_values(blank_nodes, "transformation"),
                 "properties": _map_values(blank_nodes, "properties"),
                 "methodTier": _map_values(blank_nodes, "methodTier"),
                 "methodModel": _map_values(blank_nodes, "methodModel"),
@@ -292,14 +301,22 @@ def _group_cycle_blank_nodes(
     end_year: int = None,
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, Dict[str, BlankNodeFormatted]]:
+        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(cycle, list_key, start_year, end_year)
         values = reduce(
             _group_cycle_blank_node(cycle, product, product_value), blank_nodes, {}
         )
         # after combining all values, need to compute the final statistical values
         group[list_key] = {
-            k: v | _compute_blank_node_stats(v) for k, v in values.items()
+            k: v | _compute_blank_node_stats(v)
+            for k, v in values.items()
+            if list_key != "emissions" or has_value_without_transformation(v)
         }
+        logger.debug(
+            "function=_group_cycle_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return grouper
@@ -405,6 +422,7 @@ def _combine_cycle_blank_nodes(
     cycles: List[CycleFormatted], completeness: Dict[str, int], cycle_count: int
 ):
     def combine(group: dict, list_key: str):
+        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(cycle.get(list_key, {}).keys()) for cycle in cycles]))
         group[list_key] = reduce(
@@ -412,6 +430,11 @@ def _combine_cycle_blank_nodes(
             keys,
             {},
         )
+        logger.debug(
+            "function=_combine_cycle_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return combine
@@ -511,8 +534,14 @@ def _group_site_blank_nodes(
     site: SiteJSONLD, start_year: int = None, end_year: int = None
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, BlankNodeFormatted]:
+        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(site, list_key, start_year, end_year)
         group[list_key] = reduce(_group_site_blank_node(), blank_nodes, {})
+        logger.debug(
+            "function=_group_site_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return grouper
@@ -559,9 +588,15 @@ def _combine_site_blank_node(sites: List[SiteFormatted], list_key: str):
 
 def _combine_site_blank_nodes(sites: List[SiteFormatted]):
     def combine(group: dict, list_key: str):
+        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(site.get(list_key, {}).keys()) for site in sites]))
         group[list_key] = reduce(_combine_site_blank_node(sites, list_key), keys, {})
+        logger.debug(
+            "function=_combine_site_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return combine
@@ -647,9 +682,16 @@ def _aggregate_formatted(
     data: Union[CycleFormatted, SiteFormatted], aggregated_keys: List[str]
 ):
     def aggregate(key: str):
+        now = current_time_ms()
         values = data.get(key, {}).values()
         logger.debug(f"Aggregating {len(values)} {key}...")
-        return list(map(_aggregate_blank_node, values))
+        values = list(map(_aggregate_blank_node, values))
+        logger.debug(
+            "function=_aggregate_formatted, key=%s, time=%s",
+            key,
+            current_time_ms() - now,
+        )
+        return values
 
     return reduce(lambda group, key: group | {key: aggregate(key)}, aggregated_keys, {})
 

hestia_earth/aggregation/utils/aggregate_weighted.py

@@ -6,9 +6,11 @@ from hestia_earth.utils.tools import (
     list_sum,
     list_average,
     is_boolean,
+    current_time_ms,
 )
 from hestia_earth.utils.blank_node import get_node_value
 
+from hestia_earth.aggregation.log import logger
 from . import weighted_average, _min, _max, _sd, format_evs, pick
 from .emission import get_method_tier, get_method_model
 from .property import aggregate_properties
@@ -188,14 +190,19 @@ def _aggregate_nodes(
     aggregate_keys: List[str], data: dict, weights: dict, missing_weights_node_id_func
 ):
     def aggregate_single(key: str):
+        now = current_time_ms()
         aggregates_map: dict = data.get(key)
         terms = aggregates_map.keys()
-        return non_empty_list(
+        values = non_empty_list(
             map(
                 _aggregate_term(aggregates_map, weights, missing_weights_node_id_func),
                 terms,
             )
         )
+        logger.debug(
+            "function=_aggregate_nodes, key=%s, time=%s", key, current_time_ms() - now
+        )
+        return values
 
     return reduce(
         lambda prev, curr: prev | {curr: aggregate_single(curr)}, aggregate_keys, {}

hestia_earth/aggregation/utils/completeness.py

@@ -9,7 +9,6 @@ from hestia_earth.schema import (
 )
 from hestia_earth.utils.tools import non_empty_list
 
-
 _ANIMAL_FEED_INPUT_MAPPING = {
     SchemaType.INPUT.value: {
         TermTermType.ANIMALPRODUCT.value: CompletenessField.ANIMALFEED.value,

hestia_earth/aggregation/utils/emission.py

@@ -69,3 +69,18 @@ def get_method_model(emissions: list):
     values = non_empty_list(flatten([e.get("methodModel", []) for e in emissions]))
     values = list({v["@id"]: v for v in values}.values())
     return values[0] if len(values) == 1 else None
+
+
+def has_value_without_transformation(blank_node: dict):
+    values = blank_node.get("value", [])
+    transformations = blank_node.get("transformation", [])
+    return (
+        not transformations
+        or len(transformations) != len(values)
+        or any(
+            [
+                value is not None and transformations[index] is None
+                for index, value in enumerate(values)
+            ]
+        )
+    )

hestia_earth/aggregation/version.py

@@ -1 +1 @@
-VERSION = "0.21.13"
+VERSION = "0.21.15"

{hestia_earth_aggregation-0.21.13.dist-info → hestia_earth_aggregation-0.21.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_aggregation
-Version: 0.21.13
+Version: 0.21.15
 Summary: HESTIA's aggregation engine.
 Home-page: https://gitlab.com/hestia-earth/hestia-aggregation-engine
 Author: HESTIA Team

{hestia_earth_aggregation-0.21.13.dist-info → hestia_earth_aggregation-0.21.15.dist-info}/RECORD

@@ -2,18 +2,18 @@ hestia_earth/aggregation/__init__.py,sha256=5r0icTYP5F4DFIX-Xm1Ms3inpAUfpXQwOmO2
 hestia_earth/aggregation/aggregate_cycles.py,sha256=9BRX41d0uLhKLUPBlALxAoRMfPriDj3Dhy0sTgmTS5Y,7812
 hestia_earth/aggregation/log.py,sha256=0CibXDtKqFfE-R86qziM2uH7ATUZ6I9p_bVpRSn_e_o,2351
 hestia_earth/aggregation/recalculate_cycles.py,sha256=xCO_tj5BAC407vZ36EVuxeOBmX2y82EXRrTx5W6g4CY,484
-hestia_earth/aggregation/version.py,sha256=IZwzxHxCNgUUuwZLTtt70K8Col8ceGMlViNLT2Mpr3M,20
+hestia_earth/aggregation/version.py,sha256=Z3Ng7JCS2UZEW-r2srjuHslYzJkIywa_nSkx0uI3j84,20
 hestia_earth/aggregation/config/Cycle/processedFood.json,sha256=JWelgL-ZLnurS9R8Y-HOXjiWZ5k3kAuUQHDoMcukEq8,1127
 hestia_earth/aggregation/utils/__init__.py,sha256=nVVLgVFZ4415CgQAqgv4jDamA7gDmzOWk5SY4vshAQs,6583
-hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=LXk2jQQVUSX_5Ut6NxJG12jd3ZxW7pLiwH3fYMwhOcs,27453
-hestia_earth/aggregation/utils/aggregate_weighted.py,sha256=OeNylBN6RPbZtq4zmf5j8YNywci7Pd4MLIMYaaLmPm0,6793
+hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=U39M2clD9_6FE0yFYvamP7EI8FrCQ350pzD0y_9Rw_Y,28806
+hestia_earth/aggregation/utils/aggregate_weighted.py,sha256=CUTqR8z3_MRnGhzAOiFswy68Twe1GaNCyrSd5x7qf_Q,7037
 hestia_earth/aggregation/utils/blank_node.py,sha256=f8JBcd5_SLqW8juzFAfbZysAESaZXj8JX6C7rDkLEoc,16812
 hestia_earth/aggregation/utils/combine.py,sha256=EmXUZksdkrtQ6o1gMOG5WMivUhL28NKSM9D4cPohJiE,4904
-hestia_earth/aggregation/utils/completeness.py,sha256=vBwT25r7d_CsAicjwII4LvhUFykNnYDzYlbpuxhq5AQ,4634
+hestia_earth/aggregation/utils/completeness.py,sha256=eZ759PAdOSshhys-YKKPEpAYxR3sTrEjI--WJTEwmus,4633
 hestia_earth/aggregation/utils/covariance.py,sha256=2VaNGPvyD4ub3rR-OBn8VIEIeAz4JOefl6ZlAtXKj4U,5884
 hestia_earth/aggregation/utils/cycle.py,sha256=gzU2KofOVGBZvOPTymJGpsbbouABjSzUM35OJ42sFl8,15954
 hestia_earth/aggregation/utils/distribution.py,sha256=2XQKXVu-1XUy1zEHgThERupaj4RizXKO5F-VY-QQlMo,6935
-hestia_earth/aggregation/utils/emission.py,sha256=6MpDdf_izru0PGmkr2y9gOf9x_zfafyvSJSWALsseos,2554
+hestia_earth/aggregation/utils/emission.py,sha256=2roXjGmDuS7a3muPJoEOqg7E-TZyQsnoavhU6b3B-Z8,2983
 hestia_earth/aggregation/utils/group.py,sha256=qU9sAuJu0RG80yLs90nsyoOcU57Pb6M1WKwT1TlJEkM,4765
 hestia_earth/aggregation/utils/input.py,sha256=pZ1bcdHR3y4S-3b0JZllbEUWdEyVZbltxv-07j7SucI,1092
 hestia_earth/aggregation/utils/lookup.py,sha256=osJiXTRns5KeBMz3mRsanmFQTATrwcMVBzB07xrb71M,2046
@@ -28,8 +28,8 @@ hestia_earth/aggregation/utils/site.py,sha256=nc1N4IRiTki69IbCX4G6lagQ5U7IpsOsqi
 hestia_earth/aggregation/utils/source.py,sha256=SOiE-jB5WNtbRxreUWQ8c04ywzrwICy1aGyRL3-L0RY,686
 hestia_earth/aggregation/utils/term.py,sha256=uVAfBYnxNkcTNfzuwwHur6JGBahNrPwR2pN-__sj9zk,2758
 hestia_earth/aggregation/utils/weights.py,sha256=3WO2_26bj1RB2h524Gknu0btvTFJucqMtn-tag339aY,6843
-hestia_earth_aggregation-0.21.13.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
-hestia_earth_aggregation-0.21.13.dist-info/METADATA,sha256=4krfzpbdrJPRoVMR1zs8ANpt_dwksZKsWqWp7HECXbg,2481
-hestia_earth_aggregation-0.21.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hestia_earth_aggregation-0.21.13.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
-hestia_earth_aggregation-0.21.13.dist-info/RECORD,,
+hestia_earth_aggregation-0.21.15.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
+hestia_earth_aggregation-0.21.15.dist-info/METADATA,sha256=XgOXgQA87fTsZYpB8xXvgcz1Mp5LP7Kkec3Ik5wlLZo,2481
+hestia_earth_aggregation-0.21.15.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+hestia_earth_aggregation-0.21.15.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
+hestia_earth_aggregation-0.21.15.dist-info/RECORD,,

{hestia_earth_aggregation-0.21.13.dist-info → hestia_earth_aggregation-0.21.15.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any