hestia-earth-aggregation 0.21.13__tar.gz → 0.21.15__tar.gz

{hestia_earth_aggregation-0.21.13/hestia_earth_aggregation.egg-info → hestia_earth_aggregation-0.21.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_aggregation
-Version: 0.21.13
+Version: 0.21.15
 Summary: HESTIA's aggregation engine.
 Home-page: https://gitlab.com/hestia-earth/hestia-aggregation-engine
 Author: HESTIA Team

{hestia_earth_aggregation-0.21.13 → hestia_earth_aggregation-0.21.15}/hestia_earth/aggregation/utils/aggregate_country_nodes.py

@@ -10,6 +10,7 @@ from hestia_earth.utils.tools import (
     flatten,
     is_number,
     is_boolean,
+    current_time_ms,
 )
 from hestia_earth.utils.blank_node import (
     ArrayTreatment,
@@ -60,7 +61,11 @@ from .site import (
     format_site_description,
     site_parent_region_id,
 )
-from .emission import get_method_tier, get_method_model
+from .emission import (
+    get_method_tier,
+    get_method_model,
+    has_value_without_transformation,
+)
 from .property import aggregate_properties
 from .distribution import generate_blank_node_distribution
 from .covariance import add_covariance_cycles, generate_covariance_cycles
@@ -109,6 +114,8 @@ class BlankNodeFormatted(TypedDict, total=False):
     completeness_key: Optional[str]
     start_date: Optional[str]
     end_date: str
+    inputs: List[dict]
+    transformation: List[dict]
 
 
 # --- Cycle
@@ -242,6 +249,8 @@ def _group_cycle_blank_node(cycle: dict, product: dict, product_value: float):
                 "economicValueShare": non_empty_list([_blank_node_evs(blank_nodes)]),
                 # needed for background emissions
                 "inputs": _map_values(blank_nodes, "inputs"),
+                # needed to exclude emissions with only transformation value
+                "transformation": _map_values(blank_nodes, "transformation"),
                 "properties": _map_values(blank_nodes, "properties"),
                 "methodTier": _map_values(blank_nodes, "methodTier"),
                 "methodModel": _map_values(blank_nodes, "methodModel"),
@@ -292,14 +301,22 @@ def _group_cycle_blank_nodes(
     end_year: int = None,
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, Dict[str, BlankNodeFormatted]]:
+        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(cycle, list_key, start_year, end_year)
         values = reduce(
             _group_cycle_blank_node(cycle, product, product_value), blank_nodes, {}
         )
         # after combining all values, need to compute the final statistical values
         group[list_key] = {
-            k: v | _compute_blank_node_stats(v) for k, v in values.items()
+            k: v | _compute_blank_node_stats(v)
+            for k, v in values.items()
+            if list_key != "emissions" or has_value_without_transformation(v)
         }
+        logger.debug(
+            "function=_group_cycle_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return grouper
@@ -405,6 +422,7 @@ def _combine_cycle_blank_nodes(
     cycles: List[CycleFormatted], completeness: Dict[str, int], cycle_count: int
 ):
     def combine(group: dict, list_key: str):
+        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(cycle.get(list_key, {}).keys()) for cycle in cycles]))
         group[list_key] = reduce(
@@ -412,6 +430,11 @@ def _combine_cycle_blank_nodes(
             keys,
             {},
         )
+        logger.debug(
+            "function=_combine_cycle_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return combine
@@ -511,8 +534,14 @@ def _group_site_blank_nodes(
     site: SiteJSONLD, start_year: int = None, end_year: int = None
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, BlankNodeFormatted]:
+        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(site, list_key, start_year, end_year)
         group[list_key] = reduce(_group_site_blank_node(), blank_nodes, {})
+        logger.debug(
+            "function=_group_site_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return grouper
@@ -559,9 +588,15 @@ def _combine_site_blank_node(sites: List[SiteFormatted], list_key: str):
 
 def _combine_site_blank_nodes(sites: List[SiteFormatted]):
     def combine(group: dict, list_key: str):
+        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(site.get(list_key, {}).keys()) for site in sites]))
         group[list_key] = reduce(_combine_site_blank_node(sites, list_key), keys, {})
+        logger.debug(
+            "function=_combine_site_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return combine
@@ -647,9 +682,16 @@ def _aggregate_formatted(
     data: Union[CycleFormatted, SiteFormatted], aggregated_keys: List[str]
 ):
     def aggregate(key: str):
+        now = current_time_ms()
         values = data.get(key, {}).values()
         logger.debug(f"Aggregating {len(values)} {key}...")
-        return list(map(_aggregate_blank_node, values))
+        values = list(map(_aggregate_blank_node, values))
+        logger.debug(
+            "function=_aggregate_formatted, key=%s, time=%s",
+            key,
+            current_time_ms() - now,
+        )
+        return values
 
     return reduce(lambda group, key: group | {key: aggregate(key)}, aggregated_keys, {})
 

{hestia_earth_aggregation-0.21.13 → hestia_earth_aggregation-0.21.15}/hestia_earth/aggregation/utils/aggregate_weighted.py

@@ -6,9 +6,11 @@ from hestia_earth.utils.tools import (
     list_sum,
     list_average,
     is_boolean,
+    current_time_ms,
 )
 from hestia_earth.utils.blank_node import get_node_value
 
+from hestia_earth.aggregation.log import logger
 from . import weighted_average, _min, _max, _sd, format_evs, pick
 from .emission import get_method_tier, get_method_model
 from .property import aggregate_properties
@@ -188,14 +190,19 @@ def _aggregate_nodes(
     aggregate_keys: List[str], data: dict, weights: dict, missing_weights_node_id_func
 ):
     def aggregate_single(key: str):
+        now = current_time_ms()
         aggregates_map: dict = data.get(key)
         terms = aggregates_map.keys()
-        return non_empty_list(
+        values = non_empty_list(
             map(
                 _aggregate_term(aggregates_map, weights, missing_weights_node_id_func),
                 terms,
             )
         )
+        logger.debug(
+            "function=_aggregate_nodes, key=%s, time=%s", key, current_time_ms() - now
+        )
+        return values
 
     return reduce(
         lambda prev, curr: prev | {curr: aggregate_single(curr)}, aggregate_keys, {}

{hestia_earth_aggregation-0.21.13 → hestia_earth_aggregation-0.21.15}/hestia_earth/aggregation/utils/completeness.py

@@ -9,7 +9,6 @@ from hestia_earth.schema import (
 )
 from hestia_earth.utils.tools import non_empty_list
 
-
 _ANIMAL_FEED_INPUT_MAPPING = {
     SchemaType.INPUT.value: {
         TermTermType.ANIMALPRODUCT.value: CompletenessField.ANIMALFEED.value,

{hestia_earth_aggregation-0.21.13 → hestia_earth_aggregation-0.21.15}/hestia_earth/aggregation/utils/emission.py

@@ -69,3 +69,18 @@ def get_method_model(emissions: list):
     values = non_empty_list(flatten([e.get("methodModel", []) for e in emissions]))
     values = list({v["@id"]: v for v in values}.values())
     return values[0] if len(values) == 1 else None
+
+
+def has_value_without_transformation(blank_node: dict):
+    values = blank_node.get("value", [])
+    transformations = blank_node.get("transformation", [])
+    return (
+        not transformations
+        or len(transformations) != len(values)
+        or any(
+            [
+                value is not None and transformations[index] is None
+                for index, value in enumerate(values)
+            ]
+        )
+    )

hestia_earth_aggregation-0.21.15/hestia_earth/aggregation/version.py

@@ -0,0 +1 @@
+VERSION = "0.21.15"

{hestia_earth_aggregation-0.21.13 → hestia_earth_aggregation-0.21.15/hestia_earth_aggregation.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_aggregation
-Version: 0.21.13
+Version: 0.21.15
 Summary: HESTIA's aggregation engine.
 Home-page: https://gitlab.com/hestia-earth/hestia-aggregation-engine
 Author: HESTIA Team

hestia_earth_aggregation-0.21.13/hestia_earth/aggregation/version.py

@@ -1 +0,0 @@
-VERSION = "0.21.13"