hestia-earth-aggregation 0.21.12__py3-none-any.whl → 0.21.14__py3-none-any.whl

hestia_earth/aggregation/aggregate_cycles.py

@@ -1,38 +1,38 @@
 from typing import Tuple
 from hestia_earth.utils.tools import non_empty_list
 
-from hestia_earth.aggregation.log import logger, log_memory_usage
-from hestia_earth.aggregation.utils import CYCLE_AGGREGATION_KEYS, SITE_AGGREGATION_KEYS
-from hestia_earth.aggregation.utils.term import is_global
-from hestia_earth.aggregation.utils.queries import (
+from .log import logger, log_memory_usage
+from .utils import CYCLE_AGGREGATION_KEYS, SITE_AGGREGATION_KEYS
+from .utils.term import is_global
+from .utils.queries import (
     find_global_nodes,
     find_country_nodes,
     download_site,
 )
-from hestia_earth.aggregation.utils.group import group_blank_nodes
-from hestia_earth.aggregation.utils.blank_node import cleanup_node_blank_nodes
-from hestia_earth.aggregation.utils.aggregate_weighted import (
+from .utils.group import group_blank_nodes
+from .utils.blank_node import cleanup_node_blank_nodes
+from .utils.aggregate_weighted import (
     aggregate as aggregate_weighted,
 )
-from hestia_earth.aggregation.utils.aggregate_country_nodes import aggregate_cycles
-from hestia_earth.aggregation.utils.weights import (
+from .utils.aggregate_country_nodes import aggregate_cycles
+from .utils.weights import (
     country_weights,
     country_weight_node_id,
     world_weights,
     world_weight_node_id,
 )
-from hestia_earth.aggregation.utils.site import (
+from .utils.site import (
     format_site,
     format_site_description_weighted,
 )
-from hestia_earth.aggregation.utils.cycle import (
+from .utils.cycle import (
     aggregate_with_matrix,
     format_for_grouping,
     format_terms_results,
     format_country_results,
     update_cycle,
 )
-from hestia_earth.aggregation.utils.covariance import (
+from .utils.covariance import (
     init_covariance_files,
     remove_covariance_files,
     generate_covariance_country,
@@ -83,7 +83,9 @@ def _aggregate_country(
         sites, weights, use_description=not is_global(country)
     )
 
-    cycle_data = format_country_results(cycle_data, product, country, aggregated_site)
+    cycle_data = format_country_results(
+        cycle_data, product, country, aggregated_site, weights
+    )
     aggregated_cycle = update_cycle(
         country, start_year, end_year, source, functional_unit, False
     )(cycle_data)

hestia_earth/aggregation/utils/aggregate_country_nodes.py

@@ -10,6 +10,7 @@ from hestia_earth.utils.tools import (
     flatten,
     is_number,
     is_boolean,
+    current_time_ms,
 )
 from hestia_earth.utils.blank_node import (
     ArrayTreatment,
@@ -292,6 +293,7 @@ def _group_cycle_blank_nodes(
     end_year: int = None,
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, Dict[str, BlankNodeFormatted]]:
+        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(cycle, list_key, start_year, end_year)
         values = reduce(
             _group_cycle_blank_node(cycle, product, product_value), blank_nodes, {}
@@ -300,6 +302,11 @@ def _group_cycle_blank_nodes(
         group[list_key] = {
             k: v | _compute_blank_node_stats(v) for k, v in values.items()
         }
+        logger.debug(
+            "function=_group_cycle_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return grouper
@@ -405,6 +412,7 @@ def _combine_cycle_blank_nodes(
     cycles: List[CycleFormatted], completeness: Dict[str, int], cycle_count: int
 ):
     def combine(group: dict, list_key: str):
+        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(cycle.get(list_key, {}).keys()) for cycle in cycles]))
         group[list_key] = reduce(
@@ -412,6 +420,11 @@ def _combine_cycle_blank_nodes(
             keys,
             {},
         )
+        logger.debug(
+            "function=_combine_cycle_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return combine
@@ -511,8 +524,14 @@ def _group_site_blank_nodes(
     site: SiteJSONLD, start_year: int = None, end_year: int = None
 ):
     def grouper(group: dict, list_key: str) -> Dict[str, BlankNodeFormatted]:
+        now = current_time_ms()
         blank_nodes = _filter_blank_nodes(site, list_key, start_year, end_year)
         group[list_key] = reduce(_group_site_blank_node(), blank_nodes, {})
+        logger.debug(
+            "function=_group_site_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return grouper
@@ -559,9 +578,15 @@ def _combine_site_blank_node(sites: List[SiteFormatted], list_key: str):
 
 def _combine_site_blank_nodes(sites: List[SiteFormatted]):
     def combine(group: dict, list_key: str):
+        now = current_time_ms()
         # get all possible keys first, then group each key values into a single blank node
         keys = set(flatten([list(site.get(list_key, {}).keys()) for site in sites]))
         group[list_key] = reduce(_combine_site_blank_node(sites, list_key), keys, {})
+        logger.debug(
+            "function=_combine_site_blank_nodes, list_key=%s, time=%s",
+            list_key,
+            current_time_ms() - now,
+        )
         return group
 
     return combine
@@ -647,9 +672,16 @@ def _aggregate_formatted(
     data: Union[CycleFormatted, SiteFormatted], aggregated_keys: List[str]
 ):
     def aggregate(key: str):
+        now = current_time_ms()
         values = data.get(key, {}).values()
         logger.debug(f"Aggregating {len(values)} {key}...")
-        return list(map(_aggregate_blank_node, values))
+        values = list(map(_aggregate_blank_node, values))
+        logger.debug(
+            "function=_aggregate_formatted, key=%s, time=%s",
+            key,
+            current_time_ms() - now,
+        )
+        return values
 
     return reduce(lambda group, key: group | {key: aggregate(key)}, aggregated_keys, {})
 

hestia_earth/aggregation/utils/aggregate_weighted.py

@@ -6,18 +6,17 @@ from hestia_earth.utils.tools import (
     list_sum,
     list_average,
     is_boolean,
+    current_time_ms,
 )
 from hestia_earth.utils.blank_node import get_node_value
 
+from hestia_earth.aggregation.log import logger
 from . import weighted_average, _min, _max, _sd, format_evs, pick
 from .emission import get_method_tier, get_method_model
 from .property import aggregate_properties
 from .completeness import blank_node_completeness_key
 from .distribution import sample_weighted_distributions, _nb_iterations
-
-
-def _format_weight_key(key: str):
-    return ", ".join(key.split("_")).capitalize()
+from .weights import format_weights
 
 
 def _format_description(completeness_key: str, weights: dict):
@@ -26,14 +25,7 @@ def _format_description(completeness_key: str, weights: dict):
         for w in weights.values()
         if w.get("completeness", {}).get(completeness_key, False)
     ]
-    total = list_sum(non_empty_list([w.get("weight") for w in complete_weights]))
-    description = ";".join(
-        [
-            f"{_format_weight_key(weight.get('key'))}: {round(weight.get('weight') * 100 / total, 2)}"
-            for weight in complete_weights
-            if weight.get("key")
-        ]
-    )
+    description = format_weights(complete_weights)
     return {"description": description}
 
 
@@ -198,14 +190,19 @@ def _aggregate_nodes(
     aggregate_keys: List[str], data: dict, weights: dict, missing_weights_node_id_func
 ):
     def aggregate_single(key: str):
+        now = current_time_ms()
         aggregates_map: dict = data.get(key)
         terms = aggregates_map.keys()
-        return non_empty_list(
+        values = non_empty_list(
             map(
                 _aggregate_term(aggregates_map, weights, missing_weights_node_id_func),
                 terms,
             )
         )
+        logger.debug(
+            "function=_aggregate_nodes, key=%s, time=%s", key, current_time_ms() - now
+        )
+        return values
 
     return reduce(
         lambda prev, curr: prev | {curr: aggregate_single(curr)}, aggregate_keys, {}

hestia_earth/aggregation/utils/completeness.py

@@ -9,7 +9,6 @@ from hestia_earth.schema import (
 )
 from hestia_earth.utils.tools import non_empty_list
 
-
 _ANIMAL_FEED_INPUT_MAPPING = {
     SchemaType.INPUT.value: {
         TermTermType.ANIMALPRODUCT.value: CompletenessField.ANIMALFEED.value,

hestia_earth/aggregation/utils/cycle.py

@@ -39,6 +39,7 @@ from .emission import new_emission
 from .input import new_input
 from .practice import new_practice, organic_practice
 from .product import new_product
+from .weights import format_weights
 
 _MEAN_DATE_DESCRIPTION = (
     "Additional notes: the mean endDate of all aggregated Cycles is "
@@ -61,6 +62,11 @@ def _combine_mean_dates(cycles: list):
     return _mean_date(dates)
 
 
+def _aggregated_weights(weights: dict):
+    description = format_weights(weights.values())
+    return ("Sub-aggregation weighting: " + description) if description else ""
+
+
 def is_organic(cycle: dict):
     term_type = TermTermType.STANDARDSLABELS
     practices = filter_list_term_type(cycle.get("practices", []), term_type)
@@ -234,7 +240,9 @@ def format_terms_results(data: dict, product: dict, country: dict):
     return None
 
 
-def format_country_results(data: dict, product: dict, country: dict, site: dict):
+def format_country_results(
+    data: dict, product: dict, country: dict, site: dict, weights: dict
+):
     inputs = data.get("inputs", [])
     practices = data.get("practices", [])
     products = data.get("products", [])
@@ -247,7 +255,14 @@ def format_country_results(data: dict, product: dict, country: dict, site: dict)
         completeness = aggregate_completeness(data.get("node-completeness", []))
         return remove_empty_fields(
             {
-                "description": _MEAN_DATE_DESCRIPTION + _combine_mean_dates(cycles),
+                "description": ". ".join(
+                    non_empty_list(
+                        [
+                            _aggregated_weights(weights),
+                            _MEAN_DATE_DESCRIPTION + _combine_mean_dates(cycles),
+                        ]
+                    )
+                ),
                 "name": _cycle_name(cycle, primary_product, False, False, False),
                 "id": _cycle_id(cycle, primary_product, False, False, False),
                 **_format_results(

hestia_earth/aggregation/utils/weights.py

@@ -1,6 +1,6 @@
 from functools import reduce, lru_cache
 from hestia_earth.utils.lookup import download_lookup, get_table_value
-from hestia_earth.utils.tools import to_precision, non_empty_list
+from hestia_earth.utils.tools import to_precision, non_empty_list, list_sum
 
 from hestia_earth.aggregation.log import debugWeights, debugRequirements
 from .lookup import (
@@ -16,6 +16,21 @@ from .term import (
 )
 
 
+def _format_weight_key(key: str):
+    return ", ".join(key.split("_")).capitalize()
+
+
+def format_weights(weights: list):
+    total = list_sum(non_empty_list([w.get("weight") for w in weights]))
+    return ";".join(
+        [
+            f"{_format_weight_key(weight.get('key'))}: {round(weight.get('weight') * 100 / total, 2)}"
+            for weight in weights
+            if weight.get("key")
+        ]
+    )
+
+
 def _country_organic_weight(country_id: str, start_year: int, end_year: int):
     lookup = download_lookup("region-standardsLabels-isOrganic.csv")
     data = get_table_value(lookup, "term.id", country_id, "organic")

hestia_earth/aggregation/version.py

@@ -1 +1 @@
-VERSION = "0.21.12"
+VERSION = "0.21.14"

{hestia_earth_aggregation-0.21.12.dist-info → hestia_earth_aggregation-0.21.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_aggregation
-Version: 0.21.12
+Version: 0.21.14
 Summary: HESTIA's aggregation engine.
 Home-page: https://gitlab.com/hestia-earth/hestia-aggregation-engine
 Author: HESTIA Team

{hestia_earth_aggregation-0.21.12.dist-info → hestia_earth_aggregation-0.21.14.dist-info}/RECORD

@@ -1,17 +1,17 @@
 hestia_earth/aggregation/__init__.py,sha256=5r0icTYP5F4DFIX-Xm1Ms3inpAUfpXQwOmO2y_DV3zs,3202
-hestia_earth/aggregation/aggregate_cycles.py,sha256=olCgWclxJ0sjh93ZRS6ete3iyAXUhKUOpI5aU6UpzUI,8077
+hestia_earth/aggregation/aggregate_cycles.py,sha256=9BRX41d0uLhKLUPBlALxAoRMfPriDj3Dhy0sTgmTS5Y,7812
 hestia_earth/aggregation/log.py,sha256=0CibXDtKqFfE-R86qziM2uH7ATUZ6I9p_bVpRSn_e_o,2351
 hestia_earth/aggregation/recalculate_cycles.py,sha256=xCO_tj5BAC407vZ36EVuxeOBmX2y82EXRrTx5W6g4CY,484
-hestia_earth/aggregation/version.py,sha256=nHbnpcq6XZSWMiVNW064FoAdba7JReYQDKDgFFjIdoM,20
+hestia_earth/aggregation/version.py,sha256=zqCwaEmEyCPuIxRLzAP6Ds1U0koeoYfXnJ0e800-MWo,20
 hestia_earth/aggregation/config/Cycle/processedFood.json,sha256=JWelgL-ZLnurS9R8Y-HOXjiWZ5k3kAuUQHDoMcukEq8,1127
 hestia_earth/aggregation/utils/__init__.py,sha256=nVVLgVFZ4415CgQAqgv4jDamA7gDmzOWk5SY4vshAQs,6583
-hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=LXk2jQQVUSX_5Ut6NxJG12jd3ZxW7pLiwH3fYMwhOcs,27453
-hestia_earth/aggregation/utils/aggregate_weighted.py,sha256=oexmkQ6z4loqGpAN7tl0nKIRxw1yqZfdsg3sbmGRiEA,7107
+hestia_earth/aggregation/utils/aggregate_country_nodes.py,sha256=c2rosOtI7B8vwb2CbDye-YPx4xyPc-o4kKKPsyAVoyU,28456
+hestia_earth/aggregation/utils/aggregate_weighted.py,sha256=CUTqR8z3_MRnGhzAOiFswy68Twe1GaNCyrSd5x7qf_Q,7037
 hestia_earth/aggregation/utils/blank_node.py,sha256=f8JBcd5_SLqW8juzFAfbZysAESaZXj8JX6C7rDkLEoc,16812
 hestia_earth/aggregation/utils/combine.py,sha256=EmXUZksdkrtQ6o1gMOG5WMivUhL28NKSM9D4cPohJiE,4904
-hestia_earth/aggregation/utils/completeness.py,sha256=vBwT25r7d_CsAicjwII4LvhUFykNnYDzYlbpuxhq5AQ,4634
+hestia_earth/aggregation/utils/completeness.py,sha256=eZ759PAdOSshhys-YKKPEpAYxR3sTrEjI--WJTEwmus,4633
 hestia_earth/aggregation/utils/covariance.py,sha256=2VaNGPvyD4ub3rR-OBn8VIEIeAz4JOefl6ZlAtXKj4U,5884
-hestia_earth/aggregation/utils/cycle.py,sha256=Wq5HTYLXxuUgs7jEVLL4wa-ebww93c8Yg8-9k0Loq0s,15498
+hestia_earth/aggregation/utils/cycle.py,sha256=gzU2KofOVGBZvOPTymJGpsbbouABjSzUM35OJ42sFl8,15954
 hestia_earth/aggregation/utils/distribution.py,sha256=2XQKXVu-1XUy1zEHgThERupaj4RizXKO5F-VY-QQlMo,6935
 hestia_earth/aggregation/utils/emission.py,sha256=6MpDdf_izru0PGmkr2y9gOf9x_zfafyvSJSWALsseos,2554
 hestia_earth/aggregation/utils/group.py,sha256=qU9sAuJu0RG80yLs90nsyoOcU57Pb6M1WKwT1TlJEkM,4765
@@ -27,9 +27,9 @@ hestia_earth/aggregation/utils/queries.py,sha256=kzob6CHL1a60CRJdGICiWS98LKJRyq3
 hestia_earth/aggregation/utils/site.py,sha256=nc1N4IRiTki69IbCX4G6lagQ5U7IpsOsqiPuHwbl1HM,5455
 hestia_earth/aggregation/utils/source.py,sha256=SOiE-jB5WNtbRxreUWQ8c04ywzrwICy1aGyRL3-L0RY,686
 hestia_earth/aggregation/utils/term.py,sha256=uVAfBYnxNkcTNfzuwwHur6JGBahNrPwR2pN-__sj9zk,2758
-hestia_earth/aggregation/utils/weights.py,sha256=jBgrv3SBRpUSAruEPfo_aQyINcMprFpAvnAS84LTUGk,6420
-hestia_earth_aggregation-0.21.12.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
-hestia_earth_aggregation-0.21.12.dist-info/METADATA,sha256=o7xTa1oo6B5HJm07eEvcWaY__ls8ECuGZ6sgeerwaNQ,2481
-hestia_earth_aggregation-0.21.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hestia_earth_aggregation-0.21.12.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
-hestia_earth_aggregation-0.21.12.dist-info/RECORD,,
+hestia_earth/aggregation/utils/weights.py,sha256=3WO2_26bj1RB2h524Gknu0btvTFJucqMtn-tag339aY,6843
+hestia_earth_aggregation-0.21.14.dist-info/licenses/LICENSE,sha256=TD25LoiRJsA5CPUNrcyt1PXlGcbUGFMAeZoBcfCrCNE,1154
+hestia_earth_aggregation-0.21.14.dist-info/METADATA,sha256=DId0r59jDje3k-jfnNbBa1G9mcBRvi2rAbEjyvZQGnU,2481
+hestia_earth_aggregation-0.21.14.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+hestia_earth_aggregation-0.21.14.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
+hestia_earth_aggregation-0.21.14.dist-info/RECORD,,

{hestia_earth_aggregation-0.21.12.dist-info → hestia_earth_aggregation-0.21.14.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any