hestia-earth-utils 0.15.13__tar.gz → 0.15.15__tar.gz

{hestia_earth_utils-0.15.13 → hestia_earth_utils-0.15.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_utils
-Version: 0.15.13
+Version: 0.15.15
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.15.13 → hestia_earth_utils-0.15.15}/hestia_earth/utils/emission.py

@@ -9,7 +9,11 @@ from .lookup_utils import (
     is_product_termType_allowed,
     is_product_id_allowed,
     is_input_termType_allowed,
-    is_input_id_allowed
+    is_input_id_allowed,
+    is_practice_termType_allowed,
+    is_practice_id_allowed,
+    is_transformation_termType_allowed,
+    is_transformation_id_allowed,
 )
 
 
@@ -35,7 +39,11 @@ def cycle_emission_is_in_system_boundary(cycle: dict):
             is_product_termType_allowed,
             is_product_id_allowed,
             is_input_termType_allowed,
-            is_input_id_allowed
+            is_input_id_allowed,
+            is_practice_termType_allowed,
+            is_practice_id_allowed,
+            is_transformation_termType_allowed,
+            is_transformation_id_allowed,
         ]))
 
     return filter_term

{hestia_earth_utils-0.15.13 → hestia_earth_utils-0.15.15}/hestia_earth/utils/lookup_utils.py

@@ -1,5 +1,6 @@
 from functools import lru_cache
 import json
+from hestia_earth.schema import SchemaType
 
 from .lookup import _download_lookup_data, download_lookup, get_table_value, column_name
 from .api import download_hestia
@@ -35,6 +36,10 @@ def _get_site_measurements(node: dict):
     return flatten([non_empty_list(site.get('measurements', [])) for site in sites])
 
 
+def _blank_node_term_values(blank_nodes: list, key: str = '@id'):
+    return non_empty_list([v.get('term', {}).get(key) for v in blank_nodes])
+
+
 @lru_cache()
 def _allowed_model_mapping(model: str, term_id: str, column: str):
     mapping = _allowed_mapping_data()
@@ -45,22 +50,23 @@ def _allowed_model_mapping(model: str, term_id: str, column: str):
     return (value or _ALLOW_ALL).split(';') if isinstance(value, str) else _ALLOW_ALL
 
 
+def _is_model_value_allowed(model: str, term_id: str, values: list, lookup_column: str):
+    allowed_values = _allowed_model_mapping(model, term_id, lookup_column)
+    return any([
+        _ALLOW_ALL in allowed_values,
+        len(values) == 0
+    ]) or any([value in allowed_values for value in values])
+
+
 def is_model_siteType_allowed(model: str, term_id: str, data: dict):
-    site_types = _get_site_types(data)
-    allowed_values = _allowed_model_mapping(model, term_id, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_types else any([
-        (site_type in allowed_values) for site_type in site_types
-    ])
+    values = _get_site_types(data)
+    return _is_model_value_allowed(model, term_id, values, 'siteTypesAllowed')
 
 
 def is_model_product_id_allowed(model: str, term_id: str, data: dict):
     products = data.get('products', [])
-    values = non_empty_list([p.get('term', {}).get('@id') for p in products])
-    allowed_values = _allowed_model_mapping(model, term_id, 'productTermIdsAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
+    values = _blank_node_term_values(products)
+    return _is_model_value_allowed(model, term_id, values, 'productTermIdsAllowed')
 
 
 @lru_cache()
@@ -73,68 +79,76 @@ def _allowed_mapping(term_id: str, column: str):
     return (value or _ALLOW_ALL).split(';') if isinstance(value, str) else _ALLOW_ALL
 
 
+def _is_term_value_allowed(term_id: str, values: list, lookup_column: str):
+    allowed_values = _allowed_mapping(term_id, lookup_column)
+    return any([
+        _ALLOW_ALL in allowed_values,
+        len(values) == 0
+    ]) or any([value in allowed_values for value in values])
+
+
 def is_siteType_allowed(data: dict, term_id: str):
-    site_types = _get_site_types(data)
-    allowed_values = _allowed_mapping(term_id, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_types else any([
-        (site_type in allowed_values) for site_type in site_types
-    ])
+    values = _get_site_types(data)
+    return _is_term_value_allowed(term_id, values, 'siteTypesAllowed')
 
 
 def is_site_measurement_id_allowed(data: dict, term_id: str):
     measurements = _get_site_measurements(data)
-    values = non_empty_list([v.get('term', {}).get('@id') for v in measurements])
-    allowed_values = _allowed_mapping(term_id, 'siteMeasurementIdsAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
+    values = _blank_node_term_values(measurements, key='@id')
+    return _is_term_value_allowed(term_id, values, 'siteMeasurementIdsAllowed')
 
 
 def is_product_termType_allowed(data: dict, term_id: str):
     products = data.get('products', [])
-    values = non_empty_list([p.get('term', {}).get('termType') for p in products])
-    allowed_values = _allowed_mapping(term_id, 'productTermTypesAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
+    values = _blank_node_term_values(products, key='termType')
+    return _is_term_value_allowed(term_id, values, 'productTermTypesAllowed')
 
 
 def is_product_id_allowed(data: dict, term_id: str):
     products = data.get('products', [])
-    values = non_empty_list([p.get('term', {}).get('@id') for p in products])
-    allowed_values = _allowed_mapping(term_id, 'productTermIdsAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
+    values = _blank_node_term_values(products, key='@id')
+    return _is_term_value_allowed(term_id, values, 'productTermIdsAllowed')
 
 
 def is_input_termType_allowed(data: dict, term_id: str):
     inputs = data.get('inputs', [])
-    values = non_empty_list([p.get('term', {}).get('termType') for p in inputs])
-    allowed_values = _allowed_mapping(term_id, 'inputTermTypesAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
+    values = _blank_node_term_values(inputs, key='termType')
+    return _is_term_value_allowed(term_id, values, 'inputTermTypesAllowed')
 
 
 def is_input_id_allowed(data: dict, term_id: str):
     inputs = data.get('inputs', [])
-    values = non_empty_list([p.get('term', {}).get('@id') for p in inputs])
-    allowed_values = _allowed_mapping(term_id, 'inputTermIdsAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
+    values = _blank_node_term_values(inputs, key='@id')
+    return _is_term_value_allowed(term_id, values, 'inputTermIdsAllowed')
+
+
+def is_practice_termType_allowed(data: dict, term_id: str):
+    practices = data.get('practices', [])
+    values = _blank_node_term_values(practices, key='termType')
+    return _is_term_value_allowed(term_id, values, 'practiceTermTypesAllowed')
+
+
+def is_practice_id_allowed(data: dict, term_id: str):
+    practices = data.get('practices', [])
+    values = _blank_node_term_values(practices, key='@id')
+    return _is_term_value_allowed(term_id, values, 'practiceTermIdsAllowed')
+
+
+def is_transformation_termType_allowed(data: dict, term_id: str):
+    is_transformation = data.get('@type', data.get('type')) == SchemaType.TRANSFORMATION.value
+    values = non_empty_list([data.get('term', {}).get('termType')])
+    return not is_transformation or _is_term_value_allowed(term_id, values, 'transformationTermTypesAllowed')
+
+
+def is_transformation_id_allowed(data: dict, term_id: str):
+    is_transformation = data.get('@type', data.get('type')) == SchemaType.TRANSFORMATION.value
+    values = non_empty_list([data.get('term', {}).get('@id')])
+    return not is_transformation or _is_term_value_allowed(term_id, values, 'transformationTermIdsAllowed')
 
 
 def is_node_type_allowed(data: dict, term_id: str):
-    node_type = data.get('@type', data.get('type'))
-    allowed_types = _allowed_mapping(term_id, 'typesAllowed')
-    return True if _ALLOW_ALL in allowed_types or not node_type else node_type in allowed_types
+    values = non_empty_list([data.get('@type', data.get('type'))])
+    return _is_term_value_allowed(term_id, values, 'typesAllowed')
 
 
 @lru_cache()

{hestia_earth_utils-0.15.13 → hestia_earth_utils-0.15.15}/hestia_earth/utils/pivot/_shared.py

@@ -1,5 +1,6 @@
 import json
 import numpy as np
+from hestia_earth.schema import EmissionMethodTier
 
 
 EXCLUDE_FIELDS = ["@type", "type", "@context"]
@@ -33,3 +34,18 @@ def _is_scalar_list(value):
             all_scalar = False
             break
     return all_scalar
+
+
+def _filter_not_relevant(blank_node: dict):
+    return blank_node.get('methodTier') != EmissionMethodTier.NOT_RELEVANT.value
+
+
+def _filter_emissions_not_relevant(node: dict):
+    """
+    Ignore all emissions where `methodTier=not relevant` so save space.
+    """
+    return node | ({
+        key: list(filter(_filter_not_relevant, node[key]))
+        for key in ['emissions', 'emissionsResourceUse']
+        if key in node
+    })

{hestia_earth_utils-0.15.13 → hestia_earth_utils-0.15.15}/hestia_earth/utils/pivot/pivot_csv.py

@@ -8,7 +8,7 @@ from flatten_json import flatten as flatten_json
 
 # __package__ = "hestia_earth.utils" # required to run interactively in vscode
 from ..api import find_term_ids_by_names
-from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting
+from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting, _filter_emissions_not_relevant
 
 
 PANDAS_IMPORT_ERROR_MSG = "Run `pip install pandas>=1.2` to use this functionality"
@@ -293,7 +293,7 @@ def pivot_nodes(nodes: list[dict]):
     Pivot array of nodes in dict format (e.g under the 'nodes' key of a .hestia file)
     The nodes json should first be parsed using _with_csv_formatting for output as csv.
     """
-    df_in = nodes_to_df(nodes)
+    df_in = nodes_to_df(list(map(_filter_emissions_not_relevant, nodes)))
 
     regex = rf".*\.({'|'.join(EXCLUDE_FIELDS + EXCLUDE_PRIVATE_FIELDS)})$"
     df_in.drop(df_in.filter(regex=regex).columns, axis=1, inplace=True)

{hestia_earth_utils-0.15.13 → hestia_earth_utils-0.15.15}/hestia_earth/utils/pivot/pivot_json.py

@@ -5,7 +5,8 @@ from flatten_json import flatten, unflatten_list
 from collections import defaultdict
 from copy import deepcopy
 
-from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting
+from hestia_earth.utils.pipeline import _node_type
+from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting, _filter_emissions_not_relevant
 
 pivot_exclude_fields = Term().fields
 pivot_exclude_fields.update({k: "" for k in EXCLUDE_FIELDS} | {k: "" for k in EXCLUDE_PRIVATE_FIELDS})
@@ -52,7 +53,7 @@ def _base_pivoted_value(key: str, value, is_top_level: bool):
 
 def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa: C901
     # print('\ninput node', level, node, '\n')
-    node_type = node.get("@type", node.get("type"))
+    node_type = _node_type(node)
     if node_type not in ADAPTED_UNIQUENESS_FIELDS:
         return node
     pivoted_node = {
@@ -206,6 +207,13 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
     return pivoted_node
 
 
+def pivot_node(node: dict):
+    """
+    Pivot single node in dict format parsed with object_hook=_with_csv_formatting
+    """
+    return _do_pivot(_filter_emissions_not_relevant(node))
+
+
 def pivot_json_node(json_node: str):
     """
     Pivot single schema-compliant unparsed json string node
@@ -227,11 +235,3 @@ def pivot_nodes(nodes: list[dict]):
     Pivot multiple nodes in dict format parsed with object_hook=_with_csv_formatting
     """
     return [pivot_node(node) for node in nodes]
-
-
-def pivot_node(node: dict):
-    """
-    Pivot single node in dict format parsed with object_hook=_with_csv_formatting
-    """
-    pivoted_node = _do_pivot(node)
-    return pivoted_node

hestia_earth_utils-0.15.15/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = '0.15.15'

{hestia_earth_utils-0.15.13 → hestia_earth_utils-0.15.15}/hestia_earth_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-utils
-Version: 0.15.13
+Version: 0.15.15
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

hestia_earth_utils-0.15.13/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = '0.15.13'