hestia-earth-utils 0.15.12__tar.gz → 0.15.14__tar.gz

{hestia_earth_utils-0.15.12 → hestia_earth_utils-0.15.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_utils
-Version: 0.15.12
+Version: 0.15.14
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.15.12 → hestia_earth_utils-0.15.14}/hestia_earth/utils/pivot/_shared.py

@@ -1,5 +1,6 @@
 import json
 import numpy as np
+from hestia_earth.schema import EmissionMethodTier
 
 
 EXCLUDE_FIELDS = ["@type", "type", "@context"]
@@ -33,3 +34,18 @@ def _is_scalar_list(value):
             all_scalar = False
             break
     return all_scalar
+
+
+def _filter_not_relevant(blank_node: dict):
+    return blank_node.get('methodTier') != EmissionMethodTier.NOT_RELEVANT.value
+
+
+def _filter_emissions_not_relevant(node: dict):
+    """
+    Ignore all emissions where `methodTier=not relevant` so save space.
+    """
+    return node | ({
+        key: list(filter(_filter_not_relevant, node[key]))
+        for key in ['emissions', 'emissionsResourceUse']
+        if key in node
+    })

{hestia_earth_utils-0.15.12 → hestia_earth_utils-0.15.14}/hestia_earth/utils/pivot/pivot_csv.py

@@ -8,7 +8,7 @@ from flatten_json import flatten as flatten_json
 
 # __package__ = "hestia_earth.utils" # required to run interactively in vscode
 from ..api import find_term_ids_by_names
-from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting
+from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting, _filter_emissions_not_relevant
 
 
 PANDAS_IMPORT_ERROR_MSG = "Run `pip install pandas>=1.2` to use this functionality"
@@ -293,7 +293,7 @@ def pivot_nodes(nodes: list[dict]):
     Pivot array of nodes in dict format (e.g under the 'nodes' key of a .hestia file)
     The nodes json should first be parsed using _with_csv_formatting for output as csv.
     """
-    df_in = nodes_to_df(nodes)
+    df_in = nodes_to_df(list(map(_filter_emissions_not_relevant, nodes)))
 
     regex = rf".*\.({'|'.join(EXCLUDE_FIELDS + EXCLUDE_PRIVATE_FIELDS)})$"
     df_in.drop(df_in.filter(regex=regex).columns, axis=1, inplace=True)

{hestia_earth_utils-0.15.12 → hestia_earth_utils-0.15.14}/hestia_earth/utils/pivot/pivot_json.py

@@ -5,7 +5,8 @@ from flatten_json import flatten, unflatten_list
 from collections import defaultdict
 from copy import deepcopy
 
-from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting
+from hestia_earth.utils.pipeline import _node_type
+from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting, _filter_emissions_not_relevant
 
 pivot_exclude_fields = Term().fields
 pivot_exclude_fields.update({k: "" for k in EXCLUDE_FIELDS} | {k: "" for k in EXCLUDE_PRIVATE_FIELDS})
@@ -52,7 +53,7 @@ def _base_pivoted_value(key: str, value, is_top_level: bool):
 
 def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa: C901
     # print('\ninput node', level, node, '\n')
-    node_type = node.get("@type", node.get("type"))
+    node_type = _node_type(node)
     if node_type not in ADAPTED_UNIQUENESS_FIELDS:
         return node
     pivoted_node = {
@@ -206,6 +207,13 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
     return pivoted_node
 
 
+def pivot_node(node: dict):
+    """
+    Pivot single node in dict format parsed with object_hook=_with_csv_formatting
+    """
+    return _do_pivot(_filter_emissions_not_relevant(node))
+
+
 def pivot_json_node(json_node: str):
     """
     Pivot single schema-compliant unparsed json string node
@@ -227,11 +235,3 @@ def pivot_nodes(nodes: list[dict]):
     Pivot multiple nodes in dict format parsed with object_hook=_with_csv_formatting
     """
     return [pivot_node(node) for node in nodes]
-
-
-def pivot_node(node: dict):
-    """
-    Pivot single node in dict format parsed with object_hook=_with_csv_formatting
-    """
-    pivoted_node = _do_pivot(node)
-    return pivoted_node

{hestia_earth_utils-0.15.12 → hestia_earth_utils-0.15.14}/hestia_earth/utils/storage/_s3_client.py

@@ -100,12 +100,16 @@ def _list_bucket_objects(bucket: str, folder: str = ''):
 def _delete_objects(bucket: str, objects: list):
     from botocore.exceptions import ClientError
     try:
-        return _get_s3_client().delete_objects(
-            Bucket=bucket,
-            Delete={
-                'Objects': objects,
-                'Quiet': True
-            }
-        )
+        # delete in batch of 1000 max allowed
+        batch_size = 1000
+        for i in range(0, len(objects), batch_size):
+            batch_objects = objects[i:i + batch_size]
+            _get_s3_client().delete_objects(
+                Bucket=bucket,
+                Delete={
+                    'Objects': batch_objects,
+                    'Quiet': True
+                }
+            )
     except ClientError:
         return None

hestia_earth_utils-0.15.14/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = '0.15.14'

{hestia_earth_utils-0.15.12 → hestia_earth_utils-0.15.14}/hestia_earth_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-utils
-Version: 0.15.12
+Version: 0.15.14
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

hestia_earth_utils-0.15.12/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = '0.15.12'