hestia-earth-utils 0.15.6__py3-none-any.whl → 0.15.8__py3-none-any.whl

hestia_earth/utils/blank_node.py

@@ -6,7 +6,7 @@ from statistics import mode, mean
 from hestia_earth.schema import TermTermType
 
 from .lookup import download_lookup, get_table_value, column_name
-from .tools import non_empty_list, flatten
+from .tools import non_empty_list, non_empty_value, flatten
 from .emission import cycle_emissions_in_system_boundary
 from .model import filter_list_term_type
 
@@ -145,7 +145,7 @@ def get_node_value(
 
     return reducer(value) if reducer else (
         value if any([isinstance(value, float), isinstance(value, int), isinstance(value, bool)]) else
-        default if value is None else
+        default if not non_empty_value(value) else
         value
     )
 

hestia_earth/utils/pipeline.py

@@ -4,7 +4,9 @@ import numpy as np
 
 from .tools import current_time_ms, non_empty_list, flatten
 from .api import find_related
-from .storage._s3_client import _load_from_bucket, _upload_to_bucket, _last_modified, _read_metadata, _update_metadata
+from .storage._s3_client import (
+    _load_from_bucket, _upload_to_bucket, _last_modified, _read_metadata, _update_metadata, _exists_in_bucket
+)
 from .storage._sns_client import _get_sns_client
 
 PROGRESS_EXT = '.progress'
@@ -156,6 +158,15 @@ def _node_id(node: dict): return node.get('@id', node.get('id'))
 def _node_path(node: dict, folder: str = ''): return join(folder, _node_type(node), f"{_node_id(node)}.jsonld")
 
 
+def _load_node(bucket: str, file_key: str): return json.loads(_load_from_bucket(bucket, file_key))
+
+
+def _cache_path(node: dict): return join(_node_type(node), f"{_node_id(node)}.cache")
+
+
+def _has_cache(bucket: str, node: dict): return _exists_in_bucket(bucket, _cache_path(node))
+
+
 def is_calculating(bucket: str, node: dict, folder: str = ''):
     return _read_metadata(bucket, _node_path(node, folder)).get(METADATA_PROGRESS_KEY, 'false') == 'true'
 

hestia_earth/utils/storage/_s3_client.py

@@ -95,3 +95,17 @@ def _list_bucket_objects(bucket: str, folder: str = ''):
         return contents
     except ClientError:
         return []
+
+
+def _delete_objects(bucket: str, objects: list):
+    from botocore.exceptions import ClientError
+    try:
+        return _get_s3_client().delete_objects(
+            Bucket=bucket,
+            Delete={
+                'Objects': objects,
+                'Quiet': True
+            }
+        )
+    except ClientError:
+        return None

hestia_earth/utils/version.py

@@ -1 +1 @@
-VERSION = '0.15.6'
+VERSION = '0.15.8'

{hestia_earth_utils-0.15.6.dist-info → hestia_earth_utils-0.15.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-utils
-Version: 0.15.6
+Version: 0.15.8
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.15.6.dist-info → hestia_earth_utils-0.15.8.dist-info}/RECORD

@@ -1,7 +1,7 @@
 hestia_earth/__init__.py,sha256=G-d438vPx7m_ks5e9XTtM3u7LDRO5dSSukibukWmyPM,56
 hestia_earth/utils/__init__.py,sha256=qEFeq3yuf3lQKVseALmL8aPM8fpCS54B_5pry00M3hk,76
 hestia_earth/utils/api.py,sha256=y0gw5pCCHNnFIhM62Hok_5eDtH3QDAZdkye_1mANMNs,9654
-hestia_earth/utils/blank_node.py,sha256=COmhDWc9mdk2EyXJIQHw_reGM_0Drs1Onh4lNP8rxa4,7345
+hestia_earth/utils/blank_node.py,sha256=1wc9zUkOvFhJS-YmuKexfIdYxfsp5KyJczLmHlW559Q,7375
 hestia_earth/utils/calculation_status.py,sha256=X7lbgVMD9luH1gj9lEcxd3_P2-u7e8ZPGCvX1czPZUo,2238
 hestia_earth/utils/cycle.py,sha256=rFLRL9X4KQ1UrE6fEPA_gV8KmwzrZpR3Ce56zg41lRk,1326
 hestia_earth/utils/date.py,sha256=SPQ69uxHiv1o3BqIkBKkM5XX_CmS20CB7g6u2rhsdh8,1807
@@ -10,13 +10,13 @@ hestia_earth/utils/emission.py,sha256=crHAJOQRI0NRHht0fFOFjEqcarsFqzbV4pDgk90dfM
 hestia_earth/utils/lookup.py,sha256=0RLqy3HPzkbhkRaO7fYoHU0jKhAYzI6QHMptMEbqTlg,10344
 hestia_earth/utils/lookup_utils.py,sha256=9pkJUKo3NEBspoN9yRPoDCzkxcbpnbLpC7Z0I6HiP7I,5369
 hestia_earth/utils/model.py,sha256=uUcrF07XmBzqLni8VSaP0HoebJnQ57kk0EOmhwYMbfI,4637
-hestia_earth/utils/pipeline.py,sha256=KDA1PBimFIUJKVytbxaXh33QGFW55I-2kcVDCRbYxn4,9277
+hestia_earth/utils/pipeline.py,sha256=O-6DPtK0U1lJ51LFGa1gM6pjkBJUfxOjNjY8LxQPXV0,9588
 hestia_earth/utils/request.py,sha256=bu7hkWKmFdXl2_Feawiam_x32whlclA9oP0asJyC69k,626
 hestia_earth/utils/stats.py,sha256=4t3op10xDJbGxWJEY1Jtyl302PYWyMFwLpsSkMlzQn8,34667
 hestia_earth/utils/table.py,sha256=RrTt-KF_QzjKiCpaAueoG6La1FG-Iusxw5NMDpoRBpQ,2861
 hestia_earth/utils/term.py,sha256=6LiUSc6KX3IOkfWF6fYkQ2tENCO8ENljcdDypxU6WtA,1060
 hestia_earth/utils/tools.py,sha256=eCppZ0gFR6RPF7TEraMLwLJnUCJb93xP0iEzFPcYpK0,4685
-hestia_earth/utils/version.py,sha256=hcbBawDU79tFjoFeJoCgvYJcMG4vzlMkCMPvi0NU6JE,19
+hestia_earth/utils/version.py,sha256=5hi4E88hS3MsyLqNGhaIdxzoN1dAL-AFFGZ8QV5HmWw,19
 hestia_earth/utils/pivot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hestia_earth/utils/pivot/_shared.py,sha256=sRE_PGe9u0-sFnCb0mdeM7vPOkFWt3-kDs8WSGQTQGk,875
 hestia_earth/utils/pivot/pivot_csv.py,sha256=Ed-sCKEqVIFJu71AncS7zlMkHbw5V15QLd5c5B_uxiE,12296
@@ -24,14 +24,14 @@ hestia_earth/utils/pivot/pivot_json.py,sha256=Rm4_mNaEgx-Jmq8sGIq_-jYMtZO78YXGxw
 hestia_earth/utils/storage/__init__.py,sha256=uNX6_EHWWnNUIm4Ng7L43-cQmuc6NGFAxXye85saIXQ,922
 hestia_earth/utils/storage/_azure_client.py,sha256=sevCZni04eknMql2DgUsWG23f7u0KvsXP7me1ZUBy00,1274
 hestia_earth/utils/storage/_local_client.py,sha256=IbzziUKY0QS3ybHFfgEpELqvafa7hQnZ-DdGdjQuypE,515
-hestia_earth/utils/storage/_s3_client.py,sha256=y-4rdqrAN45_W-pl5Zr7M4PL89JSjFaeq6lJiOFl3qQ,2637
+hestia_earth/utils/storage/_s3_client.py,sha256=aWt4i_vObcTDWcwyzOOnBBnMv1rxPZV3AJZQMn96Hok,2975
 hestia_earth/utils/storage/_sns_client.py,sha256=LowUatj78Egu6_Id6Rr7hZjfZx1WguS3lozB3yAwSps,347
-hestia_earth_utils-0.15.6.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
-hestia_earth_utils-0.15.6.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
+hestia_earth_utils-0.15.8.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
+hestia_earth_utils-0.15.8.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
 tests/pivot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/pivot/test_pivot_csv.py,sha256=aYni7o3QDPSgtVxVCspEetotgpYHY7Lz5VHf-DR89gw,8131
 tests/pivot/test_pivot_json.py,sha256=UYTAN4AZhzVicIYsU1A2VgJcctUXohjHppg6s-pqwcg,8287
-hestia_earth_utils-0.15.6.dist-info/METADATA,sha256=UOzNkiXzlimsJgwkjiWm7uVW_MBa9CFiZV7u-0dE63o,1757
-hestia_earth_utils-0.15.6.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-hestia_earth_utils-0.15.6.dist-info/top_level.txt,sha256=1dqA9TqpOLTEgpqa-YBsmbCmmNU1y56AtfFGEceZ2A0,19
-hestia_earth_utils-0.15.6.dist-info/RECORD,,
+hestia_earth_utils-0.15.8.dist-info/METADATA,sha256=Gs1yR4BSYz4ytA17VCFUOg8ZUJVAb0vmgC39LHIpo9g,1757
+hestia_earth_utils-0.15.8.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+hestia_earth_utils-0.15.8.dist-info/top_level.txt,sha256=1dqA9TqpOLTEgpqa-YBsmbCmmNU1y56AtfFGEceZ2A0,19
+hestia_earth_utils-0.15.8.dist-info/RECORD,,