hestia-earth-utils 0.15.5__tar.gz → 0.15.7__tar.gz

{hestia_earth_utils-0.15.5 → hestia_earth_utils-0.15.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_utils
-Version: 0.15.5
+Version: 0.15.7
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.15.5 → hestia_earth_utils-0.15.7}/hestia_earth/utils/pipeline.py

@@ -4,7 +4,9 @@ import numpy as np
 
 from .tools import current_time_ms, non_empty_list, flatten
 from .api import find_related
-from .storage._s3_client import _load_from_bucket, _upload_to_bucket, _last_modified, _read_metadata, _update_metadata
+from .storage._s3_client import (
+    _load_from_bucket, _upload_to_bucket, _last_modified, _read_metadata, _update_metadata, _exists_in_bucket
+)
 from .storage._sns_client import _get_sns_client
 
 PROGRESS_EXT = '.progress'
@@ -156,6 +158,15 @@ def _node_id(node: dict): return node.get('@id', node.get('id'))
 def _node_path(node: dict, folder: str = ''): return join(folder, _node_type(node), f"{_node_id(node)}.jsonld")
 
 
+def _load_node(bucket: str, file_key: str): return json.loads(_load_from_bucket(bucket, file_key))
+
+
+def _cache_path(node: dict): return join(_node_type(node), f"{_node_id(node)}.cache")
+
+
+def _has_cache(bucket: str, node: dict): return _exists_in_bucket(bucket, _cache_path(node))
+
+
 def is_calculating(bucket: str, node: dict, folder: str = ''):
     return _read_metadata(bucket, _node_path(node, folder)).get(METADATA_PROGRESS_KEY, 'false') == 'true'
 

{hestia_earth_utils-0.15.5 → hestia_earth_utils-0.15.7}/hestia_earth/utils/storage/_s3_client.py

@@ -81,3 +81,31 @@ def _upload_to_bucket(bucket: str, key: str, body, content_type: str):
         )
     except ClientError:
         return None
+
+
+def _list_bucket_objects(bucket: str, folder: str = ''):
+    from botocore.exceptions import ClientError
+    try:
+        paginator = _get_s3_client().get_paginator('list_objects_v2')
+        pages = paginator.paginate(Bucket=bucket, Prefix=folder)
+
+        contents = []
+        for page in pages:
+            contents.extend(page.get('Contents', []))
+        return contents
+    except ClientError:
+        return []
+
+
+def _delete_objects(bucket: str, objects: list):
+    from botocore.exceptions import ClientError
+    try:
+        return _get_s3_client().delete_objects(
+            Bucket=bucket,
+            Delete={
+                'Objects': objects,
+                'Quiet': True
+            }
+        )
+    except ClientError:
+        return None

hestia_earth_utils-0.15.7/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = '0.15.7'

{hestia_earth_utils-0.15.5 → hestia_earth_utils-0.15.7}/hestia_earth_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-utils
-Version: 0.15.5
+Version: 0.15.7
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

hestia_earth_utils-0.15.5/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = '0.15.5'