dao-scripts 1.4.5__py3-none-any.whl → 1.5.0__py3-none-any.whl

dao_analyzer/cache_scripts/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.5'
-__version_tuple__ = version_tuple = (1, 4, 5)
+__version__ = version = '1.5.0'
+__version_tuple__ = version_tuple = (1, 5, 0)

dao_analyzer/cache_scripts/common/thegraph.py

@@ -2,6 +2,7 @@ from typing import Optional, Callable, Any
 from abc import ABC, abstractmethod
 
 from gql.dsl import DSLField
+from graphql.language import visit, Visitor
 
 import pandas as pd
 
@@ -32,6 +33,34 @@ def partial_query(q: Callable[..., DSLField], w: dict[str, Any]) -> Callable[...
         return q(**add_where(kwargs, **w))
     return wrapper
 
+class ColumnsVisitor(Visitor):
+    def __init__(self):
+        super().__init__()
+        self.columns = []
+        self._bases = []
+
+    def enter_field(self, node, *args):
+        self._bases.append(node)
+
+    def leave_field(self, node, *args):
+        self._bases.pop()
+
+    def leave_selection_set(self, node, *_args):
+        base = ".".join([x.name.value for x in self._bases])
+        for s in node.selections:
+            # Skip non-leaf nodes
+            if s.selection_set:
+                continue
+            name = s.name.value
+            fullname = ".".join([base, name]) if base else name
+            self.columns.append(fullname)
+    
+def get_columns_from_query(q: DSLField) -> list[str]:
+    c = ColumnsVisitor()
+    # We use selection_set directly to avoid putting the name of the query
+    # i.e: returning id instead of moloches.id
+    visit(q.ast_field.selection_set, c)
+    return c.columns
 
 class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
     def __init__(
@@ -83,7 +112,12 @@ class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
         return df
 
     def transform_to_df(self, data: list[dict[str, Any]], skip_post: bool=False) -> pd.DataFrame:
-        df = pd.DataFrame.from_dict(pd.json_normalize(data))
+        if data:
+            df = pd.DataFrame.from_dict(pd.json_normalize(data))
+        else:
+            df = pd.DataFrame(columns=get_columns_from_query(self.query()))
+
+        assert set(df.columns) == set(get_columns_from_query(self.query()))
 
         # For compatibility reasons we change from . to snake case
         def dotsToSnakeCase(str: str) -> str:
@@ -147,7 +181,7 @@ class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
 
         return no_errors
 
-    def check_subgraph_health(self) -> bool:
+    def check_subgraph_health(self, check_deployment: bool = True) -> bool:
         ds = self.schema
         q = ds.Query._meta().select(
             ds._Meta_.deployment,
@@ -165,8 +199,10 @@ class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
             self.logger.error('Subgraph has indexing errors')
             return False
         
-        # TODO: Save the block info to use it later in run
         self._indexer_block = Block(r['block'])
+
+        if not check_deployment:
+            return True
         
         return self.check_deployment_health(r['deployment'])
 
@@ -188,6 +224,7 @@ class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
 
     def run(self, force=False, block: Optional[Block] = None, prev_block: Optional[Block] = None):
         self.logger.info(f"Running The Graph collector with block: {block}, prev_block: {prev_block}")
+        assert self.check_subgraph_health(check_deployment=False) # Just update the _indexer_block
         if block and self._indexer_block:
             assert self._indexer_block >= block, f"Block number {block} is not indexed yet ({self._indexer_block})"
         

dao_analyzer/cache_scripts/utils/uploadDataWarehouse.py

@@ -80,6 +80,7 @@ def archiveToZenodo(tmpdir, max_retries: int, sleep_seconds: int = 60):
                     print(f"Retrying upload to Zenodo {i}/{max_retries}")
                     sleep(sleep_seconds)
                 else:
+                    print("Errored response:", e.response.content, file=sys.stderr)
                     raise e
             i += 1
 

{dao_scripts-1.4.5.dist-info → dao_scripts-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dao-scripts
-Version: 1.4.5
+Version: 1.5.0
 Summary: "A tool to download data to monitor DAO activity"
 Home-page: https://github.com/Grasia/dao-scripts
 Author: David Davó

{dao_scripts-1.4.5.dist-info → dao_scripts-1.5.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-dao_scripts-1.4.5-py3.12-nspkg.pth,sha256=_0DzrHlnOxaQL5l1mMdHT1BtXZwXuB6MitMF5phpOhk,497
+dao_scripts-1.5.0-py3.12-nspkg.pth,sha256=_0DzrHlnOxaQL5l1mMdHT1BtXZwXuB6MitMF5phpOhk,497
 dao_analyzer/cache_scripts/__init__.py,sha256=Urg-J2hvyzGcgE2Z-ba71XpcHi1YlPZgex86OFSd_0A,116
 dao_analyzer/cache_scripts/__main__.py,sha256=aNVhgUXjB4rzTefrX3d6dKeLOX3G_Ezs82pI8QCpwx0,61
-dao_analyzer/cache_scripts/_version.py,sha256=HhvBOPJ1g3k0gtuZ-SLO8_784kPPejMh80IxITX8Cvc,411
+dao_analyzer/cache_scripts/_version.py,sha256=OYzqgMEgfFG0au4hzbEdgYI-c7Hxo3wdBtrpEjK1RoY,411
 dao_analyzer/cache_scripts/argparser.py,sha256=uOBVdRxmgR8H0dZMu2AQyvro6WvkZFKUrf7vJ6Mvw48,3574
 dao_analyzer/cache_scripts/config.py,sha256=6Ooh-SpTtNCHcl2sE637-XMCCCNZBmATJRyD_eguWr0,2180
 dao_analyzer/cache_scripts/endpoints.json,sha256=JLkXgE7P7ZXpAsorIq0jCVCoPoK4_YeyfTgCmtPdo14,1278
@@ -16,17 +16,17 @@ dao_analyzer/cache_scripts/common/api_requester.py,sha256=objbyqKVVEfNFmffds3BC2
 dao_analyzer/cache_scripts/common/blockscout.py,sha256=8-Xz7yhG8jtZsWBA2Ik3T3xqzDB0Bw89-EI8vcIB8Yk,5041
 dao_analyzer/cache_scripts/common/common.py,sha256=Vj_UHqcCQRiKBwCV1skVTZoDl-eGOtclI8ps28BrD6A,10910
 dao_analyzer/cache_scripts/common/cryptocompare.py,sha256=jaHdJ-X_AXEUdVRMVtyK6xwHNJdhTs2Ps44QMD0iuIY,2203
-dao_analyzer/cache_scripts/common/thegraph.py,sha256=Q2932YDKXXgN6_03PyIwABCGYD-IfKVY6OOx5RxWlGU,6883
+dao_analyzer/cache_scripts/common/thegraph.py,sha256=XoGh7WbLkRiGyQQ-f70xh9vqU-9FenIBGjBhmap1PFo,8159
 dao_analyzer/cache_scripts/daohaus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dao_analyzer/cache_scripts/daohaus/runner.py,sha256=nQ7YN_Fbfx0KsHkq47hShwTS82jbo285x60Z-31vj8M,7549
 dao_analyzer/cache_scripts/daostack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dao_analyzer/cache_scripts/daostack/runner.py,sha256=emjzhZzGMTRGHKljll_lX-uMsGwaIsZBSi4jV1Lsh8Y,10144
 dao_analyzer/cache_scripts/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dao_analyzer/cache_scripts/utils/uploadDataWarehouse.py,sha256=ID5AQxfDMKkatiZnRQRWeRm5ztQhbLutMqPveWMuzOM,4614
-dao_scripts-1.4.5.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-dao_scripts-1.4.5.dist-info/METADATA,sha256=8qzsNOrJEzwm_2QAxm0-I9nvdY4TEY0GPkOO0Us3KYc,8193
-dao_scripts-1.4.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dao_scripts-1.4.5.dist-info/entry_points.txt,sha256=dHhgA8pcQ2YbIlMuqPPSh7fiQkfHE8ioFPUcbBvXWDc,207
-dao_scripts-1.4.5.dist-info/namespace_packages.txt,sha256=gbsNupMgtHQXOyBinbeLMo06f278s8My_9N-CrZh3Bk,13
-dao_scripts-1.4.5.dist-info/top_level.txt,sha256=gbsNupMgtHQXOyBinbeLMo06f278s8My_9N-CrZh3Bk,13
-dao_scripts-1.4.5.dist-info/RECORD,,
+dao_analyzer/cache_scripts/utils/uploadDataWarehouse.py,sha256=dBI_XKCWqblhCh-w0-ewoP82zHZBLxAH8SH44y8nxoU,4698
+dao_scripts-1.5.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dao_scripts-1.5.0.dist-info/METADATA,sha256=Wkg6MlLUhusCvLT-y7buIUb8a0Qg19_YB1_WvNJVsy0,8193
+dao_scripts-1.5.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dao_scripts-1.5.0.dist-info/entry_points.txt,sha256=dHhgA8pcQ2YbIlMuqPPSh7fiQkfHE8ioFPUcbBvXWDc,207
+dao_scripts-1.5.0.dist-info/namespace_packages.txt,sha256=gbsNupMgtHQXOyBinbeLMo06f278s8My_9N-CrZh3Bk,13
+dao_scripts-1.5.0.dist-info/top_level.txt,sha256=gbsNupMgtHQXOyBinbeLMo06f278s8My_9N-CrZh3Bk,13
+dao_scripts-1.5.0.dist-info/RECORD,,