dao-scripts 1.4.6__tar.gz → 1.5.1__tar.gz

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## 1.5.1 - 2024-06-05
+- Multiple small bugfixes
+
+## 1.5.0 - 2024-06-04
+- Avoid errors when no new data is available
+
 ## 1.4.1 - 2024-06-03
 - Added debug logging to `dao-utils-upload-dw`
 

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dao-scripts
-Version: 1.4.6
+Version: 1.5.1
 Summary: "A tool to download data to monitor DAO activity"
 Home-page: https://github.com/Grasia/dao-scripts
 Author: David Davó

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/dao_scripts.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dao-scripts
-Version: 1.4.6
+Version: 1.5.1
 Summary: "A tool to download data to monitor DAO activity"
 Home-page: https://github.com/Grasia/dao-scripts
 Author: David Davó

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/src/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.4.6'
-__version_tuple__ = version_tuple = (1, 4, 6)
+__version__ = version = '1.5.1'
+__version_tuple__ = version_tuple = (1, 5, 1)

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/src/common/blockscout.py

@@ -44,7 +44,7 @@ class BlockscoutBallancesCollector(NetworkCollector):
     def endpoint(self) -> str:
         return ENDPOINTS[self.network]['blockscout']
 
-    def _get_from_address(self, addr: str, retry: int = 0, maxretries: int = 3, block: Union[int, Block, None] = None, ignore_errors=False) -> pd.DataFrame: # noqa: C901
+    def _get_from_address(self, addr: str, retry: int = 0, maxretries: int = 5, block: Union[int, Block, None] = None, ignore_errors=False) -> pd.DataFrame: # noqa: C901
         if retry >= maxretries:
             raise ValueError(f"Too many retries {retry}/{maxretries}")
 

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/src/common/thegraph.py

@@ -2,6 +2,7 @@ from typing import Optional, Callable, Any
 from abc import ABC, abstractmethod
 
 from gql.dsl import DSLField
+from graphql.language import visit, Visitor
 
 import pandas as pd
 
@@ -32,6 +33,34 @@ def partial_query(q: Callable[..., DSLField], w: dict[str, Any]) -> Callable[...
         return q(**add_where(kwargs, **w))
     return wrapper
 
+class ColumnsVisitor(Visitor):
+    def __init__(self):
+        super().__init__()
+        self.columns = []
+        self._bases = []
+
+    def enter_field(self, node, *args):
+        self._bases.append(node)
+
+    def leave_field(self, node, *args):
+        self._bases.pop()
+
+    def leave_selection_set(self, node, *_args):
+        base = ".".join([x.name.value for x in self._bases])
+        for s in node.selections:
+            # Skip non-leaf nodes
+            if s.selection_set:
+                continue
+            name = s.name.value
+            fullname = ".".join([base, name]) if base else name
+            self.columns.append(fullname)
+    
+def get_columns_from_query(q: DSLField) -> list[str]:
+    c = ColumnsVisitor()
+    # We use selection_set directly to avoid putting the name of the query
+    # i.e: returning id instead of moloches.id
+    visit(q.ast_field.selection_set, c)
+    return c.columns
 
 class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
     def __init__(
@@ -83,7 +112,13 @@ class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
         return df
 
     def transform_to_df(self, data: list[dict[str, Any]], skip_post: bool=False) -> pd.DataFrame:
-        df = pd.DataFrame.from_dict(pd.json_normalize(data))
+        if data:
+            df = pd.DataFrame.from_dict(pd.json_normalize(data))
+        else:
+            df = pd.DataFrame(columns=get_columns_from_query(self.query()))
+
+        if (s1 := set(df.columns)) != (s2 := set(get_columns_from_query(self.query()))):
+            raise ValueError(f"Received columns are not the expected columns: {s1} != {s2}")
 
         # For compatibility reasons we change from . to snake case
         def dotsToSnakeCase(str: str) -> str:
@@ -165,7 +200,6 @@ class TheGraphCollector(NetworkCollector, UpdatableCollector, ABC):
             self.logger.error('Subgraph has indexing errors')
             return False
         
-        # TODO: Save the block info to use it later in run
         self._indexer_block = Block(r['block'])
 
         if not check_deployment:

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/src/daohaus/runner.py

@@ -156,7 +156,7 @@ class TokenBalancesCollector(TheGraphCollector):
         def coalesce_bank_type(df: pd.DataFrame) -> pd.DataFrame:
             bank_idx = ['guildBank', 'memberBank', 'ecrowBank']
 
-            df['bank'] = df[bank_idx].idxmax(1)
+            df['bank'] = df[bank_idx].idxmax(1).astype(str)
             df['bank'] = df['bank'].str.lower()
             df['bank'] = df['bank'].str.replace('bank', '')
             df = df.drop(columns=bank_idx)

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/src/metadata.py

@@ -25,7 +25,7 @@ class Block:
             self.number = int(init["number"]) if "number" in init else self.number
             self.id = init["id"] if "id" in init else self.id
 
-            if "timestamp" in init:
+            if init.get("timestamp", None):
                 if isinstance(init['timestamp'], int) or init["timestamp"].isdigit():
                     self.timestamp = datetime.fromtimestamp(int(init["timestamp"]))
                 else:

{dao_scripts-1.4.6 → dao_scripts-1.5.1}/src/utils/uploadDataWarehouse.py

@@ -80,6 +80,7 @@ def archiveToZenodo(tmpdir, max_retries: int, sleep_seconds: int = 60):
                     print(f"Retrying upload to Zenodo {i}/{max_retries}")
                     sleep(sleep_seconds)
                 else:
+                    print("Errored response:", e.response.content, file=sys.stderr)
                     raise e
             i += 1