dao-scripts 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

dao_analyzer/cache_scripts/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.2.1'
-__version_tuple__ = version_tuple = (1, 2, 1)
+__version__ = version = '1.3.0'
+__version_tuple__ = version_tuple = (1, 3, 0)

dao_analyzer/cache_scripts/aragon/runner.py

@@ -16,13 +16,13 @@ import json
 
 from ..aragon import __name__ as aragon_module_name
 from ..common.cryptocompare import CCPricesCollector
-from ..common import ENDPOINTS, Collector
-from ..common.graphql import GraphQLCollector, GraphQLRunner
+from ..common import ENDPOINTS, Collector, NetworkRunner
+from ..common.thegraph import TheGraphCollector
 from ..common.blockscout import BlockscoutBallancesCollector
 
-class AppsCollector(GraphQLCollector):
+class AppsCollector(TheGraphCollector):
     def __init__(self, runner, network: str):
-        super().__init__('apps', runner, endpoint=ENDPOINTS[network]['aragon'], network=network)
+        super().__init__('apps', network, ENDPOINTS[network]['aragon'], runner)
 
     def query(self, **kwargs) -> DSLField:
         ds = self.schema
@@ -39,9 +39,9 @@ class BalancesCollector(BlockscoutBallancesCollector):
     def __init__(self, runner, base, network: str):
         super().__init__(runner, addr_key='recoveryVault', base=base, network=network)
 
-class CastsCollector(GraphQLCollector):
+class CastsCollector(TheGraphCollector):
     def __init__(self, runner, network: str):
-        super().__init__('casts', runner, endpoint=ENDPOINTS[network]['aragon_voting'], network=network, pbar_enabled=False)
+        super().__init__('casts', network, ENDPOINTS[network]['aragon_voting'], runner, pbar_enabled=False)
 
         @self.postprocessor
         def changeColumnNames(df: pd.DataFrame) -> pd.DataFrame:
@@ -66,11 +66,11 @@ class CastsCollector(GraphQLCollector):
             )
         )
 
-class OrganizationsCollector(GraphQLCollector):
+class OrganizationsCollector(TheGraphCollector):
     DAO_NAMES=pkgutil.get_data(aragon_module_name, 'dao_names.json')
 
     def __init__(self, runner, network: str):
-        super().__init__('organizations', runner, endpoint=ENDPOINTS[network]['aragon'], network=network)
+        super().__init__('organizations', network, ENDPOINTS[network]['aragon'], runner)
 
         @self.postprocessor
         def set_dead_recoveryVault(df: pd.DataFrame) -> pd.DataFrame:
@@ -113,9 +113,9 @@ class OrganizationsCollector(GraphQLCollector):
             ds.Organization.recoveryVault
         )
 
-class MiniMeTokensCollector(GraphQLCollector):
+class MiniMeTokensCollector(TheGraphCollector):
     def __init__(self, runner, network: str):
-        super().__init__('miniMeTokens', runner, endpoint=ENDPOINTS[network]['aragon_tokens'], network=network, pbar_enabled=False)
+        super().__init__('miniMeTokens', network, ENDPOINTS[network]['aragon_tokens'], runner, pbar_enabled=False)
 
     def query(self, **kwargs) -> DSLField:
         ds = self.schema
@@ -131,9 +131,9 @@ class MiniMeTokensCollector(GraphQLCollector):
             ds.MiniMeToken.lastUpdateAt
         )
 
-class TokenHoldersCollector(GraphQLCollector):
-    def __init__(self, runner: GraphQLRunner, network: str):
-        super().__init__('tokenHolders', runner, endpoint=ENDPOINTS[network]['aragon_tokens'], network=network)
+class TokenHoldersCollector(TheGraphCollector):
+    def __init__(self, runner: NetworkRunner, network: str):
+        super().__init__('tokenHolders', network, ENDPOINTS[network]['aragon_tokens'], runner)
 
         @self.postprocessor
         def add_minitokens(df: pd.DataFrame) -> pd.DataFrame:
@@ -156,9 +156,9 @@ class TokenHoldersCollector(GraphQLCollector):
 class TokenPricesCollector(CCPricesCollector):
     pass
 
-class ReposCollector(GraphQLCollector):
+class ReposCollector(TheGraphCollector):
     def __init__(self, runner, network: str):
-        super().__init__('repos', runner, network=network, endpoint=ENDPOINTS[network]['aragon'])
+        super().__init__('repos', network, ENDPOINTS[network]['aragon'], runner)
 
     def query(self, **kwargs) -> DSLField:
         ds = self.schema
@@ -170,9 +170,9 @@ class ReposCollector(GraphQLCollector):
             ds.Repo.appCount
         )
 
-class TransactionsCollector(GraphQLCollector):
+class TransactionsCollector(TheGraphCollector):
     def __init__(self, runner, network: str):
-        super().__init__('transactions', runner, network=network, endpoint=ENDPOINTS[network]['aragon_finance'])
+        super().__init__('transactions', network, ENDPOINTS[network]['aragon_finance'], runner)
 
     def query(self, **kwargs) -> DSLField:
         ds = self.schema
@@ -188,9 +188,9 @@ class TransactionsCollector(GraphQLCollector):
             ds.Transaction.reference
         )
 
-class VotesCollector(GraphQLCollector):
+class VotesCollector(TheGraphCollector):
     def __init__(self, runner, network: str):
-        super().__init__('votes', runner, network=network, endpoint=ENDPOINTS[network]['aragon_voting'])
+        super().__init__('votes', network, ENDPOINTS[network]['aragon_voting'], runner)
 
     def query(self, **kwargs) -> DSLField:
         ds = self.schema
@@ -214,14 +214,12 @@ class VotesCollector(GraphQLCollector):
             ds.Vote.metadata,
         )
 
-class AragonRunner(GraphQLRunner):
+class AragonRunner(NetworkRunner):
     name: str = 'aragon'
 
     def __init__(self, dw=None):
         super().__init__(dw)
         self._collectors: List[Collector] = []
-        ## TODO: Fix aragon-tokens xdai subgraph and redeploy
-        self.networks = ['mainnet']
 
         for n in self.networks: 
             self._collectors.extend([
@@ -231,11 +229,10 @@ class AragonRunner(GraphQLRunner):
                 ReposCollector(self, n),
                 TransactionsCollector(self, n),
                 TokenHoldersCollector(self, n),
-                VotesCollector(self, n)
+                VotesCollector(self, n),
+                oc := OrganizationsCollector(self, n),
+                BalancesCollector(self, oc, n),
             ])
-            oc = OrganizationsCollector(self, n)
-            bc = BalancesCollector(self, oc, n)
-            self._collectors += [oc, bc]
         
         self._collectors.append(CCPricesCollector(self))
 

dao_analyzer/cache_scripts/argparser.py

@@ -3,7 +3,6 @@ from typing import List
 
 from datetime import datetime
 import pathlib
-import os
 
 from . import config
 
@@ -26,13 +25,13 @@ class CacheScriptsArgParser(ArgumentParser):
             help="The platforms to update. Every platform is updated by default."
         )
         self.add_argument(
-            "--ignore-errors",
-            default=True,
+            "--raise-runner-errors",
+            default=False,
             action=BooleanOptionalAction,
             help="Whether to ignore errors and continue")
         self.add_argument(
             "-d", "--debug",
-            action='store_true', default=False,
+            dest="DEBUG", action='store_true', default=False,
             help="Shows debug info"
         )
         self.add_argument(
@@ -45,11 +44,6 @@ class CacheScriptsArgParser(ArgumentParser):
             action="store_true", default=False,
             help="Removes the datawarehouse folder before doing anything"
         )
-        self.add_argument(
-            "--skip-daohaus-names",
-            action="store_true", default=False,
-            help="Skips the step of getting Daohaus Moloch's names, which takes some time"
-        )
         self.add_argument(
             "--skip-token-balances",
             action="store_true", default=False,
@@ -70,7 +64,7 @@ class CacheScriptsArgParser(ArgumentParser):
             help="Collectors to run. For example: aragon/casts"
         )
         self.add_argument(
-            "--block-datetime",
+            "-B", "--block-datetime",
             required=False,
             type=datetime.fromisoformat,
             help="Get data up to a block datetime (input in ISO format)"
@@ -82,15 +76,9 @@ class CacheScriptsArgParser(ArgumentParser):
             type=pathlib.Path,
             default=config.DEFAULT_DATAWAREHOUSE
         )
-        self.add_argument(
-            "--cc-api-key",
-            help="Set the CryptoCompare API key (overrides environment variable)",
-            required=False,
-            type=str,
-            default=os.getenv('DAOA_CC_API_KEY')
-        )
         self.add_argument(
             "--only-updatable",
+            dest='run_only_updatable',
             help=SUPPRESS, # "Run only updatable collectors (only for testing)",
             action='store_true',
             required=False,
@@ -99,7 +87,14 @@ class CacheScriptsArgParser(ArgumentParser):
         self.add_argument(
             "--daostack-all",
             help="Obtain all DAOs in DAOstack, not only registered ones",
-            action='store_true',
+            action='store_false',
             required=False,
-            default=False,
+            # For the user is "daostack_all", for us is "registered only"
+            dest='daostack__registered_only',
+            default=True,
+        )
+        self.add_argument(
+            "--daohaus-skip-names",
+            action="store_true", default=False, dest='daohaus__skip_names',
+            help="Skips the step of getting Daohaus Moloch's names, which takes some time"
         )

dao_analyzer/cache_scripts/common/__init__.py

@@ -7,10 +7,12 @@
         <david@ddavo.me>
 """
 
-from .common import Collector, Runner, ENDPOINTS
+from .common import Collector, Runner, ENDPOINTS, NetworkRunner, NetworkCollector
 
 __all__ = [
     'Collector',
     'Runner',
     'ENDPOINTS',
+    'NetworkRunner',
+    'NetworkCollector',
 ]

dao_analyzer/cache_scripts/common/api_requester.py

@@ -17,7 +17,7 @@ from functools import partial
 import logging
 import sys
 from tqdm import tqdm
-from typing import Dict, List, Union, Iterable
+from typing import Optional, Union, Iterable, Callable
 
 class GQLQueryException(Exception):
     def __init__(self, errors, msg="Errors in GraphQL Query"):
@@ -84,22 +84,23 @@ class GQLRequester:
         self.__transport = RequestsHTTPTransport(endpoint)
         self.__client: Client = Client(transport=self.__transport, fetch_schema_from_transport=introspection)
         self.pbar = IndexProgressBar if pbar_enabled else RequestProgressSpinner
+        self.logger = logging.getLogger('dao-scripts.gql-requester')
 
-        logging.debug(f"Invoked ApiRequester with endpoint: {endpoint}")
+        self.logger.debug(f"Invoked ApiRequester with endpoint: {endpoint}")
 
     def get_schema(self) -> DSLSchema:
         with self.__client:
             assert(self.__client.schema is not None)
             return DSLSchema(self.__client.schema)
 
-    def request(self, query: Union[DSLQuery, DSLField, str]) -> Dict:
+    def request(self, query: Union[DSLQuery, DSLField, str]) -> dict:
         """
         Requests data from endpoint.
         """
         if isinstance(query, DSLField):
             query = DSLQuery(query)
 
-        logging.debug(f"Requesting: {query}")
+        self.logger.debug(f"Requesting: {query}")
 
         if isinstance(query, DSLQuery):
             result = self.__client.execute(dsl_gql(query))
@@ -111,14 +112,14 @@ class GQLRequester:
 
         return result
 
-    def request_single(self, q: Union[DSLQuery, DSLField, str]) -> Dict:
+    def request_single(self, q: Union[DSLQuery, DSLField, str]) -> dict:
         result = self.request(q)
         if result and len(result.values()) == 1:
             return next(iter(result.values()))
         else:
             raise 
 
-    def n_requests(self, query:DSLType, index='id', last_index: str = "", block_hash: str = None) -> List[Dict]:
+    def n_requests(self, query: Callable[..., DSLField], index='id', last_index: str = "", block_hash: Optional[str] = None) -> list[dict]:
         """
         Requests all chunks from endpoint.
 
@@ -128,8 +129,8 @@ class GQLRequester:
             * last_index: used to continue the request
             * block_hash: make the request to that block hash
         """
-        elements: List[Dict] = list()
-        result = Dict
+        elements: list[dict] = list()
+        result = dict()
         
         # do-while structure
         exit: bool = False
@@ -171,18 +172,18 @@ class CryptoCompareQueryException(Exception):
 class CryptoCompareRequester:
     BASEURL = 'https://min-api.cryptocompare.com/data/'
 
-    def __init__(self, api_key: str = None, pbar_enabled: bool = True):
-        self.logger = logging.getLogger('ccrequester')
+    def __init__(self, api_key: Optional[str] = None, pbar_enabled: bool = True):
+        self.logger = logging.getLogger('dao-scripts.ccrequester')
         self.pbar = partial(tqdm, delay=1, file=sys.stdout, desc="Requesting",
             dynamic_ncols=True)
         
         if not api_key:
-            logging.warning(f'Invalid api key: {api_key}')
+            self.logger.warning('CryptoCompare API key is not set')
             api_key = ""
 
         self.api_key = api_key
 
-    def _build_headers(self) -> Dict[str, str]:
+    def _build_headers(self) -> dict[str, str]:
         return {
           'Authorization': 'Apikey ' + self.api_key
         }
@@ -204,7 +205,7 @@ class CryptoCompareRequester:
             if 'Data' not in j:
                 return j
             if "HasWarning" in j and j["HasWarning"]:
-                logging.warning("Warning in query", r.url, ":", j["Message"])
+                self.logger.warning("Warning in query", r.url, ":", j["Message"])
             if j["Type"] == 100:
                 return j['Data']
         

dao_analyzer/cache_scripts/common/blockscout.py

@@ -1,10 +1,9 @@
 import pandas as pd
 import requests
-import logging
 from functools import partial
 from tqdm import tqdm
 from time import sleep
-from typing import Union
+from typing import Union, Optional
 
 import numpy as np
 
@@ -14,14 +13,13 @@ from .. import config
 from . import ENDPOINTS
 from .common import NetworkCollector, solve_decimals
 from .cryptocompare import cc_postprocessor
-from .graphql import GraphQLCollector
 
 MSG_NO_TOKENS_FOUND = "No tokens found"
 
 class BlockscoutBallancesCollector(NetworkCollector):
     ERR_SLEEP = 60
 
-    def __init__(self, runner, base: GraphQLCollector, name: str='tokenBalances', network: str='mainnet', addr_key: str='id'):
+    def __init__(self, runner, base: NetworkCollector, name: str='tokenBalances', network: str='mainnet', addr_key: str='id'):
         """ Initializes a ballance collector that uses blockscout
         
         Parameters:
@@ -37,7 +35,7 @@ class BlockscoutBallancesCollector(NetworkCollector):
 
     def verify(self) -> bool:
         if config.skip_token_balances:
-            logging.warning('Skipping token balances because --skip-token-balances flag was set')
+            self.logger.warning('Skipping token balances because --skip-token-balances flag was set')
             return False
         else:
             return super().verify()
@@ -46,7 +44,7 @@ class BlockscoutBallancesCollector(NetworkCollector):
     def endpoint(self) -> str:
         return ENDPOINTS[self.network]['blockscout']
 
-    def _get_from_address(self, addr: str, retry: int = 0, maxretries: int = 3, block: Union[int, Block] = None, ignore_errors=False) -> pd.DataFrame: # noqa: C901
+    def _get_from_address(self, addr: str, retry: int = 0, maxretries: int = 3, block: Union[int, Block, None] = None, ignore_errors=False) -> pd.DataFrame: # noqa: C901
         if retry >= maxretries:
             raise ValueError(f"Too many retries {retry}/{maxretries}")
 
@@ -83,32 +81,32 @@ class BlockscoutBallancesCollector(NetworkCollector):
             elif j['message'] == MSG_NO_TOKENS_FOUND:
                 return pd.DataFrame()
             else:
-                logging.warning(f"Status {j['status']}, message: {j['message']}")
+                self.logger.warning(f"Status {j['status']}, message: {j['message']}")
                 return pd.DataFrame()
         elif r.status_code == 429: # Too many requests
-            logging.warning(f"Too many requests, sleep and retry {retry}/{maxretries} time")
+            self.logger.warning(f"Too many requests, sleep and retry {retry}/{maxretries} time")
             sleep(self.ERR_SLEEP)
             return self._get_from_address(addr, retry=retry+1, maxretries=maxretries)
         elif r.status_code == 503:
-            logging.warning(f"Service unavailable, sleep and retry {retry}/{maxretries} time")
+            self.logger.warning(f"Service unavailable, sleep and retry {retry}/{maxretries} time")
             sleep(self.ERR_SLEEP)
             return self._get_from_address(addr, retry=retry+1, maxretries=maxretries)
         elif r.status_code == 504: # Gateway Time-out (Response too large)
-            logging.warning(f"Requests returned Gateway Time-out, ignoring response for addr {addr}")
+            self.logger.warning(f"Requests returned Gateway Time-out, ignoring response for addr {addr}")
             return pd.DataFrame() 
         else:
-            logging.error(f'Requests failed for address "{addr}" with status code {r.status_code}: {r.reason}')
+            self.logger.error(f'Requests failed for address "{addr}" with status code {r.status_code}: {r.reason}')
             if ignore_errors:
                 return pd.DataFrame()
             else:
                 raise ValueError(f"Requests failed for address {addr[:12]}... with status code {r.status_code}: {r.reason}")
 
-    def run(self, force=False, block: Block = None, prev_block: Block = None, **kwargs):
+    def run(self, force=False, block: Optional[Block] = None, prev_block: Optional[Block] = None, **kwargs):
         # For each of the DAOs in the df, get the token balance
         addresses = self.base.df[self.addr_key].drop_duplicates()
 
         if addresses.empty:
-            logging.warning("No addresses returned, not running blockscout collector")
+            self.logger.warning("No addresses returned, not running blockscout collector")
             return
 
         ptqdm = partial(tqdm, delay=1, desc="Requesting token balances", 

dao_analyzer/cache_scripts/common/common.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import List, Dict, Iterable
+from typing import Optional, Iterable
 import logging
 import sys
 import json
@@ -19,7 +19,24 @@ from .. import config
 from dao_analyzer import cache_scripts
 
 # To be able to obtain endpoints.json
-ENDPOINTS: Dict = json.loads(pkgutil.get_data(cache_scripts.__name__, 'endpoints.json'))
+ENDPOINTS: dict = json.loads(pkgutil.get_data(cache_scripts.__name__, 'endpoints.json'))
+THE_GRAPH_URL_TEMPLATE = 'https://gateway-arbitrum.network.thegraph.com/api/{api_key}/subgraphs/id/{subgraph_id}'
+THE_GRAPH_DEPLOYMENT_TEMPLATE = 'https://gateway-arbitrum.network.thegraph.com/api/{api_key}/deployments/id/{deployment_id}'
+
+def get_graph_url(subgraph_id: str) -> str:
+    if subgraph_id.startswith("http"):
+        return subgraph_id
+
+    if subgraph_id.startswith("Qm"):
+        return THE_GRAPH_DEPLOYMENT_TEMPLATE.format(
+            api_key=config.THE_GRAPH_API_KEY,
+            deployment_id=subgraph_id,
+        )
+
+    return THE_GRAPH_URL_TEMPLATE.format(
+        api_key=config.THE_GRAPH_API_KEY,
+        subgraph_id=subgraph_id,
+    )
 
 def solve_decimals(df: pd.DataFrame) -> pd.DataFrame:
     """ Adds the balanceFloat column to the dataframe
@@ -36,10 +53,14 @@ def solve_decimals(df: pd.DataFrame) -> pd.DataFrame:
 class Collector(ABC):
     INDEX = ['network', 'id']
     
-    def __init__(self, name:str, runner):
+    def __init__(self, name:str, runner: 'Runner'):
         self.name: str = name
         self.runner = runner
 
+    @property
+    def logger(self):
+        return logging.getLogger(f'dao_analyzer.collectors.{self.collectorid}')
+
     @property
     def data_path(self) -> Path:
         return self.runner.basedir / (self.name + '.arr')
@@ -67,7 +88,7 @@ class Collector(ABC):
         """ Updates the dataframe in `self.data_path` with the new data.
         """
         if df.empty:
-            logging.warning("Empty dataframe, not updating file")
+            self.logger.warning("Empty dataframe, not updating file")
             return
 
         if not self.data_path.is_file():
@@ -106,25 +127,31 @@ class UpdatableCollector(Collector): # Flag class
     pass
 
 class Runner(ABC):
-    def __init__(self, dw: Path = Path()):
+    def __init__(self, dw: Path):
         self.__dw: Path = dw
 
-    def set_dw(self, dw) -> Path:
-        self.__dw = dw
+    @property
+    def logger(self):
+        return logging.getLogger(f'dao_analyzer.runner.{self.name}')
+
+    @property
+    def cache(self) -> Path:
+        # Common cache folder for everyone
+        return self.__dw / '.cache'
 
     @property
     def basedir(self) -> Path:
         return self.__dw / self.name
 
     @property
-    def collectors(self) -> List[Collector]:
+    def collectors(self) -> list[Collector]:
         return []
 
     def run(self, **kwargs):
         raise NotImplementedError
 
 class NetworkRunner(Runner, ABC):
-    def __init__(self, dw = None):
+    def __init__(self, dw):
         super().__init__(dw)
         self.networks = {n for n,v in ENDPOINTS.items() if self.name in v and not n.startswith('_')}
 
@@ -133,9 +160,9 @@ class NetworkRunner(Runner, ABC):
         names: Iterable[str] = [],
         long_names: Iterable[str] = []
     ) -> Iterable[Collector]:
-        result = self.collectors
+        result: Iterable[Collector] = self.collectors
 
-        if config.only_updatable:
+        if config.run_only_updatable:
             result = filter(lambda c: isinstance(c, UpdatableCollector), result)
 
         # networks ^ (names v long_names)
@@ -150,11 +177,11 @@ class NetworkRunner(Runner, ABC):
         return result
 
     def filterCollector(self,
-        collector_id: str = None,
-        network: str = None,
-        name: str = None,
-        long_name: str = None,
-    ) -> Collector:
+        collector_id: Optional[str] = None,
+        network: Optional[str] = None,
+        name: Optional[str] = None,
+        long_name: Optional[str] = None,
+    ) -> Optional[Collector]:
         if collector_id:
             return next((c for c in self.collectors if c.collectorid == collector_id), None)
 
@@ -164,10 +191,9 @@ class NetworkRunner(Runner, ABC):
             long_names=[long_name] if long_name else []
         ), None)
 
-    @staticmethod
     @retry(retry=retry_if_exception_type(TransportQueryError), wait=wait_exponential(max=10), stop=stop_after_attempt(3))
-    def validated_block(network: str, prev_block: Block = None) -> Block:
-        requester = GQLRequester(ENDPOINTS[network]["_blocks"])
+    def validated_block(self, network: str, prev_block: Optional[Block] = None, until_date: Optional[datetime] = None) -> Optional[Block]:
+        requester = GQLRequester(get_graph_url(ENDPOINTS[network]['_blocks']))
         ds = requester.get_schema()
 
         number_gte = prev_block.number if prev_block else 0
@@ -182,10 +208,11 @@ class NetworkRunner(Runner, ABC):
             }
         }
 
-        if config.block_datetime:
+        # TODO: SET THE UNTIL_DATE
+        if until_date:
             del args["skip"]
             del args["where"]["number_gte"]
-            args["where"]["timestamp_lte"] = int(config.block_datetime.timestamp())
+            args["where"]["timestamp_lte"] = int(until_date.timestamp())
 
         response = requester.request(ds.Query.blocks(**args).select(
             ds.Block.id,
@@ -194,13 +221,13 @@ class NetworkRunner(Runner, ABC):
         ))["blocks"]
     
         if len(response) == 0:
-            logging.warning(f"Blocks query returned no response with args {args}")
+            self.logger.warning(f"Blocks query returned no response with args {args}")
             return prev_block
 
         return Block(response[0])
 
     @staticmethod
-    def _verifyCollectors(tocheck: Iterable[Collector]):
+    def _verifyCollectors(tocheck: Iterable[Collector]) -> Iterable[Collector]:
         verified = []
         for c in tqdm(list(tocheck), desc="Verifying"):
             try:
@@ -213,7 +240,7 @@ class NetworkRunner(Runner, ABC):
                 traceback.print_exc()
         return verified
 
-    def run(self, networks: List[str] = [], force=False, collectors=None):
+    def run(self, networks: list[str] = [], force=False, collectors=None, until_date: Optional[datetime]=None):
         self.basedir.mkdir(parents=True, exist_ok=True)
 
         print("Verifying collectors")
@@ -234,14 +261,18 @@ class NetworkRunner(Runner, ABC):
                         if c.network not in blocks:
                             # Getting a block more recent than the one in the metadata (just to narrow down the search)
                             print("Requesting a block number...", end='\r')
-                            blocks[c.network] = self.validated_block(c.network, None if force else metadata[c.collectorid].block)
+                            blocks[c.network] = self.validated_block(
+                                network=c.network, 
+                                prev_block=None if force else metadata[c.collectorid].block,
+                                until_date=until_date,
+                            )
                             print(f"Using block {blocks[c.network].id} for {c.network} (ts: {blocks[c.network].timestamp.isoformat()})")
 
                         print(f"Running collector {c.long_name} ({c.network})")
                         olderBlock = blocks[c.network] < metadata[c.collectorid].block
                         if not force and olderBlock:
                             print("Warning: Forcing because requesting an older block")
-                            logging.warning("Forcing because using an older block")
+                            self.logger.warning("Forcing because using an older block")
 
                         # Running the collector
                         c.run(
@@ -261,8 +292,9 @@ class NetworkRunner(Runner, ABC):
                     metadata[c.collectorid].last_update = datetime.now(timezone.utc)
                 except Exception as e:
                     metadata.errors[c.collectorid] = e.__str__()
-                    if config.ignore_errors:
-                        print(traceback.format_exc())
-                    else:
+                    if config.raise_runner_errors:
                         raise e
+                    else:
+                        # TODO: Use a logger instead
+                        print(traceback.format_exc(), file=sys.stderr)
             print(f'--- {self.name}\'s datawarehouse updated ---')

dao_analyzer/cache_scripts/common/cryptocompare.py

@@ -11,11 +11,11 @@ import logging
 EMPTY_KEY_MSG = \
 """\
 Empty CryptoCompare API key. You can obtain one from https://www.cryptocompare.com/cryptopian/api-keys
-You can set the API key using --cc-api-key argument or the DAOA_CC_API_KEY env variable.
+You can set the API key using the DAOA_CC_API_KEY env variable.
 """
 
 def cc_postprocessor(df: pd.DataFrame) -> pd.DataFrame:
-    ccrequester = CryptoCompareRequester(api_key=config.cc_api_key)
+    ccrequester = CryptoCompareRequester(api_key=config.CC_API_KEY)
 
     tokenSymbols = df['symbol'].drop_duplicates()
 
@@ -41,11 +41,11 @@ def cc_postprocessor(df: pd.DataFrame) -> pd.DataFrame:
 class CCPricesCollector(Collector):
     def __init__(self, runner: NetworkRunner, name: str='tokenPrices'):
         super().__init__(name, runner)
-        self.requester = CryptoCompareRequester(api_key=config.cc_api_key)
+        self.requester = CryptoCompareRequester(api_key=config.CC_API_KEY)
 
     def verify(self) -> bool:
         if not self.requester.api_key:
-            logging.warning(EMPTY_KEY_MSG)
+            self.logger.warning(EMPTY_KEY_MSG)
             return False
 
         return super().verify()