dao-scripts 1.2.2__py3-none-any.whl → 1.3.0.post1__py3-none-any.whl

dao_analyzer/cache_scripts/logging.py

@@ -0,0 +1,98 @@
+import sys
+import logging
+from pathlib import Path
+import datetime as dt
+
+from . import config
+
+LOG_FILE_FORMAT = "[%(levelname)s] - %(asctime)s - %(name)s - : %(message)s in %(pathname)s:%(lineno)d"
+LOG_STREAM_FORMAT = "%(levelname)s: %(message)s"
+
+class AuxDatawarehouseHandler(logging.Handler):
+    # Inspired by MemoryHandler
+
+    def __init__(self, aux_dw: Path, real_dw: Path, name: str, level=logging.NOTSET):
+        super().__init__(level=level)
+        self._aux_dw = aux_dw
+        self._real_dw = real_dw
+        self._fname = name
+        
+        self._auxHandler = logging.FileHandler(self._aux_dw / 'logs' / f'{self._fname}.log')
+        self._buffer: list[logging.LogRecord] = []
+
+        self.set_name(f'aux_dw_{name}')
+
+    def setFormatter(self, fmt):
+        self._auxHandler.setFormatter(fmt)
+        return super().setFormatter(fmt)
+
+    def setLevel(self, level):
+        super().setLevel(level)
+        self._auxHandler.setLevel(level)
+
+    def emit(self, record):
+        self._auxHandler.emit(record)
+        self._buffer.append(record)
+
+    def dump(self, errors: bool):
+        if errors:
+            target: logging.Handler = logging.handlers.RotatingFileHandler(
+                self._real_dw / 'logs' / f'{self._fname}_error_{dt.datetime.now().isoformat()}.log',
+            )
+        else:
+            target: logging.Handler = logging.handlers.RotatingFileHandler(
+                self._real_dw / 'logs' / f'{self._fname}.log',
+                maxBytes=int(config.LOGGING_MAX_SIZE),
+                backupCount=int(config.LOGGING_BACKUP_COUNT),
+            )
+
+        target.setLevel(self.level)
+        target.setFormatter(self.formatter)
+
+        self.acquire()
+        try:
+            for record in self._buffer:
+                target.handle(record)
+            self._buffer.clear()
+        finally:
+            self.release()
+
+_all_dw_handlers: list[AuxDatawarehouseHandler] = []
+def _setup_handler_in_logger(logger: str | logging.Logger, aux_dw, real_dw, name):
+    _all_dw_handlers.append(h := AuxDatawarehouseHandler(aux_dw, real_dw, name))
+    h.setFormatter(logging.Formatter(LOG_FILE_FORMAT))
+
+    if isinstance(logger, logging.Logger):
+        logger.addHandler(h)
+    else:
+        logging.getLogger(logger).addHandler(h)
+
+    return h
+
+def setup_logging(aux_dw: Path, real_dw: Path, debug: bool):
+    (aux_dw / 'logs').mkdir(exist_ok=True)
+    (real_dw / 'logs').mkdir(exist_ok=True)
+    
+    logger = logging.getLogger('dao_analyzer')
+    logger.propagate = True
+
+    gqlLogger = logging.getLogger('gql.transport.requests')
+
+    _setup_handler_in_logger(logger, aux_dw, real_dw, 'cache_scripts')
+    _setup_handler_in_logger(gqlLogger, aux_dw, real_dw, 'gql_requests')
+
+    streamhandler = logging.StreamHandler(sys.stderr)
+    streamhandler.setLevel(logging.WARNING if debug else logging.ERROR)
+    streamhandler.setFormatter(logging.Formatter(LOG_STREAM_FORMAT))
+
+    logger.addHandler(streamhandler)
+    gqlLogger.addHandler(streamhandler)
+
+    if debug:
+        logger.setLevel(logging.DEBUG)
+        gqlLogger.setLevel(logging.DEBUG)
+
+def finish_logging(errors: bool):
+    for h in _all_dw_handlers:
+        h.dump(errors)
+        h.close()

dao_analyzer/cache_scripts/main.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
-from typing import Dict
-
 from datetime import datetime
+import logging.handlers
 from pathlib import Path
 import portalocker as pl
 import os
@@ -11,19 +10,19 @@ import sys
 from sys import stderr
 
 import logging
-from logging.handlers import RotatingFileHandler
+
+from argparse import Namespace
 
 from .aragon.runner import AragonRunner
 from .daohaus.runner import DaohausRunner
 from .daostack.runner import DaostackRunner
-from .common import Runner, ENDPOINTS
+from .common import ENDPOINTS, NetworkRunner
 from .argparser import CacheScriptsArgParser
+from ._version import __version__
+from .logging import setup_logging, finish_logging
 from . import config
 
-LOG_FILE_FORMAT = "[%(levelname)s] - %(asctime)s - %(name)s - : %(message)s in %(pathname)s:%(lineno)d"
-LOG_STREAM_FORMAT = "%(levelname)s: %(message)s"
-
-AVAILABLE_PLATFORMS: Dict[str, Runner] = {
+AVAILABLE_PLATFORMS: dict[str, type[NetworkRunner]] = {
     AragonRunner.name: AragonRunner,
     DaohausRunner.name: DaohausRunner,
     DaostackRunner.name: DaostackRunner
@@ -32,10 +31,9 @@ AVAILABLE_PLATFORMS: Dict[str, Runner] = {
 # Get available networks from Runners
 AVAILABLE_NETWORKS = {n for n in ENDPOINTS.keys() if not n.startswith('_')}
 
-def _call_platform(platform: str, datawarehouse: Path, force: bool=False, networks=None, collectors=None):
-    p = AVAILABLE_PLATFORMS[platform]()
-    p.set_dw(datawarehouse)
-    p.run(networks=networks, force=force, collectors=collectors)
+def _call_platform(platform: str, datawarehouse: Path, force: bool=False, networks=None, collectors=None, block_datetime=None):
+    p = AVAILABLE_PLATFORMS[platform](datawarehouse)
+    p.run(networks=networks, force=force, collectors=collectors, until_date=block_datetime)
 
 def _is_good_version(datawarehouse: Path) -> bool:
     versionfile = datawarehouse / 'version.txt'
@@ -43,64 +41,36 @@ def _is_good_version(datawarehouse: Path) -> bool:
         return False
 
     with open(versionfile, 'r') as vf:
-        l = vf.readline().strip()
-        return l == config.CACHE_SCRIPTS_VERSION
-
-def main_aux(datawarehouse: Path):
-    if config.delete_force or not _is_good_version(datawarehouse):
-        if not config.delete_force:
-            print(f"datawarehouse version is not version {config.CACHE_SCRIPTS_VERSION}, upgrading")
-
-        # We skip the dotfiles like .lock
-        for p in datawarehouse.glob('[!.]*'):
-            if p.is_dir():
-                shutil.rmtree(p)
-            else:
-                p.unlink()
-
-    logger = logging.getLogger()
-    logger.propagate = True
-    filehandler = RotatingFileHandler(
-        filename=config.datawarehouse / 'cache_scripts.log',
-        maxBytes=config.LOGGING_MAX_MB * 2**20,
-        backupCount=config.LOGGING_BACKUP_COUNT,
-    )
-
-    filehandler.setFormatter(logging.Formatter(LOG_FILE_FORMAT))
-    logger.addHandler(filehandler)
-    logger.setLevel(level=logging.DEBUG if config.debug else logging.INFO)
-
-    logging.getLogger('gql.transport.requests').setLevel(level=logging.DEBUG if config.debug else logging.WARNING)
-
-    # Log errors to STDERR
-    streamhandler = logging.StreamHandler(stderr)
-    streamhandler.setLevel(logging.WARNING if config.debug else logging.ERROR)
-    streamhandler.setFormatter(logging.Formatter(LOG_STREAM_FORMAT))
-    logger.addHandler(streamhandler)
-
-    logging.info("Running dao-scripts with arguments: %s", sys.orig_argv)
+        return vf.readline().strip() == __version__
+
+def run_all(
+    datawarehouse: Path, delete_force: bool, 
+    platforms: list[str], networks: list[str], collectors: list[str], 
+    block_datetime: datetime, force: bool
+):
 
     # The default config is every platform
-    if not config.platforms:
-        config.platforms = AVAILABLE_PLATFORMS.keys()
+    if not platforms:
+        platforms = list(AVAILABLE_PLATFORMS.keys())
 
     # Now calling the platform and deleting if needed
-    for p in config.platforms:
-        _call_platform(p, datawarehouse, config.force, config.networks, config.collectors)
+    for platform in platforms:
+        _call_platform(platform, datawarehouse, force, networks, collectors, block_datetime)
 
     # write date
     data_date: str = str(datetime.now().isoformat())
 
-    if config.block_datetime:
-        data_date = config.block_datetime.isoformat()
+    if block_datetime:
+        data_date = block_datetime.isoformat()
 
     with open(datawarehouse / 'update_date.txt', 'w') as f:
         print(data_date, file=f)
 
     with open(datawarehouse / 'version.txt', 'w') as f:
-        print(config.CACHE_SCRIPTS_VERSION, file=f)
+        print(__version__, file=f)
 
-def main_lock(datawarehouse: Path):
+def lock_and_run(args: Namespace):
+    datawarehouse: Path = args.datawarehouse
     datawarehouse.mkdir(exist_ok=True)
     
     # Lock for the datawarehouse (also used by the dash)
@@ -111,28 +81,63 @@ def main_lock(datawarehouse: Path):
 
     try:
         with pl.Lock(cs_lock, 'w', timeout=1) as lock, \
-             tempfile.TemporaryDirectory(prefix="datawarehouse_") as tmp_dw:
+             tempfile.TemporaryDirectory(prefix="datawarehouse_") as tmp_dw_str:
 
             # Writing pid and dir name to lock (debugging)
-            tmp_dw = Path(tmp_dw)
+            tmp_dw = Path(tmp_dw_str)
             print(os.getpid(), file=lock)
             print(tmp_dw, file=lock)
             lock.flush()
-
-            ignore = shutil.ignore_patterns('*.log', '.lock*')
-
-            # We want to copy the dw, so we open it as readers
-            p_lock.touch(exist_ok=True)
-            with pl.Lock(p_lock, 'r', timeout=1, flags=pl.LOCK_SH | pl.LOCK_NB):
-                shutil.copytree(datawarehouse, tmp_dw, dirs_exist_ok=True, ignore=ignore)
-
-            main_aux(datawarehouse=tmp_dw)
-
-            with pl.Lock(p_lock, 'w', timeout=10):
-                shutil.copytree(tmp_dw, datawarehouse, dirs_exist_ok=True, ignore=ignore)
-
-            # Removing pid from lock
-            lock.truncate(0)
+            (datawarehouse / '.running').symlink_to(tmp_dw)
+
+            # Used to tell the loggers to use errors.log or the main logs
+            copied_dw = False
+
+            try:
+                ignore = shutil.ignore_patterns('.lock*', 'logs/*')
+
+                # We want to copy the dw, so we open it as readers
+                p_lock.touch(exist_ok=True)
+                with pl.Lock(p_lock, 'r', timeout=1, flags=pl.LOCK_SH | pl.LOCK_NB):
+                    shutil.copytree(datawarehouse, tmp_dw, dirs_exist_ok=True, ignore=ignore)
+
+                if args.delete_force or not _is_good_version(datawarehouse):
+                    if not args.delete_force:
+                        print(f"datawarehouse version is not version {__version__}, upgrading")
+
+                    # We skip the dotfiles like .lock
+                    for p in datawarehouse.glob('[!.]*'):
+                        if p.is_dir():
+                            shutil.rmtree(p)
+                        else:
+                            p.unlink()
+
+                setup_logging(tmp_dw, datawarehouse, config.DEBUG)
+                logger = logging.getLogger('dao_analyzer.main')
+                logger.info(">>> Running dao-scripts with arguments: %s", sys.orig_argv)
+
+                # Execute the scripts in the aux datawarehouse
+                run_all(
+                    datawarehouse=tmp_dw,
+                    delete_force=args.delete_force,
+                    platforms=args.platforms,
+                    networks=args.networks,
+                    collectors=args.collectors,
+                    block_datetime=args.block_datetime,
+                    force=args.force,
+                )
+
+                # Copying back the dw
+                logger.info(f"<<< Copying back the datawarehouse from {tmp_dw} to {datawarehouse}")
+                with pl.Lock(p_lock, 'w', timeout=10):
+                    shutil.copytree(tmp_dw, datawarehouse, dirs_exist_ok=True, ignore=ignore)
+                
+                copied_dw = True
+            finally:
+                # Removing pid from lock
+                lock.truncate(0)
+                (datawarehouse / '.running').unlink()
+                finish_logging(errors=not copied_dw)
     except pl.LockException:
         with open(cs_lock, 'r') as f:
             pid = int(f.readline())
@@ -145,13 +150,14 @@ def main():
         available_platforms=list(AVAILABLE_PLATFORMS.keys()),
         available_networks=AVAILABLE_NETWORKS)
 
-    config.populate_args(parser.parse_args())
+    args = parser.parse_args()
+    config.args2config(args)
 
-    if config.display_version:
-        print(config.CACHE_SCRIPTS_VERSION)
+    if args.display_version:
+        print(__version__)
         exit(0)
 
-    main_lock(config.datawarehouse)
+    lock_and_run(args)
 
 if __name__ == '__main__':
     main()

dao_analyzer/cache_scripts/metadata.py

@@ -7,7 +7,7 @@
         <david@ddavo.me>
 """
 from json.encoder import JSONEncoder
-from typing import Dict
+from typing import Optional
 import json
 from functools import total_ordering
 from datetime import datetime, timezone
@@ -26,7 +26,7 @@ class Block:
             self.id = init["id"] if "id" in init else self.id
 
             if "timestamp" in init:
-                if init["timestamp"].isdigit():
+                if isinstance(init['timestamp'], int) or init["timestamp"].isdigit():
                     self.timestamp = datetime.fromtimestamp(int(init["timestamp"]))
                 else:
                     self.timestamp = datetime.fromisoformat(init["timestamp"])
@@ -56,7 +56,7 @@ class Block:
 
 class CollectorMetaData:
     def __init__(self, c: str, d = None):
-        self.block = Block()
+        self.block: Optional[Block] = Block()
         self._collector: str = c
         self.last_update: datetime = datetime.now(timezone.utc)
 
@@ -89,8 +89,8 @@ class MetadataEncoder(JSONEncoder):
 class RunnerMetadata:
     def __init__(self, runner):
         self._path = runner.basedir / 'metadata.json'
-        self.collectorMetaData: Dict[str, CollectorMetaData] = {}
-        self.errors: Dict[str, str] = {}
+        self.collectorMetaData: dict[str, CollectorMetaData] = {}
+        self.errors: dict[str, str] = {}
         self._setPrev()
 
     def _setPrev(self):
@@ -125,7 +125,7 @@ class RunnerMetadata:
                 "metadata": self.collectorMetaData,
                 "errors": self.errors
             }, f, 
-                indent=2 if config.debug else None, 
+                indent=2 if config.DEBUG else None, 
                 cls=MetadataEncoder)
 
     def ifdump(self):

dao_scripts-1.3.0.post1-py3.12-nspkg.pth

@@ -0,0 +1 @@
+import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('dao_analyzer',));importlib = __import__('importlib.util');__import__('importlib.machinery');m = sys.modules.setdefault('dao_analyzer', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('dao_analyzer', [os.path.dirname(p)])));m = m or sys.modules.setdefault('dao_analyzer', types.ModuleType('dao_analyzer'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)