eth-portfolio 0.5.8__cp310-cp310-win_amd64.whl

eth_portfolio_scripts/docker/.grafana/dashboards/dashboards.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+- name: 'Portfolio dashboards'
+  folder: 'Portfolio'
+  type: file
+  disableDeletion: true
+  editable: false
+  options:
+    path: /etc/grafana/provisioning/dashboards/Portfolio

eth_portfolio_scripts/docker/.grafana/datasources/datasources.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+- name: 'PROMETHEUS'
+  type: 'prometheus'
+  access: 'proxy'
+  org_id: 1
+  url: 'http://victoria-metrics:8428'
+  is_default: true
+  version: 1
+  editable: false

eth_portfolio_scripts/docker/__init__.py

@@ -0,0 +1,16 @@
+from eth_portfolio_scripts.docker.check import check_docker, check_docker_compose, check_system
+from eth_portfolio_scripts.docker.docker_compose import build, down, ensure_containers, stop, up
+
+__all__ = [
+    # commands
+    "build",
+    "up",
+    "down",
+    "stop",
+    # decorators
+    "ensure_containers",
+    # checks
+    "check_docker",
+    "check_docker_compose",
+    "check_system",
+]

eth_portfolio_scripts/docker/check.py

@@ -0,0 +1,66 @@
+from functools import lru_cache
+from subprocess import CalledProcessError, check_output
+
+
+def check_docker() -> None:
+    """
+    Check that docker is installed on the user's system.
+
+    Raises:
+        RuntimeError: If docker is not installed.
+    """
+    print(" 🔍 checking your computer for docker")
+    try:
+        check_output(["docker", "--version"])
+    except (CalledProcessError, FileNotFoundError):
+        raise RuntimeError(
+            "Docker is not installed. You must install Docker before using dao-treasury."
+        ) from None
+    else:
+        print(" ✔️ eth-portfolio found docker!")
+
+
+def check_docker_compose() -> list[str]:
+    """
+    Check that either `docker-compose` or `docker compose` is installed on the user's system.
+
+    Returns:
+        A valid compose command.
+
+    Raises:
+        RuntimeError: If docker-compose is not installed.
+    """
+    for cmd in ["docker-compose", "docker compose"]:
+        print(f" 🔍 checking your computer for {cmd}")
+
+        try:
+            check_output([*cmd.split(" "), "--version"])
+        except (CalledProcessError, FileNotFoundError):
+            print(f" ❌ {cmd} not found")
+            continue
+        else:
+            print(f" ✔️ eth-portfolio found {cmd}!")
+            return cmd.split(" ")
+
+    raise RuntimeError(
+        "Docker Compose is not installed. You must install Docker Compose before using dao-treasury."
+    ) from None
+
+
+@lru_cache(maxsize=None)
+def check_system() -> list[str]:
+    """
+    Check that docker and docker-compose is installed on the user's system.
+
+    Returns:
+        A valid compose command.
+
+    Raises:
+        RuntimeError: If docker-compose is not installed.
+    """
+    print("eth-portfolio is checking for the required docker dependencies...")
+    check_docker()
+    return check_docker_compose()
+
+
+__all__ = ["check_docker", "check_docker_compose", "check_system"]

eth_portfolio_scripts/docker/docker-compose.yaml

@@ -0,0 +1,61 @@
+networks:
+  eth_portfolio:
+
+services:
+  grafana:
+    image: grafana/grafana:12.3.1
+    ports:
+      - 127.0.0.1:${GRAFANA_PORT:-3000}:3000
+    environment:
+      - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin}
+      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin}
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/portfolio/Balances.json
+      - GF_SERVER_ROOT_URL
+      - GF_RENDERING_SERVER_URL=http://renderer:8091/render
+      - GF_RENDERING_CALLBACK_URL=http://grafana:3000/
+      - GF_LOG_FILTERS=rendering:debug
+      #- GF_INSTALL_PLUGINS=volkovlabs-variable-panel
+    #command: >
+    #  sh -c "grafana-cli plugins install frser-sqlite-datasource && /run.sh"
+    volumes:
+      - ./.grafana/:/etc/grafana/provisioning/
+    networks:
+      - eth_portfolio
+    restart: always
+
+  renderer:
+    platform: linux/amd64
+    image: grafana/grafana-image-renderer:latest
+    ports:
+      - 127.0.0.1:${RENDERER_PORT:-8091}:8091
+    environment:
+      - ENABLE_METRICS=true
+      - HTTP_PORT=8091
+    networks:
+      - eth_portfolio
+    restart: always
+  
+  vmagent:
+    image: victoriametrics/vmagent:heads-public-single-node-0-g52eb9c99e
+    command:
+      - "-remoteWrite.url=http://victoria-metrics:8428/api/v1/write"
+    depends_on:
+      - victoria-metrics
+    networks:
+      - eth_portfolio
+    restart: always
+
+  victoria-metrics:
+    image: victoriametrics/victoria-metrics:v1.134.0
+    volumes:
+      - ~/.eth-portfolio/data/victoria/:/victoria-metrics-data
+    command:
+      - '-memory.allowedBytes=3GB'
+      - "-retentionPeriod=10y"
+      - "-search.disableAutoCacheReset=true"
+    ports:
+      - 127.0.0.1:${VICTORIA_PORT:-8428}:8428
+    networks:
+      - eth_portfolio
+    restart: always

eth_portfolio_scripts/docker/docker_compose.py

@@ -0,0 +1,97 @@
+import logging
+from collections.abc import Callable
+from functools import wraps
+from importlib import resources
+from subprocess import CalledProcessError, check_output
+from typing import Final, Literal, TypeVar
+
+from typing_extensions import ParamSpec
+
+from eth_portfolio_scripts.docker.check import check_system
+
+logger: Final = logging.getLogger(__name__)
+
+COMPOSE_FILE: Final = str(
+    resources.files("eth_portfolio_scripts").joinpath("docker/docker-compose.yaml")
+)
+
+
+def up(*services: str) -> None:
+    """Build and start the specified docker-compose services."""
+    build(*services)
+    _print_notice("starting", services)
+    _exec_command(["up", "-d", *services])
+
+
+def down() -> None:
+    """Stop all of eth-portfolio's docker-compose services."""
+    _exec_command(["down"])
+
+
+def build(*services: str) -> None:
+    """Build the specified docker-compose services."""
+    _print_notice("building", services)
+    _exec_command(["build", *services])
+
+
+def stop(*services: str) -> None:
+    """Stop the specified docker-compose services, if running."""
+    _print_notice("stopping", services)
+    _exec_command(["stop", *services])
+
+
+_P = ParamSpec("_P")
+_T = TypeVar("_T")
+
+
+def ensure_containers(fn: Callable[_P, _T]) -> Callable[_P, _T]:
+    @wraps(fn)
+    async def compose_wrap(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+        # register shutdown sequence
+        # TODO: argument to leave them up
+        # NOTE: do we need both this and the finally?
+        # signal.signal(signal.SIGINT, down)
+
+        # start Grafana containers
+        up()
+
+        try:
+            # attempt to run `fn`
+            return await fn(*args, **kwargs)
+        finally:
+            # stop and remove containers
+            # down()
+            pass
+
+    return compose_wrap
+
+
+def _print_notice(
+    doing: Literal["building", "starting", "stopping"],
+    services: tuple[str, ...],
+) -> None:
+    if len(services) == 0:
+        print(f"{doing} the backend containers")
+    elif len(services) == 1:
+        container = services[0]
+        print(f"{doing} the {container} container")
+    elif len(services) == 2:
+        first, second = services
+        print(f"{doing} the {first} and {second} containers")
+    else:
+        *all_but_last, last = services
+        print(f"{doing} the {', '.join(all_but_last)}, and {last} containers")
+
+
+def _exec_command(
+    command: list[str],
+    *,
+    compose_file: str = COMPOSE_FILE,
+    compose_options: tuple[str, ...] = (),
+) -> None:
+    compose = check_system()
+    full_command = [*compose, *compose_options, "-f", compose_file, *command]
+    try:
+        check_output(full_command)
+    except (CalledProcessError, FileNotFoundError) as e:
+        raise RuntimeError(f"Error occurred while running `{' '.join(full_command)}`: {e}") from e

eth_portfolio_scripts/main.py

@@ -0,0 +1,118 @@
+import asyncio
+from argparse import ArgumentParser
+from os import environ
+
+import brownie
+
+from eth_portfolio_scripts import docker
+from eth_portfolio_scripts._args import add_infra_port_args
+from eth_portfolio_scripts.balances import export_balances
+
+parser = ArgumentParser(description="eth-portfolio")
+
+subparsers = parser.add_subparsers(title="Commands", dest="command", required=True)
+
+export_parser = subparsers.add_parser("export", help="Export a specific dataset for your portfolio")
+export_parser.add_argument("target", help="Choose an exporter to run")
+add_infra_port_args(export_parser)
+export_parser.set_defaults(func=export_balances)
+
+infra_parser = subparsers.add_parser(
+    "infra", help="Start the docker containers required to run any eth-portfolio service"
+)
+infra_parser.add_argument("cmd", help="What do you want to do?")
+add_infra_port_args(infra_parser)
+infra_parser.add_argument(
+    "--start-renderer",
+    action="store_true",
+    help="If set, starts the renderer container in addition to the default containers. By default, only grafana, victoria-metrics, and vmagent are started.",
+)
+
+export_parser.add_argument(
+    "--wallet",
+    type=str,
+    help="The address of a wallet to export. You can pass multiple, ie. `--wallet 0x123 0x234 0x345`",
+    required=True,
+    nargs="+",
+)
+export_parser.add_argument(
+    "--network",
+    type=str,
+    help="The brownie network identifier for the rpc you wish to use. default: mainnet",
+    default="mainnet",
+)
+export_parser.add_argument(
+    "--label",
+    type=str,
+    help='The label for this portfolio, if you want one. Defaults to "MyPortfolio".',
+    default="My Portfolio",
+)
+export_parser.add_argument(
+    "--interval",
+    type=str,
+    help="The time interval between datapoints. default: 6h",
+    default="6h",
+)
+export_parser.add_argument(
+    "--concurrency",
+    type=int,
+    help="The max number of historical blocks to export concurrently. default: 30",
+    default=30,
+)
+export_parser.add_argument(
+    "--first-tx-block",
+    type=int,
+    help=(
+        "The block of your portfolio's first transaction, if known. "
+        "This value, if provided, allows us to speed up processing of your data by limiting the block range we need to query. "
+        "If not provided, the whole blockchain will be scanned."
+    ),
+    default=0,
+)
+export_parser.add_argument(
+    "--export-start-block",
+    type=int,
+    help="The first block in the range you wish to export.",
+    default=0,
+)
+export_parser.add_argument(
+    "--daemon",
+    action="store_true",
+    help="TODO: If True, starts a daemon process instead of running in your terminal. Not currently supported.",
+)
+
+args = parser.parse_args()
+
+if hasattr(args, "network"):
+    environ["BROWNIE_NETWORK_ID"] = args.network
+
+environ["GRAFANA_PORT"] = str(args.grafana_port)
+environ["RENDERER_PORT"] = str(args.renderer_port)
+environ["VICTORIA_PORT"] = str(args.victoria_port)
+
+
+# TODO: run forever arg
+def main():
+    command = args.command
+    if command == "infra":
+        if args.cmd == "start":
+            # Start the backend containers
+            if getattr(args, "start_renderer", False):
+                docker.up()
+            else:
+                docker.up("grafana", "victoria-metrics", "vmagent")
+        elif args.cmd == "stop":
+            docker.down()
+        else:
+            raise ValueError(f"{args.target} is not a valid command")
+
+    else:
+        # The user's command is `export`
+        if args.target == "balances":
+            asyncio.run(args.func(args))
+        else:
+            raise ValueError(f"{args.target} is not a valid command")
+
+
+if __name__ == "__main__":
+    brownie.project.run(__file__)

eth_portfolio_scripts/py.typed

@@ -0,0 +1 @@
+

eth_portfolio_scripts/victoria/__init__.py

@@ -0,0 +1,72 @@
+from gzip import compress
+from logging import getLogger
+from os import environ
+from typing import Final, List
+
+import a_sync
+from aiohttp import ClientError, ClientSession, ServerDisconnectedError
+from msgspec import json
+
+from eth_portfolio_scripts.victoria import types
+from eth_portfolio_scripts.victoria.types import Metric
+
+BASE_URL: Final = environ.get("VM_URL", "http://127.0.0.1:8430")
+
+# this will be populated later
+session: ClientSession = None  # type: ignore [assignment]
+
+logger: Final = getLogger("eth_portfolio.victoria")
+
+encode: Final = json.encode
+decode: Final = json.decode
+
+
+class VictoriaMetricsError(ValueError): ...
+
+
+def get_session() -> ClientSession:
+    sesh = session
+    if sesh is None:
+        sesh = ClientSession(BASE_URL, raise_for_status=True)
+        __set_session(sesh)
+    return sesh
+
+
+async def get(url: str) -> bytes:
+    session = get_session()
+    while True:
+        try:
+            async with session.get(url=url, headers={"Connection": "close"}) as response:
+                return await response.read()
+        except ServerDisconnectedError:
+            continue
+
+
+@a_sync.Semaphore(2)
+async def post_data(metrics_to_export: list["Metric"]) -> None:
+    """Post all metrics at once."""
+    data = compress(b"\n".join(encode(metric) for metric in metrics_to_export))
+    attempts = 0
+    session = get_session()
+    while True:
+        try:
+            async with session.post(
+                "/api/v1/import",
+                headers={"Connection": "close", "Content-Encoding": "gzip"},
+                data=data,
+            ):
+                logger.debug(f"posted {len(data)} datas")
+                return
+        except ClientError as e:
+            attempts += 1
+            logger.debug("You had a ClientError: %s", e)
+            if attempts >= 10:
+                raise e
+
+
+def __set_session(sesh: ClientSession) -> None:
+    global session
+    session = sesh
+
+
+__all__ = ["Metric", "get", "post_data"]

eth_portfolio_scripts/victoria/types.py

@@ -0,0 +1,38 @@
+"""
+These 2 helpers enable us to decode only the relevant data in the json response and discard the rest
+They're in a separate file so mypyc doesn't try to compile them
+"""
+
+from typing import TypedDict, final
+
+from eth_typing import ChecksumAddress
+from msgspec import Raw, Struct
+
+
+@final
+class Metric(TypedDict):
+    param: str
+    wallet: ChecksumAddress
+    token_address: ChecksumAddress
+    token: str
+    bucket: str
+    network: str
+    __name__: str
+
+
+@final
+class PrometheusItem(TypedDict):
+    metric: Metric
+    values: list[float]
+    timestamps: list[float]
+
+
+@final
+class Data(Struct):
+    result: tuple[Raw, ...]
+
+
+@final
+class Response(Struct):
+    status: str
+    data: Data