eth-portfolio-temp 0.2.12__cp313-cp313-win32.whl

eth_portfolio/typing/balance/single.py

@@ -0,0 +1,176 @@
+from typing import Literal, Optional, Union, final
+
+from dictstruct import DictStruct
+from eth_typing import BlockNumber, ChecksumAddress
+from mypy_extensions import mypyc_attr
+
+from eth_portfolio._decimal import Decimal
+
+
+@final
+@mypyc_attr(native_class=False)
+class Balance(
+    DictStruct, frozen=True, omit_defaults=True, repr_omit_defaults=True, forbid_unknown_fields=True
+):
+    """
+    Represents the balance of a single token, including its token amount and equivalent USD value.
+
+    Example:
+        >>> balance1 = Balance(Decimal('100'), Decimal('2000'))
+        >>> balance2 = Balance(Decimal('50'), Decimal('1000'))
+        >>> combined_balance = balance1 + balance2
+        >>> combined_balance.balance
+        Decimal('150')
+        >>> combined_balance.usd_value
+        Decimal('3000')
+    """
+
+    balance: Decimal = Decimal(0)
+    """
+    The amount of the token.
+    """
+
+    usd_value: Decimal = Decimal(0)
+    """
+    The USD equivalent value of the token amount.
+    """
+
+    token: Optional[ChecksumAddress] = None
+    """
+    The token the balance is of, if known.
+    """
+
+    block: Optional[BlockNumber] = None
+    """
+    The block from which the balance was taken, if known.
+    """
+
+    @property
+    def usd(self) -> Decimal:
+        """
+        An alias for `usd_value`. Returns the USD value of the token amount.
+        """
+        return self.usd_value
+
+    def __add__(self, other: "Balance") -> "Balance":
+        """
+        Adds two :class:`~eth_portfolio.typing.Balance` objects together. It is the user's responsibility to ensure that the two
+        :class:`~eth_portfolio.typing.Balance` instances represent the same token.
+
+        Args:
+            other: Another :class:`~eth_portfolio.typing.Balance` object.
+
+        Returns:
+            A new :class:`~eth_portfolio.typing.Balance` object with the summed values.
+
+        Raises:
+            TypeError: If the other object is not a :class:`~eth_portfolio.typing.Balance`.
+            Exception: If any other error occurs during addition.
+
+        Example:
+            >>> balance1 = Balance(Decimal('100'), Decimal('2000'))
+            >>> balance2 = Balance(Decimal('50'), Decimal('1000'))
+            >>> combined_balance = balance1 + balance2
+            >>> combined_balance.balance
+            Decimal('150')
+            >>> combined_balance.usd_value
+            Decimal('3000')
+        """
+        if not isinstance(other, Balance):
+            raise TypeError(f"{other} is not a `Balance` object")
+        if self.token != other.token:
+            raise ValueError(
+                f"These Balance objects represent balances of different tokens ({self.token} and {other.token})"
+            )
+        if self.block != other.block:
+            raise ValueError(
+                f"These Balance objects represent balances from different blocks ({self.block} and {other.block})"
+            )
+        try:
+            return Balance(
+                balance=self.balance + other.balance,
+                usd_value=self.usd_value + other.usd_value,
+                token=self.token,
+                block=self.block,
+            )
+        except Exception as e:
+            e.args = (f"Cannot add {self} and {other}: {e}", *e.args)
+            raise
+
+    def __radd__(self, other: Union["Balance", Literal[0]]) -> "Balance":
+        """
+        Supports the addition operation from the right side to enable use of `sum`.
+
+        Args:
+            other: Another :class:`~eth_portfolio.typing.Balance` object or zero.
+
+        Returns:
+            A new :class:`~eth_portfolio.typing.Balance` object with the summed values.
+
+        Raises:
+            TypeError: If the other object is not a :class:`~eth_portfolio.typing.Balance`.
+            Exception: If any other error occurs during addition.
+
+        Example:
+            >>> balance = Balance(Decimal('100'), Decimal('2000'))
+            >>> sum_balance = sum([balance, Balance()])
+            >>> sum_balance.balance
+            Decimal('100')
+        """
+        return self if other == 0 else self.__add__(other)  # type: ignore
+
+    def __sub__(self, other: "Balance") -> "Balance":
+        """
+        Subtracts one :class:`~eth_portfolio.typing.Balance` object from another. It is the user's responsibility to ensure that
+        the two :class:`~eth_portfolio.typing.Balance` instances represent the same token.
+
+        Args:
+            other: Another :class:`~eth_portfolio.typing.Balance` object.
+
+        Returns:
+            A new :class:`~eth_portfolio.typing.Balance` object with the subtracted values.
+
+        Raises:
+            TypeError: If the other object is not a :class:`~eth_portfolio.typing.Balance`.
+            Exception: If any other error occurs during subtraction.
+
+        Example:
+            >>> balance1 = Balance(Decimal('100'), Decimal('2000'))
+            >>> balance2 = Balance(Decimal('50'), Decimal('1000'))
+            >>> result_balance = balance1 - balance2
+            >>> result_balance.balance
+            Decimal('50')
+        """
+        if not isinstance(other, Balance):
+            raise TypeError(f"{other} is not a `Balance` object.")
+        if self.token != other.token:
+            raise ValueError(
+                f"These Balance objects represent balances of different tokens ({self.token} and {other.token})"
+            )
+        if self.block != other.block:
+            raise ValueError(
+                f"These Balance objects represent balances from different blocks ({self.block} and {other.block})"
+            )
+        try:
+            return Balance(
+                balance=self.balance - other.balance,
+                usd_value=self.usd_value - other.usd_value,
+                token=self.token,
+                block=self.block,
+            )
+        except Exception as e:
+            raise e.__class__(f"Cannot subtract {self} and {other}: {e}") from e
+
+    def __bool__(self) -> bool:
+        """
+        Evaluates the truth value of the :class:`~eth_portfolio.typing.Balance` object.
+
+        Returns:
+            True if either the balance or the USD value is non-zero, otherwise False.
+
+        Example:
+            >>> balance = Balance(Decimal('0'), Decimal('0'))
+            >>> bool(balance)
+            False
+        """
+        return self.balance != 0 or self.usd_value != 0

eth_portfolio_scripts/__init__.py

@@ -0,0 +1,20 @@
+from os import environ
+
+
+environ["DANKMIDS_GANACHE_FORK"] = "0"
+environ["DANKMIDS_COLLECT_STATS"] = "0"
+
+
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+
+from eth_portfolio_scripts._logging import logger, setup_logging
+
+
+setup_logging()
+
+
+__all__ = ["logger"]

eth_portfolio_scripts/_args.py

@@ -0,0 +1,26 @@
+from argparse import ArgumentParser
+
+
+def get_arg_parser(description: str) -> ArgumentParser:
+    return ArgumentParser(description)
+
+
+def add_infra_port_args(parser: ArgumentParser) -> None:
+    parser.add_argument(
+        "--grafana-port",
+        type=int,
+        help="The port that will be used by grafana",
+        default=3000,
+    )
+    parser.add_argument(
+        "--renderer-port",
+        type=int,
+        help="The port that will be used by grafana",
+        default=8091,
+    )
+    parser.add_argument(
+        "--victoria-port",
+        type=int,
+        help="The port that will be used by victoria metrics",
+        default=8428,
+    )

eth_portfolio_scripts/_logging.py

@@ -0,0 +1,15 @@
+import warnings
+from logging import INFO, basicConfig, getLogger
+
+from brownie.exceptions import BrownieCompilerWarning, BrownieEnvironmentWarning
+
+
+logger = getLogger(__name__)
+
+
+def setup_logging() -> None:
+    basicConfig(level=INFO)
+
+
+warnings.simplefilter("ignore", BrownieCompilerWarning)
+warnings.simplefilter("ignore", BrownieEnvironmentWarning)

eth_portfolio_scripts/_portfolio.py

@@ -0,0 +1,194 @@
+import asyncio
+from datetime import datetime, timezone
+from logging import getLogger
+from math import floor
+from typing import Awaitable, Callable, Final, Iterator, List, Optional, Tuple
+
+import a_sync
+import eth_retry
+import y
+from a_sync.functools import cached_property_unsafe as cached_property
+from eth_typing import BlockNumber, ChecksumAddress
+from msgspec import ValidationError, json
+from y import ERC20, Network, NonStandardERC20
+from y.constants import CHAINID
+from y.time import NoBlockFound
+
+from eth_portfolio import Portfolio
+from eth_portfolio.buckets import get_token_bucket
+from eth_portfolio.portfolio import _DEFAULT_LABEL
+from eth_portfolio.typing import (
+    Addresses,
+    Balance,
+    RemoteTokenBalances,
+    PortfolioBalances,
+    TokenBalances,
+)
+from eth_portfolio_scripts import victoria
+
+
+NETWORK_LABEL: Final = Network.label(CHAINID)
+
+decode: Final = json.decode
+
+logger: Final = getLogger("eth_portfolio")
+log_debug: Final = logger.debug
+log_error: Final = logger.error
+
+_block_at_timestamp_semaphore: Final = a_sync.Semaphore(
+    50, name="eth-portfolio get_block_at_timestamp"
+)
+
+
+async def get_block_at_timestamp(dt: datetime) -> BlockNumber:
+    async with _block_at_timestamp_semaphore:
+        while True:
+            try:
+                return await y.get_block_at_timestamp(dt, sync=False)
+            except NoBlockFound:
+                await asyncio.sleep(10)
+
+
+class ExportablePortfolio(Portfolio):
+    """Adds methods to export full portoflio data."""
+
+    def __init__(
+        self,
+        addresses: Addresses,
+        *,
+        start_block: int = 0,
+        label: str = _DEFAULT_LABEL,
+        concurrency: int = 40,
+        load_prices: bool = True,
+        get_bucket: Callable[[ChecksumAddress], Awaitable[str]] = get_token_bucket,
+        num_workers_transactions: int = 1000,
+        asynchronous: bool = False,
+    ):
+        super().__init__(
+            addresses, start_block, label, load_prices, num_workers_transactions, asynchronous
+        )
+        self.get_bucket = get_bucket
+        self._semaphore = a_sync.Semaphore(concurrency)
+
+    @cached_property
+    def _data_queries(self) -> Tuple[str, str]:
+        label = self.label.lower().replace(" ", "_")
+        return f"{label}_assets", f"{label}_debts"
+
+    @eth_retry.auto_retry
+    @a_sync.Semaphore(16)
+    async def data_exists(self, dt: datetime) -> bool:
+        # sourcery skip: use-contextlib-suppress
+        async for data in a_sync.as_completed(list(self.__get_data_exists_coros(dt)), aiter=True):
+            try:
+                result = decode(data, type=victoria.types.Response)
+            except ValidationError:
+                raise victoria.VictoriaMetricsError(data.decode()) from None
+            if result.status == "success" and len(result.data.result) > 0:
+                print(f"{dt} already loaded")
+                return True
+        return False
+
+    async def export_snapshot(self, dt: datetime) -> None:
+        log_debug("checking data at %s for %s", dt, self.label)
+        try:
+            if await self.data_exists(dt, sync=False):
+                return
+            block = await get_block_at_timestamp(dt)
+            log_debug("block at %s: %s", dt, block)
+            data = await self.get_data_for_export(block, dt, sync=False)
+            await victoria.post_data(data)
+        except Exception as e:
+            log_error("Error processing %s:", dt, exc_info=True)
+
+    async def get_data_for_export(self, block: BlockNumber, ts: datetime) -> List[victoria.Metric]:
+        async with self._semaphore:
+            print(f"exporting {ts} for {self.label}")
+            start = datetime.now(tz=timezone.utc)
+
+            metrics_to_export = []
+            data: PortfolioBalances = await self.describe(block, sync=False)
+
+            for wallet, wallet_data in dict.items(data):
+                for section, section_data in wallet_data.items():
+                    if isinstance(section_data, TokenBalances):
+                        for token, bals in dict.items(section_data):
+                            metrics_to_export.extend(
+                                await self.__process_token(ts, section, wallet, token, bals)
+                            )
+                    elif isinstance(section_data, RemoteTokenBalances):
+                        if section == "external":
+                            section = "assets"
+                        for protocol, token_bals in section_data.items():
+                            for token, bals in dict.items(token_bals):
+                                metrics_to_export.extend(
+                                    await self.__process_token(
+                                        ts, section, wallet, token, bals, protocol=protocol
+                                    )
+                                )
+                    else:
+                        raise NotImplementedError()
+
+            print(f"got data for {ts} in {datetime.now(tz=timezone.utc) - start}")
+            return metrics_to_export
+
+    def __get_data_exists_coros(self, dt: datetime) -> Iterator[str]:
+        for query in self._data_queries:
+            yield victoria.get(f"/api/v1/query?query={query}&time={int(dt.timestamp())}")
+
+    async def __process_token(
+        self,
+        ts: datetime,
+        section: str,
+        wallet: ChecksumAddress,
+        token: ChecksumAddress,
+        bal: Balance,
+        protocol: Optional[str] = None,
+    ) -> Tuple[victoria.types.PrometheusItem, victoria.types.PrometheusItem]:
+        # TODO wallet nicknames in grafana
+        # wallet = KNOWN_ADDRESSES[wallet] if wallet in KNOWN_ADDRESSES else wallet
+        if protocol is not None:
+            wallet = f"{protocol} | {wallet}"
+
+        label_and_section = f"{self.label}_{section}".lower().replace(" ", "_")
+        symbol = await _get_symbol(token)
+        bucket = await self.get_bucket(token)
+        ts_millis = floor(ts.timestamp()) * 1000
+
+        return (
+            victoria.types.PrometheusItem(
+                metric=victoria.Metric(
+                    param="balance",
+                    wallet=wallet,
+                    token_address=token,
+                    token=symbol,
+                    bucket=bucket,
+                    network=NETWORK_LABEL,
+                    __name__=label_and_section,
+                ),
+                values=[float(bal.balance)],
+                timestamps=[ts_millis],
+            ),
+            victoria.types.PrometheusItem(
+                metric=victoria.Metric(
+                    param="usd value",
+                    wallet=wallet,
+                    token_address=token,
+                    token=symbol,
+                    bucket=bucket,
+                    network=NETWORK_LABEL,
+                    __name__=label_and_section,
+                ),
+                values=[float(bal.usd)],
+                timestamps=[ts_millis],
+            ),
+        )
+
+
+async def _get_symbol(token: str) -> str:
+    if token == "ETH":
+        return "ETH"
+    try:
+        return await ERC20(token, asynchronous=True).symbol
+    except NonStandardERC20:
+        return "<NonStandardERC20>"

eth_portfolio_scripts/_utils.py

@@ -0,0 +1,106 @@
+import re
+from asyncio import Task, create_task, sleep
+from datetime import datetime, timedelta, timezone
+from typing import Any, AsyncGenerator, Dict, Final, List, Optional
+
+from brownie import chain
+
+
+timedelta_pattern: Final = re.compile(r"(\d+)([dhms]?)")
+
+
+def parse_timedelta(value: str) -> timedelta:
+    days, hours, minutes, seconds = 0, 0, 0, 0
+
+    for val, unit in timedelta_pattern.findall(value):
+        val = int(val)
+        if unit == "d":
+            days = val
+        elif unit == "h":
+            hours = val
+        elif unit == "m":
+            minutes = val
+        elif unit == "s":
+            seconds = val
+
+    return timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds)
+
+
+async def aiter_timestamps(
+    *,
+    start: Optional[datetime] = None,
+    interval: timedelta = timedelta(days=1),
+    run_forever: bool = False,
+) -> AsyncGenerator[datetime, None]:
+    """
+    Generates the timestamps to be queried based on the specified range and interval.
+    """
+    if not isinstance(run_forever, bool):
+        raise TypeError(f"`run_forever` must be boolean. You passed {run_forever}")
+
+    if start is None:
+        start = datetime.now(tz=timezone.utc)
+
+    block0_ts = datetime.fromtimestamp(chain[0].timestamp, tz=timezone.utc)
+    helper = datetime(
+        year=block0_ts.year,
+        month=block0_ts.month,
+        day=block0_ts.day,
+        hour=block0_ts.hour,
+        minute=block0_ts.minute,
+        tzinfo=timezone.utc,
+    )
+    while helper + interval < start:
+        helper += interval
+    start = helper
+    if start < block0_ts:
+        start += interval
+
+    timestamp = start
+
+    timestamps = []
+    while timestamp <= datetime.now(tz=timezone.utc):
+        timestamps.append(timestamp)
+        timestamp = timestamp + interval
+
+    # cycle between yielding earliest, latest, and middle from `timestamps` until complete
+    while timestamps:
+        # yield the earliest timestamp
+        yield timestamps.pop(0)
+        # yield the most recent timestamp if there is one
+        if timestamps:
+            yield timestamps.pop(-1)
+            # yield the most middle timestamp if there is one
+            if timestamps:
+                yield timestamps.pop(len(timestamps) // 2)
+
+    del timestamps
+
+    while run_forever:
+        while timestamp > datetime.now(tz=timezone.utc):
+            await _get_waiter(timestamp)
+        yield timestamp
+        timestamp += interval
+
+
+_waiters: Dict[datetime, "Task[None]"] = {}
+
+
+def _get_waiter(timestamp: datetime) -> "Task[None]":
+    if timestamp not in _waiters:
+        waiter = create_task(sleep_until(timestamp))
+        waiter.add_done_callback(_sleep_done_callback)
+        _waiters[timestamp] = waiter
+    return _waiters[timestamp]
+
+
+async def sleep_until(until: datetime) -> None:
+    now = datetime.now(tz=timezone.utc)
+    await sleep((until - now).total_seconds())
+
+
+def _sleep_done_callback(t: "Task[Any]") -> None:
+    low_to_hi = sorted(_waiters)
+    for k in low_to_hi:
+        if _waiters[k] is t:
+            _waiters.pop(k)

eth_portfolio_scripts/balances.py

@@ -0,0 +1,52 @@
+from argparse import Namespace
+from datetime import datetime, timezone
+from typing import Final
+
+import a_sync
+import a_sync.asyncio
+
+from eth_portfolio_scripts import docker
+from eth_portfolio_scripts._utils import aiter_timestamps, parse_timedelta
+
+
+_UTC: Final = timezone.utc
+
+create_task: Final = a_sync.create_task
+yield_to_loop: Final = a_sync.asyncio.sleep0
+
+
+@docker.ensure_containers
+async def export_balances(args: Namespace) -> None:
+    import dank_mids
+
+    from eth_portfolio_scripts._portfolio import ExportablePortfolio
+
+    if args.daemon is True:
+        raise NotImplementedError("This feature must be implemented")
+
+    interval = parse_timedelta(args.interval)
+    portfolio = ExportablePortfolio(
+        args.wallet,
+        label=args.label,
+        start_block=args.first_tx_block,
+        concurrency=args.concurrency,
+        load_prices=False,
+    )
+
+    if export_start_block := args.export_start_block or args.first_tx_block:
+        start_ts = await dank_mids.eth.get_block_timestamp(export_start_block)
+        start = datetime.fromtimestamp(start_ts, tz=timezone.utc)
+    else:
+        start = None
+
+    print(f"Exporting {portfolio}")
+    async for ts in aiter_timestamps(start=start, interval=interval, run_forever=True):
+        create_task(
+            coro=portfolio.export_snapshot(ts, sync=False),
+            name=f"eth-portfolio export snapshot {ts}",
+            skip_gc_until_done=True,
+        )
+        # get some work in before yielding the next task
+        await yield_to_loop()
+        await yield_to_loop()
+        await yield_to_loop()