eth-portfolio 0.5.8__cp310-cp310-win_amd64.whl

eth_portfolio/_utils.py

@@ -0,0 +1,229 @@
+import logging
+import sqlite3
+from abc import abstractmethod
+from collections.abc import AsyncGenerator, AsyncIterator
+from datetime import datetime
+from functools import cached_property
+from typing import TYPE_CHECKING, Final, Generic, TypeVar
+
+import dank_mids
+from a_sync import ASyncGenericBase, ASyncIterable, ASyncIterator, as_yielded
+from a_sync.asyncio import sleep0 as yield_to_loop
+from brownie import chain
+from brownie.exceptions import ContractNotFound
+from eth_typing import ChecksumAddress
+from faster_async_lru import alru_cache
+from faster_eth_abi.exceptions import InsufficientDataBytes
+from pandas import DataFrame
+from y import ERC20, Contract, Network
+from y.constants import CHAINID, NETWORK_NAME
+from y.datatypes import AddressOrContract, Block
+from y.exceptions import (
+    CantFetchParam,
+    ContractNotVerified,
+    NodeNotSynced,
+    NonStandardERC20,
+    PriceError,
+    reraise_excs_with_extra_context,
+    yPriceMagicError,
+)
+from y.prices.magic import get_price
+
+from eth_portfolio import _config, _decimal
+from eth_portfolio.typing import _T
+
+if TYPE_CHECKING:
+    from eth_portfolio.structs import LedgerEntry
+
+
+logger: Final = logging.getLogger(__name__)
+
+NON_STANDARD_ERC721: Final[list[ChecksumAddress]] = {
+    Network.Mainnet: ["0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB"],  # CryptoPunks
+}.get(CHAINID, [])
+
+SUPPRESS_ERROR_LOGS: Final[list[ChecksumAddress]] = {
+    # put tokens here when you don't expect them to price successfully and do not want to see the associated error logs
+}.get(CHAINID, [])
+"""Put tokens here when you don't expect them to price successfully and do not want to see the associated error logs."""
+
+
+async def get_buffered_chain_height() -> int:
+    """Returns an int equal to the current height of the chain minus `_config.REORG_BUFFER`."""
+    return await dank_mids.eth.get_block_number() - _config.REORG_BUFFER
+
+
+class PandableList(list[_T]):
+    @cached_property
+    def df(self) -> DataFrame:
+        return self._df()
+
+    def _df(self) -> DataFrame:
+        """Override this method if you need to manipulate your dataframe before returning it."""
+        return DataFrame(self)
+
+
+class Decimal(_decimal.Decimal):
+    """
+    I'm in the process of moving from floats to decimals, this will help be as I buidl.
+    """
+
+    # TODO i forget why I had this lol, should I delete?
+
+
+async def _describe_err(token: AddressOrContract, block: Block | None) -> str:
+    """
+    Assembles a string used to provide as much useful information as possible in PriceError messages
+    """
+    try:
+        symbol = await ERC20(token, asynchronous=True).symbol
+    except NonStandardERC20:
+        symbol = None
+
+    if block is None:
+        if symbol:
+            return f"{symbol} {token} on {NETWORK_NAME}"
+
+        return f"malformed token {token} on {NETWORK_NAME}"
+
+    if symbol:
+        return f"{symbol} {token} on {NETWORK_NAME} at {block}"
+
+    return f"malformed token {token} on {NETWORK_NAME} at {block}"
+
+
+_to_raise: Final = (
+    OSError,
+    FileNotFoundError,
+    NodeNotSynced,
+    NotImplementedError,
+    sqlite3.OperationalError,
+    InsufficientDataBytes,
+    UnboundLocalError,
+    RuntimeError,
+)
+
+
+async def _get_price(token: AddressOrContract, block: int | None = None) -> _decimal.Decimal:
+    token = str(token)
+    with reraise_excs_with_extra_context(token, block):
+        try:
+            if await is_erc721(token):
+                return _decimal.Decimal(0)
+            maybe_float = await get_price(token, block, silent=True, sync=False)
+            dprice = _decimal.Decimal(maybe_float)
+            return round(dprice, 18)
+        except CantFetchParam as e:
+            logger.warning("CantFetchParam %s", e)
+        except yPriceMagicError as e:
+            # Raise these exceptions
+            if isinstance(e.exception, _to_raise) and not isinstance(e.exception, RecursionError):
+                raise e.exception
+            # The exceptions below are acceptable enough
+            elif isinstance(e.exception, NonStandardERC20):
+                # Can't get symbol for handling like other excs
+                logger.warning(f"NonStandardERC20 while fetching price for {token}")
+            elif isinstance(e.exception, PriceError):
+                if token not in SUPPRESS_ERROR_LOGS:
+                    logger.warning(
+                        f"PriceError while fetching price for {await _describe_err(token, block)}"
+                    )
+            else:
+                logger.warning(f"{e} while fetching price for {await _describe_err(token, block)}")
+                logger.warning(e, exc_info=True)
+    return _decimal.Decimal(0)
+
+
+@alru_cache(maxsize=None)
+async def is_erc721(token: ChecksumAddress) -> bool:
+    # This can probably be improved
+    try:
+        contract = await Contract.coroutine(token)
+    except (ContractNotFound, ContractNotVerified):
+        return False
+    attrs = "setApprovalForAll", "getApproved", "isApprovedForAll"
+    if all(hasattr(contract, attr) for attr in attrs):
+        return True
+    elif contract.address in NON_STANDARD_ERC721:
+        return True
+    return False
+
+
+def _unpack_indicies(indicies: Block | tuple[Block, Block]) -> tuple[Block, Block]:
+    """Unpacks indicies and returns a tuple (start_block, end_block)."""
+    if isinstance(indicies, tuple):
+        start_block, end_block = indicies
+    else:
+        start_block = 0
+        end_block = indicies
+    return start_block, end_block
+
+
+class _YieldEvery:
+    __slots__ = ("_every", "_count")
+
+    def __init__(self, every: int) -> None:
+        self._every: Final = every
+        self._count = 0
+
+    async def tick(self) -> None:
+        self._count += 1
+        if self._count % self._every == 0:
+            await yield_to_loop()
+
+
+class _AiterMixin(ASyncIterable[_T]):
+    def __aiter__(self) -> AsyncIterator[_T]:
+        return self[self._start_block : chain.height].__aiter__()
+
+    def __getitem__(self, slice: slice) -> ASyncIterator[_T]:
+        if slice.start is not None and not isinstance(slice.start, (int, datetime)):
+            raise TypeError(f"start must be int or datetime. you passed {slice.start}")
+        if slice.stop and not isinstance(slice.stop, (int, datetime)):
+            raise TypeError(f"start must be int or datetime. you passed {slice.start}")
+        if slice.step is not None:
+            raise ValueError("You cannot use a step here.")
+        return ASyncIterator(self._get_and_yield(slice.start or 0, slice.stop, True))  # type: ignore [truthy-function]
+
+    def yield_forever(self, mem_cache: bool = True) -> ASyncIterator[_T]:
+        return ASyncIterator(self._get_and_yield(self._start_block or 0, None, mem_cache))
+
+    @abstractmethod
+    async def _get_and_yield(
+        self, start_block: Block, end_block: Block, mem_cache: bool
+    ) -> AsyncGenerator[_T, None]:
+        yield
+
+    @property
+    @abstractmethod
+    def _start_block(self) -> int: ...
+
+
+_LT = TypeVar("_LT")
+
+
+class _LedgeredBase(ASyncGenericBase, _AiterMixin["LedgerEntry"], Generic[_LT]):
+    """A mixin class for things with ledgers"""
+
+    transactions: _LT
+    internal_transfers: _LT
+    token_transfers: _LT
+
+    def __init__(self, start_block: Block) -> None:
+        self.start_block = start_block
+        super().__init__()
+
+    @property
+    def _start_block(self) -> Block:
+        # in the middle of refactoring this
+        return self.start_block
+
+    @cached_property
+    def _ledgers(self) -> tuple[_LT, _LT, _LT]:
+        """A tuple with the 3 ledgers (transactions, internal transfers, token transfers)"""
+        return self.transactions, self.internal_transfers, self.token_transfers
+
+    def _get_and_yield(
+        self, start_block: Block, end_block: Block, mem_cache: bool
+    ) -> AsyncGenerator["LedgerEntry", None]:
+        return as_yielded(*(ledger._get_and_yield(start_block, end_block, mem_cache) for ledger in self._ledgers))  # type: ignore [return-value,index]

eth_portfolio/_ydb/token_transfers.py

@@ -0,0 +1,144 @@
+from abc import abstractmethod
+from asyncio import Task, create_task
+from collections.abc import AsyncIterator
+from logging import DEBUG, getLogger
+from typing import Any, Final, cast
+
+import dank_mids
+import evmspec
+import y._db.log
+from a_sync import ASyncIterable, ASyncIterator, as_yielded
+from brownie import chain
+from eth_typing import BlockNumber, ChecksumAddress
+from faster_eth_abi import encode
+from faster_eth_utils import encode_hex
+from y.utils.events import ProcessedEvents
+
+from eth_portfolio._loaders import load_token_transfer
+from eth_portfolio._shitcoins import SHITCOINS
+from eth_portfolio._utils import _YieldEvery
+from eth_portfolio.constants import TRANSFER_SIGS
+from eth_portfolio.structs import TokenTransfer
+
+logger: Final = getLogger(__name__)
+_logger_is_enabled_for: Final = logger.isEnabledFor
+_logger_log: Final = logger._log
+
+
+def encode_address(address: Any) -> bytes:
+    return encode_hex(encode(["address"], [str(address)]))
+
+
+class _TokenTransfers(ProcessedEvents["Task[TokenTransfer]"]):
+    """A helper mixin that contains all logic for fetching token transfers for a particular wallet address"""
+
+    __slots__ = "address", "_load_prices"
+
+    def __init__(
+        self,
+        address: ChecksumAddress,
+        from_block: BlockNumber,
+        load_prices: bool = False,
+    ) -> None:
+        self.address: Final = address
+        self._load_prices: Final = load_prices
+        super().__init__(topics=self._topics, from_block=from_block)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(address={self.address})"
+
+    @property
+    @abstractmethod
+    def _topics(self) -> list: ...
+
+    @ASyncIterator.wrap  # type: ignore [call-overload]
+    async def yield_thru_block(self, block: BlockNumber) -> AsyncIterator["Task[TokenTransfer]"]:
+        if not _logger_is_enabled_for(DEBUG):
+            async for task in self._objects_thru(block=block):
+                yield task
+            return
+
+        _logger_log(DEBUG, "%s yielding all objects thru block %s", (self, block))
+        async for task in self._objects_thru(block=block):
+            _logger_log(
+                DEBUG,
+                "yielding %s at block %s [thru: %s, lock: %s]",
+                (task, task.block, block, self._lock.value),
+            )
+            yield task
+        _logger_log(DEBUG, "%s yield thru %s complete", (self, block))
+
+    async def _extend(self, objs: list[evmspec.Log]) -> None:
+        shitcoins = SHITCOINS.get(chain.id, set())
+        append_loader_task = self._objects.append
+        yielder = _YieldEvery(100)
+        for log in objs:
+            if cast(ChecksumAddress, log.address) in shitcoins:
+                continue
+            # save i/o
+            array_encodable_log = y._db.log.Log(**log)
+            task = create_task(load_token_transfer(array_encodable_log, self._load_prices))
+            task.block = log.block  # type: ignore [attr-defined]
+            append_loader_task(task)
+            # Make sure the event loop doesn't get blocked
+            await yielder.tick()
+
+    def _get_block_for_obj(self, task: "Task[TokenTransfer]") -> int:
+        return task.block  # type: ignore [attr-defined]
+
+    def _process_event(self, task: "Task[TokenTransfer]") -> "Task[TokenTransfer]":
+        return task
+
+    def _done_callback(self, task: Task) -> None:
+        if e := task.exception():
+            self._exc = e
+            logger.exception(e)
+            raise e
+
+
+class InboundTokenTransfers(_TokenTransfers):
+    """A container that fetches and iterates over all inbound token transfers for a particular wallet address"""
+
+    @property
+    def _topics(self) -> list:
+        return [TRANSFER_SIGS, None, encode_address(self.address)]
+
+
+class OutboundTokenTransfers(_TokenTransfers):
+    """A container that fetches and iterates over all outbound token transfers for a particular wallet address"""
+
+    @property
+    def _topics(self) -> list:
+        return [TRANSFER_SIGS, encode_address(self.address)]
+
+
+class TokenTransfers(ASyncIterable[TokenTransfer]):
+    """
+    A container that fetches and iterates over all token transfers for a particular wallet address.
+    NOTE: These do not come back in chronologcal order.
+    """
+
+    def __init__(
+        self,
+        address: ChecksumAddress,
+        from_block: BlockNumber,
+        load_prices: bool = False,
+    ) -> None:
+        self.transfers_in: Final = InboundTokenTransfers(
+            address, from_block, load_prices=load_prices
+        )
+        self.transfers_out: Final = OutboundTokenTransfers(
+            address, from_block, load_prices=load_prices
+        )
+
+    async def __aiter__(self) -> AsyncIterator["Task[TokenTransfer]"]:
+        async for transfer in self.__yield_thru_block(await dank_mids.eth.block_number):
+            yield transfer
+
+    def yield_thru_block(self, block: BlockNumber) -> ASyncIterator["Task[TokenTransfer]"]:
+        return ASyncIterator(self.__yield_thru_block(block))
+
+    def __yield_thru_block(self, block: BlockNumber) -> AsyncIterator["Task[TokenTransfer]"]:
+        return as_yielded(
+            self.transfers_in.yield_thru_block(block), self.transfers_out.yield_thru_block(block)
+        )