eth-portfolio 1.1.0__py3-none-any.whl

eth_portfolio/_ledgers/portfolio.py

@@ -0,0 +1,327 @@
+import logging
+from typing import TYPE_CHECKING, AsyncIterator, Dict, Generic, Optional, TypeVar
+
+import a_sync
+from pandas import DataFrame, concat  # type: ignore
+from y.datatypes import Address, Block
+
+from eth_portfolio._decorators import set_end_block_if_none
+from eth_portfolio._ledgers.address import (
+    AddressLedgerBase,
+    InternalTransfersList,
+    TokenTransfersList,
+    TransactionsList,
+    _LedgerEntryList,
+)
+from eth_portfolio._utils import _AiterMixin
+from eth_portfolio.structs import InternalTransfer, TokenTransfer, Transaction
+
+if TYPE_CHECKING:
+    from eth_portfolio.portfolio import Portfolio
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class PortfolioLedgerBase(a_sync.ASyncGenericBase, _AiterMixin[T], Generic[_LedgerEntryList, T]):
+    property_name: str
+    object_caches: Dict[Address, AddressLedgerBase[_LedgerEntryList, T]]
+
+    def __init__(self, portfolio: "Portfolio"):  # type: ignore
+        assert hasattr(self, "property_name"), "Subclasses must define a property_name"
+        self.object_caches = {
+            address.address: getattr(address, self.property_name) for address in portfolio
+        }
+        self.portfolio = portfolio
+        super().__init__()
+
+    @property
+    def _start_block(self) -> int:
+        """Returns the start block for analysis of this portfolio."""
+        return self.portfolio._start_block
+
+    def _get_and_yield(self, start_block: int, end_block: int) -> AsyncIterator[T]:
+        """
+        Asynchronously yields ledger entries for each address in the portfolio for the specified block range.
+
+        This method is crucial for efficient data retrieval across multiple addresses, as it:
+        - Utilizes asynchronous iteration to process addresses concurrently.
+        - Reduces memory usage by yielding entries one at a time.
+
+        Args:
+            start_block: The starting block for the ledger query.
+            end_block: The ending block for the ledger query.
+
+        Yields:
+            Individual ledger entries of type T for each address in the portfolio.
+
+        Example:
+            >>> async for entry in ledger._get_and_yield(start_block=1000000, end_block=1100000):
+            ...     print(entry)
+        """
+        aiterators = [
+            getattr(address, self.property_name)[start_block:end_block]
+            for address in self.portfolio
+        ]
+        return a_sync.as_yielded(*aiterators)
+
+    @property
+    def asynchronous(self) -> bool:
+        """Returns `True` if the portfolio associated with this ledger is asynchronous, `False` if not."""
+        return self.portfolio.asynchronous
+
+    @set_end_block_if_none
+    async def get(self, start_block: Block, end_block: Block) -> Dict[Address, _LedgerEntryList]:
+        """
+        Fetches ledger entries for all portfolio addresses within the specified block range.
+
+        Args:
+            start_block: The starting block number for the query.
+            end_block: The ending block number for the query.
+
+        Returns:
+            A dictionary mapping each portfolio address to its corresponding ledger entries within the specified block range.
+
+        Note:
+            The @set_end_block_if_none decorator ensures that if end_block is not provided,
+            it defaults to the latest block.
+
+        Example:
+            >>> ledger = PortfolioTransactionsLedger(portfolio=portfolio)
+            >>> ledger_entries = await ledger.get(start_block=1000000, end_block=1100000)
+            >>> print("\n".join(f"Address {addr}: {len(entries)} entries" for addr, entries in ledger_entries.items()))
+        """
+        coros = {
+            address: cache.get(start_block, end_block, sync=False)
+            for address, cache in self.object_caches.items()
+        }
+        return await a_sync.gather(coros)
+
+    @set_end_block_if_none
+    async def df(self, start_block: Block, end_block: Block) -> DataFrame:
+        """
+        Returns a DataFrame containing all entries for this ledger.
+
+        This method provides an easy way to access your data in a standardized, clean format
+        for further analysis and reporting.
+
+        NOTE: Subclasses may override this method for type-specific DataFrame processing.
+
+        Args:
+            start_block: The starting block for the query.
+            end_block: The ending block for the query.
+
+        Returns:
+            A DataFrame containing processed ledger entries.
+
+        Example:
+            >>> df = await ledger.df(start_block=1000000, end_block=1100000)
+            >>> print(df)
+        """
+        df = await self._df_base(start_block, end_block)
+        if len(df) > 0:
+            df = self._cleanup_df(df)
+        return df
+
+    async def sent(
+        self, start_block: Optional[Block] = None, end_block: Optional[Block] = None
+    ) -> AsyncIterator[T]:
+        portfolio_addresses = set(self.portfolio.addresses.keys())
+        async for obj in self[start_block:end_block]:
+            if (
+                obj.from_address in portfolio_addresses
+                and obj.to_address not in portfolio_addresses
+            ):
+                yield obj
+
+    async def received(
+        self, start_block: Optional[Block] = None, end_block: Optional[Block] = None
+    ) -> AsyncIterator[T]:
+        portfolio_addresses = set(self.portfolio.addresses.keys())
+        async for obj in self[start_block:end_block]:
+            if (
+                obj.to_address in portfolio_addresses
+                and obj.from_address not in portfolio_addresses
+            ):
+                yield obj
+
+    async def _df_base(self, start_block: Block, end_block: Block) -> DataFrame:
+        """
+        Fetches and concatenates raw ledger data into a :class:`~DataFrame` for all addresses in the portfolio.
+
+        This method is a crucial part of the data processing pipeline, as it:
+        - Retrieves ledger entries for the specified block range across all portfolio addresses.
+        - Combines the data from multiple addresses into a single DataFrame.
+        - Serves as the foundation for further data cleaning and analysis.
+
+        Args:
+            start_block: The starting block number for the query.
+            end_block: The ending block number for the query.
+
+        Returns:
+            DataFrame: A concatenated DataFrame containing raw ledger entries from all addresses.
+
+        Example:
+            >>> df_base = await ledger._df_base(start_block=1000000, end_block=1100000)
+            >>> print(f"Total entries: {len(df_base)}", df_base.head(), sep="\n")
+
+        Note:
+            This method returns raw data that may contain duplicates or require further processing.
+            For cleaned and deduplicated data, use the `df()` method instead.
+        """
+        data: Dict[Address, _LedgerEntryList] = await self.get(start_block, end_block, sync=False)
+        return concat(pandable.df for pandable in data.values())
+
+    @classmethod
+    def _deduplicate_df(cls, df: DataFrame) -> DataFrame:
+        """
+        Deduplicate the DataFrame to prevent double-counting of transfers within the portfolio.
+
+        This method is crucial for ensuring accurate portfolio analysis by removing duplicate entries
+        where transfers between owned addresses appear once in each result set.
+
+        Note:
+            - This method can be overridden in subclasses if needed.
+            - If the DataFrame contains columns with list-type values, they are converted to strings for comparison.
+
+        Args:
+            df: The DataFrame to deduplicate.
+
+        Returns:
+            A deduplicated version of the input DataFrame.
+
+        Example:
+            >>> original_df = pd.DataFrame(...)  # Your original DataFrame
+            >>> deduped_df = PortfolioLedgerBase._deduplicate_df(original_df)
+            >>> print(f"Original rows: {len(original_df)}, Deduplicated rows: {len(deduped_df)}")
+        """
+        return df.loc[df.astype(str).drop_duplicates().index]
+
+    @classmethod
+    def _cleanup_df(cls, df: DataFrame) -> DataFrame:
+        """
+        Cleans up the DataFrame by deduplicating and sorting it by block number.
+
+        Args:
+            df: The DataFrame to clean up.
+
+        Returns:
+            A cleaned and deduplicated DataFrame sorted by block number.
+
+        Example:
+            >>> cleaned_df = ledger._cleanup_df(df)
+            >>> print(cleaned_df)
+        """
+        df = cls._deduplicate_df(df)
+        return df.sort_values(["blockNumber"]).reset_index(drop=True)
+
+
+class PortfolioTransactionsLedger(PortfolioLedgerBase[TransactionsList, Transaction]):
+    """
+    The :class:`~eth_portfolio._ledgers.PortfolioTransactionsLedger` class manages Ethereum
+    transaction entries across all addresses in a portfolio. It aggregates and processes
+    transactions for the entire portfolio within specified block ranges.
+
+    In the eth-portfolio ecosystem, this class is essential for:
+    - Providing a comprehensive view of all Ethereum transactions across multiple addresses.
+    - Supporting portfolio-wide analysis and reporting of transaction history.
+    - Enabling efficient querying of transaction data for specific time periods or block ranges.
+
+    Example:
+        >>> ledger = PortfolioTransactionsLedger(portfolio=Portfolio(addresses=["0x1234...", "0xABCD..."]))
+        >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+        >>> print(df)
+    """
+
+    property_name = "transactions"
+
+
+class PortfolioTokenTransfersLedger(PortfolioLedgerBase[TokenTransfersList, TokenTransfer]):
+    """
+    The :class:`~eth_portfolio._ledgers.PortfolioTokenTransfersLedger` class manages ERC20 token
+    transfer entries across all addresses in a portfolio. It aggregates and processes
+    token transfers for the entire portfolio within specified block ranges.
+
+    In the eth-portfolio ecosystem, this class is crucial for:
+    - Tracking all ERC20 token movements across multiple addresses in a portfolio.
+    - Facilitating token balance calculations and portfolio valuation.
+    - Supporting analysis of token transfer patterns and history.
+
+    Example:
+        >>> ledger = PortfolioTokenTransfersLedger(portfolio=Portfolio(addresses=["0x1234...", "0xABCD..."]))
+        >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+        >>> print(df)
+    """
+
+    property_name = "token_transfers"
+
+
+class PortfolioInternalTransfersLedger(
+    PortfolioLedgerBase[InternalTransfersList, InternalTransfer]
+):
+    """
+    The :class:`~eth_portfolio._ledgers.PortfolioInternalTransfersLedger` class manages internal
+    transfer entries across all addresses in a portfolio. It aggregates and processes internal transfers
+    for the entire portfolio within specified block ranges.
+
+    In the eth-portfolio ecosystem, this class plays a vital role in:
+    - Tracking internal Ethereum transfers between addresses within the same portfolio.
+    - Providing insights into complex transactions that involve multiple internal transfers.
+    - Supporting accurate portfolio analysis of internal transfer data from EVM traces.
+
+    Example:
+        >>> ledger = PortfolioInternalTransfersLedger(portfolio=Portfolio(addresses=["0x1234...", "0xABCD..."]))
+        >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+        >>> print(df)
+    """
+
+    property_name = "internal_transfers"
+
+    @set_end_block_if_none
+    async def df(self, start_block: Block, end_block: Block) -> DataFrame:
+        """
+        Returns a DataFrame containing all internal transfers to or from any of the addresses in the portfolio.
+
+        Args:
+            start_block: The starting block for the query.
+            end_block: The ending block for the query.
+
+        Returns:
+            A DataFrame containing processed internal transfer entries.
+
+        Example:
+            >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+            >>> print(df)
+        """
+        df = await self._df_base(start_block, end_block)
+        if len(df) > 0:
+            df.rename(
+                columns={"transactionHash": "hash", "transactionPosition": "transactionIndex"},
+                inplace=True,
+            )
+            df = self._cleanup_df(df)
+        return df
+
+    @classmethod
+    def _deduplicate_df(cls, df: DataFrame) -> DataFrame:
+        """
+        Deduplicates the DataFrame.
+
+        This deduplication is essential for ensuring accurate portfolio analysis by removing
+        duplicate entries due to transfers between addresses within the same portfolio.
+
+        Args:
+            df: The DataFrame to deduplicate.
+
+        Returns:
+            A deduplicated DataFrame.
+
+        Example:
+            >>> deduped_df = PortfolioInternalTransfersLedger._deduplicate_df(df)
+            >>> print(deduped_df)
+        """
+        df = df.reset_index(drop=True)
+        # We cant use drop_duplicates when one of the columns, `traceAddress`, contains lists.
+        # We must first convert the lists to strings
+        return df.loc[df.astype(str).drop_duplicates().index]

eth_portfolio/_loaders/__init__.py

@@ -0,0 +1,33 @@
+"""
+This module initializes the `_loaders` package within the `eth_portfolio` library.
+It imports key functions responsible for loading blockchain data related to transactions
+and token transfers for use within the package.
+
+The functions imported here are designed to facilitate the retrieval and processing of
+Ethereum blockchain data, enabling efficient data handling and storage for portfolio analysis.
+
+Imported Functions:
+    - :func:`~eth_portfolio._loaders.transaction.load_transaction`:
+      Loads transaction data by address and nonce, with optional price data retrieval.
+    - :func:`~eth_portfolio._loaders.token_transfer.load_token_transfer`:
+      Processes and loads token transfer data from log entries, with optional price fetching.
+
+Examples:
+    These functions can be used to load and process blockchain data for portfolio analysis.
+    For example, you might use them as follows:
+
+    >>> from eth_portfolio._loaders import load_transaction, load_token_transfer
+    >>> nonce, transaction = await load_transaction(address="0x1234567890abcdef1234567890abcdef12345678", nonce=5, load_prices=True)
+    >>> print(transaction)
+
+    >>> transfer_log = {"address": "0xTokenAddress", "data": "0xData", "removed": False}
+    >>> token_transfer = await load_token_transfer(transfer_log, load_prices=True)
+    >>> print(token_transfer)
+
+See Also:
+    - :mod:`eth_portfolio._loaders.transaction`: Contains functions for loading transaction data.
+    - :mod:`eth_portfolio._loaders.token_transfer`: Contains functions for processing token transfer logs.
+"""
+
+from eth_portfolio._loaders.transaction import load_transaction
+from eth_portfolio._loaders.token_transfer import load_token_transfer

eth_portfolio/_loaders/balances.py

@@ -0,0 +1,78 @@
+import logging
+from decimal import InvalidOperation
+
+import y
+from y._decorators import stuck_coro_debugger
+from y.datatypes import Address, Block
+
+from eth_portfolio._decimal import Decimal
+from eth_portfolio._utils import _get_price
+from eth_portfolio.typing import Balance
+
+logger = logging.getLogger(__name__)
+
+
+@stuck_coro_debugger
+async def load_token_balance(token: y.ERC20, address: Address, block: Block) -> Balance:
+    """
+    Asynchronously fetch the ERC20 token balance and its USD value for a given address at a specific block.
+
+    Args:
+        token: The ERC20 token contract to query.
+        address: The address holding the ERC20 tokens.
+        block: The block number for the balance query.
+
+    Returns:
+        :class:`~eth_portfolio.typing.Balance`: A custom object containing:
+            - balance: The token balance (in token's smallest unit).
+            - value: The USD value of the balance (18 decimal places).
+            - token: The ERC20 token which was checked.
+            - block: The block number where the balance was taken.
+
+    Note:
+        Non-standard ERC20 tokens are handled gracefully, returning a zero balance.
+
+    Example:
+        >>> balance = await load_token_balance(token=dai_contract, address='0x1234...', block=12345678)
+        >>> print(f"Token: {balance.token}, Value: {balance.balance}, USD: {balance.usd_value}")
+    """
+    try:
+        balance = await token.balance_of_readable(address, block, sync=False)
+    except y.NonStandardERC20:
+        logger.warning("NonStandardERC20 exc for %s", token)
+        balance = 0
+    if not balance:
+        return Balance(token=token.address, block=block)
+    price = await _get_price(token, block)
+    return Balance(
+        round(Decimal(balance), 18), _calc_value(balance, price), token=token.address, block=block
+    )
+
+
+def _calc_value(balance, price) -> Decimal:
+    """
+    Calculate the USD value of a token balance based on its price.
+
+    Args:
+        balance: The token balance.
+        price: The token price in USD.
+
+    Returns:
+        The total USD value, rounded to 18 decimal places if possible.
+        If rounding is not possible due to high precision, returns the unrounded value.
+
+    Note:
+        Returns :class:`~decimal.Decimal(0)` if the price is None, handling cases where price data is unavailable.
+
+    Example:
+        >>> value = _calc_value(balance=1000, price=0.50)
+        >>> print(f"USD Value: {value}")
+    """
+    if price is None:
+        return Decimal(0)
+    # NOTE If balance * price returns a Decimal with precision < 18, rounding is both impossible and unnecessary.
+    value = Decimal(balance) * Decimal(price)
+    try:
+        return round(value, 18)
+    except InvalidOperation:
+        return value

eth_portfolio/_loaders/token_transfer.py

@@ -0,0 +1,214 @@
+"""
+This module orchestrates the process of loading and processing token transfers within the eth_portfolio ecosystem.
+"""
+
+import decimal
+from logging import getLogger
+from typing import Optional
+
+from a_sync import create_task, gather
+from async_lru import alru_cache
+from dank_mids import BlockSemaphore
+from evmspec.data import TransactionIndex
+from msgspec import Struct, ValidationError
+from msgspec.json import decode
+from pony.orm import TransactionIntegrityError, UnexpectedError
+from y import ERC20
+from y._db.log import Log
+from y._decorators import stuck_coro_debugger
+from y.exceptions import NonStandardERC20, reraise_excs_with_extra_context
+
+from eth_portfolio._cache import cache_to_disk
+from eth_portfolio._db import utils as db
+from eth_portfolio._decimal import Decimal
+from eth_portfolio._loaders.utils import get_transaction_receipt
+from eth_portfolio._utils import _get_price
+from eth_portfolio.structs import TokenTransfer
+
+
+logger = getLogger(__name__)
+
+token_transfer_semaphore = BlockSemaphore(
+    20_000,  # Some arbitrary number
+    name="eth_portfolio.token_transfers",
+)
+"""A semaphore that regulates the concurrent processing of token transfers by processing lower blocks first."""
+
+
+@stuck_coro_debugger
+async def load_token_transfer(
+    transfer_log: "Log", load_prices: bool
+) -> Optional[TokenTransfer]:  # sourcery skip: simplify-boolean-comparison
+    """
+    Processes and loads a token transfer from a log entry, with comprehensive error handling and optional price fetching.
+
+    This function employs a multi-step process:
+    1. Validates the transfer against a known set of 'shitcoins', skipping processing if matched.
+    2. Checks for existing database entries, potentially deleting and reprocessing if price data is requested but missing.
+    3. Decodes the transfer log and retrieves associated metadata (e.g., token scale, symbol, transaction index).
+    4. Optionally fetches the token price at the time of the transaction.
+    5. Constructs and persists a :class:`~eth_portfolio.structs.TokenTransfer` object in the database.
+
+    The function handles various exceptions, including :class:`~y.exceptions.NonStandardERC20` for non-compliant tokens and :class:`decimal.InvalidOperation` for extreme :class:`~Decimal` values.
+
+    Args:
+        transfer_log: A dictionary containing the raw log entry of the token transfer.
+        load_prices: A flag indicating whether to fetch and include price data for the token at the time of transfer.
+
+    Returns:
+        A processed TokenTransfer object if successful, or None if the transfer cannot be processed due to various constraints or errors.
+
+    Note:
+        This function employs caching mechanisms and database operations to optimize performance.
+    """
+    if transfer_log.removed:
+        if transfer := await db.get_token_transfer(transfer_log):
+            await db.delete_token_transfer(transfer)
+        return None
+
+    if transfer := await db.get_token_transfer(transfer_log):
+        if load_prices is False or transfer.price:
+            return transfer
+        await db.delete_token_transfer(transfer)
+
+    if transfer_log.address in _non_standard_erc20:
+        logger.debug("%s is not a standard ERC20 token, skipping.", log.address)
+        return None
+
+    async with token_transfer_semaphore[transfer_log.block]:
+        token = ERC20(transfer_log.address, asynchronous=True)
+        try:
+            try:
+                # This will be mem cached so no need to gather and add a bunch of overhead
+                scale = await token.scale
+            except NonStandardERC20 as e:
+                # NOTE: if we cant fetch scale, this is probably either a shitcoin or an NFT (which we don't support at this time)
+                logger.debug("%s for %s, skipping.", e, transfer_log)
+                _non_standard_erc20.add(transfer_log.address)
+                return None
+
+            # This will be mem cached so no need to include it in the gather and add a bunch of overhead
+            symbol = await get_symbol(token)
+
+            tx_index_coro = get_transaction_index(transfer_log.transactionHash.hex())
+            coro_results = {"token": symbol}
+
+            if load_prices:
+                coro_results.update(
+                    await gather(
+                        {
+                            "transaction_index": tx_index_coro,
+                            "price": _get_price(token.address, transfer_log.blockNumber),
+                        }
+                    )
+                )
+            else:
+                coro_results["transaction_index"] = await tx_index_coro
+
+        except Exception as e:
+            logger.error(
+                f"%s %s for %s %s at block %s:",
+                e.__class__.__name__,
+                e,
+                await get_symbol(token) or token.address,
+                transfer_log.address,
+                transfer_log.blockNumber,
+            )
+            logger.exception(e)
+            return None
+
+        value = Decimal(transfer_log.data.as_uint256) / scale
+
+        if price := coro_results.get("price"):
+            coro_results["value_usd"] = round(value * price, 18)
+
+        transfer = TokenTransfer(log=transfer_log, value=value, **coro_results)
+
+        create_task(
+            _insert_to_db(transfer, load_prices),
+            skip_gc_until_done=True,
+        )
+
+        return transfer
+
+
+async def _insert_to_db(transfer: TokenTransfer, load_prices: bool) -> None:
+    with reraise_excs_with_extra_context(transfer):
+        try:
+            await db.insert_token_transfer(transfer)
+        except TransactionIntegrityError:
+            if load_prices:
+                await db.delete_token_transfer(transfer)
+                await db.insert_token_transfer(transfer)
+        except UnexpectedError:
+            digits_before_decimal = str(transfer.value).split(".")[0]
+            if len(digits_before_decimal) <= 20:
+                raise
+        except decimal.InvalidOperation:
+            # TODO: debug why this happens sometimes
+            pass
+
+
+_non_standard_erc20 = set()
+
+
+@stuck_coro_debugger
+async def get_symbol(token: ERC20) -> Optional[str]:
+    """
+    Retrieves the symbol of a given ERC20 token, with error handling for non-standard implementations.
+
+    This function attempts to access the token's symbol through the standard ERC20 symbol method. If the token contract
+    does not adhere to the standard ERC20 interface, indicated by a :class:`~y.exceptions.NonStandardERC20` exception, the function
+    returns `None` instead of propagating the error.
+
+    Args:
+        token: An ERC20 token object representing the token whose symbol is to be retrieved.
+
+    Returns:
+        The token's symbol as a string if successfully retrieved, or None if the token does not implement a standard symbol method.
+    """
+    if token.address in _non_standard_erc20:
+        return None
+    try:
+        return await token.__symbol__
+    except NonStandardERC20:
+        _non_standard_erc20.add(token.address)
+        return None
+
+
+@alru_cache(ttl=60 * 60)
+@stuck_coro_debugger
+@cache_to_disk
+async def get_transaction_index(hash: str) -> int:
+    """
+    Retrieves the transaction index for a given transaction hash, with results cached to disk.
+
+    This function fetches the transaction receipt corresponding to the provided hash and extracts the transaction index,
+    which represents the position of the transaction within its containing block. The result is cached to disk to
+    optimize performance for future runs.
+
+    Args:
+        hash: The hexadecimal string representation of the transaction hash.
+
+    Returns:
+        The zero-based index of the transaction within its block.
+    """
+    while True:
+        receipt_bytes = await get_transaction_receipt(hash)
+        if receipt_bytes is not None:
+            # TODO: debug why this happens, its something inside of dank_mids
+            break
+        logger.info("get_transaction_index failed, retrying...")
+
+    try:
+        return decode(
+            receipt_bytes, type=HasTxIndex, dec_hook=TransactionIndex._decode_hook
+        ).transactionIndex
+    except ValidationError as e:
+        new = TypeError(e, receipt_bytes, decode(receipt_bytes))
+        logger.exception(new)
+        raise new from e
+
+
+class HasTxIndex(Struct):
+    transactionIndex: TransactionIndex