eth-portfolio-temp 0.2.12__cp313-cp313-win32.whl

eth_portfolio/_ledgers/portfolio.py

@@ -0,0 +1,327 @@
+import logging
+from typing import TYPE_CHECKING, AsyncIterator, Dict, Generic, Optional, TypeVar
+
+import a_sync
+from pandas import DataFrame, concat  # type: ignore
+from y.datatypes import Address, Block
+
+from eth_portfolio._decorators import set_end_block_if_none
+from eth_portfolio._ledgers.address import (
+    AddressLedgerBase,
+    InternalTransfersList,
+    TokenTransfersList,
+    TransactionsList,
+    _LedgerEntryList,
+)
+from eth_portfolio._utils import _AiterMixin
+from eth_portfolio.structs import InternalTransfer, TokenTransfer, Transaction
+
+if TYPE_CHECKING:
+    from eth_portfolio.portfolio import Portfolio
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+class PortfolioLedgerBase(a_sync.ASyncGenericBase, _AiterMixin[T], Generic[_LedgerEntryList, T]):
+    property_name: str
+    object_caches: Dict[Address, AddressLedgerBase[_LedgerEntryList, T]]
+
+    def __init__(self, portfolio: "Portfolio"):  # type: ignore
+        assert hasattr(self, "property_name"), "Subclasses must define a property_name"
+        self.object_caches = {
+            address.address: getattr(address, self.property_name) for address in portfolio
+        }
+        self.portfolio = portfolio
+        super().__init__()
+
+    @property
+    def _start_block(self) -> int:
+        """Returns the start block for analysis of this portfolio."""
+        return self.portfolio._start_block
+
+    def _get_and_yield(self, start_block: int, end_block: int, mem_cache: bool) -> AsyncIterator[T]:
+        """
+        Asynchronously yields ledger entries for each address in the portfolio for the specified block range.
+
+        This method is crucial for efficient data retrieval across multiple addresses, as it:
+        - Utilizes asynchronous iteration to process addresses concurrently.
+        - Reduces memory usage by yielding entries one at a time.
+
+        Args:
+            start_block: The starting block for the ledger query.
+            end_block: The ending block for the ledger query.
+
+        Yields:
+            Individual ledger entries of type T for each address in the portfolio.
+
+        Example:
+            >>> async for entry in ledger._get_and_yield(start_block=1000000, end_block=1100000):
+            ...     print(entry)
+        """
+        aiterators = [
+            getattr(address, self.property_name)._get_and_yield(start_block, end_block, mem_cache)
+            for address in self.portfolio
+        ]
+        return a_sync.as_yielded(*aiterators)
+
+    @property
+    def asynchronous(self) -> bool:
+        """Returns `True` if the portfolio associated with this ledger is asynchronous, `False` if not."""
+        return self.portfolio.asynchronous
+
+    @set_end_block_if_none
+    async def get(self, start_block: Block, end_block: Block) -> Dict[Address, _LedgerEntryList]:
+        """
+        Fetches ledger entries for all portfolio addresses within the specified block range.
+
+        Args:
+            start_block: The starting block number for the query.
+            end_block: The ending block number for the query.
+
+        Returns:
+            A dictionary mapping each portfolio address to its corresponding ledger entries within the specified block range.
+
+        Note:
+            The @set_end_block_if_none decorator ensures that if end_block is not provided,
+            it defaults to the latest block.
+
+        Example:
+            >>> ledger = PortfolioTransactionsLedger(portfolio=portfolio)
+            >>> ledger_entries = await ledger.get(start_block=1000000, end_block=1100000)
+            >>> print("\n".join(f"Address {addr}: {len(entries)} entries" for addr, entries in ledger_entries.items()))
+        """
+        coros = {
+            address: cache.get(start_block, end_block, sync=False)
+            for address, cache in self.object_caches.items()
+        }
+        return await a_sync.gather(coros)
+
+    @set_end_block_if_none
+    async def df(self, start_block: Block, end_block: Block) -> DataFrame:
+        """
+        Returns a DataFrame containing all entries for this ledger.
+
+        This method provides an easy way to access your data in a standardized, clean format
+        for further analysis and reporting.
+
+        NOTE: Subclasses may override this method for type-specific DataFrame processing.
+
+        Args:
+            start_block: The starting block for the query.
+            end_block: The ending block for the query.
+
+        Returns:
+            A DataFrame containing processed ledger entries.
+
+        Example:
+            >>> df = await ledger.df(start_block=1000000, end_block=1100000)
+            >>> print(df)
+        """
+        df = await self._df_base(start_block, end_block)
+        if len(df) > 0:
+            df = self._cleanup_df(df)
+        return df
+
+    async def sent(
+        self, start_block: Optional[Block] = None, end_block: Optional[Block] = None
+    ) -> AsyncIterator[T]:
+        portfolio_addresses = set(self.portfolio.addresses.keys())
+        async for obj in self[start_block:end_block]:
+            if (
+                obj.from_address in portfolio_addresses
+                and obj.to_address not in portfolio_addresses
+            ):
+                yield obj
+
+    async def received(
+        self, start_block: Optional[Block] = None, end_block: Optional[Block] = None
+    ) -> AsyncIterator[T]:
+        portfolio_addresses = set(self.portfolio.addresses.keys())
+        async for obj in self[start_block:end_block]:
+            if (
+                obj.to_address in portfolio_addresses
+                and obj.from_address not in portfolio_addresses
+            ):
+                yield obj
+
+    async def _df_base(self, start_block: Block, end_block: Block) -> DataFrame:
+        """
+        Fetches and concatenates raw ledger data into a :class:`~DataFrame` for all addresses in the portfolio.
+
+        This method is a crucial part of the data processing pipeline, as it:
+        - Retrieves ledger entries for the specified block range across all portfolio addresses.
+        - Combines the data from multiple addresses into a single DataFrame.
+        - Serves as the foundation for further data cleaning and analysis.
+
+        Args:
+            start_block: The starting block number for the query.
+            end_block: The ending block number for the query.
+
+        Returns:
+            DataFrame: A concatenated DataFrame containing raw ledger entries from all addresses.
+
+        Example:
+            >>> df_base = await ledger._df_base(start_block=1000000, end_block=1100000)
+            >>> print(f"Total entries: {len(df_base)}", df_base.head(), sep="\n")
+
+        Note:
+            This method returns raw data that may contain duplicates or require further processing.
+            For cleaned and deduplicated data, use the `df()` method instead.
+        """
+        data: Dict[Address, _LedgerEntryList] = await self.get(start_block, end_block, sync=False)
+        return concat(pandable.df for pandable in data.values())
+
+    @classmethod
+    def _deduplicate_df(cls, df: DataFrame) -> DataFrame:
+        """
+        Deduplicate the DataFrame to prevent double-counting of transfers within the portfolio.
+
+        This method is crucial for ensuring accurate portfolio analysis by removing duplicate entries
+        where transfers between owned addresses appear once in each result set.
+
+        Note:
+            - This method can be overridden in subclasses if needed.
+            - If the DataFrame contains columns with list-type values, they are converted to strings for comparison.
+
+        Args:
+            df: The DataFrame to deduplicate.
+
+        Returns:
+            A deduplicated version of the input DataFrame.
+
+        Example:
+            >>> original_df = pd.DataFrame(...)  # Your original DataFrame
+            >>> deduped_df = PortfolioLedgerBase._deduplicate_df(original_df)
+            >>> print(f"Original rows: {len(original_df)}, Deduplicated rows: {len(deduped_df)}")
+        """
+        return df.loc[df.astype(str).drop_duplicates().index]
+
+    @classmethod
+    def _cleanup_df(cls, df: DataFrame) -> DataFrame:
+        """
+        Cleans up the DataFrame by deduplicating and sorting it by block number.
+
+        Args:
+            df: The DataFrame to clean up.
+
+        Returns:
+            A cleaned and deduplicated DataFrame sorted by block number.
+
+        Example:
+            >>> cleaned_df = ledger._cleanup_df(df)
+            >>> print(cleaned_df)
+        """
+        df = cls._deduplicate_df(df)
+        return df.sort_values(["blockNumber"]).reset_index(drop=True)
+
+
+class PortfolioTransactionsLedger(PortfolioLedgerBase[TransactionsList, Transaction]):
+    """
+    The :class:`~eth_portfolio._ledgers.PortfolioTransactionsLedger` class manages Ethereum
+    transaction entries across all addresses in a portfolio. It aggregates and processes
+    transactions for the entire portfolio within specified block ranges.
+
+    In the eth-portfolio ecosystem, this class is essential for:
+    - Providing a comprehensive view of all Ethereum transactions across multiple addresses.
+    - Supporting portfolio-wide analysis and reporting of transaction history.
+    - Enabling efficient querying of transaction data for specific time periods or block ranges.
+
+    Example:
+        >>> ledger = PortfolioTransactionsLedger(portfolio=Portfolio(addresses=["0x1234...", "0xABCD..."]))
+        >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+        >>> print(df)
+    """
+
+    property_name = "transactions"
+
+
+class PortfolioTokenTransfersLedger(PortfolioLedgerBase[TokenTransfersList, TokenTransfer]):
+    """
+    The :class:`~eth_portfolio._ledgers.PortfolioTokenTransfersLedger` class manages ERC20 token
+    transfer entries across all addresses in a portfolio. It aggregates and processes
+    token transfers for the entire portfolio within specified block ranges.
+
+    In the eth-portfolio ecosystem, this class is crucial for:
+    - Tracking all ERC20 token movements across multiple addresses in a portfolio.
+    - Facilitating token balance calculations and portfolio valuation.
+    - Supporting analysis of token transfer patterns and history.
+
+    Example:
+        >>> ledger = PortfolioTokenTransfersLedger(portfolio=Portfolio(addresses=["0x1234...", "0xABCD..."]))
+        >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+        >>> print(df)
+    """
+
+    property_name = "token_transfers"
+
+
+class PortfolioInternalTransfersLedger(
+    PortfolioLedgerBase[InternalTransfersList, InternalTransfer]
+):
+    """
+    The :class:`~eth_portfolio._ledgers.PortfolioInternalTransfersLedger` class manages internal
+    transfer entries across all addresses in a portfolio. It aggregates and processes internal transfers
+    for the entire portfolio within specified block ranges.
+
+    In the eth-portfolio ecosystem, this class plays a vital role in:
+    - Tracking internal Ethereum transfers between addresses within the same portfolio.
+    - Providing insights into complex transactions that involve multiple internal transfers.
+    - Supporting accurate portfolio analysis of internal transfer data from EVM traces.
+
+    Example:
+        >>> ledger = PortfolioInternalTransfersLedger(portfolio=Portfolio(addresses=["0x1234...", "0xABCD..."]))
+        >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+        >>> print(df)
+    """
+
+    property_name = "internal_transfers"
+
+    @set_end_block_if_none
+    async def df(self, start_block: Block, end_block: Block) -> DataFrame:
+        """
+        Returns a DataFrame containing all internal transfers to or from any of the addresses in the portfolio.
+
+        Args:
+            start_block: The starting block for the query.
+            end_block: The ending block for the query.
+
+        Returns:
+            A DataFrame containing processed internal transfer entries.
+
+        Example:
+            >>> df = await ledger.df(start_block=10000000, end_block=12000000)
+            >>> print(df)
+        """
+        df = await self._df_base(start_block, end_block)
+        if len(df) > 0:
+            df.rename(
+                columns={"transactionHash": "hash", "transactionPosition": "transactionIndex"},
+                inplace=True,
+            )
+            df = self._cleanup_df(df)
+        return df
+
+    @classmethod
+    def _deduplicate_df(cls, df: DataFrame) -> DataFrame:
+        """
+        Deduplicates the DataFrame.
+
+        This deduplication is essential for ensuring accurate portfolio analysis by removing
+        duplicate entries due to transfers between addresses within the same portfolio.
+
+        Args:
+            df: The DataFrame to deduplicate.
+
+        Returns:
+            A deduplicated DataFrame.
+
+        Example:
+            >>> deduped_df = PortfolioInternalTransfersLedger._deduplicate_df(df)
+            >>> print(deduped_df)
+        """
+        df = df.reset_index(drop=True)
+        # We cant use drop_duplicates when one of the columns, `traceAddress`, contains lists.
+        # We must first convert the lists to strings
+        return df.loc[df.astype(str).drop_duplicates().index]

eth_portfolio/_loaders/__init__.py

@@ -0,0 +1,33 @@
+"""
+This module initializes the `_loaders` package within the `eth_portfolio` library.
+It imports key functions responsible for loading blockchain data related to transactions
+and token transfers for use within the package.
+
+The functions imported here are designed to facilitate the retrieval and processing of
+Ethereum blockchain data, enabling efficient data handling and storage for portfolio analysis.
+
+Imported Functions:
+    - :func:`~eth_portfolio._loaders.transaction.load_transaction`:
+      Loads transaction data by address and nonce, with optional price data retrieval.
+    - :func:`~eth_portfolio._loaders.token_transfer.load_token_transfer`:
+      Processes and loads token transfer data from log entries, with optional price fetching.
+
+Examples:
+    These functions can be used to load and process blockchain data for portfolio analysis.
+    For example, you might use them as follows:
+
+    >>> from eth_portfolio._loaders import load_transaction, load_token_transfer
+    >>> nonce, transaction = await load_transaction(address="0x1234567890abcdef1234567890abcdef12345678", nonce=5, load_prices=True)
+    >>> print(transaction)
+
+    >>> transfer_log = {"address": "0xTokenAddress", "data": "0xData", "removed": False}
+    >>> token_transfer = await load_token_transfer(transfer_log, load_prices=True)
+    >>> print(token_transfer)
+
+See Also:
+    - :mod:`eth_portfolio._loaders.transaction`: Contains functions for loading transaction data.
+    - :mod:`eth_portfolio._loaders.token_transfer`: Contains functions for processing token transfer logs.
+"""
+
+from eth_portfolio._loaders.transaction import load_transaction
+from eth_portfolio._loaders.token_transfer import load_token_transfer

eth_portfolio/_loaders/_nonce.py

@@ -0,0 +1,196 @@
+import asyncio
+import logging
+from collections import defaultdict
+from time import time
+from typing import ClassVar, DefaultDict, Dict, Final, Optional, Tuple, final
+
+import a_sync
+import dank_mids
+from eth_typing import BlockNumber, ChecksumAddress
+
+from eth_portfolio._loaders import utils
+
+
+logger: Final = logging.getLogger("eth_portfolio.nonces")
+logger_is_enabled: Final = logger.isEnabledFor
+__log: Final = logger._log
+
+DEBUG: Final = logging.DEBUG
+
+Nonce = int
+AccountNonces = DefaultDict[Nonce, BlockNumber]
+GlobalNonces = DefaultDict[ChecksumAddress, AccountNonces]
+
+nonces: Final[GlobalNonces] = defaultdict(lambda: defaultdict(int))  # type: ignore [arg-type]
+locks: Final[DefaultDict[ChecksumAddress, asyncio.Lock]] = defaultdict(asyncio.Lock)
+
+get_transaction_count: Final = dank_mids.eth.get_transaction_count
+
+igather: Final = a_sync.igather
+
+now: Final = time
+
+
+async def get_nonce_at_block(address: ChecksumAddress, block: BlockNumber) -> int:
+    """
+    Retrieves the nonce of an address at a specific block.
+
+    This function gets the transaction count (nonce) for the given address at the specified block. It also includes a special case to handle known issues on certain networks like Arbitrum.
+
+    Args:
+        address: The address of the account.
+        block: The block number at which to retrieve the nonce.
+
+    Returns:
+        The nonce of the address at the given block.
+
+    Example:
+        >>> block = 12345678
+        >>> nonce = await get_nonce_at_block("0x1234567890abcdef1234567890abcdef12345678", block)
+        >>> print(f"The nonce at block {block} is {nonce}.")
+
+    """
+    try:
+        nonce = await get_transaction_count(address, block_identifier=block) - 1
+        _update_nonces(address, nonce, block)
+        return nonce
+    except ValueError as e:
+        # NOTE this is known to occur on Arbitrum
+        if "error creating execution cursor" in str(e) and block == 0:
+            return -1
+        raise ValueError(f"For {address} at {block}: {e}") from e
+
+
+async def get_block_for_nonce(address: ChecksumAddress, nonce: Nonce) -> int:
+    highest_known_nonce_lt_query: Optional[int]
+    lowest_known_nonce_gt_query: Optional[int]
+
+    async with locks[address]:
+        highest_known_nonce_lt_query = None
+        lowest_known_nonce_gt_query = None
+
+        # it is impossible for n to == nonce
+        for n in nonces[address]:
+            if n < nonce:
+                if highest_known_nonce_lt_query is None or n > highest_known_nonce_lt_query:
+                    highest_known_nonce_lt_query = n
+            elif n == nonce:
+                continue
+            elif lowest_known_nonce_gt_query is None or n < lowest_known_nonce_gt_query:
+                lowest_known_nonce_gt_query = n
+
+        if highest_known_nonce_lt_query is not None:
+            lo = nonces[address][highest_known_nonce_lt_query]
+        else:
+            lo = BlockNumber(0)
+
+        if lowest_known_nonce_gt_query is not None:
+            hi = nonces[address][lowest_known_nonce_gt_query]
+        else:
+            hi = await get_block_number()
+
+        # lets find the general area first before we proceed with our binary search
+        range_size = hi - lo + 1
+        if range_size > 4:
+            lo, hi = await _get_area(address, nonce, lo, hi, range_size)
+
+    debug_logs_enabled = logger_is_enabled(DEBUG)
+    while True:
+        _nonce = await get_nonce_at_block(address, lo)
+
+        if _nonce < nonce:
+            old_lo = lo
+            lo += int((hi - lo) / 2) or 1  # type: ignore [assignment]
+            if debug_logs_enabled:
+                __log(
+                    DEBUG,
+                    "Nonce for %s at %s is %s, checking higher block %s",
+                    (address, old_lo, _nonce, lo),
+                )
+            continue
+
+        prev_block_nonce: int = await get_nonce_at_block(address, lo - 1)  # type: ignore [arg-type]
+        if prev_block_nonce >= nonce:
+            hi = lo
+            lo = int(lo / 2)  # type: ignore [assignment]
+            if debug_logs_enabled:
+                __log(
+                    DEBUG,
+                    "Nonce for %s at %s is %s, checking lower block %s",
+                    (address, hi, _nonce, lo),
+                )
+            continue
+
+        if debug_logs_enabled:
+            __log(DEBUG, "Found nonce %s for %s at block %s", (nonce, address, lo))
+
+        return lo
+
+
+async def _get_area(
+    address: ChecksumAddress,
+    nonce: Nonce,
+    lo: BlockNumber,
+    hi: BlockNumber,
+    range_size: int,
+) -> Tuple[BlockNumber, BlockNumber]:
+    num_chunks = _get_num_chunks(range_size)
+    chunk_size = range_size // num_chunks
+    points = [BlockNumber(lo + i * chunk_size) for i in range(num_chunks)]
+    nonces = await igather(get_nonce_at_block(address, point) for point in points)
+    for block, n in zip(points, nonces):
+        if n >= nonce:
+            return lo, block
+        lo = block
+    return lo, hi
+
+
+def _update_nonces(address: ChecksumAddress, nonce: Nonce, block: BlockNumber) -> None:
+    # if you are searching for `nonce` and you verified it occurs AT or ABOVE `block` call this fn.
+    if block > nonces[address][nonce]:
+        nonces[address][nonce] = block
+
+
+def _get_num_chunks(range_size: int) -> int:
+    if range_size >= 4096:
+        return 100
+    elif range_size >= 2048:
+        return 80
+    elif range_size >= 1024:
+        return 40
+    elif range_size >= 512:
+        return 20
+    elif range_size >= 256:
+        return 10
+    elif range_size >= 128:
+        return 8
+    elif range_size >= 64:
+        return 6
+    elif range_size >= 32:
+        return 5
+    elif range_size >= 16:
+        return 4
+    elif range_size >= 8:
+        return 3
+    else:
+        return 2
+
+
+@final
+class BlockCache:
+    block: ClassVar[BlockNumber] = 0
+    updated_at: ClassVar = 0.0
+    lock: Final = asyncio.Lock()
+    ttl: Final = 5.0
+
+
+async def get_block_number() -> BlockNumber:
+    if now() - BlockCache.updated_at < BlockCache.ttl:
+        return BlockCache.block
+    async with BlockCache.lock:
+        if now() - BlockCache.updated_at < BlockCache.ttl:
+            return BlockCache.block
+        ts = now()
+        block = BlockCache.block = await dank_mids.eth.block_number
+        BlockCache.updated_at = ts
+        return block

eth_portfolio/_loaders/balances.py

@@ -0,0 +1,94 @@
+import logging
+from decimal import InvalidOperation
+from typing import Final
+
+import y
+from y._decorators import stuck_coro_debugger
+from y.datatypes import Address, Block
+
+from eth_portfolio._decimal import Decimal
+from eth_portfolio._utils import _get_price
+from eth_portfolio.typing import Balance
+
+
+_ZERO: Final = Decimal(0)
+
+logger: Final = logging.getLogger(__name__)
+
+
+@stuck_coro_debugger
+async def load_token_balance(token: y.ERC20, address: Address, block: Block) -> Balance:
+    """
+    Asynchronously fetch the ERC20 token balance and its USD value for a given address at a specific block.
+
+    Args:
+        token: The ERC20 token contract to query.
+        address: The address holding the ERC20 tokens.
+        block: The block number for the balance query.
+
+    Returns:
+        :class:`~eth_portfolio.typing.Balance`: A custom object containing:
+            - balance: The token balance (in token's smallest unit).
+            - value: The USD value of the balance (18 decimal places).
+            - token: The ERC20 token which was checked.
+            - block: The block number where the balance was taken.
+
+    Note:
+        Non-standard ERC20 tokens are handled gracefully, returning a zero balance.
+
+    Example:
+        >>> balance = await load_token_balance(token=dai_contract, address='0x1234...', block=12345678)
+        >>> print(f"Token: {balance.token}, Value: {balance.balance}, USD: {balance.usd_value}")
+    """
+    try:
+        balance = await token.balance_of_readable(address, block, sync=False)
+    except y.NonStandardERC20:
+        logger.warning("NonStandardERC20 exc for %s", token)
+        balance = _ZERO
+    token_address = token.address
+    if not balance:
+        return Balance(token=token_address, block=block)
+    price = await _get_price(token_address, block)
+    return Balance(
+        balance=round(balance, 18),
+        usd_value=_calc_value(balance, price),
+        token=token_address,
+        block=block,
+    )
+
+
+def _calc_value(balance, price) -> Decimal:
+    """
+    Calculate the USD value of a token balance based on its price.
+
+    Args:
+        balance: The token balance.
+        price: The token price in USD.
+
+    Returns:
+        The total USD value, rounded to 18 decimal places if possible.
+        If rounding is not possible due to high precision, returns the unrounded value.
+
+    Note:
+        Returns :class:`~decimal.Decimal(0)` if the price is None, handling cases where price data is unavailable.
+
+    Example:
+        >>> value = _calc_value(balance=1000, price=0.50)
+        >>> print(f"USD Value: {value}")
+    """
+    if price is None:
+        return _ZERO
+    # NOTE If balance * price returns a Decimal with precision < 18, rounding is both impossible and unnecessary.
+    value = Decimal(balance) * Decimal(price)
+    return round(value, 18)
+
+
+_builtin_round: Final = round
+
+
+def round(value: Decimal, digits: int) -> Decimal:
+    # For a Decimal with precision < 18, rounding is both impossible and unnecessary.
+    try:
+        return _builtin_round(value, digits)  # type: ignore [return-value]
+    except InvalidOperation:
+        return value