eth-portfolio-temp 0.2.12__cp313-cp313-win32.whl

eth_portfolio/_loaders/token_transfer.py

@@ -0,0 +1,217 @@
+"""
+This module orchestrates the process of loading and processing token transfers within the eth_portfolio ecosystem.
+"""
+
+import decimal
+from logging import getLogger
+from typing import Final, Optional, Set
+
+from a_sync import create_task, gather
+from dank_mids import BlockSemaphore
+from eth_typing import ChecksumAddress
+from evmspec.data import TransactionIndex
+from faster_async_lru import alru_cache
+from msgspec import Struct, ValidationError
+from msgspec.json import decode
+from pony.orm import TransactionIntegrityError, UnexpectedError
+from y import ERC20
+from y._db.log import Log
+from y._decorators import stuck_coro_debugger
+from y.exceptions import NonStandardERC20, reraise_excs_with_extra_context
+
+from eth_portfolio._cache import cache_to_disk
+from eth_portfolio._db import utils as db
+from eth_portfolio._decimal import Decimal
+from eth_portfolio._loaders.utils import get_transaction_receipt
+from eth_portfolio._utils import _get_price
+from eth_portfolio.structs import TokenTransfer
+
+
+logger = getLogger(__name__)
+
+token_transfer_semaphore: Final = BlockSemaphore(
+    20_000,  # Some arbitrary number
+    name="eth_portfolio.token_transfers",
+)
+"""A semaphore that regulates the concurrent processing of token transfers by processing lower blocks first."""
+
+
+@stuck_coro_debugger
+async def load_token_transfer(
+    transfer_log: "Log", load_prices: bool
+) -> Optional[TokenTransfer]:  # sourcery skip: simplify-boolean-comparison
+    """
+    Processes and loads a token transfer from a log entry, with comprehensive error handling and optional price fetching.
+
+    This function employs a multi-step process:
+    1. Validates the transfer against a known set of 'shitcoins', skipping processing if matched.
+    2. Checks for existing database entries, potentially deleting and reprocessing if price data is requested but missing.
+    3. Decodes the transfer log and retrieves associated metadata (e.g., token scale, symbol, transaction index).
+    4. Optionally fetches the token price at the time of the transaction.
+    5. Constructs and persists a :class:`~eth_portfolio.structs.TokenTransfer` object in the database.
+
+    The function handles various exceptions, including :class:`~y.exceptions.NonStandardERC20` for non-compliant tokens and :class:`decimal.InvalidOperation` for extreme :class:`~Decimal` values.
+
+    Args:
+        transfer_log: A dictionary containing the raw log entry of the token transfer.
+        load_prices: A flag indicating whether to fetch and include price data for the token at the time of transfer.
+
+    Returns:
+        A processed TokenTransfer object if successful, or None if the transfer cannot be processed due to various constraints or errors.
+
+    Note:
+        This function employs caching mechanisms and database operations to optimize performance.
+    """
+    if transfer_log.removed:
+        if transfer := await db.get_token_transfer(transfer_log):
+            await db.delete_token_transfer(transfer)
+        return None
+
+    if transfer := await db.get_token_transfer(transfer_log):
+        if load_prices is False or transfer.price:
+            return transfer
+        await db.delete_token_transfer(transfer)
+
+    token_address: ChecksumAddress = transfer_log.address
+    if token_address in _non_standard_erc20:
+        logger.debug("%s is not a standard ERC20 token, skipping.", token_address)
+        return None
+
+    async with token_transfer_semaphore[transfer_log.block]:
+        token = ERC20(token_address, asynchronous=True)
+        try:
+            try:
+                # This will be mem cached so no need to gather and add a bunch of overhead
+                scale = await token.scale
+            except NonStandardERC20 as e:
+                # NOTE: if we cant fetch scale, this is probably either a shitcoin or an NFT (which we don't support at this time)
+                logger.debug("%s for %s, skipping.", e, transfer_log)
+                _non_standard_erc20.add(token_address)
+                return None
+
+            # This will be mem cached so no need to include it in the gather and add a bunch of overhead
+            symbol = await get_symbol(token)
+
+            tx_index_coro = get_transaction_index(transfer_log.transactionHash.hex())
+            coro_results = {"token": symbol}
+
+            if load_prices:
+                coro_results.update(
+                    await gather(
+                        {
+                            "transaction_index": tx_index_coro,
+                            "price": _get_price(token.address, transfer_log.blockNumber),
+                        }
+                    )
+                )
+            else:
+                coro_results["transaction_index"] = await tx_index_coro
+
+        except Exception as e:
+            logger.error(
+                f"%s %s for %s %s at block %s:",
+                e.__class__.__name__,
+                e,
+                await get_symbol(token) or token.address,
+                token_address,
+                transfer_log.blockNumber,
+            )
+            logger.exception(e)
+            return None
+
+        value = Decimal(transfer_log.data.as_uint256) / scale
+
+        if price := coro_results.get("price"):
+            coro_results["value_usd"] = round(value * price, 18)
+
+        transfer = TokenTransfer(log=transfer_log, value=value, **coro_results)
+
+        create_task(
+            _insert_to_db(transfer, load_prices),
+            skip_gc_until_done=True,
+        )
+
+        return transfer
+
+
+async def _insert_to_db(transfer: TokenTransfer, load_prices: bool) -> None:
+    with reraise_excs_with_extra_context(transfer):
+        try:
+            await db.insert_token_transfer(transfer)
+        except TransactionIntegrityError:
+            if load_prices:
+                await db.delete_token_transfer(transfer)
+                await db.insert_token_transfer(transfer)
+        except UnexpectedError:
+            digits_before_decimal = str(transfer.value).split(".")[0]
+            if len(digits_before_decimal) <= 20:
+                raise
+        except decimal.InvalidOperation as e:
+            # TODO: debug why this happens sometimes
+            logger.exception("%s %s", e, transfer)
+            pass
+
+
+_non_standard_erc20: Final[Set[ChecksumAddress]] = set()
+
+
+@stuck_coro_debugger
+async def get_symbol(token: ERC20) -> Optional[str]:
+    """
+    Retrieves the symbol of a given ERC20 token, with error handling for non-standard implementations.
+
+    This function attempts to access the token's symbol through the standard ERC20 symbol method. If the token contract
+    does not adhere to the standard ERC20 interface, indicated by a :class:`~y.exceptions.NonStandardERC20` exception, the function
+    returns `None` instead of propagating the error.
+
+    Args:
+        token: An ERC20 token object representing the token whose symbol is to be retrieved.
+
+    Returns:
+        The token's symbol as a string if successfully retrieved, or None if the token does not implement a standard symbol method.
+    """
+    if token.address in _non_standard_erc20:
+        return None
+    try:
+        return await token.__symbol__
+    except NonStandardERC20:
+        _non_standard_erc20.add(token.address)
+        return None
+
+
+@alru_cache(ttl=60 * 60)
+@stuck_coro_debugger
+@cache_to_disk
+async def get_transaction_index(hash: str) -> int:
+    """
+    Retrieves the transaction index for a given transaction hash, with results cached to disk.
+
+    This function fetches the transaction receipt corresponding to the provided hash and extracts the transaction index,
+    which represents the position of the transaction within its containing block. The result is cached to disk to
+    optimize performance for future runs.
+
+    Args:
+        hash: The hexadecimal string representation of the transaction hash.
+
+    Returns:
+        The zero-based index of the transaction within its block.
+    """
+    while True:
+        receipt_bytes = await get_transaction_receipt(hash)
+        if receipt_bytes is not None:
+            # TODO: debug why this happens, its something inside of dank_mids
+            break
+        logger.info("get_transaction_index failed, retrying...")
+
+    try:
+        return decode(
+            receipt_bytes, type=HasTxIndex, dec_hook=TransactionIndex._decode_hook
+        ).transactionIndex
+    except ValidationError as e:
+        new = TypeError(e, receipt_bytes, decode(receipt_bytes))
+        logger.exception(new)
+        raise new from e
+
+
+class HasTxIndex(Struct):
+    transactionIndex: TransactionIndex

eth_portfolio/_loaders/transaction.py

@@ -0,0 +1,240 @@
+"""
+This module contains loader functions that facilitate the retrieval and loading of blockchain transaction data.
+
+The functions defined here use various async operations to retrieve transaction data from Ethereum-like blockchains, and process them, to then store them in a local database. The module leverages utilities like retry mechanisms, caching, and efficient async processing to handle potentially large amounts of blockchain data.
+
+The primary focus of this module is to support eth-portfolio's internal operations such as loading transactions by address and nonce, retrieving transaction details from specific blocks, and managing transaction-related data.
+"""
+
+from logging import getLogger
+from typing import Awaitable, Callable, Final, List, Optional, Tuple
+
+import a_sync
+import dank_mids
+import eth_retry
+import evmspec
+import msgspec
+from a_sync import SmartProcessingQueue
+from eth_typing import ChecksumAddress
+from evmspec import data
+from faster_async_lru import alru_cache
+from pony.orm import TransactionIntegrityError
+from y import get_price
+from y._decorators import stuck_coro_debugger
+from y.constants import EEE_ADDRESS
+from y.datatypes import Block
+from y.exceptions import reraise_excs_with_extra_context
+from y.utils.events import decode_logs
+
+from eth_portfolio.structs import structs
+from eth_portfolio._cache import cache_to_disk
+from eth_portfolio._db import utils as db
+from eth_portfolio._decimal import Decimal
+from eth_portfolio._loaders._nonce import Nonce, get_block_for_nonce
+from eth_portfolio._loaders._nonce import get_nonce_at_block as _get_nonce_at_block
+from eth_portfolio._loaders.utils import get_transaction_receipt
+
+
+Transactions = List[evmspec.Transaction]
+
+
+logger: Final = getLogger(__name__)
+
+
+@eth_retry.auto_retry
+@stuck_coro_debugger
+async def load_transaction(
+    address: ChecksumAddress, nonce: Nonce, load_prices: bool
+) -> Tuple[Nonce, Optional[structs.Transaction]]:
+    """
+    Loads a transaction by address and nonce.
+
+    This function attempts to load a transaction from the local database based on the given address and nonce.
+
+    If the transaction is not found in the local database or if price data is missing but requested by the user, it retrieves the transaction data from the blockchain using a binary search approach, processes it, and stores it back in the local database.
+
+    Args:
+        address: The address of the account involved in the transaction.
+        nonce: The nonce associated with the transaction.
+        load_prices: Whether to load and calculate price-related information for the transaction.
+
+    Returns:
+        A tuple containing the nonce and the retrieved or newly created :class:`~eth_portfolio.structs.Transaction` object, or None if the transaction could not be found.
+
+    Example:
+        >>> print(await load_transaction(address="0x1234567890abcdef1234567890abcdef12345678", nonce=5, load_prices=True))
+        (5, Transaction(...))
+    """
+    if transaction := await db.get_transaction(address, nonce):
+        if load_prices and transaction.price is None:
+            await db.delete_transaction(transaction)
+        else:
+            return nonce, transaction
+
+    block = await get_block_for_nonce(address, nonce)
+    tx = await get_transaction_by_nonce_and_block(address, nonce, block)
+    if tx is None:
+        return nonce, None
+
+    if load_prices:
+        # TODO: debug why `tx.value` isnt already a Wei obj
+        scaled = data.Wei(tx.value).scaled
+        price = Decimal(await get_price(EEE_ADDRESS, block=tx.blockNumber, sync=False))
+        transaction = structs.Transaction.from_rpc_response(
+            tx, price=round(price, 18), value_usd=round(scaled * price, 18)
+        )
+    else:
+        transaction = structs.Transaction.from_rpc_response(tx)
+
+    a_sync.create_task(
+        coro=_insert_to_db(transaction, load_prices),
+        skip_gc_until_done=True,
+    )
+
+    return nonce, transaction
+
+
+async def _insert_to_db(transaction: structs.Transaction, load_prices: bool) -> None:
+    with reraise_excs_with_extra_context(transaction):
+        try:
+            await db.insert_transaction(transaction)
+        except TransactionIntegrityError:
+            if load_prices:
+                await db.delete_transaction(transaction)
+                await db.insert_transaction(transaction)
+
+
+@eth_retry.auto_retry
+@stuck_coro_debugger
+@cache_to_disk
+async def get_transaction_by_nonce_and_block(
+    address: ChecksumAddress, nonce: int, block: Block
+) -> Optional[evmspec.Transaction]:
+    """
+    This function retrieves a transaction for a specifified address by its nonce and block, if any match.
+
+    It also handles special cases:
+
+        1. Contract creation transactions where the transaction's 'to' field is None.
+        2. Gnosis Safe deployments where the transaction's 'to' field is a specific address.
+
+    Args:
+        address: The addresses of the accounts that sent the transaction.
+        nonce: The nonce associated with the transaction creator.
+        block: The block number from which to retrieve the transactions.
+
+    Returns:
+        The transaction data if found, otherwise None.
+
+    Example:
+        >>> transaction = await get_transaction_by_nonce_and_block(address="0x1234567890abcdef1234567890abcdef12345678", nonce=5, block=1234567)
+        >>> if transaction:
+        ...     print(transaction.hash)
+    """
+    for tx in await get_block_transactions(block):
+        if tx.sender == address and tx.nonce == nonce:
+            return tx
+
+        receipt_bytes: msgspec.Raw
+        # Special handler for contract creation transactions
+        if tx.to is None:
+            receipt_bytes = await get_transaction_receipt(tx.hash.hex())
+            receipt_0 = msgspec.json.decode(
+                receipt_bytes,
+                type=ReceiptContractAddress,
+                dec_hook=data.Address._decode_hook,
+            )
+            if receipt_0.contractAddress == address:
+                return tx
+        # Special handler for Gnosis Safe deployments
+        elif tx.to == "0xa6B71E26C5e0845f74c812102Ca7114b6a896AB2":
+            receipt_bytes = await get_transaction_receipt(tx.hash.hex())
+            receipt_1 = msgspec.json.decode(
+                receipt_bytes,
+                type=ReceiptLogs,
+                dec_hook=data._decode_hook,
+            )
+            events = decode_logs(receipt_1.logs)
+            if (
+                "SafeSetup" in events
+                and "ProxyCreation" in events
+                and any(event["proxy"] == address for event in events["ProxyCreation"])
+            ):
+                return tx
+    return None
+
+
+class ReceiptContractAddress(msgspec.Struct):
+    """We only decode what we need and immediately discard the rest of the receipt."""
+
+    contractAddress: data.Address
+
+
+class ReceiptLogs(msgspec.Struct):
+    logs: List[evmspec.Log]
+
+
+get_transaction_count: Final = dank_mids.eth.get_transaction_count
+
+
+@alru_cache(maxsize=None)
+@eth_retry.auto_retry
+@stuck_coro_debugger
+@cache_to_disk
+async def get_nonce_at_block(address: ChecksumAddress, block: Block) -> int:
+    """
+    Retrieves the nonce of an address at a specific block.
+
+    This function gets the transaction count (nonce) for the given address at the specified block. It also includes a special case to handle known issues on certain networks like Arbitrum.
+
+    Args:
+        address: The address of the account.
+        block: The block number at which to retrieve the nonce.
+
+    Returns:
+        The nonce of the address at the given block.
+
+    Example:
+        >>> block = 12345678
+        >>> nonce = await get_nonce_at_block("0x1234567890abcdef1234567890abcdef12345678", block)
+        >>> print(f"The nonce at block {block} is {nonce}.")
+
+    """
+    return await _get_nonce_at_block(address, block)
+
+
+_get_block_transactions: Final[Callable[[Block], Awaitable[Transactions]]] = alru_cache(
+    ttl=60 * 60
+)(eth_retry.auto_retry(stuck_coro_debugger(dank_mids.eth.get_transactions)))
+"""
+Retrieves all transactions from a specific block.
+
+This function fetches the full transaction data for a block using async caching to optimize repeated requests.
+
+The cache has a time-to-live (TTL) of 1 hour to avoid memory leaks for services or long-running scripts.
+
+Args:
+    block: The block number from which to retrieve the transactions.
+
+Returns:
+    A list of transaction data objects from the block.
+
+Example:
+    >>> transactions = await _get_block_transactions(block=12345678)
+    >>> [print(tx.hash) for tx in transactions]
+"""
+
+
+get_block_transactions: Final[SmartProcessingQueue[Block, [], Transactions]] = SmartProcessingQueue(
+    _get_block_transactions, 1_000
+)
+"""
+A smart processing queue that retrieves transactions from blocks with managable concurrency.
+
+This queue processes the retrieval of block transactions using a limited number of workers to handle potentially overwhelming volume of transactions.
+It wraps the _get_block_transactions function to provide efficient concurrent processing.
+
+Example:
+    >>> transactions = await get_block_transactions.process(block=12345678)
+    >>> [print(tx['hash']) for tx in transactions]
+"""

eth_portfolio/_loaders/utils.py

@@ -0,0 +1,68 @@
+from typing import Final
+
+import dank_mids
+import eth_retry
+import msgspec
+from a_sync import SmartProcessingQueue
+from eth_typing import HexStr
+from faster_async_lru import alru_cache
+from y._decorators import stuck_coro_debugger
+
+
+Raw: Final = msgspec.Raw
+TxReceiptQueue = SmartProcessingQueue[HexStr, [], msgspec.Raw]
+
+
+@eth_retry.auto_retry(min_sleep_time=1, max_sleep_time=3, max_retries=20, suppress_logs=1)
+@alru_cache(maxsize=None, ttl=60 * 60)
+@stuck_coro_debugger
+async def _get_transaction_receipt(txhash: HexStr) -> msgspec.Raw:
+    """
+    Fetches the transaction receipt for a given transaction hash.
+
+    This function retrieves the transaction receipt from the Ethereum network
+    using the provided transaction hash. It utilizes caching to store results
+    for a specified time-to-live (TTL) to improve performance and reduce
+    network calls. The function is also decorated to automatically retry
+    in case of failures and to debug if the coroutine gets stuck.
+
+    Args:
+        txhash: The transaction hash for which to retrieve the receipt.
+
+    Returns:
+        msgspec.Raw: The raw transaction receipt data.
+
+    Examples:
+        >>> txhash = "0x1234567890abcdef..."
+        >>> receipt = await _get_transaction_receipt(txhash)
+        >>> print(receipt)
+
+    See Also:
+        - :func:`eth_retry.auto_retry`: For automatic retry logic.
+        - :func:`async_lru.alru_cache`: For caching the results.
+        - :func:`y._decorators.stuck_coro_debugger`: For debugging stuck coroutines.
+    """
+    return await __get_transaction_receipt(txhash, decode_to=Raw, decode_hook=None)
+
+
+get_transaction_receipt: Final[TxReceiptQueue] = SmartProcessingQueue(
+    _get_transaction_receipt, 5000
+)
+"""
+A queue for processing transaction receipt requests.
+
+This queue manages the processing of transaction receipt requests, allowing
+up to 5000 concurrent requests. It uses the `_get_transaction_receipt` function
+to fetch the receipts.
+
+Examples:
+    >>> txhash = "0x1234567890abcdef..."
+    >>> receipt = await get_transaction_receipt(txhash)
+    >>> print(receipt)
+
+See Also:
+    - :class:`a_sync.SmartProcessingQueue`: For managing asynchronous processing queues.
+"""
+
+
+__get_transaction_receipt: Final = dank_mids.eth.get_transaction_receipt