eth-portfolio 1.1.0__py3-none-any.whl

eth_portfolio/__init__.py

@@ -0,0 +1,16 @@
+import a_sync._smart
+
+a_sync._smart.set_smart_task_factory()
+
+
+from eth_portfolio.portfolio import Portfolio, portfolio
+
+# make sure we init the extended db before we init ypm somewhere
+from eth_portfolio._db import utils
+from eth_portfolio._shitcoins import SHITCOINS
+
+__all__ = [
+    "Portfolio",
+    "portfolio",
+    "SHITCOINS",
+]

eth_portfolio/_argspec.py

@@ -0,0 +1,42 @@
+from inspect import getfullargspec
+from typing import Any, Callable, List, Tuple, Type
+
+# WIP:
+
+
+def get_args_type(sample: Callable) -> Tuple[Type, ...]:
+    argspec = getfullargspec(sample)
+    args = {
+        arg_name: argspec.annotations[arg_name] if arg_name in argspec.annotations else Any
+        for arg_name in argspec.args
+        if arg_name != "self"
+    }
+    return tuple(*args.values())
+
+
+def get_kwargs_type(sample: Callable) -> Tuple[Type, ...]:
+    argspec = getfullargspec(sample)
+    num_kwargs = len(argspec.args) - len(argspec.defaults or [])
+    kwarg_names = argspec.args[num_kwargs:]
+    kwargs = {
+        kwarg_name: argspec.annotations[kwarg_name] if kwarg_name in argspec.annotations else Any
+        for kwarg_name in kwarg_names
+    }
+    _kwarg_types: List[Type[object]] = list(*kwargs.values())
+    if num_kwargs == 1:
+        return Tuple[_kwarg_types[0]]  # type: ignore [valid-type,return-value]
+    elif num_kwargs == 2:
+        return Tuple[_kwarg_types[0], _kwarg_types[1]]  # type: ignore [misc,return-value]
+    elif num_kwargs == 3:
+        return Tuple[_kwarg_types[0], _kwarg_types[1], _kwarg_types[2]]  # type: ignore [misc,return-value]
+    else:
+        return Any  # type: ignore [misc,return-value]
+
+
+def get_return_type(sample: Callable) -> Type:
+    argspec = getfullargspec(sample)
+    return argspec.annotations["return"] if "return" in argspec.annotations else Any
+
+
+def get_types(sample: Callable) -> Tuple[Type, Type, Type]:
+    return get_args_type(sample), get_kwargs_type(sample), get_return_type(sample)  # type: ignore [return-value]

eth_portfolio/_cache.py

@@ -0,0 +1,116 @@
+import functools
+import inspect
+from asyncio import PriorityQueue, Task, current_task, get_event_loop
+from concurrent.futures import Executor
+from hashlib import md5
+from logging import getLogger
+from os import makedirs
+from os.path import exists, join
+from pickle import dumps, load, loads
+from random import random
+from typing import Any, Callable, List, NoReturn
+
+from a_sync import PruningThreadPoolExecutor
+from a_sync._typing import P, T
+from a_sync.asyncio import create_task
+from a_sync.primitives.queue import log_broken
+from aiofiles import open as _aio_open
+from brownie import chain
+
+BASE_PATH = f"./cache/{chain.id}/"
+_THREAD_NAME_PREFIX = "eth-portfolio-cache-decorator"
+_EXISTS_EXECUTOR = PruningThreadPoolExecutor(8, f"{_THREAD_NAME_PREFIX}-exists")
+
+
+def cache_to_disk(fn: Callable[P, T]) -> Callable[P, T]:
+    # sourcery skip: use-contextlib-suppress
+    name = fn.__name__
+    cache_path_for_fn = f"{BASE_PATH}{fn.__module__.replace('.', '/')}/{name}"
+    logger = getLogger(f"eth_portfolio.cache_to_disk.{name}")
+
+    def get_cache_file_path(args, kwargs):
+        # Create a unique filename based on the function arguments
+        key = md5(dumps((args, sorted(kwargs.items())))).hexdigest()
+        return join(cache_path_for_fn, f"{key}.json")
+
+    write_executor = PruningThreadPoolExecutor(8, f"{_THREAD_NAME_PREFIX}-{fn.__qualname__}-write")
+
+    makedirs(cache_path_for_fn, exist_ok=True)
+
+    if inspect.iscoroutinefunction(fn):
+        read_executor = PruningThreadPoolExecutor(
+            8, f"{_THREAD_NAME_PREFIX}-{fn.__qualname__}-read"
+        )
+
+        queue = PriorityQueue()
+
+        async def cache_deco_worker_coro(func) -> NoReturn:
+            try:
+                while True:
+                    _, fut, cache_path, args, kwargs = await queue.get()
+                    try:
+                        async with _aio_open(cache_path, "rb", executor=read_executor) as f:
+                            fut.set_result(loads(await f.read()))
+                    except Exception as e:
+                        fut.set_exception(e)
+            except Exception as e:
+                logger.error("%s for %s is broken!!!", current_task(), func)
+                logger.exception(e)
+                raise
+
+        workers: List[Task[NoReturn]] = []
+
+        @functools.wraps(fn)
+        async def disk_cache_wrap(*args: P.args, **kwargs: P.kwargs) -> T:
+            cache_path = get_cache_file_path(args, kwargs)
+            if await _EXISTS_EXECUTOR.run(exists, cache_path):
+                fut = get_event_loop().create_future()
+                # we intentionally mix up the order to break up heavy load block ranges
+                queue.put_nowait((random(), fut, cache_path, args, kwargs))
+                if not workers:
+                    workers.extend(create_task(cache_deco_worker_coro(fn)) for _ in range(100))
+                try:
+                    return await fut
+                except EOFError:
+                    pass
+
+            async_result: T = await fn(*args, **kwargs)
+            try:
+                await __cache_write(cache_path, async_result, write_executor)
+            except OSError as e:
+                # I was having some weird issues in docker that I don't want to debug,
+                # so I'm going to assume you have another means to let you know you're
+                # out of disk space and will pass right on through here so my script
+                # can continue
+                if e.strerror != "No space left on device":
+                    raise
+            return async_result
+
+    else:
+
+        @functools.wraps(fn)
+        def disk_cache_wrap(*args: P.args, **kwargs: P.kwargs) -> T:
+            cache_path = get_cache_file_path(args, kwargs)
+            try:
+                with open(cache_path, "rb") as f:
+                    return load(f)
+            except (FileNotFoundError, EOFError):
+                pass
+
+            sync_result: T = fn(*args, **kwargs)  # type: ignore [assignment, return-value]
+            try:
+                create_task(
+                    coro=__cache_write(cache_path, sync_result, write_executor),
+                    skip_gc_until_done=True,
+                )
+            except RuntimeError:
+                pass
+            return sync_result
+
+    return disk_cache_wrap
+
+
+async def __cache_write(cache_path: str, result: Any, executor: Executor) -> None:
+    result = dumps(result)
+    async with _aio_open(cache_path, "wb", executor=executor) as f:
+        await f.write(result)

eth_portfolio/_config.py

@@ -0,0 +1,3 @@
+import os
+
+REORG_BUFFER = int(os.environ.get("REORG_BUFFER", 30))

eth_portfolio/_db/decorators.py

@@ -0,0 +1,147 @@
+from asyncio import iscoroutinefunction
+from asyncio import sleep as aio_sleep
+from functools import wraps
+from logging import DEBUG, getLogger
+from random import random
+from time import sleep as time_sleep
+from typing import Callable, TypeVar
+
+from a_sync._typing import AnyFn
+from pony.orm import OperationalError, TransactionError
+from typing_extensions import ParamSpec
+
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+logger = getLogger(__name__)
+__logger_is_enabled_for = logger.isEnabledFor
+__logger_warning = logger.warning
+__logger_log = logger._log
+
+
+def break_locks(fn: AnyFn[P, T]) -> AnyFn[P, T]:
+    """
+    Decorator to handle database lock errors by retrying the function.
+
+    This decorator is designed to wrap functions that interact with a database
+    and may encounter `OperationalError` due to database locks. It will retry
+    the function indefinitely if a "database is locked" error occurs. After
+    5 attempts, a warning is logged, but the function will continue to retry
+    until it succeeds or a non-lock-related error occurs.
+
+    Args:
+        fn: The function to be wrapped, which may be a coroutine or a regular function.
+
+    Examples:
+        Basic usage with a regular function:
+
+        >>> @break_locks
+        ... def my_function():
+        ...     # Function logic that may encounter a database lock
+        ...     pass
+
+        Basic usage with an asynchronous function:
+
+        >>> @break_locks
+        ... async def my_async_function():
+        ...     # Async function logic that may encounter a database lock
+        ...     pass
+
+    See Also:
+        - :func:`pony.orm.db_session`: For managing database sessions.
+    """
+    if iscoroutinefunction(fn):
+
+        @wraps(fn)
+        async def break_locks_wrap(*args: P.args, **kwargs: P.kwargs) -> T:
+            debug_logs_enabled = None
+            tries = 0
+            while True:
+                try:
+                    return await fn(*args, **kwargs)
+                except OperationalError as e:
+                    if str(e) != "database is locked":
+                        raise e
+
+                    if debug_logs_enabled is None:
+                        debug_logs_enabled = __logger_is_enabled_for(DEBUG)
+
+                    if debug_logs_enabled is True:
+                        __logger_log(DEBUG, "%s.%s %s", (fn.__module__, fn.__name__, e))
+
+                    await aio_sleep(tries * random())
+                    tries += 1
+                    if tries > 5:
+                        __logger_warning("%s caught in err loop with %s", fn, e)
+
+    else:
+
+        @wraps(fn)
+        def break_locks_wrap(*args: P.args, **kwargs: P.kwargs) -> T:
+            debug_logs_enabled = None
+            tries = 0
+            while True:
+                try:
+                    return fn(*args, **kwargs)  # type: ignore [return-value]
+                except OperationalError as e:
+                    if str(e) != "database is locked":
+                        raise e
+
+                    if debug_logs_enabled is None:
+                        debug_logs_enabled = __logger_is_enabled_for(DEBUG)
+
+                    if debug_logs_enabled is True:
+                        __logger_log(DEBUG, "%s.%s %s", (fn.__module__, fn.__name__, e))
+
+                    time_sleep(tries * random())
+                    tries += 1
+                    if tries > 5:
+                        __logger_warning("%s caught in err loop with %s", fn, e)
+
+    return break_locks_wrap
+
+
+def requery_objs_on_diff_tx_err(fn: Callable[P, T]) -> Callable[P, T]:
+    """
+    Decorator to handle transaction errors by retrying the function.
+
+    This decorator is designed to wrap functions that may encounter
+    `TransactionError` due to mixing objects from different transactions.
+    It will retry the function until it succeeds or a non-transaction-related
+    error occurs.
+
+    Args:
+        fn: The function to be wrapped, which must not be a coroutine.
+
+    Raises:
+        TypeError: If the function is a coroutine.
+
+    Examples:
+        Basic usage with a function that may encounter transaction errors:
+
+        >>> @requery_objs_on_diff_tx_err
+        ... def my_function():
+        ...     # Function logic that may encounter a transaction error
+        ...     pass
+
+    See Also:
+        - :func:`pony.orm.db_session`: For managing database sessions.
+    """
+    if iscoroutinefunction(fn):
+        raise TypeError(f"{fn} must not be async")
+
+    @wraps(fn)
+    def requery_wrap(*args: P.args, **kwargs: P.kwargs) -> T:
+        while True:
+            try:
+                return fn(*args, **kwargs)
+            except TransactionError as e:
+                if str(e) != "An attempt to mix objects belonging to different transactions":
+                    raise e
+                # The error occurs if you committed new objs to the db and started a new transaction while still inside of a `db_session`,
+                # and then tried to use the newly committed objects in the next transaction. Now that the objects are in the db this will
+                # not reoccur. The next iteration will be successful.
+
+    return requery_wrap

eth_portfolio/_db/entities.py

@@ -0,0 +1,204 @@
+import typing
+from functools import cached_property
+
+from evmspec.structs.transaction import AccessListEntry
+from hexbytes import HexBytes
+from msgspec import json
+from pony.orm import Optional, PrimaryKey, Required, Set, composite_key
+from y._db.entities import Address, Block, Contract, DbEntity, Token
+
+from eth_portfolio import structs
+from eth_portfolio._decimal import Decimal
+
+
+class BlockExtended(Block):
+    if typing.TYPE_CHECKING:
+        # if we execute this code we get `TypeError: 'type' object is not subscriptable`
+        transactions: Set["Transaction"]
+        internal_transfers: Set["InternalTransfer"]
+        token_transfers: Set["TokenTransfer"]
+
+    transactions = Set("Transaction", lazy=True, reverse="block")
+    internal_transfers = Set("InternalTransfer", lazy=True, reverse="block")
+    token_transfers = Set("TokenTransfer", lazy=True, reverse="block")
+
+
+class AddressExtended(Address):
+    if typing.TYPE_CHECKING:
+        # if we execute this code we get `TypeError: 'type' object is not subscriptable`
+        transactions_sent: Set["Transaction"]
+        transactions_received: Set["Transaction"]
+        internal_transfers_sent: Set["InternalTransfer"]
+        internal_transfers_received: Set["InternalTransfer"]
+        token_transfers_sent: Set["TokenTransfer"]
+        token_transfers_received: Set["TokenTransfer"]
+
+    transactions_sent = Set("Transaction", lazy=True, reverse="from_address")
+    transactions_received = Set("Transaction", lazy=True, reverse="to_address")
+    internal_transfers_sent = Set("InternalTransfer", lazy=True, reverse="from_address")
+    internal_transfers_received = Set("InternalTransfer", lazy=True, reverse="to_address")
+    token_transfers_sent = Set("TokenTransfer", lazy=True, reverse="from_address")
+    token_transfers_received = Set("TokenTransfer", lazy=True, reverse="to_address")
+
+
+class ContractExtended(Contract, AddressExtended):
+    pass
+
+
+class TokenExtended(Token, AddressExtended):
+    if typing.TYPE_CHECKING:
+        # if we execute this code we get `TypeError: 'type' object is not subscriptable`
+        transfers: Set["TokenTransfer"]
+
+    transfers = Set("TokenTransfer", lazy=True, reverse="token")
+
+
+class Transaction(DbEntity):
+    _id = PrimaryKey(int, auto=True)
+    block = Required(BlockExtended, lazy=True, reverse="transactions")
+    transaction_index = Required(int, lazy=True)
+    hash = Required(str, index=True, lazy=True)
+    from_address = Required(AddressExtended, index=True, lazy=True, reverse="transactions_sent")
+    to_address = Optional(AddressExtended, index=True, lazy=True, reverse="transactions_received")
+    value = Required(Decimal, 38, 18, lazy=True)
+    price = Optional(Decimal, 38, 18, lazy=True)
+    value_usd = Optional(Decimal, 38, 18, lazy=True)
+
+    nonce = Required(int, lazy=True)
+    type = Optional(int, lazy=True)
+    gas = Required(Decimal, 38, 1, lazy=True)
+    gas_price = Required(Decimal, 38, 1, lazy=True)
+    max_fee_per_gas = Optional(Decimal, 38, 1, lazy=True)
+    max_priority_fee_per_gas = Optional(Decimal, 38, 1, lazy=True)
+
+    composite_key(block, transaction_index)
+
+    raw = Required(bytes, lazy=True)
+
+    @cached_property
+    def decoded(self) -> structs.Transaction:
+        return json.decode(self.raw, type=structs.Transaction)
+
+    @property
+    def input(self) -> HexBytes:
+        structs.Transaction.input.__doc__
+        return self.decoded.input
+
+    @property
+    def r(self) -> HexBytes:
+        structs.Transaction.r.__doc__
+        return self.decoded.r
+
+    @property
+    def s(self) -> HexBytes:
+        structs.Transaction.s.__doc__
+        return self.decoded.s
+
+    @property
+    def v(self) -> int:
+        structs.Transaction.v.__doc__
+        return self.decoded.v
+
+    @property
+    def access_list(self) -> typing.List[AccessListEntry]:
+        structs.Transaction.access_list.__doc__
+        return self.decoded.access_list
+
+    @property
+    def y_parity(self) -> typing.Optional[int]:
+        structs.TokenTransfer.y_parity.__doc__
+        return self.decoded.y_parity
+
+
+class InternalTransfer(DbEntity):
+    _id = PrimaryKey(int, auto=True)
+
+    # common
+    block = Required(BlockExtended, lazy=True, reverse="internal_transfers")
+    transaction_index = Required(int, lazy=True)
+    hash = Required(str, lazy=True)
+    from_address = Required(
+        AddressExtended, index=True, lazy=True, reverse="internal_transfers_sent"
+    )
+    to_address = Optional(
+        AddressExtended, index=True, lazy=True, reverse="internal_transfers_received"
+    )
+    value = Required(Decimal, 38, 18, lazy=True)
+    price = Optional(Decimal, 38, 18, lazy=True)
+    value_usd = Optional(Decimal, 38, 18, lazy=True)
+
+    # unique
+    type = Required(str, lazy=True)
+    call_type = Required(str, lazy=True)
+    trace_address = Required(str, lazy=True)
+    gas = Required(Decimal, 38, 1, lazy=True)
+    gas_used = Optional(Decimal, 38, 1, lazy=True)
+
+    composite_key(
+        block,
+        transaction_index,
+        hash,
+        from_address,
+        to_address,
+        value,
+        type,
+        call_type,
+        trace_address,
+        gas,
+        gas_used,
+    )
+
+    raw = Required(bytes, lazy=True)
+
+    @cached_property
+    def decoded(self) -> structs.InternalTransfer:
+        structs.InternalTransfer.__doc__
+        return json.decode(self.raw, type=structs.InternalTransfer)
+
+    @property
+    def code(self) -> HexBytes:
+        structs.InternalTransfer.code.__doc__
+        return self.decoded.code
+
+    @property
+    def input(self) -> HexBytes:
+        structs.InternalTransfer.input.__doc__
+        return self.decoded.input
+
+    @property
+    def output(self) -> HexBytes:
+        structs.InternalTransfer.output.__doc__
+        return self.decoded.output
+
+    @property
+    def subtraces(self) -> int:
+        structs.InternalTransfer.subtraces.__doc__
+        return self.decoded.subtraces
+
+
+class TokenTransfer(DbEntity):
+    _id = PrimaryKey(int, auto=True)
+
+    # common
+    block = Required(BlockExtended, lazy=True, reverse="token_transfers")
+    transaction_index = Required(int, lazy=True)
+    hash = Required(str, lazy=True)
+    from_address = Required(AddressExtended, index=True, lazy=True, reverse="token_transfers_sent")
+    to_address = Required(
+        AddressExtended, index=True, lazy=True, reverse="token_transfers_received"
+    )
+    value = Required(Decimal, 38, 18, lazy=True)
+    price = Optional(Decimal, 38, 18, lazy=True)
+    value_usd = Optional(Decimal, 38, 18, lazy=True)
+
+    # unique
+    log_index = Required(int, lazy=True)
+    token = Optional(TokenExtended, index=True, lazy=True, reverse="transfers")
+
+    composite_key(block, transaction_index, log_index)
+
+    raw = Required(bytes, lazy=True)
+
+    @cached_property
+    def decoded(self) -> structs.TokenTransfer:
+        return json.decode(self.raw, type=structs.TokenTransfer)