dao-treasury 0.0.34__cp312-cp312-win32.whl → 0.0.72__cp312-cp312-win32.whl

dao_treasury/db.py

@@ -3,6 +3,7 @@
 Database models and utilities for DAO treasury reporting.
 
 This module defines Pony ORM entities for:
+
 - Blockchain networks (:class:`Chain`)
 - On-chain addresses (:class:`Address`)
 - ERC-20 tokens and native coin placeholder (:class:`Token`)
@@ -17,27 +18,41 @@ and creating SQL views for reporting.
 
 import typing
 from asyncio import Semaphore
+from collections import OrderedDict
 from decimal import Decimal, InvalidOperation
 from functools import lru_cache
 from logging import getLogger
 from os import path
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, Final, Tuple, Union, final
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Coroutine,
+    Dict,
+    Final,
+    Literal,
+    Tuple,
+    Union,
+    final,
+    overload,
+)
 from datetime import date, datetime, time, timezone
 
+import eth_portfolio
 from a_sync import AsyncThreadPoolExecutor
 from brownie import chain
 from brownie.convert.datatypes import HexString
 from brownie.exceptions import EventLookupError
 from brownie.network.event import EventDict, _EventItem
 from brownie.network.transaction import TransactionReceipt
-from eth_typing import ChecksumAddress, HexAddress, HexStr
 from eth_portfolio.structs import (
     InternalTransfer,
     LedgerEntry,
     TokenTransfer,
     Transaction,
 )
+from eth_retry import auto_retry
+from eth_typing import ChecksumAddress, HexAddress, HexStr
 from pony.orm import (
     Database,
     InterfaceError,
@@ -53,6 +68,7 @@ from pony.orm import (
     select,
 )
 from y import EEE_ADDRESS, Contract, Network, convert, get_block_timestamp_async
+from y._db.decorators import retry_locked
 from y.contracts import _get_code
 from y.exceptions import ContractNotVerified
 
@@ -60,6 +76,9 @@ from dao_treasury.constants import CHAINID
 from dao_treasury.types import TxGroupDbid, TxGroupName
 
 
+EventItem = _EventItem[_EventItem[OrderedDict[str, Any]]]
+
+
 SQLITE_DIR = Path(path.expanduser("~")) / ".dao-treasury"
 """Path to the directory in the user's home where the DAO treasury SQLite database is stored."""
 
@@ -68,6 +87,7 @@ SQLITE_DIR.mkdir(parents=True, exist_ok=True)
 
 _INSERT_THREAD = AsyncThreadPoolExecutor(1)
 _SORT_THREAD = AsyncThreadPoolExecutor(1)
+_EVENTS_THREADS = AsyncThreadPoolExecutor(16)
 _SORT_SEMAPHORE = Semaphore(50)
 
 _UTC = timezone.utc
@@ -234,6 +254,10 @@ class Address(DbEntity):
     def contract(self) -> Contract:
         return Contract(self.address)
 
+    @property
+    def contract_coro(self) -> Coroutine[Any, Any, Contract]:
+        return Contract.coroutine(self.address)
+
     @staticmethod
     @lru_cache(maxsize=None)
     def get_dbid(address: HexAddress) -> int:
@@ -294,7 +318,6 @@ class Address(DbEntity):
             )
 
         commit()
-
         return entity  # type: ignore [no-any-return]
 
     @staticmethod
@@ -405,6 +428,10 @@ class Token(DbEntity):
     def contract(self) -> Contract:
         return Contract(self.address.address)
 
+    @property
+    def contract_coro(self) -> Coroutine[Any, Any, Contract]:
+        return Contract.coroutine(self.address.address)
+
     @property
     def scale(self) -> int:
         """Base for division according to `decimals`, e.g., `10**decimals`.
@@ -719,6 +746,10 @@ class TreasuryTx(DbEntity):
         """Human-readable label for the sender address."""
         return self.from_address.nickname or self.from_address.address  # type: ignore [union-attr]
 
+    @property
+    def token_address(self) -> ChecksumAddress:
+        return self.token.address.address
+
     @property
     def symbol(self) -> str:
         """Ticker symbol for the transferred token."""
@@ -729,7 +760,23 @@ class TreasuryTx(DbEntity):
         """Decoded event logs for this transaction."""
         return self._transaction.events
 
-    def get_events(self, event_name: str) -> _EventItem:
+    async def events_async(self) -> EventDict:
+        """Asynchronously fetch decoded event logs for this transaction."""
+        tx = self._transaction
+        events = tx._events
+        if events is None:
+            events = await _EVENTS_THREADS.run(getattr, tx, "events")
+        return events
+
+    @overload
+    def get_events(
+        self, event_name: str, sync: Literal[False]
+    ) -> Coroutine[Any, Any, EventItem]: ...
+    @overload
+    def get_events(self, event_name: str, sync: bool = True) -> EventItem: ...
+    def get_events(self, event_name: str, sync: bool = True) -> EventItem:
+        if not sync:
+            return _EVENTS_THREADS.run(self.get_events, event_name)
         try:
             return self.events[event_name]
         except EventLookupError:
@@ -746,6 +793,7 @@ class TreasuryTx(DbEntity):
         return get_transaction(self.hash)
 
     @staticmethod
+    @auto_retry
     async def insert(entry: LedgerEntry) -> None:
         """Asynchronously insert and sort a ledger entry.
 
@@ -863,27 +911,48 @@ class TreasuryTx(DbEntity):
                 },
             ) from e
         except InvalidOperation as e:
-            logger.error(e)
-            logger.error(
-                {
-                    "chain": Chain.get_dbid(CHAINID),
-                    "block": entry.block_number,
-                    "timestamp": ts,
-                    "hash": entry.hash.hex(),
-                    "log_index": log_index,
-                    "from_address": from_address,
-                    "to_address": to_address,
-                    "token": token,
-                    "amount": entry.value,
-                    "price": entry.price,
-                    "value_usd": entry.value_usd,
-                    # TODO: nuke db and add this column
-                    # gas = gas,
-                    "gas_used": gas_used,
-                    "gas_price": gas_price,
-                    "txgroup": txgroup_dbid,
-                }
-            )
+            with db_session:
+                from_address_entity = Address[from_address]
+                to_address_entity = Address[to_address]
+                token_entity = Token[token]
+                logger.error(e)
+                logger.error(
+                    {
+                        "chain": Chain.get_dbid(CHAINID),
+                        "block": entry.block_number,
+                        "timestamp": ts,
+                        "hash": entry.hash.hex(),
+                        "log_index": log_index,
+                        "from_address": {
+                            "dbid": from_address,
+                            "address": from_address_entity.address,
+                            "nickname": from_address_entity.nickname,
+                        },
+                        "to_address": {
+                            "dbid": to_address,
+                            "address": to_address_entity.address,
+                            "nickname": to_address_entity.nickname,
+                        },
+                        "token": {
+                            "dbid": token,
+                            "address": token_entity.address.address,
+                            "name": token_entity.name,
+                            "symbol": token_entity.symbol,
+                            "decimals": token_entity.decimals,
+                        },
+                        "amount": entry.value,
+                        "price": entry.price,
+                        "value_usd": entry.value_usd,
+                        # TODO: nuke db and add this column
+                        # gas = gas,
+                        "gas_used": gas_used,
+                        "gas_price": gas_price,
+                        "txgroup": {
+                            "dbid": txgroup_dbid,
+                            "fullname": TxGroup[txgroup_dbid].fullname,
+                        },
+                    }
+                )
             return None
         except TransactionIntegrityError as e:
             return _validate_integrity_error(entry, log_index)
@@ -902,6 +971,7 @@ class TreasuryTx(DbEntity):
             return dbid  # type: ignore [no-any-return]
 
     @staticmethod
+    @retry_locked
     def __set_txgroup(treasury_tx_dbid: int, txgroup_dbid: TxGroupDbid) -> None:
         with db_session:
             TreasuryTx[treasury_tx_dbid].txgroup = txgroup_dbid
@@ -1361,3 +1431,24 @@ def _validate_integrity_error(
         )
         else None
     )
+
+
+def _drop_shitcoin_txs() -> None:
+    """
+    Purge any shitcoin txs from the db.
+
+    These should not be frequent, and only occur if a user populated the db before a shitcoin was added to the SHITCOINS mapping.
+    """
+    shitcoins = eth_portfolio.SHITCOINS[CHAINID]
+    with db_session:
+        shitcoin_txs = select(
+            tx for tx in TreasuryTx if tx.token.address.address in shitcoins
+        )
+        if count := shitcoin_txs.count():
+            logger.info(f"Purging {count} shitcoin txs from the database...")
+            for tx in shitcoin_txs:
+                tx.delete()
+            logger.info("Shitcoin tx purge complete.")
+
+
+_drop_shitcoin_txs()

dao_treasury/docker-compose.yaml

@@ -3,12 +3,9 @@ networks:
   docker_eth_portfolio:
     external: true
 
-volumes:
-  grafana_data: {}
-
 services:
   grafana:
-    image: grafana/grafana:10.2.0
+    image: grafana/grafana:12.2.1
     ports:
       - 127.0.0.1:${DAO_TREASURY_GRAFANA_PORT:-3004}:3000
     environment:
@@ -24,7 +21,6 @@ services:
       - GF_INSTALL_PLUGINS=volkovlabs-variable-panel,frser-sqlite-datasource
     volumes:
       - ~/.dao-treasury/:/app/dao-treasury-data
-      - grafana_data:/var/lib/grafana
       - ./.grafana/provisioning/:/etc/grafana/provisioning/
     networks:
       - dao_treasury

dao_treasury/main.py

@@ -97,6 +97,12 @@ parser.add_argument(
     help="The time interval between datapoints. default: 1d",
     default="1d",
 )
+parser.add_argument(
+    "--concurrency",
+    type=int,
+    help="The max number of historical blocks to export concurrently. default: 30",
+    default=30,
+)
 parser.add_argument(
     "--daemon",
     action="store_true",
@@ -119,6 +125,18 @@ parser.add_argument(
     help="Port for the Grafana rendering service. Default: 8091",
     default=8091,
 )
+parser.add_argument(
+    "--custom-bucket",
+    type=str,
+    action="append",
+    help=(
+        "Custom bucket mapping for a wallet address. "
+        "Specify as 'address:bucket_name'. "
+        "Can be used multiple times. Example: "
+        "--custom-bucket '0x123:My Bucket' --custom-bucket '0x456:Other Bucket'"
+    ),
+    default=None,
+)
 
 args = parser.parse_args()
 
@@ -203,7 +221,27 @@ async def export(args) -> None:
             for address in addresses:
                 db.Address.set_nickname(address, nickname)
 
-    treasury = Treasury(wallets, args.sort_rules, asynchronous=True)
+    # Parse custom_buckets from --custom-bucket arguments
+    custom_buckets = None
+    if args.custom_bucket:
+        custom_buckets = {}
+        for item in args.custom_bucket:
+            if ":" not in item:
+                parser.error(
+                    f"Invalid format for --custom-bucket: '{item}'. Must be 'address:bucket_name'."
+                )
+            address, bucket = item.split(":", 1)
+            address = address.strip()
+            bucket = bucket.strip()
+            if not address or not bucket:
+                parser.error(
+                    f"Invalid format for --custom-bucket: '{item}'. Both address and bucket_name are required."
+                )
+            custom_buckets[address] = bucket
+
+    treasury = Treasury(
+        wallets, args.sort_rules, custom_buckets=custom_buckets, asynchronous=True
+    )
 
     # Start only the requested containers
     if args.start_renderer is True:

dao_treasury/sorting/rule.py

@@ -130,8 +130,6 @@ class _SortRule:
     func: Optional[SortFunction] = None
     """Custom matching function that takes a `TreasuryTx` and returns a bool or an awaitable that returns a bool."""
 
-    # __instances__: ClassVar[List[Self]] = []
-
     def __post_init__(self) -> None:
         """Validate inputs, checksum addresses, and register the rule.
 
@@ -235,7 +233,7 @@ class _InboundSortRule(_SortRule):
         return (
             tx.to_address is not None
             and TreasuryWallet.check_membership(tx.to_address.address, tx.block)
-            and await super().match(tx)
+            and await super(_InboundSortRule, self).match(tx)
         )
 
 
@@ -250,7 +248,7 @@ class _OutboundSortRule(_SortRule):
     async def match(self, tx: "TreasuryTx") -> bool:
         return TreasuryWallet.check_membership(
             tx.from_address.address, tx.block
-        ) and await super().match(tx)
+        ) and await super(_OutboundSortRule, self).match(tx)
 
 
 @mypyc_attr(native_class=False)
@@ -267,7 +265,7 @@ class RevenueSortRule(_InboundSortRule):
     def __post_init__(self) -> None:
         """Prepends `self.txgroup` with 'Revenue:'."""
         object.__setattr__(self, "txgroup", f"Revenue:{self.txgroup}")
-        super().__post_init__()
+        super(RevenueSortRule, self).__post_init__()
 
 
 @mypyc_attr(native_class=False)
@@ -280,7 +278,7 @@ class CostOfRevenueSortRule(_OutboundSortRule):
     def __post_init__(self) -> None:
         """Prepends `self.txgroup` with 'Cost of Revenue:'."""
         object.__setattr__(self, "txgroup", f"Cost of Revenue:{self.txgroup}")
-        super().__post_init__()
+        super(CostOfRevenueSortRule, self).__post_init__()
 
 
 @mypyc_attr(native_class=False)
@@ -293,7 +291,7 @@ class ExpenseSortRule(_OutboundSortRule):
     def __post_init__(self) -> None:
         """Prepends `self.txgroup` with 'Expenses:'."""
         object.__setattr__(self, "txgroup", f"Expenses:{self.txgroup}")
-        super().__post_init__()
+        super(ExpenseSortRule, self).__post_init__()
 
 
 @mypyc_attr(native_class=False)
@@ -306,7 +304,7 @@ class OtherIncomeSortRule(_InboundSortRule):
     def __post_init__(self) -> None:
         """Prepends `self.txgroup` with 'Other Income:'."""
         object.__setattr__(self, "txgroup", f"Other Income:{self.txgroup}")
-        super().__post_init__()
+        super(OtherIncomeSortRule, self).__post_init__()
 
 
 @mypyc_attr(native_class=False)
@@ -319,7 +317,7 @@ class OtherExpenseSortRule(_OutboundSortRule):
     def __post_init__(self) -> None:
         """Prepends `self.txgroup` with 'Other Expenses:'."""
         object.__setattr__(self, "txgroup", f"Other Expenses:{self.txgroup}")
-        super().__post_init__()
+        super(OtherExpenseSortRule, self).__post_init__()
 
 
 @mypyc_attr(native_class=False)
@@ -332,7 +330,7 @@ class IgnoreSortRule(_SortRule):
     def __post_init__(self) -> None:
         """Prepends `self.txgroup` with 'Ignore:'."""
         object.__setattr__(self, "txgroup", f"Ignore:{self.txgroup}")
-        super().__post_init__()
+        super(IgnoreSortRule, self).__post_init__()
 
 
 TRule = TypeVar(

dao_treasury/streams/llamapay.py

@@ -22,7 +22,7 @@ from eth_typing import BlockNumber, ChecksumAddress, HexAddress, HexStr
 from tqdm.asyncio import tqdm_asyncio
 
 import y
-from y.time import UnixTimestamp
+from y.time import NoBlockFound, UnixTimestamp
 from y.utils.events import decode_logs, get_logs_asap
 
 from dao_treasury import constants
@@ -338,7 +338,19 @@ class LlamaPayProcessor:
         check_at = date_obj + timedelta(days=1) - timedelta(seconds=1)
         if check_at > now(tz=_UTC):
             await sleep((check_at - now(tz=_UTC)).total_seconds())
-        block = await get_block_at_timestamp(check_at, sync=False)
+
+        while True:
+            try:
+                block = await get_block_at_timestamp(check_at, sync=False)
+            except NoBlockFound:
+                sleep_time = (check_at - now(tz=_UTC)).total_seconds()
+                logger.debug(
+                    "no block found for %s, sleeping %ss", check_at, sleep_time
+                )
+                await sleep(sleep_time)
+            else:
+                break
+
         price_fut = create_task(get_price(stream_token, block, sync=False))
         start_timestamp = await _get_start_timestamp(stream_id, block)
         if start_timestamp == 0:

dao_treasury/treasury.py

@@ -1,11 +1,28 @@
+"""Treasury orchestration and analytics interface.
+
+This module defines the Treasury class, which aggregates DAO wallets, sets up
+sorting rules, and manages transaction ingestion and streaming analytics.
+It coordinates the end-to-end flow from wallet configuration to database
+population and dashboard analytics.
+
+Key Responsibilities:
+    - Aggregate and manage DAO-controlled wallets.
+    - Ingest and process on-chain transactions.
+    - Apply sorting/categorization rules.
+    - Integrate with streaming protocols (e.g., LlamaPay).
+    - Populate the database for analytics and dashboards.
+
+This is the main entry point for orchestrating DAO treasury analytics.
+"""
+
 from asyncio import create_task, gather
 from logging import getLogger
 from pathlib import Path
-from typing import Final, Iterable, List, Optional, Union
+from typing import Dict, Final, Iterable, List, Optional, Union
 
 import a_sync
 from a_sync.a_sync.abstract import ASyncABC
-from eth_typing import BlockNumber
+from eth_typing import BlockNumber, HexAddress
 from eth_portfolio.structs import LedgerEntry
 from eth_portfolio.typing import PortfolioBalances
 from eth_portfolio_scripts._portfolio import ExportablePortfolio
@@ -35,6 +52,7 @@ class Treasury(a_sync.ASyncGenericBase):  # type: ignore [misc]
         sort_rules: Optional[Path] = None,
         start_block: int = 0,
         label: str = "your org's treasury",
+        custom_buckets: Optional[Dict[HexAddress, str]] = None,
         asynchronous: bool = False,
     ) -> None:
         """Initialize the Treasury singleton for managing DAO funds.
@@ -59,20 +77,22 @@ class Treasury(a_sync.ASyncGenericBase):  # type: ignore [misc]
             TypeError: If any item in `wallets` is not a str or TreasuryWallet.
 
         Examples:
-            Create a synchronous Treasury:
-            >>> treasury = Treasury(
-            ...     wallets=["0xAbc123...", TreasuryWallet("0xDef456...", start_block=1000)],
-            ...     sort_rules=Path("/path/to/rules"),
-            ...     start_block=500,
-            ...     label="DAO Treasury",
-            ...     asynchronous=False
-            ... )
-
-            Create an asynchronous Treasury:
-            >>> treasury_async = Treasury(
-            ...     wallets=["0xAbc123..."],
-            ...     asynchronous=True
-            ... )
+            .. code-block:: python
+
+                # Create a synchronous Treasury
+                treasury = Treasury(
+                    wallets=["0xAbc123...", TreasuryWallet("0xDef456...", start_block=1000)],
+                    sort_rules=Path("/path/to/rules"),
+                    start_block=500,
+                    label="DAO Treasury",
+                    asynchronous=False
+                )
+
+                # Create an asynchronous Treasury
+                treasury_async = Treasury(
+                    wallets=["0xAbc123..."],
+                    asynchronous=True
+                )
         """
         global TREASURY
         if TREASURY is not None:
@@ -105,6 +125,7 @@ class Treasury(a_sync.ASyncGenericBase):  # type: ignore [misc]
             start_block=start_block,
             label=label,
             load_prices=True,
+            custom_buckets=custom_buckets,
             asynchronous=asynchronous,
         )
         """An eth_portfolio.Portfolio object used for exporting tx and balance history"""

{dao_treasury-0.0.34.dist-info → dao_treasury-0.0.72.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dao_treasury
-Version: 0.0.34
+Version: 0.0.72
 Summary: Produce comprehensive financial reports for your on-chain org
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
@@ -14,7 +14,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10,<3.13
 Description-Content-Type: text/markdown
-Requires-Dist: eth-portfolio-temp<0.1,>=0.0.30.dev0
+Requires-Dist: eth-portfolio-temp==0.3.1
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
@@ -29,6 +29,7 @@ DAO Treasury is a comprehensive financial reporting and treasury management solu
 - **Financial Reporting for DAOs:** Extends core portfolio functionalities to generate detailed reports tailored for on-chain organizations.
 - **Dashboard Provisioning:** Utilizes [Grafana](https://grafana.com/) dashboards—defined in JSON files within the .grafana/provisioning directories—to offer real-time, dynamic visualizations of treasury data.
 - **Automated Data Export:** Features a treasury export tool that, once configured (with a supported [brownie network](https://eth-brownie.readthedocs.io/en/stable/network-management.html) and [Docker](https://www.docker.com/get-started/)), continuously captures financial snapshots at set intervals.
+- **Custom Buckets for Wallets:** Assign custom categories ("buckets") to specific wallet addresses for more granular reporting using the `--custom-bucket` CLI option.
 - **Ease of Contribution:** Non-technical users can easily update or create dashboard visuals using Grafana’s intuitive UI. The [Contributing Guidelines](https://github.com/BobTheBuidler/dao-treasury/blob/master/CONTRIBUTING.md) document provides a step-by-step guide to defining new visuals and dashboards and integrating those changes into the repository, ensuring that anyone can contribute to the visual reporting aspect of the project.
 
 ## Requirements
@@ -41,7 +42,7 @@ DAO Treasury is a comprehensive financial reporting and treasury management solu
 - First, you will need to bring your own archive node. This can be one you run yourself, or one from one of the common providers (Tenderly, Alchemy, QuickNode, etc.). Your archive node must have tracing enabled (free-tier Alchemy nodes do not support this option).
 - You must configure a [brownie network](https://eth-brownie.readthedocs.io/en/stable/network-management.html) to use your RPC.
 - You will need an auth token for [Etherscan](https://etherscan.io/)'s API. Follow their [guide](https://docs.etherscan.io/etherscan-v2/getting-an-api-key) to get your key, and set env var `ETHERSCAN_TOKEN` with its value.
-- You'll also need [Docker](https://www.docker.com/get-started/) installed on your system. If on MacOS, you will need to leave Docker Desktop open while Yearn Treasury is running.
+- You'll also need [Docker](https://www.docker.com/get-started/) installed on your system. If on MacOS, you will need to leave Docker Desktop open while DAO Treasury is running.
 
 ## Installation
 
@@ -63,14 +64,28 @@ For local development (from source installation), use:
 poetry run dao-treasury run --wallet 0x123 --network mainnet --interval 12h
 ```
 
+**Assigning Custom Buckets to Wallets:**
+
+You can assign custom categories ("buckets") to specific wallet addresses for more granular reporting. Use the `--custom-bucket` option one or more times, each with the format `address:bucket_name`:
+
+```bash
+dao-treasury run --wallet 0x123 --network mainnet --custom-bucket "0x123:Operations" --custom-bucket "0x456:Grants" --custom-bucket "0x789:Investments"
+```
+
 **CLI Options:**
+> Only optional arguments are listed here. Required arguments (such as `--wallet` or `--wallets`) are shown in the usage examples above.
+
 - `--network`: The id of the brownie network the exporter will connect to (default: mainnet)
 - `--interval`: The time interval between each data snapshot (default: 12h)
+- `--concurrency`: The max number of historical blocks to export concurrently. (default: 30)
 - `--daemon`: Run the export process in the background (default: False) (NOTE: currently unsupported)
+- `--sort-rules`: Directory containing sort rules definitions for transaction categorization.
+- `--nicknames`: File containing address nicknames for reporting.
 - `--grafana-port`: Set the port for the Grafana dashboard where you can view data (default: 3004)
 - `--renderer-port`: Set the port for the report rendering service (default: 8091)
 - `--victoria-port`: Set the port for the Victoria metrics reporting endpoint (default: 8430)
 - `--start-renderer`: If set, both the Grafana and renderer containers will be started for dashboard image export. By default, only the grafana container is started.
+- `--custom-bucket`: Assign a custom bucket/category to a wallet address for reporting. Specify as `address:bucket_name`. Can be used multiple times.
 
 After running the command, the export script will run continuously until you close your terminal.
 To view the dashboards, just open your browser and navigate to [http://localhost:3004](http://localhost:3004)!