dao-treasury 0.0.21__cp311-cp311-win_amd64.whl → 0.0.71__cp311-cp311-win_amd64.whl

dao_treasury/docker-compose.yaml

@@ -1,18 +1,18 @@
 networks:
   dao_treasury:
-
-volumes:
-  grafana_data: {}
+  docker_eth_portfolio:
+    external: true
 
 services:
   grafana:
-    image: grafana/grafana:10.2.0
+    image: grafana/grafana:12.2.1
     ports:
       - 127.0.0.1:${DAO_TREASURY_GRAFANA_PORT:-3004}:3000
     environment:
       - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin}
       - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin}
       - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
       - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/summary/Monthly.json
       - GF_SERVER_ROOT_URL
       - GF_RENDERING_SERVER_URL=http://renderer:8091/render
@@ -21,10 +21,10 @@ services:
       - GF_INSTALL_PLUGINS=volkovlabs-variable-panel,frser-sqlite-datasource
     volumes:
       - ~/.dao-treasury/:/app/dao-treasury-data
-      - grafana_data:/var/lib/grafana
       - ./.grafana/provisioning/:/etc/grafana/provisioning/
     networks:
       - dao_treasury
+      - docker_eth_portfolio
     restart: always
 
   renderer:
@@ -37,4 +37,5 @@ services:
       - HTTP_PORT=8091
     networks:
       - dao_treasury
+      - docker_eth_portfolio
     restart: always

dao_treasury/main.py

@@ -28,14 +28,19 @@ from pathlib import Path
 
 import brownie
 import yaml
+from a_sync import create_task
+from dao_treasury._wallet import load_wallets_from_yaml
+from eth_portfolio_scripts.balances import export_balances
 from eth_typing import BlockNumber
 
-from eth_portfolio_scripts.balances import export_balances
+from dao_treasury.constants import CHAINID
 
 
 logger = logging.getLogger(__name__)
+
 logging.basicConfig(level=logging.INFO)
 
+
 parser = argparse.ArgumentParser(
     description="Run a single DAO Treasury export and populate the database.",
 )
@@ -50,16 +55,29 @@ parser.add_argument(
     type=str,
     help=(
         "DAO treasury wallet address(es) to include in the export. "
-        "Specify one or more addresses separated by spaces."
+        "Specify one or more addresses separated by spaces. "
+        "Check out https://bobthebuidler.github.io/dao-treasury/wallets.html for more info."
     ),
     nargs="+",
 )
+parser.add_argument(
+    "--wallets",
+    type=Path,
+    help=(
+        "Path to a YAML file mapping wallet addresses to advanced settings. "
+        "Each address is a key, with nested 'start' and/or 'end' mappings containing "
+        "either 'block' or 'timestamp'. "
+        "Check out https://bobthebuidler.github.io/dao-treasury/wallets.html for more info."
+    ),
+    default=None,
+)
 parser.add_argument(
     "--sort-rules",
     type=Path,
     help=(
         "Directory containing sort rules definitions. "
-        "If omitted, transactions are exported without custom sorting."
+        "If omitted, transactions are exported without custom sorting. "
+        "Check out https://bobthebuidler.github.io/dao-treasury/sort_rules.html for more info."
     ),
     default=None,
 )
@@ -79,6 +97,12 @@ parser.add_argument(
     help="The time interval between datapoints. default: 1d",
     default="1d",
 )
+parser.add_argument(
+    "--concurrency",
+    type=int,
+    help="The max number of historical blocks to export concurrently. default: 30",
+    default=30,
+)
 parser.add_argument(
     "--daemon",
     action="store_true",
@@ -90,12 +114,29 @@ parser.add_argument(
     help="Port for the DAO Treasury dashboard web interface. Default: 3000",
     default=3000,
 )
+parser.add_argument(
+    "--start-renderer",
+    action="store_true",
+    help="If set, the Grafana renderer container will be started for dashboard image export. By default, only the grafana container is started.",
+)
 parser.add_argument(
     "--renderer-port",
     type=int,
     help="Port for the Grafana rendering service. Default: 8091",
     default=8091,
 )
+parser.add_argument(
+    "--custom-bucket",
+    type=str,
+    action="append",
+    help=(
+        "Custom bucket mapping for a wallet address. "
+        "Specify as 'address:bucket_name'. "
+        "Can be used multiple times. Example: "
+        "--custom-bucket '0x123:My Bucket' --custom-bucket '0x456:Other Bucket'"
+    ),
+    default=None,
+)
 
 args = parser.parse_args()
 
@@ -131,16 +172,13 @@ async def export(args) -> None:
 
     Args:
         args: Parsed command-line arguments containing:
-            wallet: Treasury wallet address strings.
+            wallet: List of simple addresses or TreasuryWallet instances.
             sort_rules: Directory of sorting rules.
             interval: Time interval for balance snapshots.
             daemon: Ignored flag.
             grafana_port: Port for Grafana (sets DAO_TREASURY_GRAFANA_PORT).
             renderer_port: Port for renderer (sets DAO_TREASURY_RENDERER_PORT).
-
-    Note:
-        Inside this coroutine, the environment variable GRAFANA_PORT is overridden to "3003"
-        to satisfy current dashboard requirements.
+            start_renderer: If True, start renderer; otherwise, only start grafana.
 
     Example:
         In code::
@@ -152,9 +190,24 @@ async def export(args) -> None:
         :func:`dao_treasury._docker.down`,
         :class:`dao_treasury.Treasury.populate_db`
     """
-    from y.constants import CHAINID
+    import eth_portfolio_scripts.docker
+
+    from dao_treasury import _docker, constants, db, Treasury
 
-    from dao_treasury import _docker, db, Treasury
+    wallets = getattr(args, "wallet", None)
+    wallets_advanced = getattr(args, "wallets", None)
+
+    # Ensure user does not supply both simple and advanced wallet inputs
+    if wallets and wallets_advanced:
+        parser.error("Cannot specify both --wallet and --wallets")
+
+    # Load advanced wallets from YAML if --wallets provided
+    if wallets_advanced:
+        wallets = load_wallets_from_yaml(wallets_advanced)
+
+    # Ensure at least one wallet source is provided
+    if not wallets:
+        parser.error("Must specify either --wallet or --wallets")
 
     # TODO: remove this after refactoring eth-port a bit so we arent required to bring up the e-p dashboards
     os.environ["GRAFANA_PORT"] = "3003"
@@ -162,19 +215,67 @@ async def export(args) -> None:
     # TODO but make the dashboard files more specific to dao treasury-ing
 
     if args.nicknames:
-        for nickname, addresses in (
-            yaml.safe_load(args.nicknames.read_bytes()).get(CHAINID, {}).items()
-        ):
+        parsed: dict = yaml.safe_load(args.nicknames.read_bytes())
+        active_network_config: dict = parsed.get(constants.CHAINID, {})
+        for nickname, addresses in active_network_config.items():
             for address in addresses:
                 db.Address.set_nickname(address, nickname)
 
-    treasury = Treasury(args.wallet, args.sort_rules, asynchronous=True)
-    _docker.up()
-    try:
-        await asyncio.gather(
+    # Parse custom_buckets from --custom-bucket arguments
+    custom_buckets = None
+    if args.custom_bucket:
+        custom_buckets = {}
+        for item in args.custom_bucket:
+            if ":" not in item:
+                parser.error(
+                    f"Invalid format for --custom-bucket: '{item}'. Must be 'address:bucket_name'."
+                )
+            address, bucket = item.split(":", 1)
+            address = address.strip()
+            bucket = bucket.strip()
+            if not address or not bucket:
+                parser.error(
+                    f"Invalid format for --custom-bucket: '{item}'. Both address and bucket_name are required."
+                )
+            custom_buckets[address] = bucket
+
+    treasury = Treasury(
+        wallets, args.sort_rules, custom_buckets=custom_buckets, asynchronous=True
+    )
+
+    # Start only the requested containers
+    if args.start_renderer is True:
+        _docker.up()
+    else:
+        _docker.up("grafana")
+
+    # eth-portfolio needs this present
+    # TODO: we need to update eth-portfolio to honor wallet join and exit times
+    if not getattr(args, "wallet", None):
+        args.wallet = [
+            wallet.address
+            for wallet in wallets
+            if wallet.networks is None or CHAINID in wallet.networks
+        ]
+
+    # TODO: make this user configurable? would require some dynamic grafana dashboard files
+    args.label = "Treasury"
+
+    export_task = create_task(
+        asyncio.gather(
             export_balances(args),
             treasury.populate_db(BlockNumber(0), brownie.chain.height),
         )
+    )
+
+    await asyncio.sleep(1)
+
+    # we don't need these containers since dao-treasury uses its own.
+    eth_portfolio_scripts.docker.stop("grafana")
+    eth_portfolio_scripts.docker.stop("renderer")
+
+    try:
+        await export_task
     finally:
         _docker.down()
 

dao_treasury/sorting/__init__.py

@@ -1,5 +1,28 @@
 """
-This module contains logic for sorting transactions into various categories.
+This module provides the core logic for sorting DAO Treasury transactions into transaction groups (categories).
+
+Sorting enables comprehensive financial reporting and categorization tailored for on-chain organizations.
+Transactions are matched against either statically defined rules or more advanced dynamic rules based on user-defined matching functions.
+
+Sorting works by attempting matches in this order:
+  1. Check if the transaction is an internal transfer (within treasury wallets).
+  2. Check if the transaction is "Out of Range" (neither sender nor receiver was a treasury wallet at the time of the tx).
+  3. Match by transaction hash using registered HashMatchers.
+  4. Match by sender address using registered FromAddressMatchers.
+  5. Match by recipient address using registered ToAddressMatchers.
+  6. Assign "Must Sort Inbound" or "Must Sort Outbound" groups if part of treasury.
+  7. Raise an error if no match is found (unexpected case).
+
+See the complete [sort rules documentation](https://bobthebuidler.github.io/dao-treasury/sort_rules.html) for detailed explanations
+and examples on defining and registering sort rules.
+
+See Also:
+    :func:`dao_treasury.sorting.sort_basic`
+    :func:`dao_treasury.sorting.sort_basic_entity`
+    :func:`dao_treasury.sorting.sort_advanced`
+    :class:`dao_treasury.sorting.HashMatcher`
+    :class:`dao_treasury.sorting.FromAddressMatcher`
+    :class:`dao_treasury.sorting.ToAddressMatcher`
 """
 
 from logging import getLogger
@@ -9,7 +32,7 @@ from eth_portfolio.structs import LedgerEntry
 from evmspec.data import TransactionHash
 from y.exceptions import ContractNotVerified
 
-from dao_treasury import db
+from dao_treasury import constants, db
 from dao_treasury._wallet import TreasuryWallet
 from dao_treasury.sorting._matchers import (
     _Matcher,
@@ -35,6 +58,7 @@ from dao_treasury.sorting.rule import (
     OtherIncomeSortRule,
     RevenueSortRule,
 )
+from dao_treasury.sorting.rules import *
 from dao_treasury.types import TxGroupDbid
 
 
@@ -64,19 +88,71 @@ __all__ = [
 
 # C constants
 TxGroup: Final = db.TxGroup
-must_sort_inbound_txgroup_dbid: Final = db.must_sort_inbound_txgroup_dbid
-must_sort_outbound_txgroup_dbid: Final = db.must_sort_outbound_txgroup_dbid
+MUST_SORT_INBOUND_TXGROUP_DBID: Final = db.must_sort_inbound_txgroup_dbid
+MUST_SORT_OUTBOUND_TXGROUP_DBID: Final = db.must_sort_outbound_txgroup_dbid
+
+INTERNAL_TRANSFER_TXGROUP_DBID: Final = TxGroup.get_dbid(
+    name="Internal Transfer",
+    parent=TxGroup.get_dbid("Ignore"),
+)
+"""Database ID for the 'Internal Transfer' transaction group.
+
+This group represents transactions that occur internally between treasury-owned wallets.
+Such internal movements of funds within the DAO's treasury do not require separate handling or reporting.
+
+See Also:
+    :class:`dao_treasury.db.TxGroup`
+"""
+
+OUT_OF_RANGE_TXGROUP_DBID = TxGroup.get_dbid(
+    name="Out of Range", parent=TxGroup.get_dbid("Ignore")
+)
+"""Database ID for the 'Out of Range' transaction group.
+
+This category is assigned to transactions where neither the sender nor the recipient
+wallet are members of the treasury at the time of the transaction.
+
+See Also:
+    :class:`dao_treasury.db.TxGroup`
+"""
 
 
 def sort_basic(entry: LedgerEntry) -> TxGroupDbid:
+    """Determine the transaction group ID for a basic ledger entry using static matching.
+
+    The function attempts to categorize the transaction by testing:
+      - If both 'from' and 'to' addresses are treasury wallets (internal transfer).
+      - If neither ‘to’ address is a treasury wallet at the time of the transaction (out of range).
+      - If the transaction hash matches a known HashMatcher.
+      - If the 'from' address matches a FromAddressMatcher.
+      - If the 'to' address matches a ToAddressMatcher.
+      - Assignment to 'Must Sort Outbound' or 'Must Sort Inbound' groups if applicable.
+      - Raises `NotImplementedError` if none of the above conditions are met (should not happen).
+
+    Args:
+        entry: A ledger entry representing a blockchain transaction.
+
+    Examples:
+        >>> from eth_portfolio.structs import Transaction
+        >>> entry = Transaction(from_address="0xabc...", to_address="0xdef...", block_number=1234567)
+        >>> group_id = sort_basic(entry)
+        >>> print(group_id)
+
+    See Also:
+        :func:`sort_basic_entity`
+        :func:`sort_advanced`
+        :class:`dao_treasury.sorting.HashMatcher`
+    """
+    from_address = entry.from_address
+    to_address = entry.to_address
+    block = entry.block_number
+
     txgroup_dbid: Optional[TxGroupDbid] = None
-    if TreasuryWallet.check_membership(
-        entry.from_address, entry.block_number
-    ) and TreasuryWallet.check_membership(entry.to_address, entry.block_number):
-        txgroup_dbid = TxGroup.get_dbid(
-            name="Internal Transfer",
-            parent=TxGroup.get_dbid("Ignore"),
-        )
+    if TreasuryWallet.check_membership(from_address, block):
+        if TreasuryWallet.check_membership(to_address, block):
+            txgroup_dbid = INTERNAL_TRANSFER_TXGROUP_DBID
+    elif not TreasuryWallet.check_membership(to_address, block):
+        txgroup_dbid = OUT_OF_RANGE_TXGROUP_DBID
 
     if txgroup_dbid is None:
         if isinstance(txhash := entry.hash, TransactionHash):
@@ -84,69 +160,122 @@ def sort_basic(entry: LedgerEntry) -> TxGroupDbid:
         txgroup_dbid = HashMatcher.match(txhash)
 
     if txgroup_dbid is None:
-        txgroup_dbid = FromAddressMatcher.match(entry.from_address)
+        txgroup_dbid = FromAddressMatcher.match(from_address)
 
     if txgroup_dbid is None:
-        txgroup_dbid = ToAddressMatcher.match(entry.to_address)
+        txgroup_dbid = ToAddressMatcher.match(to_address)
 
     if txgroup_dbid is None:
-        if TreasuryWallet.check_membership(entry.from_address, entry.block_number):
-            txgroup_dbid = must_sort_outbound_txgroup_dbid
+        if TreasuryWallet.check_membership(from_address, block):
+            txgroup_dbid = MUST_SORT_OUTBOUND_TXGROUP_DBID
 
-        elif TreasuryWallet.check_membership(entry.to_address, entry.block_number):
-            txgroup_dbid = must_sort_inbound_txgroup_dbid
+        elif TreasuryWallet.check_membership(to_address, block):
+            txgroup_dbid = MUST_SORT_INBOUND_TXGROUP_DBID
 
         else:
             raise NotImplementedError("this isnt supposed to happen")
     return txgroup_dbid  # type: ignore [no-any-return]
 
 
-def sort_basic_entity(entry: db.TreasuryTx) -> TxGroupDbid:
+def sort_basic_entity(tx: db.TreasuryTx) -> TxGroupDbid:
+    """Determine the transaction group ID for a TreasuryTx database entity using static matching.
+
+    Similar to :func:`sort_basic` but operates on a TreasuryTx entity from the database.
+    It considers additional constants such as `DISPERSE_APP` when determining whether
+    a transaction is out of range.
+
+    Args:
+        tx: A TreasuryTx database entity representing a treasury transaction.
+
+    Examples:
+        >>> from dao_treasury.db import TreasuryTx
+        >>> tx = TreasuryTx[123]
+        >>> group_id = sort_basic_entity(tx)
+        >>> print(group_id)
+
+    See Also:
+        :func:`sort_basic`
+        :func:`sort_advanced`
+    """
+    from_address = tx.from_address.address
+    to_address = tx.to_address
+    block = tx.block
+
     txgroup_dbid: Optional[TxGroupDbid] = None
-    if (
-        entry.from_address
-        and TreasuryWallet.check_membership(entry.from_address.address, entry.block)
-        and TreasuryWallet.check_membership(entry.to_address.address, entry.block)
+    if TreasuryWallet.check_membership(from_address, block):
+        if TreasuryWallet.check_membership(tx.to_address.address, block):
+            txgroup_dbid = INTERNAL_TRANSFER_TXGROUP_DBID
+    elif not (
+        TreasuryWallet.check_membership(tx.to_address.address, tx.block)
+        or from_address in constants.DISPERSE_APP
     ):
-        txgroup_dbid = TxGroup.get_dbid(
-            name="Internal Transfer",
-            parent=TxGroup.get_dbid("Ignore"),
-        )
+        txgroup_dbid = OUT_OF_RANGE_TXGROUP_DBID
 
     if txgroup_dbid is None:
-        txgroup_dbid = HashMatcher.match(entry.hash)
+        txgroup_dbid = HashMatcher.match(tx.hash)
 
     if txgroup_dbid is None:
-        txgroup_dbid = FromAddressMatcher.match(entry.from_address.address)
+        txgroup_dbid = FromAddressMatcher.match(from_address)
 
-    if txgroup_dbid is None and entry.to_address:
-        txgroup_dbid = ToAddressMatcher.match(entry.to_address.address)
+    if txgroup_dbid is None and to_address:
+        txgroup_dbid = ToAddressMatcher.match(to_address.address)
 
     if txgroup_dbid is None:
-        if TreasuryWallet.check_membership(entry.from_address.address, entry.block):
-            txgroup_dbid = must_sort_outbound_txgroup_dbid
+        if TreasuryWallet.check_membership(from_address, block):
+            txgroup_dbid = MUST_SORT_OUTBOUND_TXGROUP_DBID
+
+        elif TreasuryWallet.check_membership(to_address.address, block):
+            txgroup_dbid = MUST_SORT_INBOUND_TXGROUP_DBID
 
-        elif TreasuryWallet.check_membership(entry.to_address.address, entry.block):
-            txgroup_dbid = must_sort_inbound_txgroup_dbid
+        elif from_address in constants.DISPERSE_APP:
+            txgroup_dbid = MUST_SORT_OUTBOUND_TXGROUP_DBID
+
+        elif from_address in constants.DISPERSE_APP:
+            txgroup_dbid = MUST_SORT_OUTBOUND_TXGROUP_DBID
 
         else:
             raise NotImplementedError("this isnt supposed to happen")
 
     if txgroup_dbid not in (
-        must_sort_inbound_txgroup_dbid,
-        must_sort_outbound_txgroup_dbid,
+        MUST_SORT_INBOUND_TXGROUP_DBID,
+        MUST_SORT_OUTBOUND_TXGROUP_DBID,
     ):
-        logger.info("Sorted %s to %s", entry, TxGroup.get_fullname(txgroup_dbid))
+        logger.info("Sorted %s to %s", tx, TxGroup.get_fullname(txgroup_dbid))
 
     return txgroup_dbid  # type: ignore [no-any-return]
 
 
 async def sort_advanced(entry: db.TreasuryTx) -> TxGroupDbid:
+    """Determine the transaction group ID for a TreasuryTx entity using advanced dynamic rules.
+
+    Starts with the result of static matching via :func:`sort_basic_entity`, then
+    applies advanced asynchronous matching rules registered under :data:`SORT_RULES`.
+    Applies rules sequentially until a match is found or all rules are exhausted.
+
+    If a rule's match attempt raises a `ContractNotVerified` exception, the rule is skipped.
+
+    Updates the TreasuryTx entity's transaction group in the database when a match
+    other than 'Must Sort Inbound/Outbound' is found.
+
+    Args:
+        entry: A TreasuryTx database entity representing a treasury transaction.
+
+    Examples:
+        >>> from dao_treasury.db import TreasuryTx
+        >>> import asyncio
+        >>> tx = TreasuryTx[123]
+        >>> group_id = asyncio.run(sort_advanced(tx))
+        >>> print(group_id)
+
+    See Also:
+        :func:`sort_basic_entity`
+        :data:`SORT_RULES`
+    """
     txgroup_dbid = sort_basic_entity(entry)
 
     if txgroup_dbid in (
-        must_sort_inbound_txgroup_dbid,
-        must_sort_outbound_txgroup_dbid,
+        MUST_SORT_INBOUND_TXGROUP_DBID,
+        MUST_SORT_OUTBOUND_TXGROUP_DBID,
     ):
         for rules in SORT_RULES.values():
             for rule in rules:
@@ -157,10 +286,10 @@ async def sort_advanced(entry: db.TreasuryTx) -> TxGroupDbid:
                 except ContractNotVerified:
                     continue
     if txgroup_dbid not in (
-        must_sort_inbound_txgroup_dbid,
-        must_sort_outbound_txgroup_dbid,
+        MUST_SORT_INBOUND_TXGROUP_DBID,
+        MUST_SORT_OUTBOUND_TXGROUP_DBID,
     ):
         logger.info("Sorted %s to %s", entry, TxGroup.get_fullname(txgroup_dbid))
-        entry.txgroup = txgroup_dbid
+        await entry._set_txgroup(txgroup_dbid)
 
     return txgroup_dbid  # type: ignore [no-any-return]

dao_treasury/sorting/_rules.py

@@ -4,8 +4,8 @@ from typing import Final, Type, Union, final
 
 import yaml
 from pony.orm import db_session
-from y import constants
 
+from dao_treasury.constants import CHAINID
 from dao_treasury.sorting import (
     _Matcher,
     FromAddressMatcher,
@@ -15,8 +15,6 @@ from dao_treasury.sorting import (
 from dao_treasury.types import TopLevelCategory, TxGroupDbid
 
 
-CHAINID: Final = constants.CHAINID
-
 logger: Final = getLogger("dao_treasury.rules")
 
 

dao_treasury/sorting/factory.py

@@ -1,7 +1,6 @@
-from typing import Any, Final, Generic, Optional, TypeVar, Union, final, overload
-
-from y import constants
+from typing import Any, Final, Generic, Optional, Union, final, overload
 
+from dao_treasury.constants import CHAINID
 from dao_treasury.sorting.rule import (
     CostOfRevenueSortRule,
     ExpenseSortRule,
@@ -14,9 +13,6 @@ from dao_treasury.sorting.rule import (
 from dao_treasury.types import Networks, SortFunction, TxGroupName
 
 
-CHAINID: Final = constants.CHAINID
-
-
 def revenue(
     txgroup: TxGroupName, networks: Networks = CHAINID
 ) -> "SortRuleFactory[RevenueSortRule]":