dao-treasury 0.0.17__cp312-cp312-win32.whl → 0.0.61__cp312-cp312-win32.whl

dao_treasury/db.py

@@ -3,11 +3,13 @@
 Database models and utilities for DAO treasury reporting.
 
 This module defines Pony ORM entities for:
+
 - Blockchain networks (:class:`Chain`)
 - On-chain addresses (:class:`Address`)
 - ERC-20 tokens and native coin placeholder (:class:`Token`)
 - Hierarchical transaction grouping (:class:`TxGroup`)
 - Treasury transaction records (:class:`TreasuryTx`)
+- Streams and StreamedFunds for streaming payments
 
 It also provides helper functions for inserting ledger entries,
 resolving integrity conflicts, caching transaction receipts,
@@ -16,27 +18,44 @@ and creating SQL views for reporting.
 
 import typing
 from asyncio import Semaphore
+from collections import OrderedDict
 from decimal import Decimal, InvalidOperation
 from functools import lru_cache
 from logging import getLogger
 from os import path
 from pathlib import Path
-from typing import TYPE_CHECKING, Final, Union, final
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Coroutine,
+    Dict,
+    Final,
+    Literal,
+    Tuple,
+    Union,
+    final,
+    overload,
+)
+from datetime import date, datetime, time, timezone
 
+import eth_portfolio
 from a_sync import AsyncThreadPoolExecutor
 from brownie import chain
 from brownie.convert.datatypes import HexString
+from brownie.exceptions import EventLookupError
 from brownie.network.event import EventDict, _EventItem
 from brownie.network.transaction import TransactionReceipt
-from eth_typing import ChecksumAddress, HexAddress
 from eth_portfolio.structs import (
     InternalTransfer,
     LedgerEntry,
     TokenTransfer,
     Transaction,
 )
+from eth_retry import auto_retry
+from eth_typing import ChecksumAddress, HexAddress, HexStr
 from pony.orm import (
     Database,
+    InterfaceError,
     Optional,
     PrimaryKey,
     Required,
@@ -46,15 +65,20 @@ from pony.orm import (
     composite_key,
     composite_index,
     db_session,
+    select,
 )
 from y import EEE_ADDRESS, Contract, Network, convert, get_block_timestamp_async
-from y.constants import CHAINID
+from y._db.decorators import retry_locked
 from y.contracts import _get_code
 from y.exceptions import ContractNotVerified
 
+from dao_treasury.constants import CHAINID
 from dao_treasury.types import TxGroupDbid, TxGroupName
 
 
+EventItem = _EventItem[_EventItem[OrderedDict[str, Any]]]
+
+
 SQLITE_DIR = Path(path.expanduser("~")) / ".dao-treasury"
 """Path to the directory in the user's home where the DAO treasury SQLite database is stored."""
 
@@ -62,8 +86,11 @@ SQLITE_DIR.mkdir(parents=True, exist_ok=True)
 
 
 _INSERT_THREAD = AsyncThreadPoolExecutor(1)
+_SORT_THREAD = AsyncThreadPoolExecutor(1)
+_EVENTS_THREADS = AsyncThreadPoolExecutor(16)
 _SORT_SEMAPHORE = Semaphore(50)
 
+_UTC = timezone.utc
 
 db = Database()
 
@@ -112,13 +139,13 @@ class Chain(DbEntity):
     chainid = Required(int, unique=True)
     """Numeric chain ID matching the connected RPC via :data:`~y.constants.CHAINID`."""
 
-    addresses = Set("Address", reverse="chain")
+    addresses = Set("Address", reverse="chain", lazy=True)
     """Relationship to address records on this chain."""
 
-    tokens = Set("Token", reverse="chain")
+    tokens = Set("Token", reverse="chain", lazy=True)
     """Relationship to token records on this chain."""
 
-    treasury_txs = Set("TreasuryTx")
+    treasury_txs = Set("TreasuryTx", lazy=True)
     """Relationship to treasury transactions on this chain."""
 
     @staticmethod
@@ -177,7 +204,7 @@ class Address(DbEntity):
     address_id = PrimaryKey(int, auto=True)
     """Auto-incremented primary key for the addresses table."""
 
-    chain = Required(Chain, reverse="addresses")
+    chain = Required(Chain, reverse="addresses", lazy=True)
     """Reference to the chain on which this address resides."""
 
     address = Required(str, index=True)
@@ -186,7 +213,7 @@ class Address(DbEntity):
     nickname = Optional(str)
     """Optional human-readable label (e.g., contract name or token name)."""
 
-    is_contract = Required(bool, index=True)
+    is_contract = Required(bool, index=True, lazy=True)
     """Flag indicating whether the address is a smart contract."""
 
     composite_key(address, chain)
@@ -197,29 +224,40 @@ class Address(DbEntity):
         treasury_tx_from: Set["TreasuryTx"]
         treasury_tx_to: Set["TreasuryTx"]
 
-    token = Optional("Token", index=True)
+    token = Optional("Token", index=True, lazy=True)
     """Optional back-reference to a Token if this address is one."""
-    # partners_tx = Set('PartnerHarvestEvent', reverse='wrapper')
+    # partners_tx = Set('PartnerHarvestEvent', reverse='wrapper', lazy=True)
 
-    treasury_tx_from = Set("TreasuryTx", reverse="from_address")
+    treasury_tx_from = Set("TreasuryTx", reverse="from_address", lazy=True)
     """Inverse relation for transactions sent from this address."""
 
-    treasury_tx_to = Set("TreasuryTx", reverse="to_address")
+    treasury_tx_to = Set("TreasuryTx", reverse="to_address", lazy=True)
     """Inverse relation for transactions sent to this address."""
-    # streams_from = Set("Stream", reverse="from_address")
-    # streams_to = Set("Stream", reverse="to_address")
-    # streams = Set("Stream", reverse="contract")
-    # vesting_escrows = Set("VestingEscrow", reverse="address")
-    # vests_received = Set("VestingEscrow", reverse="recipient")
-    # vests_funded = Set("VestingEscrow", reverse="funder")
-
-    def __eq__(self, other: Union["Address", ChecksumAddress]) -> bool:  # type: ignore [override]
+
+    streams_from = Set("Stream", reverse="from_address", lazy=True)
+    streams_to = Set("Stream", reverse="to_address", lazy=True)
+    streams = Set("Stream", reverse="contract", lazy=True)
+    # vesting_escrows = Set("VestingEscrow", reverse="address", lazy=True)
+    # vests_received = Set("VestingEscrow", reverse="recipient", lazy=True)
+    # vests_funded = Set("VestingEscrow", reverse="funder", lazy=True)
+
+    def __eq__(self, other: Union["Address", ChecksumAddress, "Token"]) -> bool:  # type: ignore [override]
         if isinstance(other, str):
             return CHAINID == self.chain.chainid and other == self.address
+        elif isinstance(other, Token):
+            return self.address_id == other.address.address_id
         return super().__eq__(other)
 
     __hash__ = DbEntity.__hash__
 
+    @property
+    def contract(self) -> Contract:
+        return Contract(self.address)
+
+    @property
+    def contract_coro(self) -> Coroutine[Any, Any, Contract]:
+        return Contract.coroutine(self.address)
+
     @staticmethod
     @lru_cache(maxsize=None)
     def get_dbid(address: HexAddress) -> int:
@@ -256,11 +294,13 @@ class Address(DbEntity):
         if entity := Address.get(chain=chain_dbid, address=checksum_address):
             return entity  # type: ignore [no-any-return]
 
-        if _get_code(address, None).hex().removeprefix("0x"):
+        if _get_code(checksum_address, None).hex().removeprefix("0x"):
             try:
-                nickname = f"Contract: {Contract(address)._build['contractName']}"
+                nickname = (
+                    f"Contract: {Contract(checksum_address)._build['contractName']}"
+                )
             except ContractNotVerified:
-                nickname = f"Non-Verified Contract: {address}"
+                nickname = f"Non-Verified Contract: {checksum_address}"
 
             entity = Address(
                 chain=chain_dbid,
@@ -278,9 +318,34 @@ class Address(DbEntity):
             )
 
         commit()
-
         return entity  # type: ignore [no-any-return]
 
+    @staticmethod
+    def set_nickname(address: HexAddress, nickname: str) -> None:
+        if not nickname:
+            raise ValueError("You must provide an actual string")
+        with db_session:
+            entity = Address.get_or_insert(address)
+            if entity.nickname == nickname:
+                return
+            if entity.nickname:
+                old = entity.nickname
+                entity.nickname = nickname
+                commit()
+                logger.info(
+                    "%s nickname changed from %s to %s", entity.address, old, nickname
+                )
+            else:
+                entity.nickname = nickname
+                commit()
+                logger.info("%s nickname set to %s", entity.address, nickname)
+
+    @staticmethod
+    def set_nicknames(nicknames: Dict[HexAddress, str]) -> None:
+        with db_session:
+            for address, nickname in nicknames.items():
+                Address.set_nickname(address, nickname)
+
 
 UNI_V3_POS: Final = {
     Network.Mainnet: "0xC36442b4a4522E871399CD717aBDD847Ab11FE88",
@@ -325,34 +390,37 @@ class Token(DbEntity):
     token_id = PrimaryKey(int, auto=True)
     """Auto-incremented primary key for the tokens table."""
 
-    chain = Required(Chain, index=True)
+    chain = Required(Chain, index=True, lazy=True)
     """Foreign key linking to :class:`~dao_treasury.db.Chain`."""
 
-    symbol = Required(str, index=True)
+    symbol = Required(str, index=True, lazy=True)
     """Short ticker symbol for the token."""
 
-    name = Required(str)
+    name = Required(str, lazy=True)
     """Full human-readable name of the token."""
 
-    decimals = Required(int)
+    decimals = Required(int, lazy=True)
     """Number of decimals used for value scaling."""
 
     if TYPE_CHECKING:
         treasury_tx: Set["TreasuryTx"]
 
-    treasury_tx = Set("TreasuryTx", reverse="token")
+    treasury_tx = Set("TreasuryTx", reverse="token", lazy=True)
     """Inverse relation for treasury transactions involving this token."""
-    # partner_harvest_event = Set('PartnerHarvestEvent', reverse="vault")
+    # partner_harvest_event = Set('PartnerHarvestEvent', reverse="vault", lazy=True)
 
     address = Required(Address, column="address_id")
     """Foreign key to the address record for this token contract."""
-    # streams = Set('Stream', reverse="token")
-    # vesting_escrows = Set("VestingEscrow", reverse="token")
 
-    def __eq__(self, other: Union["Token", ChecksumAddress]) -> bool:  # type: ignore [override]
-        return (
-            self.address == other if isinstance(other, str) else super().__eq__(other)
-        )
+    streams = Set("Stream", reverse="token", lazy=True)
+    # vesting_escrows = Set("VestingEscrow", reverse="token", lazy=True)
+
+    def __eq__(self, other: Union["Token", Address, ChecksumAddress]) -> bool:  # type: ignore [override]
+        if isinstance(other, str):
+            return self.address == other
+        elif isinstance(other, Address):
+            return self.address.address_id == other.address_id
+        return super().__eq__(other)
 
     __hash__ = DbEntity.__hash__
 
@@ -360,6 +428,10 @@ class Token(DbEntity):
     def contract(self) -> Contract:
         return Contract(self.address.address)
 
+    @property
+    def contract_coro(self) -> Coroutine[Any, Any, Contract]:
+        return Contract.coroutine(self.address.address)
+
     @property
     def scale(self) -> int:
         """Base for division according to `decimals`, e.g., `10**decimals`.
@@ -488,7 +560,7 @@ class TxGroup(DbEntity):
     name = Required(str)
     """Name of the grouping category, e.g., 'Revenue', 'Expenses'."""
 
-    treasury_tx = Set("TreasuryTx", reverse="txgroup")
+    treasury_tx = Set("TreasuryTx", reverse="txgroup", lazy=True)
     """Inverse relation for treasury transactions assigned to this group."""
 
     parent_txgroup = Optional("TxGroup", reverse="child_txgroups")
@@ -496,11 +568,13 @@ class TxGroup(DbEntity):
 
     composite_key(name, parent_txgroup)
 
-    child_txgroups = Set("TxGroup", reverse="parent_txgroup")
+    child_txgroups = Set("TxGroup", reverse="parent_txgroup", lazy=True)
     """Set of nested child groups."""
-    # TODO: implement these
-    # streams = Set("Stream", reverse="txgroup")
-    # vesting_escrows = Set("VestingEscrow", reverse="txgroup")
+
+    streams = Set("Stream", reverse="txgroup", lazy=True)
+
+    # TODO: implement this
+    # vesting_escrows = Set("VestingEscrow", reverse="txgroup", lazy=True)
 
     @property
     def fullname(self) -> str:
@@ -576,7 +650,7 @@ class TxGroup(DbEntity):
         return txgroup  # type: ignore [no-any-return]
 
 
-@lru_cache(100)
+@lru_cache(500)
 def get_transaction(txhash: str) -> TransactionReceipt:
     """Fetch and cache a transaction receipt from the connected chain.
 
@@ -682,14 +756,32 @@ class TreasuryTx(DbEntity):
         """Decoded event logs for this transaction."""
         return self._transaction.events
 
-    def get_events(self, event_name: str) -> _EventItem:
+    async def events_async(self) -> EventDict:
+        """Asynchronously fetch decoded event logs for this transaction."""
+        tx = self._transaction
+        events = tx._events
+        if events is None:
+            events = await _EVENTS_THREADS.run(getattr, tx, "events")
+        return events
+
+    @overload
+    def get_events(
+        self, event_name: str, sync: Literal[False]
+    ) -> Coroutine[Any, Any, EventItem]: ...
+    @overload
+    def get_events(self, event_name: str, sync: bool = True) -> EventItem: ...
+    def get_events(self, event_name: str, sync: bool = True) -> EventItem:
+        if not sync:
+            return _EVENTS_THREADS.run(self.get_events, event_name)
         try:
             return self.events[event_name]
+        except EventLookupError:
+            pass
         except KeyError as e:
             # This happens sometimes due to a busted abi and hopefully shouldnt impact you
-            if str(e) == "'components'":
-                return _EventItem(event_name, None, [], ())
-            raise
+            if str(e) != "'components'":
+                raise
+        return _EventItem(event_name, None, [], ())
 
     @property
     def _transaction(self) -> TransactionReceipt:
@@ -697,6 +789,7 @@ class TreasuryTx(DbEntity):
         return get_transaction(self.hash)
 
     @staticmethod
+    @auto_retry
     async def insert(entry: LedgerEntry) -> None:
         """Asynchronously insert and sort a ledger entry.
 
@@ -717,8 +810,16 @@ class TreasuryTx(DbEntity):
             async with _SORT_SEMAPHORE:
                 from dao_treasury.sorting import sort_advanced
 
-                with db_session:
+                try:
                     await sort_advanced(TreasuryTx[txid])
+                except Exception as e:
+                    e.args = *e.args, entry
+                    raise
+
+    async def _set_txgroup(self, txgroup_dbid: TxGroupDbid) -> None:
+        await _SORT_THREAD.run(
+            TreasuryTx.__set_txgroup, self.treasury_tx_id, txgroup_dbid
+        )
 
     @staticmethod
     def __insert(entry: LedgerEntry, ts: int) -> typing.Optional[int]:
@@ -780,9 +881,74 @@ class TreasuryTx(DbEntity):
                     gas_price=gas_price,
                     txgroup=txgroup_dbid,
                 )
+                # we must commit here or else dbid below will be `None`.
+                commit()
                 dbid = entity.treasury_tx_id
+        except InterfaceError as e:
+            raise ValueError(
+                e,
+                {
+                    "chain": Chain.get_dbid(CHAINID),
+                    "block": entry.block_number,
+                    "timestamp": ts,
+                    "hash": entry.hash.hex(),
+                    "log_index": log_index,
+                    "from_address": from_address,
+                    "to_address": to_address,
+                    "token": token,
+                    "amount": entry.value,
+                    "price": entry.price,
+                    "value_usd": entry.value_usd,
+                    # TODO: nuke db and add this column
+                    # gas = gas,
+                    "gas_used": gas_used,
+                    "gas_price": gas_price,
+                    "txgroup": txgroup_dbid,
+                },
+            ) from e
         except InvalidOperation as e:
-            logger.error(e)
+            with db_session:
+                from_address_entity = Address[from_address]
+                to_address_entity = Address[to_address]
+                token_entity = Token[token]
+                logger.error(e)
+                logger.error(
+                    {
+                        "chain": Chain.get_dbid(CHAINID),
+                        "block": entry.block_number,
+                        "timestamp": ts,
+                        "hash": entry.hash.hex(),
+                        "log_index": log_index,
+                        "from_address": {
+                            "dbid": from_address,
+                            "address": from_address_entity.address,
+                            "nickname": from_address_entity.nickname,
+                        },
+                        "to_address": {
+                            "dbid": to_address,
+                            "address": to_address_entity.address,
+                            "nickname": to_address_entity.nickname,
+                        },
+                        "token": {
+                            "dbid": token,
+                            "address": token_entity.address.address,
+                            "name": token_entity.name,
+                            "symbol": token_entity.symbol,
+                            "decimals": token_entity.decimals,
+                        },
+                        "amount": entry.value,
+                        "price": entry.price,
+                        "value_usd": entry.value_usd,
+                        # TODO: nuke db and add this column
+                        # gas = gas,
+                        "gas_used": gas_used,
+                        "gas_price": gas_price,
+                        "txgroup": {
+                            "dbid": txgroup_dbid,
+                            "fullname": TxGroup[txgroup_dbid].fullname,
+                        },
+                    }
+                )
             return None
         except TransactionIntegrityError as e:
             return _validate_integrity_error(entry, log_index)
@@ -800,6 +966,158 @@ class TreasuryTx(DbEntity):
                 return None
             return dbid  # type: ignore [no-any-return]
 
+    @staticmethod
+    @retry_locked
+    def __set_txgroup(treasury_tx_dbid: int, txgroup_dbid: TxGroupDbid) -> None:
+        with db_session:
+            TreasuryTx[treasury_tx_dbid].txgroup = txgroup_dbid
+            commit()
+
+
+_stream_metadata_cache: Final[Dict[HexStr, Tuple[ChecksumAddress, date]]] = {}
+
+
+class Stream(DbEntity):
+    _table_ = "streams"
+    stream_id = PrimaryKey(str)
+
+    contract = Required("Address", reverse="streams")
+    start_block = Required(int)
+    end_block = Optional(int)
+    token = Required("Token", reverse="streams", index=True)
+    from_address = Required("Address", reverse="streams_from")
+    to_address = Required("Address", reverse="streams_to")
+    reason = Optional(str)
+    amount_per_second = Required(Decimal, 38, 1)
+    status = Required(str, default="Active")
+    txgroup = Optional("TxGroup", reverse="streams")
+
+    streamed_funds = Set("StreamedFunds", lazy=True)
+
+    scale = 10**20
+
+    @property
+    def is_alive(self) -> bool:
+        if self.end_block is None:
+            assert self.status in ["Active", "Paused"]
+            return self.status == "Active"
+        assert self.status == "Stopped"
+        return False
+
+    @property
+    def amount_per_minute(self) -> int:
+        return self.amount_per_second * 60
+
+    @property
+    def amount_per_hour(self) -> int:
+        return self.amount_per_minute * 60
+
+    @property
+    def amount_per_day(self) -> int:
+        return self.amount_per_hour * 24
+
+    @staticmethod
+    def check_closed(stream_id: HexStr) -> bool:
+        with db_session:
+            return any(sf.is_last_day for sf in Stream[stream_id].streamed_funds)
+
+    @staticmethod
+    def _get_start_and_end(stream_dbid: HexStr) -> Tuple[datetime, datetime]:
+        with db_session:
+            stream = Stream[stream_dbid]
+            start_date, end = stream.start_date, datetime.now(_UTC)
+            # convert start to datetime
+            start = datetime.combine(start_date, time(tzinfo=_UTC), tzinfo=_UTC)
+            if stream.end_block:
+                end = datetime.fromtimestamp(chain[stream.end_block].timestamp, tz=_UTC)
+            return start, end
+
+    def stop_stream(self, block: int) -> None:
+        self.end_block = block
+        self.status = "Stopped"
+
+    def pause(self) -> None:
+        self.status = "Paused"
+
+    @staticmethod
+    def _get_token_and_start_date(stream_id: HexStr) -> Tuple[ChecksumAddress, date]:
+        try:
+            return _stream_metadata_cache[stream_id]
+        except KeyError:
+            with db_session:
+                stream = Stream[stream_id]
+                token = stream.token.address.address
+                start_date = stream.start_date
+            _stream_metadata_cache[stream_id] = token, start_date
+            return token, start_date
+
+    @property
+    def stream_contract(self) -> Contract:
+        return Contract(self.contract.address)
+
+    @property
+    def start_date(self) -> date:
+        return datetime.fromtimestamp(chain[self.start_block].timestamp).date()
+
+    async def amount_withdrawable(self, block: int) -> int:
+        return await self.stream_contract.withdrawable.coroutine(
+            self.from_address.address,
+            self.to_address.address,
+            int(self.amount_per_second),
+            block_identifier=block,
+        )
+
+    def print(self) -> None:
+        symbol = self.token.symbol
+        print(f"{symbol} per second: {self.amount_per_second / self.scale}")
+        print(f"{symbol} per day: {self.amount_per_day / self.scale}")
+
+
+class StreamedFunds(DbEntity):
+    """Each object represents one calendar day of tokens streamed for a particular stream."""
+
+    _table_ = "streamed_funds"
+
+    date = Required(date)
+    stream = Required(Stream, reverse="streamed_funds")
+    PrimaryKey(stream, date)
+
+    amount = Required(Decimal, 38, 18)
+    price = Required(Decimal, 38, 18)
+    value_usd = Required(Decimal, 38, 18)
+    seconds_active = Required(int)
+    is_last_day = Required(bool)
+
+    @db_session
+    def get_entity(stream_id: str, date: datetime) -> "StreamedFunds":
+        stream = Stream[stream_id]
+        return StreamedFunds.get(date=date, stream=stream)
+
+    @classmethod
+    @db_session
+    def create_entity(
+        cls,
+        stream_id: str,
+        date: datetime,
+        price: Decimal,
+        seconds_active: int,
+        is_last_day: bool,
+    ) -> "StreamedFunds":
+        stream = Stream[stream_id]
+        amount_streamed_today = round(
+            stream.amount_per_second * seconds_active / stream.scale, 18
+        )
+        entity = StreamedFunds(
+            date=date,
+            stream=stream,
+            amount=amount_streamed_today,
+            price=round(price, 18),
+            value_usd=round(amount_streamed_today * price, 18),
+            seconds_active=seconds_active,
+            is_last_day=is_last_day,
+        )
+        return entity
+
 
 db.bind(
     provider="sqlite",  # TODO: let user choose postgres with server connection params
@@ -810,6 +1128,13 @@ db.bind(
 db.generate_mapping(create_tables=True)
 
 
+def _set_address_nicknames_for_tokens() -> None:
+    """Set address.nickname for addresses belonging to tokens."""
+    for address in select(a for a in Address if a.token and not a.nickname):
+        address.nickname = f"Token: {address.token.name}"
+        db.commit()
+
+
 def create_stream_ledger_view() -> None:
     """Create or replace the SQL view `stream_ledger` for streamed funds reporting.
 
@@ -819,19 +1144,19 @@ def create_stream_ledger_view() -> None:
     Examples:
         >>> create_stream_ledger_view()
     """
+    db.execute("""DROP VIEW IF EXISTS stream_ledger;""")
     db.execute(
         """
-        DROP VIEW IF EXISTS stream_ledger;
         create view stream_ledger as
         SELECT  'Mainnet' as chain_name,
-                cast(DATE AS timestamp) as timestamp,
+                cast(strftime('%s', date || ' 00:00:00') as INTEGER) as timestamp,
                 NULL as block, 
                 NULL as hash, 
                 NULL as log_index, 
                 symbol as token, 
                 d.address AS "from", 
                 d.nickname as from_nickname, 
-                e.address as "to", 
+                e.address AS "to", 
                 e.nickname as to_nickname, 
                 amount, 
                 price, 
@@ -850,6 +1175,31 @@ def create_stream_ledger_view() -> None:
     )
 
 
+def create_txgroup_hierarchy_view() -> None:
+    """Create or replace the SQL view `txgroup_hierarchy` for recursive txgroup hierarchy.
+
+    This view exposes txgroup_id, top_category, and parent_txgroup for all txgroups,
+    matching the recursive CTE logic used in dashboards.
+    """
+    db.execute("DROP VIEW IF EXISTS txgroup_hierarchy;")
+    db.execute(
+        """
+        CREATE VIEW txgroup_hierarchy AS
+        WITH RECURSIVE group_hierarchy (txgroup_id, top_category, parent_txgroup) AS (
+            SELECT txgroup_id, name AS top_category, parent_txgroup
+            FROM txgroups
+            WHERE parent_txgroup IS NULL
+            UNION ALL
+            SELECT child.txgroup_id, parent.top_category, child.parent_txgroup
+            FROM txgroups AS child
+            JOIN group_hierarchy AS parent
+                ON child.parent_txgroup = parent.txgroup_id
+        )
+        SELECT * FROM group_hierarchy;
+        """
+    )
+
+
 def create_vesting_ledger_view() -> None:
     """Create or replace the SQL view `vesting_ledger` for vesting escrow reporting.
 
@@ -905,7 +1255,7 @@ def create_general_ledger_view() -> None:
         create VIEW general_ledger as
         select *
         from (
-            SELECT treasury_tx_id, b.chain_name, datetime(a.timestamp, 'unixepoch') AS timestamp, a.block, a.hash, a.log_index, c.symbol AS token, d.address AS "from", d.nickname as from_nickname, e.address AS "to", e.nickname as to_nickname, a.amount, a.price, a.value_usd, f.name AS txgroup, g.name AS parent_txgroup, f.txgroup_id
+            SELECT treasury_tx_id, b.chain_name, a.timestamp, a.block, a.hash, a.log_index, c.symbol AS token, d.address AS "from", d.nickname as from_nickname, e.address AS "to", e.nickname as to_nickname, a.amount, a.price, a.value_usd, f.name AS txgroup, g.name AS parent_txgroup, f.txgroup_id
             FROM treasury_txs a
                 LEFT JOIN chains b ON a.chain = b.chain_dbid
                 LEFT JOIN tokens c ON a.token_id = c.token_id
@@ -913,9 +1263,9 @@ def create_general_ledger_view() -> None:
                 LEFT JOIN addresses e ON a."to" = e.address_id
                 LEFT JOIN txgroups f ON a.txgroup_id = f.txgroup_id
                 LEFT JOIN txgroups g ON f.parent_txgroup = g.txgroup_id
-            --UNION
-            --SELECT -1, chain_name, TIMESTAMP, cast(block AS integer) block, hash, CAST(log_index AS integer) as log_index, token, "from", from_nickname, "to", to_nickname, amount, price, value_usd, txgroup, parent_txgroup, txgroup_id
-            --FROM stream_ledger
+            UNION
+            SELECT -1, chain_name, timestamp, block, hash, log_index, token, "from", from_nickname, "to", to_nickname, amount, price, value_usd, txgroup, parent_txgroup, txgroup_id
+            FROM stream_ledger
             --UNION
             --SELECT -1, *
             --FROM vesting_ledger
@@ -991,7 +1341,8 @@ def create_monthly_pnl_view() -> None:
 
 
 with db_session:
-    # create_stream_ledger_view()
+    create_stream_ledger_view()
+    create_txgroup_hierarchy_view()
     # create_vesting_ledger_view()
     create_general_ledger_view()
     create_unsorted_txs_view()
@@ -1051,7 +1402,7 @@ def _validate_integrity_error(
             existing_object.amount,
         )
     except AssertionError:
-        logger.warning(
+        logger.debug(
             "slight rounding error in value for TreasuryTx[%s] due to sqlite decimal handling",
             existing_object.treasury_tx_id,
         )
@@ -1076,3 +1427,24 @@ def _validate_integrity_error(
         )
         else None
     )
+
+
+def _drop_shitcoin_txs() -> None:
+    """
+    Purge any shitcoin txs from the db.
+
+    These should not be frequent, and only occur if a user populated the db before a shitcoin was added to the SHITCOINS mapping.
+    """
+    shitcoins = eth_portfolio.SHITCOINS[CHAINID]
+    with db_session:
+        shitcoin_txs = select(
+            tx for tx in TreasuryTx if tx.token.address.address in shitcoins
+        )
+        if count := shitcoin_txs.count():
+            logger.info(f"Purging {count} shitcoin txs from the database...")
+            for tx in shitcoin_txs:
+                tx.delete()
+            logger.info("Shitcoin tx purge complete.")
+
+
+_drop_shitcoin_txs()