dao-treasury 0.0.35__cp311-cp311-win_amd64.whl → 0.1.1__cp311-cp311-win_amd64.whl

dao_treasury/db.py

@@ -3,6 +3,7 @@
 Database models and utilities for DAO treasury reporting.
 
 This module defines Pony ORM entities for:
+
 - Blockchain networks (:class:`Chain`)
 - On-chain addresses (:class:`Address`)
 - ERC-20 tokens and native coin placeholder (:class:`Token`)
@@ -15,29 +16,44 @@ resolving integrity conflicts, caching transaction receipts,
 and creating SQL views for reporting.
 """
 
+import os
 import typing
-from asyncio import Semaphore
+from asyncio import Lock, Semaphore
+from collections import OrderedDict
+from datetime import date, datetime, time, timezone
 from decimal import Decimal, InvalidOperation
 from functools import lru_cache
 from logging import getLogger
-from os import path
-from pathlib import Path
-from typing import TYPE_CHECKING, Dict, Final, Tuple, Union, final
-from datetime import date, datetime, time, timezone
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Coroutine,
+    Dict,
+    Final,
+    Literal,
+    Tuple,
+    TypeVar,
+    Union,
+    final,
+    overload,
+)
 
+import eth_portfolio
 from a_sync import AsyncThreadPoolExecutor
 from brownie import chain
 from brownie.convert.datatypes import HexString
 from brownie.exceptions import EventLookupError
 from brownie.network.event import EventDict, _EventItem
 from brownie.network.transaction import TransactionReceipt
-from eth_typing import ChecksumAddress, HexAddress, HexStr
 from eth_portfolio.structs import (
     InternalTransfer,
     LedgerEntry,
     TokenTransfer,
     Transaction,
 )
+from eth_retry import auto_retry
+from eth_typing import ChecksumAddress, HexAddress, HexStr
 from pony.orm import (
     Database,
     InterfaceError,
@@ -50,9 +66,12 @@ from pony.orm import (
     composite_key,
     composite_index,
     db_session,
+    rollback,
     select,
 )
+from typing_extensions import ParamSpec
 from y import EEE_ADDRESS, Contract, Network, convert, get_block_timestamp_async
+from y._db.decorators import retry_locked
 from y.contracts import _get_code
 from y.exceptions import ContractNotVerified
 
@@ -60,20 +79,34 @@ from dao_treasury.constants import CHAINID
 from dao_treasury.types import TxGroupDbid, TxGroupName
 
 
-SQLITE_DIR = Path(path.expanduser("~")) / ".dao-treasury"
-"""Path to the directory in the user's home where the DAO treasury SQLite database is stored."""
+_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+EventItem = _EventItem[_EventItem[OrderedDict[str, Any]]]
 
-SQLITE_DIR.mkdir(parents=True, exist_ok=True)
 
+# Postgres connection parameters from environment variables (with docker-compose defaults)
+POSTGRES_USER = os.getenv("DAO_TREASURY_DB_USER", "dao_treasury")
+POSTGRES_PASSWORD = os.getenv("DAO_TREASURY_DB_PASSWORD", "dao_treasury")
+POSTGRES_DB = os.getenv("DAO_TREASURY_DB_NAME", "dao_treasury")
+POSTGRES_HOST = os.getenv("DAO_TREASURY_DB_HOST", "127.0.0.1")
+POSTGRES_PORT = int(os.getenv("DAO_TREASURY_DB_PORT", "8675"))
 
 _INSERT_THREAD = AsyncThreadPoolExecutor(1)
 _SORT_THREAD = AsyncThreadPoolExecutor(1)
+_EVENTS_THREADS = AsyncThreadPoolExecutor(16)
 _SORT_SEMAPHORE = Semaphore(50)
 
 _UTC = timezone.utc
 
 db = Database()
 
+db_ready: bool = False
+startup_lock: Final = Lock()
+
+must_sort_inbound_txgroup_dbid: TxGroupDbid = None
+must_sort_outbound_txgroup_dbid: TxGroupDbid = None
+
 logger = getLogger("dao_treasury.db")
 
 
@@ -190,7 +223,7 @@ class Address(DbEntity):
     address = Required(str, index=True)
     """Checksum string of the on-chain address."""
 
-    nickname = Optional(str)
+    nickname = Optional(str, index=True)
     """Optional human-readable label (e.g., contract name or token name)."""
 
     is_contract = Required(bool, index=True, lazy=True)
@@ -234,6 +267,10 @@ class Address(DbEntity):
     def contract(self) -> Contract:
         return Contract(self.address)
 
+    @property
+    def contract_coro(self) -> Coroutine[Any, Any, Contract]:
+        return Contract.coroutine(self.address)
+
     @staticmethod
     @lru_cache(maxsize=None)
     def get_dbid(address: HexAddress) -> int:
@@ -294,7 +331,6 @@ class Address(DbEntity):
             )
 
         commit()
-
         return entity  # type: ignore [no-any-return]
 
     @staticmethod
@@ -373,7 +409,7 @@ class Token(DbEntity):
     symbol = Required(str, index=True, lazy=True)
     """Short ticker symbol for the token."""
 
-    name = Required(str, lazy=True)
+    name = Required(str, lazy=True, index=True)
     """Full human-readable name of the token."""
 
     decimals = Required(int, lazy=True)
@@ -386,12 +422,15 @@ class Token(DbEntity):
     """Inverse relation for treasury transactions involving this token."""
     # partner_harvest_event = Set('PartnerHarvestEvent', reverse="vault", lazy=True)
 
-    address = Required(Address, column="address_id")
+    address = Required(Address, column="address_id", unique=True)
     """Foreign key to the address record for this token contract."""
 
     streams = Set("Stream", reverse="token", lazy=True)
     # vesting_escrows = Set("VestingEscrow", reverse="token", lazy=True)
 
+    composite_index(chain, name)
+    composite_index(chain, symbol)
+
     def __eq__(self, other: Union["Token", Address, ChecksumAddress]) -> bool:  # type: ignore [override]
         if isinstance(other, str):
             return self.address == other
@@ -405,6 +444,10 @@ class Token(DbEntity):
     def contract(self) -> Contract:
         return Contract(self.address.address)
 
+    @property
+    def contract_coro(self) -> Coroutine[Any, Any, Contract]:
+        return Contract.coroutine(self.address.address)
+
     @property
     def scale(self) -> int:
         """Base for division according to `decimals`, e.g., `10**decimals`.
@@ -530,13 +573,13 @@ class TxGroup(DbEntity):
     txgroup_id = PrimaryKey(int, auto=True)
     """Auto-incremented primary key for transaction groups."""
 
-    name = Required(str)
+    name = Required(str, index=True)
     """Name of the grouping category, e.g., 'Revenue', 'Expenses'."""
 
     treasury_tx = Set("TreasuryTx", reverse="txgroup", lazy=True)
     """Inverse relation for treasury transactions assigned to this group."""
 
-    parent_txgroup = Optional("TxGroup", reverse="child_txgroups")
+    parent_txgroup = Optional("TxGroup", reverse="child_txgroups", index=True)
     """Optional reference to a parent group for nesting."""
 
     composite_key(name, parent_txgroup)
@@ -620,6 +663,8 @@ class TxGroup(DbEntity):
             if txgroup := TxGroup.get(name=name, parent_txgroup=parent):
                 return txgroup  # type: ignore [no-any-return]
             raise Exception(e, name, parent) from e
+        else:
+            db.execute("REFRESH MATERIALIZED VIEW txgroup_hierarchy;")
         return txgroup  # type: ignore [no-any-return]
 
 
@@ -706,6 +751,25 @@ class TreasuryTx(DbEntity):
     """Foreign key to the categorization group."""
 
     composite_index(chain, txgroup)
+    composite_index(chain, token)
+    composite_index(chain, from_address)
+    composite_index(chain, to_address)
+    composite_index(chain, from_address, to_address)
+    composite_index(timestamp, txgroup)
+    composite_index(timestamp, token)
+    composite_index(timestamp, from_address)
+    composite_index(timestamp, to_address)
+    composite_index(timestamp, from_address, to_address)
+    composite_index(timestamp, chain, txgroup)
+    composite_index(timestamp, chain, token)
+    composite_index(timestamp, chain, from_address)
+    composite_index(timestamp, chain, to_address)
+    composite_index(timestamp, chain, from_address, to_address)
+    composite_index(chain, timestamp, txgroup)
+    composite_index(chain, timestamp, token)
+    composite_index(chain, timestamp, from_address)
+    composite_index(chain, timestamp, to_address)
+    composite_index(chain, timestamp, from_address, to_address)
 
     @property
     def to_nickname(self) -> typing.Optional[str]:
@@ -719,6 +783,10 @@ class TreasuryTx(DbEntity):
         """Human-readable label for the sender address."""
         return self.from_address.nickname or self.from_address.address  # type: ignore [union-attr]
 
+    @property
+    def token_address(self) -> ChecksumAddress:
+        return self.token.address.address
+
     @property
     def symbol(self) -> str:
         """Ticker symbol for the transferred token."""
@@ -729,7 +797,23 @@ class TreasuryTx(DbEntity):
         """Decoded event logs for this transaction."""
         return self._transaction.events
 
-    def get_events(self, event_name: str) -> _EventItem:
+    async def events_async(self) -> EventDict:
+        """Asynchronously fetch decoded event logs for this transaction."""
+        tx = self._transaction
+        events = tx._events
+        if events is None:
+            events = await _EVENTS_THREADS.run(getattr, tx, "events")
+        return events
+
+    @overload
+    def get_events(
+        self, event_name: str, sync: Literal[False]
+    ) -> Coroutine[Any, Any, EventItem]: ...
+    @overload
+    def get_events(self, event_name: str, sync: bool = True) -> EventItem: ...
+    def get_events(self, event_name: str, sync: bool = True) -> EventItem:
+        if not sync:
+            return _EVENTS_THREADS.run(self.get_events, event_name)
         try:
             return self.events[event_name]
         except EventLookupError:
@@ -746,6 +830,7 @@ class TreasuryTx(DbEntity):
         return get_transaction(self.hash)
 
     @staticmethod
+    @auto_retry
     async def insert(entry: LedgerEntry) -> None:
         """Asynchronously insert and sort a ledger entry.
 
@@ -916,6 +1001,10 @@ class TreasuryTx(DbEntity):
                 must_sort_inbound_txgroup_dbid,
                 must_sort_outbound_txgroup_dbid,
             ):
+                with db_session:
+                    db.execute("REFRESH MATERIALIZED VIEW usdvalue_presum;")
+                    db.execute("REFRESH MATERIALIZED VIEW usdvalue_presum_revenue;")
+                    db.execute("REFRESH MATERIALIZED VIEW usdvalue_presum_expenses;")
                 logger.info(
                     "Sorted %s to %s", entry, TxGroup.get_fullname(txgroup_dbid)
                 )
@@ -923,29 +1012,47 @@ class TreasuryTx(DbEntity):
             return dbid  # type: ignore [no-any-return]
 
     @staticmethod
+    @retry_locked
     def __set_txgroup(treasury_tx_dbid: int, txgroup_dbid: TxGroupDbid) -> None:
         with db_session:
             TreasuryTx[treasury_tx_dbid].txgroup = txgroup_dbid
             commit()
+            db.execute("REFRESH MATERIALIZED VIEW usdvalue_presum;")
+            db.execute("REFRESH MATERIALIZED VIEW usdvalue_presum_revenue;")
+            db.execute("REFRESH MATERIALIZED VIEW usdvalue_presum_expenses;")
 
 
 _stream_metadata_cache: Final[Dict[HexStr, Tuple[ChecksumAddress, date]]] = {}
 
 
+def refresh_matview(name: str) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+    def matview_deco(fn: Callable[_P, _T]) -> Callable[_P, _T]:
+        def matview_refresh_wrap(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+            retval = fn(*args, **kwargs)
+            commit()
+            db.execute(f"REFRESH MATERIALIZED VIEW {name};")
+            commit()
+            return retval
+
+        return matview_refresh_wrap
+
+    return matview_deco
+
+
 class Stream(DbEntity):
     _table_ = "streams"
     stream_id = PrimaryKey(str)
 
-    contract = Required("Address", reverse="streams")
-    start_block = Required(int)
-    end_block = Optional(int)
+    contract = Required("Address", reverse="streams", index=True)
+    start_block = Required(int, index=True)
+    end_block = Optional(int, index=True)
     token = Required("Token", reverse="streams", index=True)
-    from_address = Required("Address", reverse="streams_from")
-    to_address = Required("Address", reverse="streams_to")
-    reason = Optional(str)
+    from_address = Required("Address", reverse="streams_from", index=True)
+    to_address = Required("Address", reverse="streams_to", index=True)
+    reason = Optional(str, index=True)
     amount_per_second = Required(Decimal, 38, 1)
-    status = Required(str, default="Active")
-    txgroup = Optional("TxGroup", reverse="streams")
+    status = Required(str, default="Active", index=True)
+    txgroup = Optional("TxGroup", reverse="streams", index=True)
 
     streamed_funds = Set("StreamedFunds", lazy=True)
 
@@ -987,10 +1094,12 @@ class Stream(DbEntity):
                 end = datetime.fromtimestamp(chain[stream.end_block].timestamp, tz=_UTC)
             return start, end
 
+    @refresh_matview("stream_ledger")
     def stop_stream(self, block: int) -> None:
         self.end_block = block
         self.status = "Stopped"
 
+    @refresh_matview("stream_ledger")
     def pause(self) -> None:
         self.status = "Paused"
 
@@ -1050,6 +1159,7 @@ class StreamedFunds(DbEntity):
 
     @classmethod
     @db_session
+    @refresh_matview("stream_ledger")
     def create_entity(
         cls,
         stream_id: str,
@@ -1074,23 +1184,54 @@ class StreamedFunds(DbEntity):
         return entity
 
 
-db.bind(
-    provider="sqlite",  # TODO: let user choose postgres with server connection params
-    filename=str(SQLITE_DIR / "dao-treasury.sqlite"),
-    create_db=True,
-)
+def init_db() -> None:
+    """Initialize the database if not yet initialized."""
+    global db_ready
+    if db_ready:
+        return
+
+    db.bind(
+        provider="postgres",
+        user=POSTGRES_USER,
+        password=POSTGRES_PASSWORD,
+        host=POSTGRES_HOST,
+        port=POSTGRES_PORT,
+        database=POSTGRES_DB,
+    )
 
-db.generate_mapping(create_tables=True)
+    db.generate_mapping(create_tables=True)
 
+    with db_session:
+        create_stream_ledger_matview()
+        create_txgroup_hierarchy_matview()
+        # create_vesting_ledger_view()
+        create_general_ledger_view()
+        create_unsorted_txs_view()
+        create_usdval_presum_matview()
+        create_usdval_presum_revenue_matview()
+        create_usdval_presum_expenses_matview()
+        # create_monthly_pnl_view()
+
+    global must_sort_inbound_txgroup_dbid
+    must_sort_inbound_txgroup_dbid = TxGroup.get_dbid(name="Sort Me (Inbound)")
 
-def _set_address_nicknames_for_tokens() -> None:
+    global must_sort_outbound_txgroup_dbid
+    must_sort_outbound_txgroup_dbid = TxGroup.get_dbid(name="Sort Me (Outbound)")
+
+    _drop_shitcoin_txs()
+
+    db_ready = True
+
+
+def set_address_nicknames_for_tokens() -> None:
     """Set address.nickname for addresses belonging to tokens."""
+    init_db()
     for address in select(a for a in Address if a.token and not a.nickname):
         address.nickname = f"Token: {address.token.name}"
         db.commit()
 
 
-def create_stream_ledger_view() -> None:
+def create_stream_ledger_matview() -> None:
     """Create or replace the SQL view `stream_ledger` for streamed funds reporting.
 
     This view joins streamed funds, streams, tokens, addresses, and txgroups
@@ -1099,60 +1240,90 @@ def create_stream_ledger_view() -> None:
     Examples:
         >>> create_stream_ledger_view()
     """
-    db.execute("""DROP VIEW IF EXISTS stream_ledger;""")
-    db.execute(
-        """
-        create view stream_ledger as
-        SELECT  'Mainnet' as chain_name,
-                cast(strftime('%s', date || ' 00:00:00') as INTEGER) as timestamp,
-                NULL as block, 
-                NULL as hash, 
-                NULL as log_index, 
-                symbol as token, 
-                d.address AS "from", 
-                d.nickname as from_nickname, 
-                e.address AS "to", 
-                e.nickname as to_nickname, 
-                amount, 
-                price, 
-                value_usd, 
-                txgroup.name as txgroup, 
-                parent.name as parent_txgroup, 
+    try:
+        db.execute(
+            """
+            DROP MATERIALIZED VIEW IF EXISTS stream_ledger CASCADE;
+            CREATE MATERIALIZED VIEW stream_ledger AS
+            SELECT
+                'Mainnet' as chain_name,
+                EXTRACT(EPOCH FROM (date::date))::integer as timestamp,
+                CAST(NULL as integer) as block,
+                NULL as hash,
+                CAST(NULL as integer) as log_index,
+                symbol as token,
+                d.address AS "from",
+                d.nickname as from_nickname,
+                e.address AS "to",
+                e.nickname as to_nickname,
+                amount,
+                price,
+                value_usd,
+                txgroup.name as txgroup,
+                parent.name as parent_txgroup,
                 txgroup.txgroup_id
-        FROM streamed_funds a
-            LEFT JOIN streams b ON a.stream = b.stream_id
-            LEFT JOIN tokens c ON b.token = c.token_id
-            LEFT JOIN addresses d ON b.from_address = d.address_id
-            LEFT JOIN addresses e ON b.to_address = e.address_id
-            LEFT JOIN txgroups txgroup ON b.txgroup = txgroup.txgroup_id
-            LEFT JOIN txgroups parent ON txgroup.parent_txgroup = parent.txgroup_id
-        """
-    )
+            FROM streamed_funds a
+                LEFT JOIN streams b ON a.stream = b.stream_id
+                LEFT JOIN tokens c ON b.token = c.token_id
+                LEFT JOIN addresses d ON b.from_address = d.address_id
+                LEFT JOIN addresses e ON b.to_address = e.address_id
+                LEFT JOIN txgroups txgroup ON b.txgroup = txgroup.txgroup_id
+                LEFT JOIN txgroups parent ON txgroup.parent_txgroup = parent.txgroup_id;
+
+            """
+        )
+    except Exception as e:
+        if '"stream_ledger" is not a materialized view' not in str(e):
+            raise
+        # we're running an old schema, lets migrate it
+        rollback()
+        db.execute("DROP VIEW IF EXISTS stream_ledger CASCADE;")
+        commit()
+        create_stream_ledger_matview()
 
 
-def create_txgroup_hierarchy_view() -> None:
+def create_txgroup_hierarchy_matview() -> None:
     """Create or replace the SQL view `txgroup_hierarchy` for recursive txgroup hierarchy.
 
     This view exposes txgroup_id, top_category, and parent_txgroup for all txgroups,
     matching the recursive CTE logic used in dashboards.
     """
-    db.execute("DROP VIEW IF EXISTS txgroup_hierarchy;")
-    db.execute(
-        """
-        CREATE VIEW txgroup_hierarchy AS
-        WITH RECURSIVE group_hierarchy (txgroup_id, top_category, parent_txgroup) AS (
-            SELECT txgroup_id, name AS top_category, parent_txgroup
-            FROM txgroups
-            WHERE parent_txgroup IS NULL
-            UNION ALL
-            SELECT child.txgroup_id, parent.top_category, child.parent_txgroup
-            FROM txgroups AS child
-            JOIN group_hierarchy AS parent
-                ON child.parent_txgroup = parent.txgroup_id
+    try:
+        db.execute(
+            """
+            DROP MATERIALIZED VIEW IF EXISTS txgroup_hierarchy CASCADE;
+            CREATE MATERIALIZED VIEW txgroup_hierarchy AS
+            WITH RECURSIVE group_hierarchy (txgroup_id, top_category, parent_txgroup) AS (
+                SELECT txgroup_id, name AS top_category, parent_txgroup
+                FROM txgroups
+                WHERE parent_txgroup IS NULL
+                UNION ALL
+                SELECT child.txgroup_id, parent.top_category, child.parent_txgroup
+                FROM txgroups AS child
+                JOIN group_hierarchy AS parent
+                    ON child.parent_txgroup = parent.txgroup_id
+            )
+            SELECT * FROM group_hierarchy;
+            
+            -- Indexes
+            CREATE UNIQUE INDEX idx_txgroup_hierarchy_txgroup_id
+                ON txgroup_hierarchy (txgroup_id);
+
+            CREATE INDEX idx_txgroup_hierarchy_top_category
+                ON txgroup_hierarchy (top_category);
+
+            CREATE INDEX idx_txgroup_hierarchy_parent
+                ON txgroup_hierarchy (parent_txgroup);
+            """
         )
-        SELECT * FROM group_hierarchy;
-        """
-    )
+    except Exception as e:
+        if '"txgroup_hierarchy" is not a materialized view' not in str(e):
+            raise
+        # we're running an old schema, lets migrate it
+        rollback()
+        db.execute("DROP VIEW IF EXISTS txgroup_hierarchy CASCADE;")
+        commit()
+        create_txgroup_hierarchy_matview()
 
 
 def create_vesting_ledger_view() -> None:
@@ -1168,11 +1339,12 @@ def create_vesting_ledger_view() -> None:
         """
         DROP VIEW IF EXISTS vesting_ledger;
         CREATE VIEW vesting_ledger AS
-        SELECT  d.chain_name, 
-            CAST(date AS timestamp) AS "timestamp",
-            cast(NULL as int) AS block,
+        SELECT
+            d.chain_name,
+            date::timestamp AS "timestamp",
+            CAST(NULL as integer) AS block,
             NULL AS "hash",
-            cast(NULL as int) AS "log_index",
+            CAST(NULL as integer) AS "log_index",
             c.symbol AS "token",
             e.address AS "from",
             e.nickname as from_nickname,
@@ -1184,14 +1356,14 @@ def create_vesting_ledger_view() -> None:
             g.name as txgroup,
             h.name AS parent_txgroup,
             g.txgroup_id
-        FROM vested_funds a 
+        FROM vested_funds a
         LEFT JOIN vesting_escrows b ON a.escrow = b.escrow_id
         LEFT JOIN tokens c ON b.token = c.token_id
         LEFT JOIN chains d ON c.chain = d.chain_dbid
         LEFT JOIN addresses e ON b.address = e.address_id
         LEFT JOIN addresses f ON b.recipient = f.address_id
         LEFT JOIN txgroups g ON b.txgroup = g.txgroup_id
-        left JOIN txgroups h ON g.parent_txgroup = h.txgroup_id
+        LEFT JOIN txgroups h ON g.parent_txgroup = h.txgroup_id;
     """
     )
 
@@ -1204,13 +1376,17 @@ def create_general_ledger_view() -> None:
     Examples:
         >>> create_general_ledger_view()
     """
-    db.execute("drop VIEW IF EXISTS general_ledger")
     db.execute(
         """
-        create VIEW general_ledger as
-        select *
-        from (
-            SELECT treasury_tx_id, b.chain_name, a.timestamp, a.block, a.hash, a.log_index, c.symbol AS token, d.address AS "from", d.nickname as from_nickname, e.address AS "to", e.nickname as to_nickname, a.amount, a.price, a.value_usd, f.name AS txgroup, g.name AS parent_txgroup, f.txgroup_id
+        DROP VIEW IF EXISTS general_ledger;
+        CREATE VIEW general_ledger AS
+        SELECT *
+        FROM (
+            SELECT
+                treasury_tx_id, b.chain_name, a.timestamp, a.block, a.hash, a.log_index,
+                c.symbol AS token, d.address AS "from", d.nickname as from_nickname,
+                e.address AS "to", e.nickname as to_nickname, a.amount, a.price, a.value_usd,
+                f.name AS txgroup, g.name AS parent_txgroup, f.txgroup_id
             FROM treasury_txs a
                 LEFT JOIN chains b ON a.chain = b.chain_dbid
                 LEFT JOIN tokens c ON a.token_id = c.token_id
@@ -1219,13 +1395,15 @@ def create_general_ledger_view() -> None:
                 LEFT JOIN txgroups f ON a.txgroup_id = f.txgroup_id
                 LEFT JOIN txgroups g ON f.parent_txgroup = g.txgroup_id
             UNION
-            SELECT -1, chain_name, timestamp, block, hash, log_index, token, "from", from_nickname, "to", to_nickname, amount, price, value_usd, txgroup, parent_txgroup, txgroup_id
+            SELECT
+                -1, chain_name, timestamp, block, hash, log_index, token, "from", from_nickname,
+                "to", to_nickname, amount, price, value_usd, txgroup, parent_txgroup, txgroup_id
             FROM stream_ledger
             --UNION
             --SELECT -1, *
             --FROM vesting_ledger
         ) a
-        ORDER BY timestamp
+        ORDER BY timestamp;
         """
     )
 
@@ -1238,14 +1416,14 @@ def create_unsorted_txs_view() -> None:
     Examples:
         >>> create_unsorted_txs_view()
     """
-    db.execute("DROP VIEW IF EXISTS unsorted_txs;")
     db.execute(
         """
-        CREATE VIEW unsorted_txs as
+        DROP VIEW IF EXISTS unsorted_txs;
+        CREATE VIEW unsorted_txs AS
         SELECT *
         FROM general_ledger
         WHERE txgroup = 'Categorization Pending'
-        ORDER BY TIMESTAMP desc
+        ORDER BY timestamp DESC;
         """
     )
 
@@ -1258,53 +1436,147 @@ def create_monthly_pnl_view() -> None:
     Examples:
         >>> create_monthly_pnl_view()
     """
-    db.execute("DROP VIEW IF EXISTS monthly_pnl;")
     sql = """
+    DROP VIEW IF EXISTS monthly_pnl;
     CREATE VIEW monthly_pnl AS
     WITH categorized AS (
-      SELECT
-        strftime('%Y-%m', datetime(t.timestamp, 'unixepoch')) AS month,
+    SELECT
+        to_char(to_timestamp(t.timestamp), 'YYYY-MM') AS month,
         CASE
-          WHEN p.name IS NOT NULL THEN p.name
-          ELSE tg.name
+        WHEN p.name IS NOT NULL THEN p.name
+        ELSE tg.name
         END AS top_category,
         --COALESCE(t.value_usd, 0) AS value_usd,
         --COALESCE(t.gas_used, 0) * COALESCE(t.gas_price, 0) AS gas_cost
-      FROM treasury_txs t
-      JOIN txgroups tg ON t.txgroup = tg.txgroup_id
-      LEFT JOIN txgroups p ON tg.parent_txgroup = p.txgroup_id
-      WHERE tg.name <> 'Ignore'
+    FROM treasury_txs t
+    JOIN txgroups tg ON t.txgroup = tg.txgroup_id
+    LEFT JOIN txgroups p ON tg.parent_txgroup = p.txgroup_id
+    WHERE tg.name <> 'Ignore'
     )
     SELECT
-      month,
-      SUM(CASE WHEN top_category = 'Revenue' THEN value_usd ELSE 0 END) AS revenue,
-      SUM(CASE WHEN top_category = 'Cost of Revenue' THEN value_usd ELSE 0 END) AS cost_of_revenue,
-      SUM(CASE WHEN top_category = 'Expenses' THEN value_usd ELSE 0 END) AS expenses,
-      SUM(CASE WHEN top_category = 'Other Income' THEN value_usd ELSE 0 END) AS other_income,
-      SUM(CASE WHEN top_category = 'Other Expenses' THEN value_usd ELSE 0 END) AS other_expense,
-      (
+    month,
+    SUM(CASE WHEN top_category = 'Revenue' THEN value_usd ELSE 0 END) AS revenue,
+    SUM(CASE WHEN top_category = 'Cost of Revenue' THEN value_usd ELSE 0 END) AS cost_of_revenue,
+    SUM(CASE WHEN top_category = 'Expenses' THEN value_usd ELSE 0 END) AS expenses,
+    SUM(CASE WHEN top_category = 'Other Income' THEN value_usd ELSE 0 END) AS other_income,
+    SUM(CASE WHEN top_category = 'Other Expenses' THEN value_usd ELSE 0 END) AS other_expense,
+    (
         SUM(CASE WHEN top_category = 'Revenue' THEN value_usd ELSE 0 END) -
         SUM(CASE WHEN top_category = 'Cost of Revenue' THEN value_usd ELSE 0 END) -
         SUM(CASE WHEN top_category = 'Expenses' THEN value_usd ELSE 0 END) +
         SUM(CASE WHEN top_category = 'Other Income' THEN value_usd ELSE 0 END) -
         SUM(CASE WHEN top_category = 'Other Expenses' THEN value_usd ELSE 0 END)
-      ) AS net_profit
+    ) AS net_profit
     FROM categorized
     GROUP BY month;
     """
     db.execute(sql)
 
 
-with db_session:
-    create_stream_ledger_view()
-    create_txgroup_hierarchy_view()
-    # create_vesting_ledger_view()
-    create_general_ledger_view()
-    create_unsorted_txs_view()
-    # create_monthly_pnl_view()
+def create_usdval_presum_matview() -> None:
+    # This view presums usd value from the general_ledger view,
+    # grouped by timestamp and txgroup
+    db.execute(
+        """
+        DROP MATERIALIZED VIEW IF EXISTS usdvalue_presum;
+        CREATE MATERIALIZED VIEW usdvalue_presum AS
+        SELECT
+            txgroup_id,
+            timestamp,
+            SUM(value_usd) AS value_usd
+        FROM general_ledger
+        GROUP BY txgroup_id, timestamp;
+        
+        -- Indexes
+        CREATE UNIQUE INDEX idx_usdvalue_presum_txgroup_id_timestamp
+            ON usdvalue_presum (txgroup_id, timestamp);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_timestamp_txgroup_id
+            ON usdvalue_presum (timestamp, txgroup_id);
+        """
+    )
+
+
+def create_usdval_presum_revenue_matview() -> None:
+    # This view is specifically for the Revenue Over Time dashboard.
+    # It presums usd value for Revenue and Other Income categories only, pre-joining txgroups and top_category.
+    db.execute(
+        """
+        DROP MATERIALIZED VIEW IF EXISTS usdvalue_presum_revenue;
+        CREATE MATERIALIZED VIEW usdvalue_presum_revenue AS
+        SELECT
+            p.txgroup_id,
+            t.name AS txgroup_name,
+            gh.top_category,
+            p.timestamp,
+            SUM(p.value_usd) AS value_usd
+        FROM general_ledger p
+        JOIN txgroup_hierarchy gh ON p.txgroup_id = gh.txgroup_id
+        JOIN txgroups t ON p.txgroup_id = t.txgroup_id
+        WHERE gh.top_category IN ('Revenue', 'Other Income')
+        GROUP BY p.txgroup_id, t.name, gh.top_category, p.timestamp;
+
+        -- Indexes
+        CREATE UNIQUE INDEX idx_usdvalue_presum_revenue_txgroup_id_timestamp
+            ON usdvalue_presum_revenue (txgroup_id, timestamp);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_revenue_timestamp_txgroup_id
+            ON usdvalue_presum_revenue (timestamp, txgroup_id);
+
+        CREATE INDEX idx_usdvalue_presum_revenue_txgroup_name_timestamp
+            ON usdvalue_presum_revenue (txgroup_name, timestamp);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_revenue_timestamp_txgroup_name
+            ON usdvalue_presum_revenue (timestamp, txgroup_name);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_revenue_top_category_txgroup_id_timestamp
+            ON usdvalue_presum_revenue (top_category, txgroup_id, timestamp);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_revenue_top_category_txgroup_name_timestamp
+            ON usdvalue_presum_revenue (top_category, txgroup_name, timestamp);
+        """
+    )
 
-    must_sort_inbound_txgroup_dbid = TxGroup.get_dbid(name="Sort Me (Inbound)")
-    must_sort_outbound_txgroup_dbid = TxGroup.get_dbid(name="Sort Me (Outbound)")
+
+def create_usdval_presum_expenses_matview() -> None:
+    # This view is specifically for the Expenses Over Time dashboard.
+    # It presums usd value for Expenses, Cost of Revenue, and Other Expense categories only, pre-joining txgroups and top_category
+    db.execute(
+        """
+        DROP MATERIALIZED VIEW IF EXISTS usdvalue_presum_expenses;
+        CREATE MATERIALIZED VIEW usdvalue_presum_expenses AS
+        SELECT
+            p.txgroup_id,
+            g.name AS txgroup_name,
+            gh.top_category,
+            p.timestamp,
+            SUM(p.value_usd) AS value_usd
+        FROM general_ledger p
+        JOIN txgroup_hierarchy gh ON p.txgroup_id = gh.txgroup_id
+        JOIN txgroups g ON p.txgroup_id = g.txgroup_id
+        WHERE gh.top_category IN ('Expenses', 'Cost of Revenue', 'Other Expense')
+        GROUP BY p.txgroup_id, g.name, gh.top_category, p.timestamp;
+
+        -- Indexes
+        CREATE UNIQUE INDEX idx_usdvalue_presum_expenses_txgroup_id_timestamp
+            ON usdvalue_presum_expenses (txgroup_id, timestamp);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_expenses_timestamp_txgroup_id
+            ON usdvalue_presum_expenses (timestamp, txgroup_id);
+
+        CREATE INDEX idx_usdvalue_presum_expenses_txgroup_name_timestamp
+            ON usdvalue_presum_expenses (txgroup_name, timestamp);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_expenses_timestamp_txgroup_name
+            ON usdvalue_presum_expenses (timestamp, txgroup_name);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_expenses_top_category_txgroup_id_timestamp
+            ON usdvalue_presum_expenses (top_category, txgroup_id, timestamp);
+
+        CREATE UNIQUE INDEX idx_usdvalue_presum_expenses_top_category_txgroup_name_timestamp
+            ON usdvalue_presum_expenses (top_category, txgroup_name, timestamp);
+        """
+    )
 
 
 @db_session
@@ -1382,3 +1654,21 @@ def _validate_integrity_error(
         )
         else None
     )
+
+
+def _drop_shitcoin_txs() -> None:
+    """
+    Purge any shitcoin txs from the db.
+
+    These should not be frequent, and only occur if a user populated the db before a shitcoin was added to the SHITCOINS mapping.
+    """
+    shitcoins = eth_portfolio.SHITCOINS[CHAINID]
+    with db_session:
+        shitcoin_txs = select(
+            tx for tx in TreasuryTx if tx.token.address.address in shitcoins
+        )
+        if count := shitcoin_txs.count():
+            logger.info(f"Purging {count} shitcoin txs from the database...")
+            for tx in shitcoin_txs:
+                tx.delete()
+            logger.info("Shitcoin tx purge complete.")