chia-blockchain 2.5.6rc2__py3-none-any.whl → 2.5.7rc2__py3-none-any.whl

chia/data_layer/data_store.py

@@ -1,18 +1,37 @@
 from __future__ import annotations
 
 import contextlib
+import copy
+import itertools
 import logging
+import shutil
+import sqlite3
 from collections import defaultdict
-from collections.abc import AsyncIterator, Awaitable
-from contextlib import asynccontextmanager
-from dataclasses import dataclass, replace
+from collections.abc import AsyncIterator, Awaitable, Iterable, Iterator, Sequence
+from contextlib import asynccontextmanager, contextmanager
+from dataclasses import dataclass, field, replace
+from hashlib import sha256
 from pathlib import Path
 from typing import Any, BinaryIO, Callable, Optional, Union
 
 import aiosqlite
+import anyio.to_thread
+import chia_rs.datalayer
+import zstd
+from chia_rs.datalayer import (
+    DeltaFileCache,
+    DeltaReader,
+    KeyAlreadyPresentError,
+    KeyId,
+    MerkleBlob,
+    ProofOfInclusion,
+    TreeIndex,
+    ValueId,
+)
 from chia_rs.sized_bytes import bytes32
+from chia_rs.sized_ints import int64
 
-from chia.data_layer.data_layer_errors import KeyNotFoundError, NodeHashError, TreeGenerationIncrementingError
+from chia.data_layer.data_layer_errors import KeyNotFoundError, MerkleBlobNotFoundError, TreeGenerationIncrementingError
 from chia.data_layer.data_layer_util import (
     DiffData,
     InsertResult,
@@ -24,8 +43,6 @@ from chia.data_layer.data_layer_util import (
     Node,
     NodeType,
     OperationType,
-    ProofOfInclusion,
-    ProofOfInclusionLayer,
     Root,
     SerializedNode,
     ServerInfo,
@@ -34,15 +51,18 @@ from chia.data_layer.data_layer_util import (
     Subscription,
     TerminalNode,
     Unspecified,
+    get_delta_filename_path,
     get_hashes_for_page,
     internal_hash,
     key_hash,
     leaf_hash,
-    row_to_node,
     unspecified,
 )
-from chia.types.blockchain_format.program import Program
+from chia.util.batches import to_batches
+from chia.util.cpu import available_logical_cores
 from chia.util.db_wrapper import SQLITE_MAX_VARIABLE_NUMBER, DBWrapper2
+from chia.util.log_exceptions import log_exceptions
+from chia.util.lru_cache import LRUCache
 
 log = logging.getLogger(__name__)
 
@@ -50,17 +70,33 @@ log = logging.getLogger(__name__)
 # TODO: review exceptions for values that shouldn't be displayed
 # TODO: pick exception types other than Exception
 
+KeyOrValueId = int64
+
+default_prefer_file_kv_blob_length: int = 4096
+
 
 @dataclass
 class DataStore:
     """A key/value store with the pairs being terminal nodes in a CLVM object tree."""
 
     db_wrapper: DBWrapper2
+    recent_merkle_blobs: LRUCache[bytes32, MerkleBlob]
+    merkle_blobs_path: Path
+    key_value_blobs_path: Path
+    unconfirmed_keys_values: dict[bytes32, list[bytes32]] = field(default_factory=dict)
+    prefer_db_kv_blob_length: int = default_prefer_file_kv_blob_length
 
     @classmethod
     @contextlib.asynccontextmanager
     async def managed(
-        cls, database: Union[str, Path], uri: bool = False, sql_log_path: Optional[Path] = None
+        cls,
+        database: Union[str, Path],
+        merkle_blobs_path: Path,
+        key_value_blobs_path: Path,
+        uri: bool = False,
+        sql_log_path: Optional[Path] = None,
+        cache_capacity: int = 1,
+        prefer_db_kv_blob_length: int = default_prefer_file_kv_blob_length,
     ) -> AsyncIterator[DataStore]:
         async with DBWrapper2.managed(
             database=database,
@@ -75,46 +111,16 @@ class DataStore:
             row_factory=aiosqlite.Row,
             log_path=sql_log_path,
         ) as db_wrapper:
-            self = cls(db_wrapper=db_wrapper)
+            recent_merkle_blobs: LRUCache[bytes32, MerkleBlob] = LRUCache(capacity=cache_capacity)
+            self = cls(
+                db_wrapper=db_wrapper,
+                recent_merkle_blobs=recent_merkle_blobs,
+                merkle_blobs_path=merkle_blobs_path,
+                key_value_blobs_path=key_value_blobs_path,
+                prefer_db_kv_blob_length=prefer_db_kv_blob_length,
+            )
 
             async with db_wrapper.writer() as writer:
-                await writer.execute(
-                    f"""
-                    CREATE TABLE IF NOT EXISTS node(
-                        hash BLOB PRIMARY KEY NOT NULL CHECK(length(hash) == 32),
-                        node_type INTEGER NOT NULL CHECK(
-                            (
-                                node_type == {int(NodeType.INTERNAL)}
-                                AND left IS NOT NULL
-                                AND right IS NOT NULL
-                                AND key IS NULL
-                                AND value IS NULL
-                            )
-                            OR
-                            (
-                                node_type == {int(NodeType.TERMINAL)}
-                                AND left IS NULL
-                                AND right IS NULL
-                                AND key IS NOT NULL
-                                AND value IS NOT NULL
-                            )
-                        ),
-                        left BLOB REFERENCES node,
-                        right BLOB REFERENCES node,
-                        key BLOB,
-                        value BLOB
-                    )
-                    """
-                )
-                await writer.execute(
-                    """
-                    CREATE TRIGGER IF NOT EXISTS no_node_updates
-                    BEFORE UPDATE ON node
-                    BEGIN
-                        SELECT RAISE(FAIL, 'updates not allowed to the node table');
-                    END
-                    """
-                )
                 await writer.execute(
                     f"""
                     CREATE TABLE IF NOT EXISTS root(
@@ -124,25 +130,7 @@ class DataStore:
                         status INTEGER NOT NULL CHECK(
                             {" OR ".join(f"status == {status}" for status in Status)}
                         ),
-                        PRIMARY KEY(tree_id, generation),
-                        FOREIGN KEY(node_hash) REFERENCES node(hash)
-                    )
-                    """
-                )
-                # TODO: Add ancestor -> hash relationship, this might involve temporarily
-                # deferring the foreign key enforcement due to the insertion order
-                # and the node table also enforcing a similar relationship in the
-                # other direction.
-                # FOREIGN KEY(ancestor) REFERENCES ancestors(ancestor)
-                await writer.execute(
-                    """
-                    CREATE TABLE IF NOT EXISTS ancestors(
-                        hash BLOB NOT NULL REFERENCES node,
-                        ancestor BLOB CHECK(length(ancestor) == 32),
-                        tree_id BLOB NOT NULL CHECK(length(tree_id) == 32),
-                        generation INTEGER NOT NULL,
-                        PRIMARY KEY(hash, tree_id, generation),
-                        FOREIGN KEY(ancestor) REFERENCES node(hash)
+                        PRIMARY KEY(tree_id, generation)
                     )
                     """
                 )
@@ -173,7 +161,34 @@ class DataStore:
                 )
                 await writer.execute(
                     """
-                    CREATE INDEX IF NOT EXISTS node_key_index ON node(key)
+                    CREATE TABLE IF NOT EXISTS ids(
+                        kv_id INTEGER PRIMARY KEY,
+                        hash BLOB NOT NULL CHECK(length(store_id) == 32),
+                        blob BLOB,
+                        store_id BLOB NOT NULL CHECK(length(store_id) == 32)
+                    )
+                    """
+                )
+                await writer.execute(
+                    """
+                    CREATE TABLE IF NOT EXISTS nodes(
+                        store_id BLOB NOT NULL CHECK(length(store_id) == 32),
+                        hash BLOB NOT NULL,
+                        root_hash BLOB NOT NULL,
+                        generation INTEGER NOT NULL CHECK(generation >= 0),
+                        idx INTEGER NOT NULL,
+                        PRIMARY KEY(store_id, hash)
+                    )
+                    """
+                )
+                await writer.execute(
+                    """
+                    CREATE UNIQUE INDEX IF NOT EXISTS ids_hash_index ON ids(hash, store_id)
+                    """
+                )
+                await writer.execute(
+                    """
+                    CREATE INDEX IF NOT EXISTS nodes_generation_index ON nodes(generation)
                     """
                 )
 
@@ -184,19 +199,228 @@ class DataStore:
         async with self.db_wrapper.writer():
             yield
 
-    async def migrate_db(self) -> None:
+    async def insert_into_data_store_from_file(
+        self,
+        store_id: bytes32,
+        root_hash: Optional[bytes32],
+        filename: Path,
+        delta_reader: Optional[DeltaReader] = None,
+    ) -> Optional[DeltaReader]:
+        async with self.db_wrapper.writer():
+            with self.manage_kv_files(store_id):
+                if root_hash is None:
+                    merkle_blob = MerkleBlob(b"")
+                else:
+                    root = await self.get_tree_root(store_id=store_id)
+                    if delta_reader is None:
+                        delta_reader = DeltaReader(internal_nodes={}, leaf_nodes={})
+                        if root.node_hash is not None:
+                            delta_reader.collect_from_merkle_blob(
+                                self.get_merkle_path(store_id=store_id, root_hash=root.node_hash),
+                                indexes=[TreeIndex(0)],
+                            )
+
+                    internal_nodes, terminal_nodes = await self.read_from_file(filename, store_id)
+                    delta_reader.add_internal_nodes(internal_nodes)
+                    delta_reader.add_leaf_nodes(terminal_nodes)
+
+                    missing_hashes = await anyio.to_thread.run_sync(delta_reader.get_missing_hashes, root_hash)
+
+                    if len(missing_hashes) > 0:
+                        # TODO: consider adding transactions around this code
+                        merkle_blob_queries = await self.build_merkle_blob_queries_for_missing_hashes(
+                            missing_hashes, store_id
+                        )
+                        if len(merkle_blob_queries) > 0:
+                            jobs = [
+                                (self.get_merkle_path(store_id=store_id, root_hash=old_root_hash), indexes)
+                                for old_root_hash, indexes in merkle_blob_queries.items()
+                            ]
+                            await anyio.to_thread.run_sync(delta_reader.collect_from_merkle_blobs, jobs)
+                        await self.build_cache_and_collect_missing_hashes(root, root_hash, store_id, delta_reader)
+
+                    merkle_blob = delta_reader.create_merkle_blob_and_filter_unused_nodes(root_hash, set())
+
+                # Don't store these blob objects into cache, since their data structures are not calculated yet.
+                await self.insert_root_from_merkle_blob(merkle_blob, store_id, Status.COMMITTED, update_cache=False)
+                return delta_reader
+
+    async def build_merkle_blob_queries_for_missing_hashes(
+        self,
+        missing_hashes: set[bytes32],
+        store_id: bytes32,
+    ) -> defaultdict[bytes32, list[TreeIndex]]:
+        queries = defaultdict[bytes32, list[TreeIndex]](list)
+
+        batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10)
+
+        async with self.db_wrapper.reader() as reader:
+            for batch in to_batches(missing_hashes, batch_size):
+                placeholders = ",".join(["?"] * len(batch.entries))
+                query = f"""
+                    SELECT hash, root_hash, idx
+                    FROM nodes
+                    WHERE store_id = ? AND hash IN ({placeholders})
+                    LIMIT {len(batch.entries)}
+                """
+
+                async with reader.execute(query, (store_id, *batch.entries)) as cursor:
+                    rows = await cursor.fetchall()
+                    for row in rows:
+                        root_hash_blob = bytes32(row["root_hash"])
+                        index = TreeIndex(row["idx"])
+                        queries[root_hash_blob].append(index)
+
+        return queries
+
+    async def build_cache_and_collect_missing_hashes(
+        self,
+        root: Root,
+        root_hash: bytes32,
+        store_id: bytes32,
+        delta_reader: DeltaReader,
+    ) -> None:
+        missing_hashes = delta_reader.get_missing_hashes(root_hash)
+
+        if len(missing_hashes) == 0:
+            return
+
+        async with self.db_wrapper.reader() as reader:
+            cursor = await reader.execute(
+                # TODO: the INDEXED BY seems like it shouldn't be needed, figure out why it is
+                #       https://sqlite.org/lang_indexedby.html: admonished to omit all use of INDEXED BY
+                #       https://sqlite.org/queryplanner-ng.html#howtofix
+                "SELECT MAX(generation) FROM nodes INDEXED BY nodes_generation_index WHERE store_id = ?",
+                (store_id,),
+            )
+            generation_row = await cursor.fetchone()
+            if generation_row is None or generation_row[0] is None:
+                current_generation = 0
+            else:
+                current_generation = generation_row[0]
+        generations: Sequence[int] = [current_generation]
+
+        while missing_hashes:
+            if current_generation >= root.generation:
+                raise Exception("Invalid delta file, cannot find all the required hashes")
+
+            current_generation = generations[-1] + 1
+
+            batch_size = available_logical_cores()
+            generations = range(
+                current_generation,
+                min(current_generation + batch_size, root.generation),
+            )
+            jobs: list[tuple[bytes32, Path]] = []
+            generations_by_root_hash: dict[bytes32, int] = {}
+            for generation in generations:
+                generation_root = await self.get_tree_root(store_id=store_id, generation=generation)
+                if generation_root.node_hash is None:
+                    # no need to process an empty generation
+                    continue
+                path = self.get_merkle_path(store_id=store_id, root_hash=generation_root.node_hash)
+                jobs.append((generation_root.node_hash, path))
+                generations_by_root_hash[generation_root.node_hash] = generation
+
+            found = await anyio.to_thread.run_sync(
+                delta_reader.collect_and_return_from_merkle_blobs,
+                jobs,
+                missing_hashes,
+            )
+            async with self.db_wrapper.writer() as writer:
+                await writer.executemany(
+                    """
+                    INSERT
+                    OR IGNORE INTO nodes(store_id, hash, root_hash, generation, idx)
+                    VALUES (?, ?, ?, ?, ?)
+                    """,
+                    (
+                        (store_id, hash, root_hash, generations_by_root_hash[root_hash], index.raw)
+                        for root_hash, map in found
+                        for hash, index in map.items()
+                    ),
+                )
+
+            missing_hashes = delta_reader.get_missing_hashes(root_hash)
+
+            log.info(f"Missing hashes: added old hashes from generation {current_generation}")
+
+    async def read_from_file(
+        self, filename: Path, store_id: bytes32
+    ) -> tuple[dict[bytes32, tuple[bytes32, bytes32]], dict[bytes32, tuple[KeyId, ValueId]]]:
+        internal_nodes: dict[bytes32, tuple[bytes32, bytes32]] = {}
+        terminal_nodes: dict[bytes32, tuple[KeyId, ValueId]] = {}
+
+        with open(filename, "rb") as reader:
+            async with self.db_wrapper.writer() as writer:
+                while True:
+                    chunk = b""
+                    while len(chunk) < 4:
+                        size_to_read = 4 - len(chunk)
+                        cur_chunk = reader.read(size_to_read)
+                        if cur_chunk is None or cur_chunk == b"":
+                            if size_to_read < 4:
+                                raise Exception("Incomplete read of length.")
+                            break
+                        chunk += cur_chunk
+                    if chunk == b"":
+                        break
+
+                    size = int.from_bytes(chunk, byteorder="big")
+                    serialize_nodes_bytes = b""
+                    while len(serialize_nodes_bytes) < size:
+                        size_to_read = size - len(serialize_nodes_bytes)
+                        cur_chunk = reader.read(size_to_read)
+                        if cur_chunk is None or cur_chunk == b"":
+                            raise Exception("Incomplete read of blob.")
+                        serialize_nodes_bytes += cur_chunk
+                    serialized_node = SerializedNode.from_bytes(serialize_nodes_bytes)
+
+                    node_type = NodeType.TERMINAL if serialized_node.is_terminal else NodeType.INTERNAL
+                    if node_type == NodeType.INTERNAL:
+                        node_hash = internal_hash(bytes32(serialized_node.value1), bytes32(serialized_node.value2))
+                        internal_nodes[node_hash] = (bytes32(serialized_node.value1), bytes32(serialized_node.value2))
+                    else:
+                        kid, vid = await self.add_key_value(
+                            serialized_node.value1,
+                            serialized_node.value2,
+                            store_id,
+                            writer=writer,
+                        )
+                        node_hash = leaf_hash(serialized_node.value1, serialized_node.value2)
+                        terminal_nodes[node_hash] = (kid, vid)
+
+        return internal_nodes, terminal_nodes
+
+    async def migrate_db(self, server_files_location: Path) -> None:
         async with self.db_wrapper.reader() as reader:
             cursor = await reader.execute("SELECT * FROM schema")
-            row = await cursor.fetchone()
-            if row is not None:
+            rows = await cursor.fetchall()
+            all_versions = {"v1.0", "v2.0"}
+
+            for row in rows:
                 version = row["version_id"]
-                if version != "v1.0":
+                if version not in all_versions:
                     raise Exception("Unknown version")
-                log.info(f"Found DB schema version {version}. No migration needed.")
-                return
+                if version == "v2.0":
+                    log.info(f"Found DB schema version {version}. No migration needed.")
+                    return
+
+        version = "v2.0"
+        old_tables = ["node", "root", "ancestors"]
+        all_stores = await self.get_store_ids()
+        all_roots: list[list[Root]] = []
+        for store_id in all_stores:
+            try:
+                root = await self.get_tree_root(store_id=store_id)
+                roots = await self.get_roots_between(store_id, 1, root.generation)
+                all_roots.append([*roots, root])
+            except Exception as e:
+                if "unable to find root for id, generation" in str(e):
+                    log.error(f"Cannot find roots for {store_id}. Skipping it.")
+
+        log.info(f"Initiating migration to version {version}. Found {len(all_roots)} stores to migrate")
 
-        version = "v1.0"
-        log.info(f"Initiating migration to version {version}")
         async with self.db_wrapper.writer(foreign_key_enforcement_enabled=False) as writer:
             await writer.execute(
                 f"""
@@ -207,16 +431,360 @@ class DataStore:
                     status INTEGER NOT NULL CHECK(
                         {" OR ".join(f"status == {status}" for status in Status)}
                     ),
-                    PRIMARY KEY(tree_id, generation),
-                    FOREIGN KEY(node_hash) REFERENCES node(hash)
+                    PRIMARY KEY(tree_id, generation)
                 )
                 """
             )
-            await writer.execute("INSERT INTO new_root SELECT * FROM root")
-            await writer.execute("DROP TABLE root")
+            for old_table in old_tables:
+                await writer.execute(f"DROP TABLE IF EXISTS {old_table}")
             await writer.execute("ALTER TABLE new_root RENAME TO root")
             await writer.execute("INSERT INTO schema (version_id) VALUES (?)", (version,))
-        log.info(f"Finished migrating DB to version {version}")
+            log.info(f"Initialized new DB schema {version}.")
+
+            total_generations = 0
+            synced_generations = 0
+            for roots in all_roots:
+                assert len(roots) > 0
+                total_generations += len(roots)
+
+            for roots in all_roots:
+                store_id = roots[0].store_id
+                await self.create_tree(store_id=store_id, status=Status.COMMITTED)
+
+                expected_synced_generations = synced_generations + len(roots)
+                for root in roots:
+                    recovery_filename: Optional[Path] = None
+
+                    for group_by_store in (True, False):
+                        filename = get_delta_filename_path(
+                            server_files_location,
+                            store_id,
+                            bytes32.zeros if root.node_hash is None else root.node_hash,
+                            root.generation,
+                            group_by_store,
+                        )
+
+                        if filename.exists():
+                            log.info(f"Found filename {filename}. Recovering data from it")
+                            recovery_filename = filename
+                            break
+
+                    if recovery_filename is None:
+                        log.error(f"Cannot find any recovery file for root {root}")
+                        break
+
+                    try:
+                        await self.insert_into_data_store_from_file(store_id, root.node_hash, recovery_filename)
+                        synced_generations += 1
+                        log.info(
+                            f"Successfully recovered root from {filename}. "
+                            f"Total roots processed: {(synced_generations / total_generations * 100):.2f}%"
+                        )
+                    except Exception as e:
+                        log.error(f"Cannot recover data from {filename}: {e}")
+                        break
+
+                if synced_generations < expected_synced_generations:
+                    log.error(
+                        f"Could not recover {expected_synced_generations - synced_generations} generations. "
+                        f"Consider resyncing the store {store_id} once the migration is complete."
+                    )
+                    # Reset the counter as if we synced correctly, so the percentages add to 100% at the end.
+                    synced_generations = expected_synced_generations
+
+    async def get_merkle_blob(
+        self,
+        store_id: bytes32,
+        root_hash: Optional[bytes32],
+        read_only: bool = False,
+        update_cache: bool = True,
+    ) -> MerkleBlob:
+        if root_hash is None:
+            return MerkleBlob(blob=b"")
+        if self.recent_merkle_blobs.get_capacity() == 0:
+            update_cache = False
+
+        existing_blob = self.recent_merkle_blobs.get(root_hash)
+        if existing_blob is not None:
+            return existing_blob if read_only else copy.deepcopy(existing_blob)
+
+        try:
+            with log_exceptions(log=log, message="Error while getting merkle blob"):
+                path = self.get_merkle_path(store_id=store_id, root_hash=root_hash)
+                # TODO: consider file-system based locking of either the file or the store directory
+                merkle_blob = MerkleBlob.from_path(path)
+        except Exception as e:
+            raise MerkleBlobNotFoundError(root_hash=root_hash) from e
+
+        if update_cache:
+            self.recent_merkle_blobs.put(root_hash, copy.deepcopy(merkle_blob))
+
+        return merkle_blob
+
+    def get_bytes_path(self, bytes_: bytes) -> Path:
+        raw = bytes_.hex()
+        segment_sizes = [2, 2, 2]
+        start = 0
+        segments = []
+        for size in segment_sizes:
+            segments.append(raw[start : start + size])
+            start += size
+
+        return Path(*segments, raw)
+
+    def get_merkle_path(self, store_id: bytes32, root_hash: Optional[bytes32]) -> Path:
+        store_root = self.merkle_blobs_path.joinpath(store_id.hex())
+        if root_hash is None:
+            return store_root
+
+        return store_root.joinpath(self.get_bytes_path(bytes_=root_hash))
+
+    def get_key_value_path(self, store_id: bytes32, blob_hash: Optional[bytes32]) -> Path:
+        store_root = self.key_value_blobs_path.joinpath(store_id.hex())
+        if blob_hash is None:
+            return store_root
+
+        return store_root.joinpath(self.get_bytes_path(bytes_=blob_hash))
+
+    async def insert_root_from_merkle_blob(
+        self,
+        merkle_blob: MerkleBlob,
+        store_id: bytes32,
+        status: Status,
+        old_root: Optional[Root] = None,
+        update_cache: bool = True,
+    ) -> Root:
+        if not merkle_blob.empty():
+            merkle_blob.calculate_lazy_hashes()
+        if self.recent_merkle_blobs.get_capacity() == 0:
+            update_cache = False
+
+        root_hash = merkle_blob.get_root_hash()
+        if old_root is not None and old_root.node_hash == root_hash:
+            raise ValueError("Changelist resulted in no change to tree data")
+
+        if root_hash is not None:
+            log.info(f"inserting merkle blob: {len(merkle_blob)} bytes {root_hash.hex()}")
+            blob_path = self.get_merkle_path(store_id=store_id, root_hash=merkle_blob.get_root_hash())
+            if not blob_path.exists():
+                blob_path.parent.mkdir(parents=True, exist_ok=True)
+                # TODO: consider file-system based locking of either the file or the store directory
+                merkle_blob.to_path(blob_path)
+
+            if update_cache:
+                self.recent_merkle_blobs.put(root_hash, copy.deepcopy(merkle_blob))
+
+        return await self._insert_root(store_id, root_hash, status)
+
+    def _use_file_for_new_kv_blob(self, blob: bytes) -> bool:
+        return len(blob) > self.prefer_db_kv_blob_length
+
+    async def get_kvid(self, blob: bytes, store_id: bytes32) -> Optional[KeyOrValueId]:
+        blob_hash = bytes32(sha256(blob).digest())
+
+        async with self.db_wrapper.reader() as reader:
+            cursor = await reader.execute(
+                "SELECT kv_id FROM ids WHERE hash = ? AND store_id = ?",
+                (
+                    blob_hash,
+                    store_id,
+                ),
+            )
+            row = await cursor.fetchone()
+
+            if row is None:
+                return None
+
+            return KeyOrValueId(row[0])
+
+    def get_blob_from_file(self, blob_hash: bytes32, store_id: bytes32) -> bytes:
+        # TODO: seems that zstd needs hinting
+        # TODO: consider file-system based locking of either the file or the store directory
+        return zstd.decompress(self.get_key_value_path(store_id=store_id, blob_hash=blob_hash).read_bytes())  # type: ignore[no-any-return]
+
+    async def get_blob_from_kvid(self, kv_id: KeyOrValueId, store_id: bytes32) -> Optional[bytes]:
+        async with self.db_wrapper.reader() as reader:
+            cursor = await reader.execute(
+                "SELECT hash, blob FROM ids WHERE kv_id = ? AND store_id = ?",
+                (
+                    kv_id,
+                    store_id,
+                ),
+            )
+            row = await cursor.fetchone()
+
+            if row is None:
+                return None
+
+        blob: bytes = row["blob"]
+        if blob is not None:
+            return blob
+
+        blob_hash = bytes32(row["hash"])
+        return self.get_blob_from_file(blob_hash, store_id)
+
+    async def get_terminal_node(self, kid: KeyId, vid: ValueId, store_id: bytes32) -> TerminalNode:
+        key = await self.get_blob_from_kvid(kid.raw, store_id)
+        value = await self.get_blob_from_kvid(vid.raw, store_id)
+        if key is None or value is None:
+            raise Exception("Cannot find the key/value pair")
+
+        return TerminalNode(hash=leaf_hash(key, value), key=key, value=value)
+
+    async def add_kvid(self, blob: bytes, store_id: bytes32, writer: aiosqlite.Connection) -> KeyOrValueId:
+        use_file = self._use_file_for_new_kv_blob(blob)
+        blob_hash = bytes32(sha256(blob).digest())
+        if use_file:
+            table_blob = None
+        else:
+            table_blob = blob
+        try:
+            row = await writer.execute_insert(
+                "INSERT INTO ids (hash, blob, store_id) VALUES (?, ?, ?)",
+                (
+                    blob_hash,
+                    table_blob,
+                    store_id,
+                ),
+            )
+        except sqlite3.IntegrityError as e:
+            if "UNIQUE constraint failed" in str(e):
+                kv_id = await self.get_kvid(blob, store_id)
+                if kv_id is None:
+                    raise Exception("Internal error") from e
+                return kv_id
+
+            raise
+
+        if row is None:
+            raise Exception("Internal error")
+        if use_file:
+            path = self.get_key_value_path(store_id=store_id, blob_hash=blob_hash)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            self.unconfirmed_keys_values[store_id].append(blob_hash)
+            # TODO: consider file-system based locking of either the file or the store directory
+            path.write_bytes(zstd.compress(blob))
+        return KeyOrValueId(row[0])
+
+    def delete_unconfirmed_kvids(self, store_id: bytes32) -> None:
+        for blob_hash in self.unconfirmed_keys_values[store_id]:
+            with log_exceptions(log=log, consume=True):
+                path = self.get_key_value_path(store_id=store_id, blob_hash=blob_hash)
+                try:
+                    path.unlink()
+                except FileNotFoundError:
+                    log.error(f"Cannot find key/value path {path} for hash {blob_hash}")
+        del self.unconfirmed_keys_values[store_id]
+
+    def confirm_all_kvids(self, store_id: bytes32) -> None:
+        del self.unconfirmed_keys_values[store_id]
+
+    @contextmanager
+    def manage_kv_files(self, store_id: bytes32) -> Iterator[None]:
+        if store_id not in self.unconfirmed_keys_values:
+            self.unconfirmed_keys_values[store_id] = []
+        else:
+            raise Exception("Internal error: unconfirmed keys values cache not cleaned")
+
+        try:
+            yield
+        except:
+            self.delete_unconfirmed_kvids(store_id)
+            raise
+        else:
+            self.confirm_all_kvids(store_id)
+
+    async def add_key_value(
+        self, key: bytes, value: bytes, store_id: bytes32, writer: aiosqlite.Connection
+    ) -> tuple[KeyId, ValueId]:
+        kid = KeyId(await self.add_kvid(key, store_id, writer=writer))
+        vid = ValueId(await self.add_kvid(value, store_id, writer=writer))
+
+        return (kid, vid)
+
+    async def get_terminal_node_by_hash(
+        self,
+        node_hash: bytes32,
+        store_id: bytes32,
+        root_hash: Union[bytes32, Unspecified] = unspecified,
+    ) -> TerminalNode:
+        resolved_root_hash: Optional[bytes32]
+        if root_hash is unspecified:
+            root = await self.get_tree_root(store_id=store_id)
+            resolved_root_hash = root.node_hash
+        else:
+            resolved_root_hash = root_hash
+
+        merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash)
+        kid, vid = merkle_blob.get_node_by_hash(node_hash)
+        return await self.get_terminal_node(kid, vid, store_id)
+
+    async def get_terminal_nodes_by_hashes(
+        self,
+        node_hashes: list[bytes32],
+        store_id: bytes32,
+        root_hash: Union[bytes32, Unspecified] = unspecified,
+    ) -> list[TerminalNode]:
+        resolved_root_hash: Optional[bytes32]
+        if root_hash is unspecified:
+            root = await self.get_tree_root(store_id=store_id)
+            resolved_root_hash = root.node_hash
+        else:
+            resolved_root_hash = root_hash
+
+        merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash)
+        kv_ids: list[tuple[KeyId, ValueId]] = []
+        for node_hash in node_hashes:
+            kid, vid = merkle_blob.get_node_by_hash(node_hash)
+            kv_ids.append((kid, vid))
+        kv_ids_unpacked = (KeyOrValueId(id.raw) for kv_id in kv_ids for id in kv_id)
+        table_blobs = await self.get_table_blobs(kv_ids_unpacked, store_id)
+
+        terminal_nodes: list[TerminalNode] = []
+        for kid, vid in kv_ids:
+            terminal_nodes.append(self.get_terminal_node_from_table_blobs(kid, vid, table_blobs, store_id))
+
+        return terminal_nodes
+
+    async def get_existing_hashes(self, node_hashes: list[bytes32], store_id: bytes32) -> set[bytes32]:
+        result: set[bytes32] = set()
+        batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10)
+
+        async with self.db_wrapper.reader() as reader:
+            for i in range(0, len(node_hashes), batch_size):
+                chunk = node_hashes[i : i + batch_size]
+                placeholders = ",".join(["?"] * len(chunk))
+                query = f"SELECT hash FROM nodes WHERE store_id = ? AND hash IN ({placeholders}) LIMIT {len(chunk)}"
+
+                async with reader.execute(query, (store_id, *chunk)) as cursor:
+                    rows = await cursor.fetchall()
+                    result.update(row["hash"] for row in rows)
+
+        return result
+
+    async def add_node_hashes(self, store_id: bytes32, generation: Optional[int] = None) -> None:
+        root = await self.get_tree_root(store_id=store_id, generation=generation)
+        if root.node_hash is None:
+            return
+
+        merkle_blob = await self.get_merkle_blob(
+            store_id=store_id, root_hash=root.node_hash, read_only=True, update_cache=False
+        )
+        hash_to_index = merkle_blob.get_hashes_indexes()
+
+        existing_hashes = await self.get_existing_hashes(list(hash_to_index.keys()), store_id)
+        async with self.db_wrapper.writer() as writer:
+            await writer.executemany(
+                """
+                INSERT INTO nodes(store_id, hash, root_hash, generation, idx)
+                VALUES (?, ?, ?, ?, ?)
+                """,
+                (
+                    (store_id, hash, root.node_hash, root.generation, index.raw)
+                    for hash, index in hash_to_index.items()
+                    if hash not in existing_hashes
+                ),
+            )
 
     async def _insert_root(
         self,
@@ -225,11 +793,7 @@ class DataStore:
         status: Status,
         generation: Optional[int] = None,
     ) -> Root:
-        # This should be replaced by an SQLite schema level check.
-        # https://github.com/Chia-Network/chia-blockchain/pull/9284
-        store_id = bytes32(store_id)
-
-        async with self.db_wrapper.writer() as writer:
+        async with self.db_wrapper.writer_maybe_transaction() as writer:
             if generation is None:
                 try:
                     existing_generation = await self.get_tree_generation(store_id=store_id)
@@ -254,173 +818,8 @@ class DataStore:
                 """,
                 new_root.to_row(),
             )
-
-            # `node_hash` is now a root, so it has no ancestor.
-            # Don't change the ancestor table unless the root is committed.
-            if node_hash is not None and status == Status.COMMITTED:
-                values = {
-                    "hash": node_hash,
-                    "tree_id": store_id,
-                    "generation": generation,
-                }
-                await writer.execute(
-                    """
-                    INSERT INTO ancestors(hash, ancestor, tree_id, generation)
-                    VALUES (:hash, NULL, :tree_id, :generation)
-                    """,
-                    values,
-                )
-
             return new_root
 
-    async def _insert_node(
-        self,
-        node_hash: bytes32,
-        node_type: NodeType,
-        left_hash: Optional[bytes32],
-        right_hash: Optional[bytes32],
-        key: Optional[bytes],
-        value: Optional[bytes],
-    ) -> None:
-        # TODO: can we get sqlite to do this check?
-        values = {
-            "hash": node_hash,
-            "node_type": node_type,
-            "left": left_hash,
-            "right": right_hash,
-            "key": key,
-            "value": value,
-        }
-
-        async with self.db_wrapper.writer() as writer:
-            try:
-                await writer.execute(
-                    """
-                    INSERT INTO node(hash, node_type, left, right, key, value)
-                    VALUES(:hash, :node_type, :left, :right, :key, :value)
-                    """,
-                    values,
-                )
-            except aiosqlite.IntegrityError as e:
-                if not e.args[0].startswith("UNIQUE constraint"):
-                    # UNIQUE constraint failed: node.hash
-                    raise
-
-                async with writer.execute(
-                    "SELECT * FROM node WHERE hash == :hash LIMIT 1",
-                    {"hash": node_hash},
-                ) as cursor:
-                    result = await cursor.fetchone()
-
-                if result is None:
-                    # some ideas for causes:
-                    #   an sqlite bug
-                    #   bad queries in this function
-                    #   unexpected db constraints
-                    raise Exception("Unable to find conflicting row") from e  # pragma: no cover
-
-                result_dict = dict(result)
-                if result_dict != values:
-                    raise Exception(
-                        f"Requested insertion of node with matching hash but other values differ: {node_hash}"
-                    ) from None
-
-    async def insert_node(self, node_type: NodeType, value1: bytes, value2: bytes) -> None:
-        if node_type == NodeType.INTERNAL:
-            left_hash = bytes32(value1)
-            right_hash = bytes32(value2)
-            node_hash = internal_hash(left_hash, right_hash)
-            await self._insert_node(node_hash, node_type, bytes32(value1), bytes32(value2), None, None)
-        else:
-            node_hash = leaf_hash(key=value1, value=value2)
-            await self._insert_node(node_hash, node_type, None, None, value1, value2)
-
-    async def _insert_internal_node(self, left_hash: bytes32, right_hash: bytes32) -> bytes32:
-        node_hash: bytes32 = internal_hash(left_hash=left_hash, right_hash=right_hash)
-
-        await self._insert_node(
-            node_hash=node_hash,
-            node_type=NodeType.INTERNAL,
-            left_hash=left_hash,
-            right_hash=right_hash,
-            key=None,
-            value=None,
-        )
-
-        return node_hash
-
-    async def _insert_ancestor_table(
-        self,
-        left_hash: bytes32,
-        right_hash: bytes32,
-        store_id: bytes32,
-        generation: int,
-    ) -> None:
-        node_hash = internal_hash(left_hash=left_hash, right_hash=right_hash)
-
-        async with self.db_wrapper.writer() as writer:
-            for hash in (left_hash, right_hash):
-                values = {
-                    "hash": hash,
-                    "ancestor": node_hash,
-                    "tree_id": store_id,
-                    "generation": generation,
-                }
-                try:
-                    await writer.execute(
-                        """
-                        INSERT INTO ancestors(hash, ancestor, tree_id, generation)
-                        VALUES (:hash, :ancestor, :tree_id, :generation)
-                        """,
-                        values,
-                    )
-                except aiosqlite.IntegrityError as e:
-                    if not e.args[0].startswith("UNIQUE constraint"):
-                        # UNIQUE constraint failed: ancestors.hash, ancestors.tree_id, ancestors.generation
-                        raise
-
-                    async with writer.execute(
-                        """
-                        SELECT *
-                        FROM ancestors
-                        WHERE hash == :hash AND generation == :generation AND tree_id == :tree_id
-                        LIMIT 1
-                        """,
-                        {"hash": hash, "generation": generation, "tree_id": store_id},
-                    ) as cursor:
-                        result = await cursor.fetchone()
-
-                    if result is None:
-                        # some ideas for causes:
-                        #   an sqlite bug
-                        #   bad queries in this function
-                        #   unexpected db constraints
-                        raise Exception("Unable to find conflicting row") from e  # pragma: no cover
-
-                    result_dict = dict(result)
-                    if result_dict != values:
-                        raise Exception(
-                            "Requested insertion of ancestor, where ancestor differ, but other values are identical: "
-                            f"{hash} {generation} {store_id}"
-                        ) from None
-
-    async def _insert_terminal_node(self, key: bytes, value: bytes) -> bytes32:
-        # forcing type hint here for:
-        # https://github.com/Chia-Network/clvm/pull/102
-        # https://github.com/Chia-Network/clvm/pull/106
-        node_hash: bytes32 = Program.to((key, value)).get_tree_hash()
-
-        await self._insert_node(
-            node_hash=node_hash,
-            node_type=NodeType.TERMINAL,
-            left_hash=None,
-            right_hash=None,
-            key=key,
-            value=value,
-        )
-
-        return node_hash
-
     async def get_pending_root(self, store_id: bytes32) -> Optional[Root]:
         async with self.db_wrapper.reader() as reader:
             cursor = await reader.execute(
@@ -478,21 +877,6 @@ class DataStore:
                     root.generation,
                 ),
             )
-            # `node_hash` is now a root, so it has no ancestor.
-            # Don't change the ancestor table unless the root is committed.
-            if root.node_hash is not None and status == Status.COMMITTED:
-                values = {
-                    "hash": root.node_hash,
-                    "tree_id": root.store_id,
-                    "generation": root.generation,
-                }
-                await writer.execute(
-                    """
-                    INSERT INTO ancestors(hash, ancestor, tree_id, generation)
-                    VALUES (:hash, NULL, :tree_id, :generation)
-                    """,
-                    values,
-                )
 
     async def check(self) -> None:
         for check in self._checks:
@@ -518,30 +902,7 @@ class DataStore:
             if len(bad_trees) > 0:
                 raise TreeGenerationIncrementingError(store_ids=bad_trees)
 
-    async def _check_hashes(self) -> None:
-        async with self.db_wrapper.reader() as reader:
-            cursor = await reader.execute("SELECT * FROM node")
-
-            bad_node_hashes: list[bytes32] = []
-            async for row in cursor:
-                node = row_to_node(row=row)
-                if isinstance(node, InternalNode):
-                    expected_hash = internal_hash(left_hash=node.left_hash, right_hash=node.right_hash)
-                elif isinstance(node, TerminalNode):
-                    expected_hash = Program.to((node.key, node.value)).get_tree_hash()
-                else:
-                    raise Exception(f"Internal error, unknown node type: {node!r}")
-
-                if node.hash != expected_hash:
-                    bad_node_hashes.append(node.hash)
-
-        if len(bad_node_hashes) > 0:
-            raise NodeHashError(node_hashes=bad_node_hashes)
-
-    _checks: tuple[Callable[[DataStore], Awaitable[None]], ...] = (
-        _check_roots_are_incrementing,
-        _check_hashes,
-    )
+    _checks: tuple[Callable[[DataStore], Awaitable[None]], ...] = (_check_roots_are_incrementing,)
 
     async def create_tree(self, store_id: bytes32, status: Status = Status.PENDING) -> bool:
         await self._insert_root(store_id=store_id, node_hash=None, status=status)
@@ -634,165 +995,62 @@ class DataStore:
 
         return roots
 
-    async def get_last_tree_root_by_hash(
-        self, store_id: bytes32, hash: Optional[bytes32], max_generation: Optional[int] = None
-    ) -> Optional[Root]:
-        async with self.db_wrapper.reader() as reader:
-            max_generation_str = "AND generation < :max_generation " if max_generation is not None else ""
-            node_hash_str = "AND node_hash == :node_hash " if hash is not None else "AND node_hash is NULL "
-            cursor = await reader.execute(
-                "SELECT * FROM root WHERE tree_id == :tree_id "
-                f"{max_generation_str}"
-                f"{node_hash_str}"
-                "ORDER BY generation DESC LIMIT 1",
-                {"tree_id": store_id, "node_hash": hash, "max_generation": max_generation},
-            )
-            row = await cursor.fetchone()
-
-        if row is None:
-            return None
-        return Root.from_row(row=row)
-
     async def get_ancestors(
         self,
         node_hash: bytes32,
         store_id: bytes32,
         root_hash: Optional[bytes32] = None,
-    ) -> list[InternalNode]:
-        async with self.db_wrapper.reader() as reader:
-            if root_hash is None:
-                root = await self.get_tree_root(store_id=store_id)
-                root_hash = root.node_hash
-            if root_hash is None:
-                raise Exception(f"Root hash is unspecified for store ID: {store_id.hex()}")
-            cursor = await reader.execute(
-                """
-                WITH RECURSIVE
-                    tree_from_root_hash(hash, node_type, left, right, key, value, depth) AS (
-                        SELECT node.*, 0 AS depth FROM node WHERE node.hash == :root_hash
-                        UNION ALL
-                        SELECT node.*, tree_from_root_hash.depth + 1 AS depth FROM node, tree_from_root_hash
-                        WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right
-                    ),
-                    ancestors(hash, node_type, left, right, key, value, depth) AS (
-                        SELECT node.*, NULL AS depth FROM node
-                        WHERE node.left == :reference_hash OR node.right == :reference_hash
-                        UNION ALL
-                        SELECT node.*, NULL AS depth FROM node, ancestors
-                        WHERE node.left == ancestors.hash OR node.right == ancestors.hash
-                    )
-                SELECT * FROM tree_from_root_hash INNER JOIN ancestors
-                WHERE tree_from_root_hash.hash == ancestors.hash
-                ORDER BY tree_from_root_hash.depth DESC
-                """,
-                {"reference_hash": node_hash, "root_hash": root_hash},
-            )
-
-            # The resulting rows must represent internal nodes.  InternalNode.from_row()
-            # does some amount of validation in the sense that it will fail if left
-            # or right can't turn into a bytes32 as expected.  There is room for more
-            # validation here if desired.
-            ancestors = [InternalNode.from_row(row=row) async for row in cursor]
-
-        return ancestors
-
-    async def get_ancestors_optimized(
-        self,
-        node_hash: bytes32,
-        store_id: bytes32,
         generation: Optional[int] = None,
-        root_hash: Optional[bytes32] = None,
     ) -> list[InternalNode]:
         async with self.db_wrapper.reader():
-            nodes = []
             if root_hash is None:
                 root = await self.get_tree_root(store_id=store_id, generation=generation)
                 root_hash = root.node_hash
-
             if root_hash is None:
-                return []
-
-            while True:
-                internal_node = await self._get_one_ancestor(node_hash, store_id, generation)
-                if internal_node is None:
-                    break
-                nodes.append(internal_node)
-                node_hash = internal_node.hash
-
-            if len(nodes) > 0:
-                if root_hash != nodes[-1].hash:
-                    raise RuntimeError("Ancestors list didn't produce the root as top result.")
-
-            return nodes
+                raise Exception(f"Root hash is unspecified for store ID: {store_id.hex()}")
 
-    async def get_internal_nodes(self, store_id: bytes32, root_hash: Optional[bytes32] = None) -> list[InternalNode]:
-        async with self.db_wrapper.reader() as reader:
-            if root_hash is None:
-                root = await self.get_tree_root(store_id=store_id)
-                root_hash = root.node_hash
-            cursor = await reader.execute(
-                """
-                WITH RECURSIVE
-                    tree_from_root_hash(hash, node_type, left, right, key, value) AS (
-                        SELECT node.* FROM node WHERE node.hash == :root_hash
-                        UNION ALL
-                        SELECT node.* FROM node, tree_from_root_hash WHERE node.hash == tree_from_root_hash.left
-                        OR node.hash == tree_from_root_hash.right
-                    )
-                SELECT * FROM tree_from_root_hash
-                WHERE node_type == :node_type
-                """,
-                {"root_hash": root_hash, "node_type": NodeType.INTERNAL},
+            merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root_hash)
+            reference_kid, _ = merkle_blob.get_node_by_hash(node_hash)
+
+        reference_index = merkle_blob.get_key_index(reference_kid)
+        lineage = merkle_blob.get_lineage_with_indexes(reference_index)
+        result: list[InternalNode] = []
+        for index, node in itertools.islice(lineage, 1, None):
+            assert isinstance(node, chia_rs.datalayer.InternalNode)
+            result.append(
+                InternalNode(
+                    hash=node.hash,
+                    left_hash=merkle_blob.get_hash_at_index(node.left),
+                    right_hash=merkle_blob.get_hash_at_index(node.right),
+                )
             )
+        return result
 
-            internal_nodes: list[InternalNode] = []
-            async for row in cursor:
-                node = row_to_node(row=row)
-                if not isinstance(node, InternalNode):
-                    raise Exception(f"Unexpected internal node found: {node.hash.hex()}")
-                internal_nodes.append(node)
+    def get_terminal_node_from_table_blobs(
+        self,
+        kid: KeyId,
+        vid: ValueId,
+        table_blobs: dict[KeyOrValueId, tuple[bytes32, Optional[bytes]]],
+        store_id: bytes32,
+    ) -> TerminalNode:
+        key = table_blobs[KeyOrValueId(kid.raw)][1]
+        if key is None:
+            key_hash = table_blobs[KeyOrValueId(kid.raw)][0]
+            key = self.get_blob_from_file(key_hash, store_id)
 
-        return internal_nodes
+        value = table_blobs[KeyOrValueId(vid.raw)][1]
+        if value is None:
+            value_hash = table_blobs[KeyOrValueId(vid.raw)][0]
+            value = self.get_blob_from_file(value_hash, store_id)
 
-    async def get_keys_values_cursor(
-        self,
-        reader: aiosqlite.Connection,
-        root_hash: Optional[bytes32],
-        only_keys: bool = False,
-    ) -> aiosqlite.Cursor:
-        select_clause = "SELECT hash, key" if only_keys else "SELECT *"
-        maybe_value = "" if only_keys else "value, "
-        select_node_clause = "node.hash, node.node_type, node.left, node.right, node.key" if only_keys else "node.*"
-        return await reader.execute(
-            f"""
-            WITH RECURSIVE
-                tree_from_root_hash(hash, node_type, left, right, key, {maybe_value}depth, rights) AS (
-                    SELECT {select_node_clause}, 0 AS depth, 0 AS rights FROM node WHERE node.hash == :root_hash
-                    UNION ALL
-                    SELECT
-                        {select_node_clause},
-                        tree_from_root_hash.depth + 1 AS depth,
-                        CASE
-                            WHEN node.hash == tree_from_root_hash.right
-                            THEN tree_from_root_hash.rights + (1 << (62 - tree_from_root_hash.depth))
-                            ELSE tree_from_root_hash.rights
-                            END AS rights
-                        FROM node, tree_from_root_hash
-                    WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right
-                )
-            {select_clause} FROM tree_from_root_hash
-            WHERE node_type == :node_type
-            ORDER BY depth ASC, rights ASC
-            """,
-            {"root_hash": root_hash, "node_type": NodeType.TERMINAL},
-        )
+        return TerminalNode(hash=leaf_hash(key, value), key=key, value=value)
 
     async def get_keys_values(
         self,
         store_id: bytes32,
         root_hash: Union[bytes32, Unspecified] = unspecified,
     ) -> list[TerminalNode]:
-        async with self.db_wrapper.reader() as reader:
+        async with self.db_wrapper.reader():
             resolved_root_hash: Optional[bytes32]
             if root_hash is unspecified:
                 root = await self.get_tree_root(store_id=store_id)
@@ -800,25 +1058,18 @@ class DataStore:
             else:
                 resolved_root_hash = root_hash
 
-            cursor = await self.get_keys_values_cursor(reader, resolved_root_hash)
+            try:
+                merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash)
+            except MerkleBlobNotFoundError:
+                return []
+
+            kv_ids = merkle_blob.get_keys_values()
+            kv_ids_unpacked = (KeyOrValueId(id.raw) for pair in kv_ids.items() for id in pair)
+            table_blobs = await self.get_table_blobs(kv_ids_unpacked, store_id)
+
             terminal_nodes: list[TerminalNode] = []
-            async for row in cursor:
-                if row["depth"] > 62:
-                    # TODO: Review the value and implementation of left-to-right order
-                    #       reporting.  Initial use is for balanced insertion with the
-                    #       work done in the query.
-
-                    # This is limited based on the choice of 63 for the maximum left
-                    # shift in the query.  This is in turn based on the SQLite integers
-                    # ranging in size up to signed 8 bytes, 64 bits.  If we exceed this then
-                    # we no longer guarantee the left-to-right ordering of the node
-                    # list.  While 63 allows for a lot of nodes in a balanced tree, in
-                    # the worst case it allows only 62 terminal nodes.
-                    raise Exception("Tree depth exceeded 62, unable to guarantee left-to-right node order.")
-                node = row_to_node(row=row)
-                if not isinstance(node, TerminalNode):
-                    raise Exception(f"Unexpected internal node found: {node.hash.hex()}")
-                terminal_nodes.append(node)
+            for kid, vid in kv_ids.items():
+                terminal_nodes.append(self.get_terminal_node_from_table_blobs(kid, vid, table_blobs, store_id))
 
         return terminal_nodes
 
@@ -827,7 +1078,7 @@ class DataStore:
         store_id: bytes32,
         root_hash: Union[bytes32, Unspecified] = unspecified,
     ) -> KeysValuesCompressed:
-        async with self.db_wrapper.reader() as reader:
+        async with self.db_wrapper.reader():
             resolved_root_hash: Optional[bytes32]
             if root_hash is unspecified:
                 root = await self.get_tree_root(store_id=store_id)
@@ -835,36 +1086,26 @@ class DataStore:
             else:
                 resolved_root_hash = root_hash
 
-            cursor = await self.get_keys_values_cursor(reader, resolved_root_hash)
             keys_values_hashed: dict[bytes32, bytes32] = {}
             key_hash_to_length: dict[bytes32, int] = {}
             leaf_hash_to_length: dict[bytes32, int] = {}
-            async for row in cursor:
-                if row["depth"] > 62:
-                    raise Exception("Tree depth exceeded 62, unable to guarantee left-to-right node order.")
-                node = row_to_node(row=row)
-                if not isinstance(node, TerminalNode):
-                    raise Exception(f"Unexpected internal node found: {node.hash.hex()}")
-                keys_values_hashed[key_hash(node.key)] = leaf_hash(node.key, node.value)
-                key_hash_to_length[key_hash(node.key)] = len(node.key)
-                leaf_hash_to_length[leaf_hash(node.key, node.value)] = len(node.key) + len(node.value)
+            if resolved_root_hash is not None:
+                try:
+                    merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash)
+                except MerkleBlobNotFoundError:
+                    return KeysValuesCompressed({}, {}, {}, resolved_root_hash)
 
-            return KeysValuesCompressed(keys_values_hashed, key_hash_to_length, leaf_hash_to_length, resolved_root_hash)
+                kv_ids = merkle_blob.get_keys_values()
+                kv_ids_unpacked = (KeyOrValueId(id.raw) for pair in kv_ids.items() for id in pair)
+                table_blobs = await self.get_table_blobs(kv_ids_unpacked, store_id)
 
-    async def get_leaf_hashes_by_hashed_key(
-        self, store_id: bytes32, root_hash: Optional[bytes32] = None
-    ) -> dict[bytes32, bytes32]:
-        result: dict[bytes32, bytes32] = {}
-        async with self.db_wrapper.reader() as reader:
-            if root_hash is None:
-                root = await self.get_tree_root(store_id=store_id)
-                root_hash = root.node_hash
+                for kid, vid in kv_ids.items():
+                    node = self.get_terminal_node_from_table_blobs(kid, vid, table_blobs, store_id)
+                    keys_values_hashed[key_hash(node.key)] = leaf_hash(node.key, node.value)
+                    key_hash_to_length[key_hash(node.key)] = len(node.key)
+                    leaf_hash_to_length[leaf_hash(node.key, node.value)] = len(node.key) + len(node.value)
 
-            cursor = await self.get_keys_values_cursor(reader, root_hash, True)
-            async for row in cursor:
-                result[key_hash(row["key"])] = bytes32(row["hash"])
-
-        return result
+            return KeysValuesCompressed(keys_values_hashed, key_hash_to_length, leaf_hash_to_length, resolved_root_hash)
 
     async def get_keys_paginated(
         self,
@@ -877,11 +1118,12 @@ class DataStore:
         pagination_data = get_hashes_for_page(page, keys_values_compressed.key_hash_to_length, max_page_size)
 
         keys: list[bytes] = []
+        leaf_hashes: list[bytes32] = []
         for hash in pagination_data.hashes:
             leaf_hash = keys_values_compressed.keys_values_hashed[hash]
-            node = await self.get_node(leaf_hash)
-            assert isinstance(node, TerminalNode)
-            keys.append(node.key)
+            leaf_hashes.append(leaf_hash)
+        nodes = await self.get_terminal_nodes_by_hashes(leaf_hashes, store_id, root_hash)
+        keys = [node.key for node in nodes]
 
         return KeysPaginationData(
             pagination_data.total_pages,
@@ -900,12 +1142,7 @@ class DataStore:
         keys_values_compressed = await self.get_keys_values_compressed(store_id, root_hash)
         pagination_data = get_hashes_for_page(page, keys_values_compressed.leaf_hash_to_length, max_page_size)
 
-        keys_values: list[TerminalNode] = []
-        for hash in pagination_data.hashes:
-            node = await self.get_node(hash)
-            assert isinstance(node, TerminalNode)
-            keys_values.append(node)
-
+        keys_values = await self.get_terminal_nodes_by_hashes(pagination_data.hashes, store_id, root_hash)
         return KeysValuesPaginationData(
             pagination_data.total_pages,
             pagination_data.total_bytes,
@@ -942,14 +1179,25 @@ class DataStore:
 
         pagination_data = get_hashes_for_page(page, lengths, max_page_size)
         kv_diff: list[DiffData] = []
-
+        insertion_hashes: list[bytes32] = []
+        deletion_hashes: list[bytes32] = []
         for hash in pagination_data.hashes:
-            node = await self.get_node(hash)
-            assert isinstance(node, TerminalNode)
             if hash in insertions:
-                kv_diff.append(DiffData(OperationType.INSERT, node.key, node.value))
+                insertion_hashes.append(hash)
             else:
-                kv_diff.append(DiffData(OperationType.DELETE, node.key, node.value))
+                deletion_hashes.append(hash)
+        if hash2 != bytes32.zeros:
+            insertion_nodes = await self.get_terminal_nodes_by_hashes(insertion_hashes, store_id, hash2)
+        else:
+            insertion_nodes = []
+        if hash1 != bytes32.zeros:
+            deletion_nodes = await self.get_terminal_nodes_by_hashes(deletion_hashes, store_id, hash1)
+        else:
+            deletion_nodes = []
+        for node in insertion_nodes:
+            kv_diff.append(DiffData(OperationType.INSERT, node.key, node.value))
+        for node in deletion_nodes:
+            kv_diff.append(DiffData(OperationType.DELETE, node.key, node.value))
 
         return KVDiffPaginationData(
             pagination_data.total_pages,
@@ -957,380 +1205,128 @@ class DataStore:
             kv_diff,
         )
 
-    async def get_node_type(self, node_hash: bytes32) -> NodeType:
-        async with self.db_wrapper.reader() as reader:
-            cursor = await reader.execute(
-                "SELECT node_type FROM node WHERE hash == :hash LIMIT 1",
-                {"hash": node_hash},
-            )
-            raw_node_type = await cursor.fetchone()
-
-        if raw_node_type is None:
-            raise Exception(f"No node found for specified hash: {node_hash.hex()}")
-
-        return NodeType(raw_node_type["node_type"])
-
-    async def get_terminal_node_for_seed(
-        self, store_id: bytes32, seed: bytes32, root_hash: Optional[bytes32] = None
-    ) -> Optional[bytes32]:
-        path = "".join(reversed("".join(f"{b:08b}" for b in seed)))
-        async with self.db_wrapper.reader() as reader:
-            if root_hash is None:
-                root = await self.get_tree_root(store_id)
-                root_hash = root.node_hash
-            if root_hash is None:
-                return None
-
-            async with reader.execute(
-                """
-                WITH RECURSIVE
-                    random_leaf(hash, node_type, left, right, depth, side) AS (
-                        SELECT
-                            node.hash AS hash,
-                            node.node_type AS node_type,
-                            node.left AS left,
-                            node.right AS right,
-                            1 AS depth,
-                            SUBSTR(:path, 1, 1) as side
-                        FROM node
-                        WHERE node.hash == :root_hash
-                        UNION ALL
-                        SELECT
-                            node.hash AS hash,
-                            node.node_type AS node_type,
-                            node.left AS left,
-                            node.right AS right,
-                            random_leaf.depth + 1 AS depth,
-                            SUBSTR(:path, random_leaf.depth + 1, 1) as side
-                        FROM node, random_leaf
-                        WHERE (
-                            (random_leaf.side == "0" AND node.hash == random_leaf.left)
-                            OR (random_leaf.side != "0" AND node.hash == random_leaf.right)
-                        )
-                    )
-                SELECT hash AS hash FROM random_leaf
-                WHERE node_type == :node_type
-                LIMIT 1
-                """,
-                {"root_hash": root_hash, "node_type": NodeType.TERMINAL, "path": path},
-            ) as cursor:
-                row = await cursor.fetchone()
-                if row is None:
-                    # No cover since this is an error state that should be unreachable given the code
-                    # above has already verified that there is a non-empty tree.
-                    raise Exception("No terminal node found for seed")  # pragma: no cover
-                return bytes32(row["hash"])
-
-    def get_side_for_seed(self, seed: bytes32) -> Side:
-        side_seed = bytes(seed)[0]
-        return Side.LEFT if side_seed < 128 else Side.RIGHT
-
     async def autoinsert(
         self,
         key: bytes,
         value: bytes,
         store_id: bytes32,
-        use_optimized: bool = True,
         status: Status = Status.PENDING,
         root: Optional[Root] = None,
     ) -> InsertResult:
-        async with self.db_wrapper.writer():
-            if root is None:
-                root = await self.get_tree_root(store_id=store_id)
-
-            was_empty = root.node_hash is None
-
-            if was_empty:
-                reference_node_hash = None
-                side = None
-            else:
-                seed = leaf_hash(key=key, value=value)
-                reference_node_hash = await self.get_terminal_node_for_seed(store_id, seed, root_hash=root.node_hash)
-                side = self.get_side_for_seed(seed)
-
-            return await self.insert(
-                key=key,
-                value=value,
-                store_id=store_id,
-                reference_node_hash=reference_node_hash,
-                side=side,
-                use_optimized=use_optimized,
-                status=status,
-                root=root,
-            )
-
-    async def get_keys_values_dict(
-        self,
-        store_id: bytes32,
-        root_hash: Union[bytes32, Unspecified] = unspecified,
-    ) -> dict[bytes, bytes]:
-        pairs = await self.get_keys_values(store_id=store_id, root_hash=root_hash)
-        return {node.key: node.value for node in pairs}
+        return await self.insert(
+            key=key,
+            value=value,
+            store_id=store_id,
+            reference_node_hash=None,
+            side=None,
+            status=status,
+            root=root,
+        )
 
     async def get_keys(
         self,
         store_id: bytes32,
         root_hash: Union[bytes32, Unspecified] = unspecified,
     ) -> list[bytes]:
-        async with self.db_wrapper.reader() as reader:
+        async with self.db_wrapper.reader():
             if root_hash is unspecified:
                 root = await self.get_tree_root(store_id=store_id)
                 resolved_root_hash = root.node_hash
             else:
                 resolved_root_hash = root_hash
-            cursor = await reader.execute(
-                """
-                WITH RECURSIVE
-                    tree_from_root_hash(hash, node_type, left, right, key) AS (
-                        SELECT node.hash, node.node_type, node.left, node.right, node.key
-                        FROM node WHERE node.hash == :root_hash
-                        UNION ALL
-                        SELECT
-                            node.hash, node.node_type, node.left, node.right, node.key FROM node, tree_from_root_hash
-                        WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right
-                    )
-                SELECT key FROM tree_from_root_hash WHERE node_type == :node_type
-                """,
-                {"root_hash": resolved_root_hash, "node_type": NodeType.TERMINAL},
-            )
-
-            keys: list[bytes] = [row["key"] async for row in cursor]
-
-        return keys
-
-    async def get_ancestors_common(
-        self,
-        node_hash: bytes32,
-        store_id: bytes32,
-        root_hash: Optional[bytes32],
-        generation: Optional[int] = None,
-        use_optimized: bool = True,
-    ) -> list[InternalNode]:
-        if use_optimized:
-            ancestors: list[InternalNode] = await self.get_ancestors_optimized(
-                node_hash=node_hash,
-                store_id=store_id,
-                generation=generation,
-                root_hash=root_hash,
-            )
-        else:
-            ancestors = await self.get_ancestors_optimized(
-                node_hash=node_hash,
-                store_id=store_id,
-                generation=generation,
-                root_hash=root_hash,
-            )
-            ancestors_2: list[InternalNode] = await self.get_ancestors(
-                node_hash=node_hash, store_id=store_id, root_hash=root_hash
-            )
-            if ancestors != ancestors_2:
-                raise RuntimeError("Ancestors optimized didn't produce the expected result.")
-
-        if len(ancestors) >= 62:
-            raise RuntimeError("Tree exceeds max height of 62.")
-        return ancestors
-
-    async def update_ancestor_hashes_on_insert(
-        self,
-        store_id: bytes32,
-        left: bytes32,
-        right: bytes32,
-        traversal_node_hash: bytes32,
-        ancestors: list[InternalNode],
-        status: Status,
-        root: Root,
-    ) -> Root:
-        # update ancestors after inserting root, to keep table constraints.
-        insert_ancestors_cache: list[tuple[bytes32, bytes32, bytes32]] = []
-        new_generation = root.generation + 1
-        # create first new internal node
-        new_hash = await self._insert_internal_node(left_hash=left, right_hash=right)
-        insert_ancestors_cache.append((left, right, store_id))
-
-        # create updated replacements for the rest of the internal nodes
-        for ancestor in ancestors:
-            if not isinstance(ancestor, InternalNode):
-                raise Exception(f"Expected an internal node but got: {type(ancestor).__name__}")
-
-            if ancestor.left_hash == traversal_node_hash:
-                left = new_hash
-                right = ancestor.right_hash
-            elif ancestor.right_hash == traversal_node_hash:
-                left = ancestor.left_hash
-                right = new_hash
-
-            traversal_node_hash = ancestor.hash
-
-            new_hash = await self._insert_internal_node(left_hash=left, right_hash=right)
-            insert_ancestors_cache.append((left, right, store_id))
-
-        new_root = await self._insert_root(
-            store_id=store_id,
-            node_hash=new_hash,
-            status=status,
-            generation=new_generation,
-        )
 
-        if status == Status.COMMITTED:
-            for left_hash, right_hash, store_id in insert_ancestors_cache:
-                await self._insert_ancestor_table(left_hash, right_hash, store_id, new_generation)
+            try:
+                merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash)
+            except MerkleBlobNotFoundError:
+                return []
 
-        return new_root
+            kv_ids = merkle_blob.get_keys_values()
+            raw_key_ids = (KeyOrValueId(id.raw) for id in kv_ids.keys())
+            table_blobs = await self.get_table_blobs(raw_key_ids, store_id)
+            keys: list[bytes] = []
+            for kid in kv_ids.keys():
+                blob_hash, blob = table_blobs[KeyOrValueId(kid.raw)]
+                if blob is None:
+                    blob = self.get_blob_from_file(blob_hash, store_id)
+                keys.append(blob)
+
+        return keys
+
+    def get_reference_kid_side(self, merkle_blob: MerkleBlob, seed: bytes32) -> tuple[KeyId, Side]:
+        side_seed = bytes(seed)[0]
+        side = Side.LEFT if side_seed < 128 else Side.RIGHT
+        reference_node = merkle_blob.get_random_leaf_node(seed)
+        kid = reference_node.key
+        return (kid, side)
+
+    async def get_terminal_node_from_kid(self, merkle_blob: MerkleBlob, kid: KeyId, store_id: bytes32) -> TerminalNode:
+        index = merkle_blob.get_key_index(kid)
+        raw_node = merkle_blob.get_raw_node(index)
+        assert isinstance(raw_node, chia_rs.datalayer.LeafNode)
+        return await self.get_terminal_node(raw_node.key, raw_node.value, store_id)
+
+    async def get_terminal_node_for_seed(self, seed: bytes32, store_id: bytes32) -> Optional[TerminalNode]:
+        root = await self.get_tree_root(store_id=store_id)
+        if root is None or root.node_hash is None:
+            return None
+
+        merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash)
+        assert not merkle_blob.empty()
+        kid, _ = self.get_reference_kid_side(merkle_blob, seed)
+        return await self.get_terminal_node_from_kid(merkle_blob, kid, store_id)
 
     async def insert(
         self,
         key: bytes,
         value: bytes,
         store_id: bytes32,
-        reference_node_hash: Optional[bytes32],
-        side: Optional[Side],
-        use_optimized: bool = True,
+        reference_node_hash: Optional[bytes32] = None,
+        side: Optional[Side] = None,
         status: Status = Status.PENDING,
         root: Optional[Root] = None,
     ) -> InsertResult:
-        async with self.db_wrapper.writer():
-            if root is None:
-                root = await self.get_tree_root(store_id=store_id)
+        async with self.db_wrapper.writer() as writer:
+            with self.manage_kv_files(store_id):
+                if root is None:
+                    root = await self.get_tree_root(store_id=store_id)
+                merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash)
 
-            try:
-                await self.get_node_by_key(key=key, store_id=store_id)
-                raise Exception(f"Key already present: {key.hex()}")
-            except KeyNotFoundError:
-                pass
-
-            was_empty = root.node_hash is None
-            if reference_node_hash is None:
-                if not was_empty:
-                    raise Exception(f"Reference node hash must be specified for non-empty tree: {store_id.hex()}")
-            else:
-                reference_node_type = await self.get_node_type(node_hash=reference_node_hash)
-                if reference_node_type == NodeType.INTERNAL:
-                    raise Exception("can not insert a new key/value on an internal node")
+                kid, vid = await self.add_key_value(key, value, store_id, writer=writer)
+                hash = leaf_hash(key, value)
+                reference_kid = None
+                if reference_node_hash is not None:
+                    reference_kid, _ = merkle_blob.get_node_by_hash(reference_node_hash)
 
-            # create new terminal node
-            new_terminal_node_hash = await self._insert_terminal_node(key=key, value=value)
+                was_empty = root.node_hash is None
+                if not was_empty and reference_kid is None:
+                    if side is not None:
+                        raise Exception("Side specified without reference node hash")
 
-            if was_empty:
-                if side is not None:
-                    raise Exception(f"Tree was empty so side must be unspecified, got: {side!r}")
+                    seed = leaf_hash(key=key, value=value)
+                    reference_kid, side = self.get_reference_kid_side(merkle_blob, seed)
 
-                new_root = await self._insert_root(
-                    store_id=store_id,
-                    node_hash=new_terminal_node_hash,
-                    status=status,
-                )
-            else:
-                if side is None:
-                    raise Exception("Tree was not empty, side must be specified.")
-                if reference_node_hash is None:
-                    raise Exception("Tree was not empty, reference node hash must be specified.")
-                if root.node_hash is None:
-                    raise Exception("Internal error.")
-
-                if side == Side.LEFT:
-                    left = new_terminal_node_hash
-                    right = reference_node_hash
-                elif side == Side.RIGHT:
-                    left = reference_node_hash
-                    right = new_terminal_node_hash
-                else:
-                    raise Exception(f"Internal error, unknown side: {side!r}")
-
-                ancestors = await self.get_ancestors_common(
-                    node_hash=reference_node_hash,
-                    store_id=store_id,
-                    root_hash=root.node_hash,
-                    generation=root.generation,
-                    use_optimized=use_optimized,
-                )
-                new_root = await self.update_ancestor_hashes_on_insert(
-                    store_id=store_id,
-                    left=left,
-                    right=right,
-                    traversal_node_hash=reference_node_hash,
-                    ancestors=ancestors,
-                    status=status,
-                    root=root,
-                )
+                merkle_blob.insert(kid, vid, hash, reference_kid, side)
 
-            return InsertResult(node_hash=new_terminal_node_hash, root=new_root)
+                new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status)
+                return InsertResult(node_hash=hash, root=new_root)
 
     async def delete(
         self,
         key: bytes,
         store_id: bytes32,
-        use_optimized: bool = True,
         status: Status = Status.PENDING,
         root: Optional[Root] = None,
     ) -> Optional[Root]:
-        root_hash = None if root is None else root.node_hash
         async with self.db_wrapper.writer():
-            try:
-                node = await self.get_node_by_key(key=key, store_id=store_id)
-                node_hash = node.hash
-                assert isinstance(node, TerminalNode)
-            except KeyNotFoundError:
-                log.debug(f"Request to delete an unknown key ignored: {key.hex()}")
-                return root
-
-            ancestors: list[InternalNode] = await self.get_ancestors_common(
-                node_hash=node_hash,
-                store_id=store_id,
-                root_hash=root_hash,
-                use_optimized=use_optimized,
-            )
-
-            if len(ancestors) == 0:
-                # the only node is being deleted
-                return await self._insert_root(
-                    store_id=store_id,
-                    node_hash=None,
-                    status=status,
-                )
-
-            parent = ancestors[0]
-            other_hash = parent.other_child_hash(hash=node_hash)
-
-            if len(ancestors) == 1:
-                # the parent is the root so the other side will become the new root
-                return await self._insert_root(
-                    store_id=store_id,
-                    node_hash=other_hash,
-                    status=status,
-                )
-
-            old_child_hash = parent.hash
-            new_child_hash = other_hash
             if root is None:
-                new_generation = await self.get_tree_generation(store_id) + 1
-            else:
-                new_generation = root.generation + 1
-            # update ancestors after inserting root, to keep table constraints.
-            insert_ancestors_cache: list[tuple[bytes32, bytes32, bytes32]] = []
-            # more parents to handle so let's traverse them
-            for ancestor in ancestors[1:]:
-                if ancestor.left_hash == old_child_hash:
-                    left_hash = new_child_hash
-                    right_hash = ancestor.right_hash
-                elif ancestor.right_hash == old_child_hash:
-                    left_hash = ancestor.left_hash
-                    right_hash = new_child_hash
-                else:
-                    raise Exception("Internal error.")
+                root = await self.get_tree_root(store_id=store_id)
+            merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash)
 
-                new_child_hash = await self._insert_internal_node(left_hash=left_hash, right_hash=right_hash)
-                insert_ancestors_cache.append((left_hash, right_hash, store_id))
-                old_child_hash = ancestor.hash
+            kid = await self.get_kvid(key, store_id)
+            if kid is not None:
+                merkle_blob.delete(KeyId(kid))
 
-            new_root = await self._insert_root(
-                store_id=store_id,
-                node_hash=new_child_hash,
-                status=status,
-                generation=new_generation,
-            )
-            if status == Status.COMMITTED:
-                for left_hash, right_hash, store_id in insert_ancestors_cache:
-                    await self._insert_ancestor_table(left_hash, right_hash, store_id, new_generation)
+            new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status)
 
         return new_root
 
@@ -1339,151 +1335,21 @@ class DataStore:
         key: bytes,
         new_value: bytes,
         store_id: bytes32,
-        use_optimized: bool = True,
         status: Status = Status.PENDING,
         root: Optional[Root] = None,
     ) -> InsertResult:
-        async with self.db_wrapper.writer():
-            if root is None:
-                root = await self.get_tree_root(store_id=store_id)
-
-            try:
-                old_node = await self.get_node_by_key(key=key, store_id=store_id)
-            except KeyNotFoundError:
-                log.debug(f"Key not found: {key.hex()}. Doing an autoinsert instead")
-                return await self.autoinsert(
-                    key=key,
-                    value=new_value,
-                    store_id=store_id,
-                    use_optimized=use_optimized,
-                    status=status,
-                    root=root,
-                )
-            if old_node.value == new_value:
-                log.debug(f"New value matches old value in upsert operation: {key.hex()}. Ignoring upsert")
-                return InsertResult(leaf_hash(key, new_value), root)
-
-            # create new terminal node
-            new_terminal_node_hash = await self._insert_terminal_node(key=key, value=new_value)
-
-            ancestors = await self.get_ancestors_common(
-                node_hash=old_node.hash,
-                store_id=store_id,
-                root_hash=root.node_hash,
-                generation=root.generation,
-                use_optimized=use_optimized,
-            )
-
-            # Store contains only the old root, replace it with a new root having the terminal node.
-            if len(ancestors) == 0:
-                new_root = await self._insert_root(
-                    store_id=store_id,
-                    node_hash=new_terminal_node_hash,
-                    status=status,
-                )
-            else:
-                parent = ancestors[0]
-                if parent.left_hash == old_node.hash:
-                    left = new_terminal_node_hash
-                    right = parent.right_hash
-                elif parent.right_hash == old_node.hash:
-                    left = parent.left_hash
-                    right = new_terminal_node_hash
-                else:
-                    raise Exception("Internal error.")
-
-                new_root = await self.update_ancestor_hashes_on_insert(
-                    store_id=store_id,
-                    left=left,
-                    right=right,
-                    traversal_node_hash=parent.hash,
-                    ancestors=ancestors[1:],
-                    status=status,
-                    root=root,
-                )
-
-            return InsertResult(node_hash=new_terminal_node_hash, root=new_root)
-
-    async def clean_node_table(self, writer: Optional[aiosqlite.Connection] = None) -> None:
-        query = """
-            WITH RECURSIVE pending_nodes AS (
-                SELECT node_hash AS hash FROM root
-                WHERE status IN (:pending_status, :pending_batch_status)
-                UNION ALL
-                SELECT n.left FROM node n
-                INNER JOIN pending_nodes pn ON n.hash = pn.hash
-                WHERE n.left IS NOT NULL
-                UNION ALL
-                SELECT n.right FROM node n
-                INNER JOIN pending_nodes pn ON n.hash = pn.hash
-                WHERE n.right IS NOT NULL
-            )
-            DELETE FROM node
-            WHERE hash IN (
-                SELECT n.hash FROM node n
-                LEFT JOIN ancestors a ON n.hash = a.hash
-                LEFT JOIN pending_nodes pn ON n.hash = pn.hash
-                WHERE a.hash IS NULL AND pn.hash IS NULL
-            )
-        """
-        params = {"pending_status": Status.PENDING.value, "pending_batch_status": Status.PENDING_BATCH.value}
-        if writer is None:
-            async with self.db_wrapper.writer(foreign_key_enforcement_enabled=False) as writer:
-                await writer.execute(query, params)
-        else:
-            await writer.execute(query, params)
-
-    async def get_nodes(self, node_hashes: list[bytes32]) -> list[Node]:
-        query_parameter_place_holders = ",".join("?" for _ in node_hashes)
-        async with self.db_wrapper.reader() as reader:
-            # TODO: handle SQLITE_MAX_VARIABLE_NUMBER
-            cursor = await reader.execute(
-                f"SELECT * FROM node WHERE hash IN ({query_parameter_place_holders})",
-                [*node_hashes],
-            )
-            rows = await cursor.fetchall()
-
-        hash_to_node = {row["hash"]: row_to_node(row=row) for row in rows}
-
-        missing_hashes = [node_hash.hex() for node_hash in node_hashes if node_hash not in hash_to_node]
-        if missing_hashes:
-            raise Exception(f"Nodes not found for hashes: {', '.join(missing_hashes)}")
-
-        return [hash_to_node[node_hash] for node_hash in node_hashes]
-
-    async def get_leaf_at_minimum_height(
-        self, root_hash: bytes32, hash_to_parent: dict[bytes32, InternalNode]
-    ) -> TerminalNode:
-        queue: list[bytes32] = [root_hash]
-        batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10)
-
-        while True:
-            assert len(queue) > 0
-            nodes = await self.get_nodes(queue[:batch_size])
-            queue = queue[batch_size:]
+        async with self.db_wrapper.writer() as writer:
+            with self.manage_kv_files(store_id):
+                if root is None:
+                    root = await self.get_tree_root(store_id=store_id)
+                merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash)
 
-            for node in nodes:
-                if isinstance(node, TerminalNode):
-                    return node
-                hash_to_parent[node.left_hash] = node
-                hash_to_parent[node.right_hash] = node
-                queue.append(node.left_hash)
-                queue.append(node.right_hash)
+                kid, vid = await self.add_key_value(key, new_value, store_id, writer=writer)
+                hash = leaf_hash(key, new_value)
+                merkle_blob.upsert(kid, vid, hash)
 
-    async def batch_upsert(
-        self,
-        hash: bytes32,
-        to_update_hashes: set[bytes32],
-        pending_upsert_new_hashes: dict[bytes32, bytes32],
-    ) -> bytes32:
-        if hash not in to_update_hashes:
-            return hash
-        node = await self.get_node(hash)
-        if isinstance(node, TerminalNode):
-            return pending_upsert_new_hashes[hash]
-        new_left_hash = await self.batch_upsert(node.left_hash, to_update_hashes, pending_upsert_new_hashes)
-        new_right_hash = await self.batch_upsert(node.right_hash, to_update_hashes, pending_upsert_new_hashes)
-        return await self._insert_internal_node(new_left_hash, new_right_hash)
+                new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status)
+                return InsertResult(node_hash=hash, root=new_root)
 
     async def insert_batch(
         self,
@@ -1492,339 +1358,90 @@ class DataStore:
         status: Status = Status.PENDING,
         enable_batch_autoinsert: bool = True,
     ) -> Optional[bytes32]:
-        async with self.transaction():
-            old_root = await self.get_tree_root(store_id)
-            pending_root = await self.get_pending_root(store_id=store_id)
-            if pending_root is None:
-                latest_local_root: Optional[Root] = old_root
-            else:
-                if pending_root.status == Status.PENDING_BATCH:
-                    # We have an unfinished batch, continue the current batch on top of it.
-                    if pending_root.generation != old_root.generation + 1:
+        async with self.db_wrapper.writer() as writer:
+            with self.manage_kv_files(store_id):
+                old_root = await self.get_tree_root(store_id=store_id)
+                pending_root = await self.get_pending_root(store_id=store_id)
+                if pending_root is not None:
+                    if pending_root.status == Status.PENDING_BATCH:
+                        # We have an unfinished batch, continue the current batch on top of it.
+                        if pending_root.generation != old_root.generation + 1:
+                            raise Exception("Internal error")
+                        old_root = pending_root
+                        await self.clear_pending_roots(store_id)
+                    else:
                         raise Exception("Internal error")
-                    await self.change_root_status(pending_root, Status.COMMITTED)
-                    await self.build_ancestor_table_for_latest_root(store_id=store_id)
-                    latest_local_root = pending_root
-                else:
-                    raise Exception("Internal error")
-
-            assert latest_local_root is not None
-
-            key_hash_frequency: dict[bytes32, int] = {}
-            first_action: dict[bytes32, str] = {}
-            last_action: dict[bytes32, str] = {}
 
-            for change in changelist:
-                key = change["key"]
-                hash = key_hash(key)
-                key_hash_frequency[hash] = key_hash_frequency.get(hash, 0) + 1
-                if hash not in first_action:
-                    first_action[hash] = change["action"]
-                last_action[hash] = change["action"]
+                merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=old_root.node_hash)
 
-            pending_autoinsert_hashes: list[bytes32] = []
-            pending_upsert_new_hashes: dict[bytes32, bytes32] = {}
-            leaf_hashes = await self.get_leaf_hashes_by_hashed_key(store_id)
+                key_hash_frequency: dict[bytes32, int] = {}
+                first_action: dict[bytes32, str] = {}
+                last_action: dict[bytes32, str] = {}
 
-            for change in changelist:
-                if change["action"] == "insert":
-                    key = change["key"]
-                    value = change["value"]
-                    reference_node_hash = change.get("reference_node_hash", None)
-                    side = change.get("side", None)
-                    if reference_node_hash is None and side is None:
-                        hash = key_hash(key)
-                        # The key is not referenced in any other operation but this autoinsert, hence the order
-                        # of performing these should not matter. We perform all these autoinserts as a batch
-                        # at the end, to speed up the tree processing operations.
-                        # Additionally, if the first action is a delete, we can still perform the autoinsert at the
-                        # end, since the order will be preserved.
-                        if enable_batch_autoinsert:
-                            if key_hash_frequency[hash] == 1 or (
-                                key_hash_frequency[hash] == 2 and first_action[hash] == "delete"
-                            ):
-                                old_node = await self.maybe_get_node_from_key_hash(leaf_hashes, hash)
-                                terminal_node_hash = await self._insert_terminal_node(key, value)
-
-                                if old_node is None:
-                                    pending_autoinsert_hashes.append(terminal_node_hash)
-                                else:
-                                    if key_hash_frequency[hash] == 1:
-                                        raise Exception(f"Key already present: {key.hex()}")
-                                    else:
-                                        pending_upsert_new_hashes[old_node.hash] = terminal_node_hash
-                                continue
-                        insert_result = await self.autoinsert(
-                            key, value, store_id, True, Status.COMMITTED, root=latest_local_root
-                        )
-                        latest_local_root = insert_result.root
-                    else:
-                        if reference_node_hash is None or side is None:
-                            raise Exception("Provide both reference_node_hash and side or neither.")
-                        insert_result = await self.insert(
-                            key,
-                            value,
-                            store_id,
-                            reference_node_hash,
-                            side,
-                            True,
-                            Status.COMMITTED,
-                            root=latest_local_root,
-                        )
-                        latest_local_root = insert_result.root
-                elif change["action"] == "delete":
-                    key = change["key"]
-                    hash = key_hash(key)
-                    if key_hash_frequency[hash] == 2 and last_action[hash] == "insert" and enable_batch_autoinsert:
-                        continue
-                    latest_local_root = await self.delete(key, store_id, True, Status.COMMITTED, root=latest_local_root)
-                elif change["action"] == "upsert":
+                for change in changelist:
                     key = change["key"]
-                    new_value = change["value"]
                     hash = key_hash(key)
-                    if key_hash_frequency[hash] == 1 and enable_batch_autoinsert:
-                        terminal_node_hash = await self._insert_terminal_node(key, new_value)
-                        old_node = await self.maybe_get_node_from_key_hash(leaf_hashes, hash)
-                        if old_node is not None:
-                            pending_upsert_new_hashes[old_node.hash] = terminal_node_hash
+                    key_hash_frequency[hash] = key_hash_frequency.get(hash, 0) + 1
+                    if hash not in first_action:
+                        first_action[hash] = change["action"]
+                    last_action[hash] = change["action"]
+
+                batch_keys_values: list[tuple[KeyId, ValueId]] = []
+                batch_hashes: list[bytes32] = []
+
+                for change in changelist:
+                    if change["action"] == "insert":
+                        key = change["key"]
+                        value = change["value"]
+
+                        reference_node_hash = change.get("reference_node_hash", None)
+                        side = change.get("side", None)
+                        reference_kid: Optional[KeyId] = None
+                        if reference_node_hash is not None:
+                            reference_kid, _ = merkle_blob.get_node_by_hash(reference_node_hash)
+
+                        key_hashed = key_hash(key)
+                        kid, vid = await self.add_key_value(key, value, store_id, writer=writer)
+                        try:
+                            merkle_blob.get_key_index(kid)
+                        except chia_rs.datalayer.UnknownKeyError:
+                            pass
                         else:
-                            pending_autoinsert_hashes.append(terminal_node_hash)
-                        continue
-                    insert_result = await self.upsert(
-                        key, new_value, store_id, True, Status.COMMITTED, root=latest_local_root
-                    )
-                    latest_local_root = insert_result.root
-                else:
-                    raise Exception(f"Operation in batch is not insert or delete: {change}")
-
-            if len(pending_upsert_new_hashes) > 0:
-                to_update_hashes: set[bytes32] = set(pending_upsert_new_hashes.keys())
-                to_update_queue: list[bytes32] = list(pending_upsert_new_hashes.keys())
-                batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10)
-
-                while len(to_update_queue) > 0:
-                    nodes = await self._get_one_ancestor_multiple_hashes(to_update_queue[:batch_size], store_id)
-                    to_update_queue = to_update_queue[batch_size:]
-                    for node in nodes:
-                        if node.hash not in to_update_hashes:
-                            to_update_hashes.add(node.hash)
-                            to_update_queue.append(node.hash)
-
-                assert latest_local_root is not None
-                assert latest_local_root.node_hash is not None
-                new_root_hash = await self.batch_upsert(
-                    latest_local_root.node_hash,
-                    to_update_hashes,
-                    pending_upsert_new_hashes,
-                )
-                latest_local_root = await self._insert_root(store_id, new_root_hash, Status.COMMITTED)
-
-            # Start with the leaf nodes and pair them to form new nodes at the next level up, repeating this process
-            # in a bottom-up fashion until a single root node remains. This constructs a balanced tree from the leaves.
-            while len(pending_autoinsert_hashes) > 1:
-                new_hashes: list[bytes32] = []
-                for i in range(0, len(pending_autoinsert_hashes) - 1, 2):
-                    internal_node_hash = await self._insert_internal_node(
-                        pending_autoinsert_hashes[i], pending_autoinsert_hashes[i + 1]
-                    )
-                    new_hashes.append(internal_node_hash)
-                if len(pending_autoinsert_hashes) % 2 != 0:
-                    new_hashes.append(pending_autoinsert_hashes[-1])
-
-                pending_autoinsert_hashes = new_hashes
-
-            if len(pending_autoinsert_hashes):
-                subtree_hash = pending_autoinsert_hashes[0]
-                if latest_local_root is None or latest_local_root.node_hash is None:
-                    await self._insert_root(store_id=store_id, node_hash=subtree_hash, status=Status.COMMITTED)
-                else:
-                    hash_to_parent: dict[bytes32, InternalNode] = {}
-                    min_height_leaf = await self.get_leaf_at_minimum_height(latest_local_root.node_hash, hash_to_parent)
-                    ancestors: list[InternalNode] = []
-                    hash = min_height_leaf.hash
-                    while hash in hash_to_parent:
-                        node = hash_to_parent[hash]
-                        ancestors.append(node)
-                        hash = node.hash
-
-                    await self.update_ancestor_hashes_on_insert(
-                        store_id=store_id,
-                        left=min_height_leaf.hash,
-                        right=subtree_hash,
-                        traversal_node_hash=min_height_leaf.hash,
-                        ancestors=ancestors,
-                        status=Status.COMMITTED,
-                        root=latest_local_root,
-                    )
-
-            root = await self.get_tree_root(store_id=store_id)
-            if root.node_hash == old_root.node_hash:
-                if len(changelist) != 0:
-                    await self.rollback_to_generation(store_id, old_root.generation)
-                raise ValueError("Changelist resulted in no change to tree data")
-            # We delete all "temporary" records stored in root and ancestor tables and store only the final result.
-            await self.rollback_to_generation(store_id, old_root.generation)
-            await self.insert_root_with_ancestor_table(store_id=store_id, node_hash=root.node_hash, status=status)
-            if status in {Status.PENDING, Status.PENDING_BATCH}:
-                new_root = await self.get_pending_root(store_id=store_id)
-                assert new_root is not None
-            elif status == Status.COMMITTED:
-                new_root = await self.get_tree_root(store_id=store_id)
-            else:
-                raise Exception(f"No known status: {status}")
-            if new_root.node_hash != root.node_hash:
-                raise RuntimeError(
-                    f"Tree root mismatches after batch update: Expected: {root.node_hash}. Got: {new_root.node_hash}"
-                )
-            if new_root.generation != old_root.generation + 1:
-                raise RuntimeError(
-                    "Didn't get the expected generation after batch update: "
-                    f"Expected: {old_root.generation + 1}. Got: {new_root.generation}"
-                )
-            return root.node_hash
-
-    async def _get_one_ancestor(
-        self,
-        node_hash: bytes32,
-        store_id: bytes32,
-        generation: Optional[int] = None,
-    ) -> Optional[InternalNode]:
-        async with self.db_wrapper.reader() as reader:
-            if generation is None:
-                generation = await self.get_tree_generation(store_id=store_id)
-            cursor = await reader.execute(
-                """
-                SELECT * from node INNER JOIN (
-                    SELECT ancestors.ancestor AS hash, MAX(ancestors.generation) AS generation
-                    FROM ancestors
-                    WHERE ancestors.hash == :hash
-                    AND ancestors.tree_id == :tree_id
-                    AND ancestors.generation <= :generation
-                    GROUP BY hash
-                ) asc on asc.hash == node.hash
-                """,
-                {"hash": node_hash, "tree_id": store_id, "generation": generation},
-            )
-            row = await cursor.fetchone()
-            if row is None:
-                return None
-            return InternalNode.from_row(row=row)
-
-    async def _get_one_ancestor_multiple_hashes(
-        self,
-        node_hashes: list[bytes32],
-        store_id: bytes32,
-        generation: Optional[int] = None,
-    ) -> list[InternalNode]:
-        async with self.db_wrapper.reader() as reader:
-            node_hashes_place_holders = ",".join("?" for _ in node_hashes)
-            if generation is None:
-                generation = await self.get_tree_generation(store_id=store_id)
-            cursor = await reader.execute(
-                f"""
-                SELECT * from node INNER JOIN (
-                    SELECT ancestors.ancestor AS hash, MAX(ancestors.generation) AS generation
-                    FROM ancestors
-                    WHERE ancestors.hash IN ({node_hashes_place_holders})
-                    AND ancestors.tree_id == ?
-                    AND ancestors.generation <= ?
-                    GROUP BY hash
-                ) asc on asc.hash == node.hash
-                """,
-                [*node_hashes, store_id, generation],
-            )
-            rows = await cursor.fetchall()
-            return [InternalNode.from_row(row=row) for row in rows]
-
-    async def build_ancestor_table_for_latest_root(self, store_id: bytes32) -> None:
-        async with self.db_wrapper.writer():
-            root = await self.get_tree_root(store_id=store_id)
-            if root.node_hash is None:
-                return
-            previous_root = await self.get_tree_root(
-                store_id=store_id,
-                generation=max(root.generation - 1, 0),
-            )
-
-            if previous_root.node_hash is not None:
-                previous_internal_nodes: list[InternalNode] = await self.get_internal_nodes(
-                    store_id=store_id,
-                    root_hash=previous_root.node_hash,
-                )
-                known_hashes: set[bytes32] = {node.hash for node in previous_internal_nodes}
-            else:
-                known_hashes = set()
-            internal_nodes: list[InternalNode] = await self.get_internal_nodes(
-                store_id=store_id,
-                root_hash=root.node_hash,
-            )
-            for node in internal_nodes:
-                # We already have the same values in ancestor tables, if we have the same internal node.
-                # Don't reinsert it so we can save DB space.
-                if node.hash not in known_hashes:
-                    await self._insert_ancestor_table(node.left_hash, node.right_hash, store_id, root.generation)
-
-    async def insert_root_with_ancestor_table(
-        self, store_id: bytes32, node_hash: Optional[bytes32], status: Status = Status.PENDING
-    ) -> None:
-        async with self.db_wrapper.writer():
-            await self._insert_root(store_id=store_id, node_hash=node_hash, status=status)
-            # Don't update the ancestor table for non-committed status.
-            if status == Status.COMMITTED:
-                await self.build_ancestor_table_for_latest_root(store_id=store_id)
-
-    async def get_node_by_key_latest_generation(self, key: bytes, store_id: bytes32) -> TerminalNode:
-        async with self.db_wrapper.reader() as reader:
-            root = await self.get_tree_root(store_id=store_id)
-            if root.node_hash is None:
-                raise KeyNotFoundError(key=key)
-
-            cursor = await reader.execute(
-                """
-                SELECT a.hash FROM ancestors a
-                JOIN node n ON a.hash = n.hash
-                WHERE n.key = :key
-                AND a.tree_id = :tree_id
-                ORDER BY a.generation DESC
-                LIMIT 1
-                """,
-                {"key": key, "tree_id": store_id},
-            )
-
-            row = await cursor.fetchone()
-            if row is None:
-                raise KeyNotFoundError(key=key)
-
-            node = await self.get_node(row["hash"])
-            node_hash = node.hash
-            while True:
-                internal_node = await self._get_one_ancestor(node_hash, store_id)
-                if internal_node is None:
-                    break
-                node_hash = internal_node.hash
-
-            if node_hash != root.node_hash:
-                raise KeyNotFoundError(key=key)
-            assert isinstance(node, TerminalNode)
-            return node
-
-    async def maybe_get_node_from_key_hash(
-        self, leaf_hashes: dict[bytes32, bytes32], hash: bytes32
-    ) -> Optional[TerminalNode]:
-        if hash in leaf_hashes:
-            leaf_hash = leaf_hashes[hash]
-            node = await self.get_node(leaf_hash)
-            assert isinstance(node, TerminalNode)
-            return node
+                            raise KeyAlreadyPresentError(kid)
+                        hash = leaf_hash(key, value)
+
+                        if reference_node_hash is None and side is None:
+                            if enable_batch_autoinsert and reference_kid is None:
+                                if key_hash_frequency[key_hashed] == 1 or (
+                                    key_hash_frequency[key_hashed] == 2 and first_action[key_hashed] == "delete"
+                                ):
+                                    batch_keys_values.append((kid, vid))
+                                    batch_hashes.append(hash)
+                                    continue
+                            if not merkle_blob.empty():
+                                seed = leaf_hash(key=key, value=value)
+                                reference_kid, side = self.get_reference_kid_side(merkle_blob, seed)
+
+                        merkle_blob.insert(kid, vid, hash, reference_kid, side)
+                    elif change["action"] == "delete":
+                        key = change["key"]
+                        deletion_kid = await self.get_kvid(key, store_id)
+                        if deletion_kid is not None:
+                            merkle_blob.delete(KeyId(deletion_kid))
+                    elif change["action"] == "upsert":
+                        key = change["key"]
+                        new_value = change["value"]
+                        kid, vid = await self.add_key_value(key, new_value, store_id, writer=writer)
+                        hash = leaf_hash(key, new_value)
+                        merkle_blob.upsert(kid, vid, hash)
+                    else:
+                        raise Exception(f"Operation in batch is not insert or delete: {change}")
 
-        return None
+                if len(batch_keys_values) > 0:
+                    merkle_blob.batch_insert(batch_keys_values, batch_hashes)
 
-    async def maybe_get_node_by_key(self, key: bytes, store_id: bytes32) -> Optional[TerminalNode]:
-        try:
-            node = await self.get_node_by_key_latest_generation(key, store_id)
-            return node
-        except KeyNotFoundError:
-            return None
+                new_root = await self.insert_root_from_merkle_blob(merkle_blob, store_id, status, old_root)
+                return new_root.node_hash
 
     async def get_node_by_key(
         self,
@@ -1832,56 +1449,49 @@ class DataStore:
         store_id: bytes32,
         root_hash: Union[bytes32, Unspecified] = unspecified,
     ) -> TerminalNode:
-        if root_hash is unspecified:
-            return await self.get_node_by_key_latest_generation(key, store_id)
-
-        nodes = await self.get_keys_values(store_id=store_id, root_hash=root_hash)
-
-        for node in nodes:
-            if node.key == key:
-                return node
-
-        raise KeyNotFoundError(key=key)
-
-    async def get_node(self, node_hash: bytes32) -> Node:
-        async with self.db_wrapper.reader() as reader:
-            cursor = await reader.execute("SELECT * FROM node WHERE hash == :hash LIMIT 1", {"hash": node_hash})
-            row = await cursor.fetchone()
+        async with self.db_wrapper.reader():
+            resolved_root_hash: Optional[bytes32]
+            if root_hash is unspecified:
+                root = await self.get_tree_root(store_id=store_id)
+                resolved_root_hash = root.node_hash
+            else:
+                resolved_root_hash = root_hash
 
-        if row is None:
-            raise Exception(f"Node not found for requested hash: {node_hash.hex()}")
+            try:
+                merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=resolved_root_hash)
+            except MerkleBlobNotFoundError:
+                raise KeyNotFoundError(key=key)
 
-        node = row_to_node(row=row)
-        return node
+            kvid = await self.get_kvid(key, store_id)
+            if kvid is None:
+                raise KeyNotFoundError(key=key)
+            kid = KeyId(kvid)
+            return await self.get_terminal_node_from_kid(merkle_blob, kid, store_id)
 
     async def get_tree_as_nodes(self, store_id: bytes32) -> Node:
-        async with self.db_wrapper.reader() as reader:
+        async with self.db_wrapper.reader():
             root = await self.get_tree_root(store_id=store_id)
             # TODO: consider actual proper behavior
             assert root.node_hash is not None
-            root_node = await self.get_node(node_hash=root.node_hash)
 
-            cursor = await reader.execute(
-                """
-                WITH RECURSIVE
-                    tree_from_root_hash(hash, node_type, left, right, key, value) AS (
-                        SELECT node.* FROM node WHERE node.hash == :root_hash
-                        UNION ALL
-                        SELECT node.* FROM node, tree_from_root_hash
-                        WHERE node.hash == tree_from_root_hash.left OR node.hash == tree_from_root_hash.right
-                    )
-                SELECT * FROM tree_from_root_hash
-                """,
-                {"root_hash": root_node.hash},
-            )
-            nodes = [row_to_node(row=row) async for row in cursor]
+            merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash)
+
+            nodes = merkle_blob.get_nodes_with_indexes()
             hash_to_node: dict[bytes32, Node] = {}
-            for node in reversed(nodes):
-                if isinstance(node, InternalNode):
-                    node = replace(node, left=hash_to_node[node.left_hash], right=hash_to_node[node.right_hash])
-                hash_to_node[node.hash] = node
+            tree_node: Node
+            for _, node in reversed(nodes):
+                if isinstance(node, chia_rs.datalayer.InternalNode):
+                    left_hash = merkle_blob.get_hash_at_index(node.left)
+                    right_hash = merkle_blob.get_hash_at_index(node.right)
+                    tree_node = InternalNode.from_child_nodes(
+                        left=hash_to_node[left_hash], right=hash_to_node[right_hash]
+                    )
+                else:
+                    assert isinstance(node, chia_rs.datalayer.LeafNode)
+                    tree_node = await self.get_terminal_node(node.key, node.value, store_id)
+                hash_to_node[node.hash] = tree_node
 
-            root_node = hash_to_node[root_node.hash]
+            root_node = hash_to_node[root.node_hash]
 
         return root_node
 
@@ -1890,66 +1500,86 @@ class DataStore:
         node_hash: bytes32,
         store_id: bytes32,
         root_hash: Optional[bytes32] = None,
-        use_optimized: bool = False,
     ) -> ProofOfInclusion:
-        """Collect the information for a proof of inclusion of a hash in the Merkle
-        tree.
-        """
-
-        # Ideally this would use get_ancestors_common, but this _common function has this interesting property
-        # when used with use_optimized=False - it will compare both methods in this case and raise an exception.
-        # this is undesirable in the DL Offers flow where PENDING roots can cause the optimized code to fail.
-        if use_optimized:
-            ancestors = await self.get_ancestors_optimized(node_hash=node_hash, store_id=store_id, root_hash=root_hash)
-        else:
-            ancestors = await self.get_ancestors(node_hash=node_hash, store_id=store_id, root_hash=root_hash)
-
-        layers: list[ProofOfInclusionLayer] = []
-        child_hash = node_hash
-        for parent in ancestors:
-            layer = ProofOfInclusionLayer.from_internal_node(internal_node=parent, traversal_child_hash=child_hash)
-            layers.append(layer)
-            child_hash = parent.hash
-
-        proof_of_inclusion = ProofOfInclusion(node_hash=node_hash, layers=layers)
-
-        if len(ancestors) > 0:
-            expected_root = ancestors[-1].hash
-        else:
-            expected_root = node_hash
-
-        if expected_root != proof_of_inclusion.root_hash:
-            raise Exception(
-                f"Incorrect root, expected: {expected_root.hex()}"
-                f"\n                     has: {proof_of_inclusion.root_hash.hex()}"
-            )
-
-        return proof_of_inclusion
+        if root_hash is None:
+            root = await self.get_tree_root(store_id=store_id)
+            root_hash = root.node_hash
+        merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root_hash)
+        kid, _ = merkle_blob.get_node_by_hash(node_hash)
+        return merkle_blob.get_proof_of_inclusion(kid)
 
     async def get_proof_of_inclusion_by_key(
         self,
         key: bytes,
         store_id: bytes32,
     ) -> ProofOfInclusion:
-        """Collect the information for a proof of inclusion of a key and its value in
-        the Merkle tree.
-        """
-        async with self.db_wrapper.reader():
-            node = await self.get_node_by_key(key=key, store_id=store_id)
-            return await self.get_proof_of_inclusion_by_hash(node_hash=node.hash, store_id=store_id)
+        root = await self.get_tree_root(store_id=store_id)
+        merkle_blob = await self.get_merkle_blob(store_id=store_id, root_hash=root.node_hash)
+        kvid = await self.get_kvid(key, store_id)
+        if kvid is None:
+            raise Exception(f"Cannot find key: {key.hex()}")
+        kid = KeyId(kvid)
+        return merkle_blob.get_proof_of_inclusion(kid)
+
+    async def get_nodes_for_file(
+        self,
+        root: Root,
+        node_hash: bytes32,
+        store_id: bytes32,
+        deltas_only: bool,
+        delta_file_cache: DeltaFileCache,
+        tree_nodes: list[SerializedNode],
+    ) -> None:
+        if deltas_only:
+            if delta_file_cache.seen_previous_hash(node_hash):
+                return
 
-    async def get_first_generation(self, node_hash: bytes32, store_id: bytes32) -> int:
-        async with self.db_wrapper.reader() as reader:
-            cursor = await reader.execute(
-                "SELECT MIN(generation) AS generation FROM ancestors WHERE hash == :hash AND tree_id == :tree_id",
-                {"hash": node_hash, "tree_id": store_id},
+        raw_index = delta_file_cache.get_index(node_hash)
+        raw_node = delta_file_cache.get_raw_node(raw_index)
+
+        if isinstance(raw_node, chia_rs.datalayer.InternalNode):
+            left_hash = delta_file_cache.get_hash_at_index(raw_node.left)
+            right_hash = delta_file_cache.get_hash_at_index(raw_node.right)
+            await self.get_nodes_for_file(root, left_hash, store_id, deltas_only, delta_file_cache, tree_nodes)
+            await self.get_nodes_for_file(root, right_hash, store_id, deltas_only, delta_file_cache, tree_nodes)
+            tree_nodes.append(SerializedNode(False, bytes(left_hash), bytes(right_hash)))
+        elif isinstance(raw_node, chia_rs.datalayer.LeafNode):
+            tree_nodes.append(
+                SerializedNode(
+                    True,
+                    raw_node.key.to_bytes(),
+                    raw_node.value.to_bytes(),
+                )
             )
-            row = await cursor.fetchone()
-            if row is None:
-                raise RuntimeError("Hash not found in ancestor table.")
+        else:
+            raise Exception(f"Node is neither InternalNode nor TerminalNode: {raw_node}")
+
+    async def get_table_blobs(
+        self, kv_ids_iter: Iterable[KeyOrValueId], store_id: bytes32
+    ) -> dict[KeyOrValueId, tuple[bytes32, Optional[bytes]]]:
+        result: dict[KeyOrValueId, tuple[bytes32, Optional[bytes]]] = {}
+        batch_size = min(500, SQLITE_MAX_VARIABLE_NUMBER - 10)
+        kv_ids = list(dict.fromkeys(kv_ids_iter))
+
+        async with self.db_wrapper.reader() as reader:
+            for i in range(0, len(kv_ids), batch_size):
+                chunk = kv_ids[i : i + batch_size]
+                placeholders = ",".join(["?"] * len(chunk))
+                query = f"""
+                    SELECT hash, blob, kv_id
+                    FROM ids
+                    WHERE store_id = ? AND kv_id IN ({placeholders})
+                    LIMIT {len(chunk)}
+                """
+
+                async with reader.execute(query, (store_id, *chunk)) as cursor:
+                    rows = await cursor.fetchall()
+                    result.update({row["kv_id"]: (row["hash"], row["blob"]) for row in rows})
 
-            generation = row["generation"]
-            return int(generation)
+        if len(result) != len(kv_ids):
+            raise Exception("Cannot retrieve all the requested kv_ids")
+
+        return result
 
     async def write_tree_to_file(
         self,
@@ -1962,24 +1592,42 @@ class DataStore:
         if node_hash == bytes32.zeros:
             return
 
-        if deltas_only:
-            generation = await self.get_first_generation(node_hash, store_id)
-            # Root's generation is not the first time we see this hash, so it's not a new delta.
-            if root.generation != generation:
-                return
-        node = await self.get_node(node_hash)
-        to_write = b""
-        if isinstance(node, InternalNode):
-            await self.write_tree_to_file(root, node.left_hash, store_id, deltas_only, writer)
-            await self.write_tree_to_file(root, node.right_hash, store_id, deltas_only, writer)
-            to_write = bytes(SerializedNode(False, bytes(node.left_hash), bytes(node.right_hash)))
-        elif isinstance(node, TerminalNode):
-            to_write = bytes(SerializedNode(True, node.key, node.value))
-        else:
-            raise Exception(f"Node is neither InternalNode nor TerminalNode: {node}")
+        with log_exceptions(log=log, message="Error while getting merkle blob"):
+            root_path = self.get_merkle_path(store_id=store_id, root_hash=root.node_hash)
+        delta_file_cache = DeltaFileCache(root_path)
 
-        writer.write(len(to_write).to_bytes(4, byteorder="big"))
-        writer.write(to_write)
+        if root.generation > 0:
+            previous_root = await self.get_tree_root(store_id=store_id, generation=root.generation - 1)
+            if previous_root.node_hash is not None:
+                with log_exceptions(log=log, message="Error while getting previous merkle blob"):
+                    previous_root_path = self.get_merkle_path(store_id=store_id, root_hash=previous_root.node_hash)
+                delta_file_cache.load_previous_hashes(previous_root_path)
+
+        tree_nodes: list[SerializedNode] = []
+
+        await self.get_nodes_for_file(root, node_hash, store_id, deltas_only, delta_file_cache, tree_nodes)
+        kv_ids = (
+            KeyOrValueId.from_bytes(raw_id)
+            for node in tree_nodes
+            if node.is_terminal
+            for raw_id in (node.value1, node.value2)
+        )
+        table_blobs = await self.get_table_blobs(kv_ids, store_id)
+
+        for node in tree_nodes:
+            if node.is_terminal:
+                blobs = []
+                for raw_id in (node.value1, node.value2):
+                    id = KeyOrValueId.from_bytes(raw_id)
+                    blob_hash, blob = table_blobs[id]
+                    if blob is None:
+                        blob = self.get_blob_from_file(blob_hash, store_id)
+                    blobs.append(blob)
+                to_write = bytes(SerializedNode(True, blobs[0], blobs[1]))
+            else:
+                to_write = bytes(node)
+            writer.write(len(to_write).to_bytes(4, byteorder="big"))
+            writer.write(to_write)
 
     async def update_subscriptions_from_wallet(self, store_id: bytes32, new_urls: list[str]) -> None:
         async with self.db_wrapper.writer() as writer:
@@ -2064,94 +1712,36 @@ class DataStore:
                     },
                 )
 
-    async def delete_store_data(self, store_id: bytes32) -> None:
-        async with self.db_wrapper.writer(foreign_key_enforcement_enabled=False) as writer:
-            await self.clean_node_table(writer)
-            cursor = await writer.execute(
-                """
-                WITH RECURSIVE all_nodes AS (
-                    SELECT a.hash, n.left, n.right
-                    FROM ancestors AS a
-                    JOIN node AS n ON a.hash = n.hash
-                    WHERE a.tree_id = :tree_id
-                ),
-                pending_nodes AS (
-                    SELECT node_hash AS hash FROM root
-                    WHERE status IN (:pending_status, :pending_batch_status)
-                    UNION ALL
-                    SELECT n.left FROM node n
-                    INNER JOIN pending_nodes pn ON n.hash = pn.hash
-                    WHERE n.left IS NOT NULL
-                    UNION ALL
-                    SELECT n.right FROM node n
-                    INNER JOIN pending_nodes pn ON n.hash = pn.hash
-                    WHERE n.right IS NOT NULL
-                )
-
-                SELECT hash, left, right
-                FROM all_nodes
-                WHERE hash NOT IN (SELECT hash FROM ancestors WHERE tree_id != :tree_id)
-                AND hash NOT IN (SELECT hash from pending_nodes)
-                """,
-                {
-                    "tree_id": store_id,
-                    "pending_status": Status.PENDING.value,
-                    "pending_batch_status": Status.PENDING_BATCH.value,
-                },
-            )
-            to_delete: dict[bytes, tuple[bytes, bytes]] = {}
-            ref_counts: dict[bytes, int] = {}
-            async for row in cursor:
-                hash = row["hash"]
-                left = row["left"]
-                right = row["right"]
-                if hash in to_delete:
-                    prev_left, prev_right = to_delete[hash]
-                    assert prev_left == left
-                    assert prev_right == right
-                    continue
-                to_delete[hash] = (left, right)
-                if left is not None:
-                    ref_counts[left] = ref_counts.get(left, 0) + 1
-                if right is not None:
-                    ref_counts[right] = ref_counts.get(right, 0) + 1
-
-            await writer.execute("DELETE FROM ancestors WHERE tree_id == ?", (store_id,))
-            await writer.execute("DELETE FROM root WHERE tree_id == ?", (store_id,))
-            queue = [hash for hash in to_delete if ref_counts.get(hash, 0) == 0]
-            while queue:
-                hash = queue.pop(0)
-                if hash not in to_delete:
-                    continue
-                await writer.execute("DELETE FROM node WHERE hash == ?", (hash,))
-
-                left, right = to_delete[hash]
-                if left is not None:
-                    ref_counts[left] -= 1
-                    if ref_counts[left] == 0:
-                        queue.append(left)
-
-                if right is not None:
-                    ref_counts[right] -= 1
-                    if ref_counts[right] == 0:
-                        queue.append(right)
-
     async def unsubscribe(self, store_id: bytes32) -> None:
         async with self.db_wrapper.writer() as writer:
             await writer.execute(
                 "DELETE FROM subscriptions WHERE tree_id == :tree_id",
                 {"tree_id": store_id},
             )
+            await writer.execute(
+                "DELETE FROM ids WHERE store_id == :store_id",
+                {"store_id": store_id},
+            )
+            await writer.execute(
+                "DELETE FROM nodes WHERE store_id == :store_id",
+                {"store_id": store_id},
+            )
+
+            with contextlib.suppress(FileNotFoundError):
+                shutil.rmtree(self.get_merkle_path(store_id=store_id, root_hash=None))
+
+            with contextlib.suppress(FileNotFoundError):
+                shutil.rmtree(self.get_key_value_path(store_id=store_id, blob_hash=None))
 
     async def rollback_to_generation(self, store_id: bytes32, target_generation: int) -> None:
         async with self.db_wrapper.writer() as writer:
             await writer.execute(
-                "DELETE FROM ancestors WHERE tree_id == :tree_id AND generation > :target_generation",
+                "DELETE FROM root WHERE tree_id == :tree_id AND generation > :target_generation",
                 {"tree_id": store_id, "target_generation": target_generation},
             )
             await writer.execute(
-                "DELETE FROM root WHERE tree_id == :tree_id AND generation > :target_generation",
-                {"tree_id": store_id, "target_generation": target_generation},
+                "DELETE FROM nodes WHERE store_id == :store_id AND generation > :target_generation",
+                {"store_id": store_id, "target_generation": target_generation},
             )
 
     async def update_server_info(self, store_id: bytes32, server_info: ServerInfo) -> None:
@@ -2228,13 +1818,12 @@ class DataStore:
                         )
                     else:
                         subscriptions.append(Subscription(store_id, []))
-                else:
-                    if url is not None and num_consecutive_failures is not None and ignore_till is not None:
-                        new_servers_info = subscription.servers_info
-                        new_servers_info.append(ServerInfo(url, num_consecutive_failures, ignore_till))
-                        new_subscription = replace(subscription, servers_info=new_servers_info)
-                        subscriptions.remove(subscription)
-                        subscriptions.append(new_subscription)
+                elif url is not None and num_consecutive_failures is not None and ignore_till is not None:
+                    new_servers_info = subscription.servers_info
+                    new_servers_info.append(ServerInfo(url, num_consecutive_failures, ignore_till))
+                    new_subscription = replace(subscription, servers_info=new_servers_info)
+                    subscriptions.remove(subscription)
+                    subscriptions.append(new_subscription)
 
         return subscriptions