chia-blockchain 2.5.6rc2__py3-none-any.whl → 2.5.7__py3-none-any.whl

chia/data_layer/data_layer.py

@@ -15,6 +15,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union, cast, final
 
 import aiohttp
+from chia_rs.datalayer import ProofOfInclusion, ProofOfInclusionLayer
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint32, uint64
 
@@ -32,10 +33,9 @@ from chia.data_layer.data_layer_util import (
     PluginRemote,
     PluginStatus,
     Proof,
-    ProofOfInclusion,
-    ProofOfInclusionLayer,
     Root,
     ServerInfo,
+    Side,
     Status,
     StoreProofs,
     Subscription,
@@ -43,18 +43,16 @@ from chia.data_layer.data_layer_util import (
     TerminalNode,
     Unspecified,
     UnsubscribeData,
+    calculate_sibling_sides_integer,
+    collect_sibling_hashes,
+    get_delta_filename_path,
+    get_full_tree_filename_path,
     leaf_hash,
     unspecified,
 )
 from chia.data_layer.data_layer_wallet import DataLayerWallet, Mirror, verify_offer
 from chia.data_layer.data_store import DataStore
-from chia.data_layer.download_data import (
-    delete_full_file_if_exists,
-    get_delta_filename_path,
-    get_full_tree_filename_path,
-    insert_from_delta_file,
-    write_files_for_root,
-)
+from chia.data_layer.download_data import delete_full_file_if_exists, insert_from_delta_file, write_files_for_root
 from chia.data_layer.singleton_record import SingletonRecord
 from chia.protocols.outbound_message import NodeType
 from chia.rpc.rpc_server import StateChangedProtocol, default_get_connections
@@ -120,6 +118,8 @@ class DataLayer:
         _protocol_check: ClassVar[RpcServiceProtocol] = cast("DataLayer", None)
 
     db_path: Path
+    merkle_blobs_path: Path
+    key_value_blobs_path: Path
     config: dict[str, Any]
     root_path: Path
     log: logging.Logger
@@ -189,6 +189,12 @@ class DataLayer:
         server_files_replaced = server_files_path_from_config(config, root_path)
 
         db_path_replaced: str = config["database_path"].replace("CHALLENGE", config["selected_network"])
+        merkle_blobs_path_replaced: str = config.get(
+            "merkle_blobs_path", "data_layer/db/merkle_blobs_CHALLENGE"
+        ).replace("CHALLENGE", config["selected_network"])
+        key_value_blobs_path_replaced: str = config.get(
+            "key_value_blobs_path", "data_layer/db/key_value_blobs_CHALLENGE"
+        ).replace("CHALLENGE", config["selected_network"])
 
         self = cls(
             config=config,
@@ -196,6 +202,8 @@ class DataLayer:
             wallet_rpc_init=wallet_rpc_init,
             log=logging.getLogger(name if name is None else __name__),
             db_path=path_from_root(root_path, db_path_replaced),
+            merkle_blobs_path=path_from_root(root_path, merkle_blobs_path_replaced),
+            key_value_blobs_path=path_from_root(root_path, key_value_blobs_path_replaced),
             server_files_location=server_files_replaced,
             downloaders=downloaders,
             uploaders=uploaders,
@@ -209,6 +217,8 @@ class DataLayer:
         )
 
         self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.merkle_blobs_path.mkdir(parents=True, exist_ok=True)
+        self.key_value_blobs_path.mkdir(parents=True, exist_ok=True)
         self.server_files_location.mkdir(parents=True, exist_ok=True)
 
         return self
@@ -219,11 +229,18 @@ class DataLayer:
         if self.config.get("log_sqlite_cmds", False):
             sql_log_path = path_from_root(self.root_path, "log/data_sql.log")
             self.log.info(f"logging SQL commands to {sql_log_path}")
-
-        async with DataStore.managed(database=self.db_path, sql_log_path=sql_log_path) as self._data_store:
+        cache_capacity = self.config.get("merkle_blobs_cache_size", 1)
+
+        async with DataStore.managed(
+            database=self.db_path,
+            merkle_blobs_path=self.merkle_blobs_path,
+            key_value_blobs_path=self.key_value_blobs_path,
+            sql_log_path=sql_log_path,
+            cache_capacity=cache_capacity,
+        ) as self._data_store:
             self._wallet_rpc = await self.wallet_rpc_init
 
-            await self._data_store.migrate_db()
+            await self._data_store.migrate_db(self.server_files_location)
             self.periodically_manage_data_task = create_referenced_task(self.periodically_manage_data())
             try:
                 yield
@@ -278,7 +295,6 @@ class DataLayer:
     ) -> Optional[TransactionRecord]:
         status = Status.PENDING if submit_on_chain else Status.PENDING_BATCH
         await self.batch_insert(store_id=store_id, changelist=changelist, status=status)
-        await self.data_store.clean_node_table()
 
         if submit_on_chain:
             return await self.publish_update(store_id=store_id, fee=fee)
@@ -312,8 +328,6 @@ class DataLayer:
             status = Status.PENDING if submit_on_chain else Status.PENDING_BATCH
             await self.batch_insert(store_id=store_id, changelist=changelist, status=status)
 
-        await self.data_store.clean_node_table()
-
         if submit_on_chain:
             updates: list[LauncherRootPair] = []
             for store_id in store_ids:
@@ -570,7 +584,6 @@ class DataLayer:
                     and pending_root.status == Status.PENDING
                 ):
                     await self.data_store.change_root_status(pending_root, Status.COMMITTED)
-                    await self.data_store.build_ancestor_table_for_latest_root(store_id=store_id)
             await self.data_store.clear_pending_roots(store_id=store_id)
 
     async def fetch_and_validate(self, store_id: bytes32) -> None:
@@ -877,8 +890,6 @@ class DataLayer:
         # stop tracking first, then unsubscribe from the data store
         await self.wallet_rpc.dl_stop_tracking(DLStopTracking(store_id))
         await self.data_store.unsubscribe(store_id)
-        if not retain_data:
-            await self.data_store.delete_store_data(store_id)
 
         self.log.info(f"Unsubscribed to {store_id}")
         for file_path in paths:
@@ -1116,7 +1127,7 @@ class DataLayer:
                         node_hash=proof_of_inclusion.node_hash,
                         layers=tuple(
                             Layer(
-                                other_hash_side=layer.other_hash_side,
+                                other_hash_side=Side(layer.other_hash_side),
                                 other_hash=layer.other_hash,
                                 combined_hash=layer.combined_hash,
                             )
@@ -1183,7 +1194,6 @@ class DataLayer:
 
             verify_offer(maker=offer.maker, taker=offer.taker, summary=summary)
 
-        await self.data_store.clean_node_table()
         return offer
 
     async def take_offer(
@@ -1216,12 +1226,12 @@ class DataLayer:
                         for layer in proof.layers
                     ]
                     proof_of_inclusion = ProofOfInclusion(node_hash=proof.node_hash, layers=layers)
-                    sibling_sides_integer = proof_of_inclusion.sibling_sides_integer()
+                    sibling_sides_integer = calculate_sibling_sides_integer(proof_of_inclusion)
                     proofs_of_inclusion.append(
                         (
                             root.hex(),
                             str(sibling_sides_integer),
-                            ["0x" + sibling_hash.hex() for sibling_hash in proof_of_inclusion.sibling_hashes()],
+                            ["0x" + sibling_hash.hex() for sibling_hash in collect_sibling_hashes(proof_of_inclusion)],
                         )
                     )
 
@@ -1242,8 +1252,6 @@ class DataLayer:
                 },
             }
 
-        await self.data_store.clean_node_table()
-
         # Excluding wallet from transaction since failures in the wallet may occur
         # after the transaction is submitted to the chain.  If we roll back data we
         # may lose published data.

chia/data_layer/data_layer_errors.py

@@ -38,6 +38,11 @@ class KeyNotFoundError(Exception):
         super().__init__(f"Key not found: {key.hex()}")
 
 
+class MerkleBlobNotFoundError(Exception):
+    def __init__(self, root_hash: bytes32) -> None:
+        super().__init__(f"Cannot find merkle blob for root hash {root_hash.hex()}")
+
+
 class OfferIntegrityError(Exception):
     pass
 

chia/data_layer/data_layer_rpc_api.py

@@ -609,7 +609,7 @@ class DataLayerRpcApi:
         for key in request.keys:
             node = await self.service.data_store.get_node_by_key(store_id=request.store_id, key=key)
             pi = await self.service.data_store.get_proof_of_inclusion_by_hash(
-                store_id=request.store_id, node_hash=node.hash, use_optimized=True
+                store_id=request.store_id, node_hash=node.hash
             )
 
             proof = HashOnlyProof.from_key_value(

chia/data_layer/data_layer_service.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from chia.data_layer.data_layer import DataLayer
+from chia.data_layer.data_layer_api import DataLayerAPI
+from chia.data_layer.data_layer_rpc_api import DataLayerRpcApi
+from chia.server.start_service import Service
+
+DataLayerService = Service[DataLayer, DataLayerAPI, DataLayerRpcApi]

chia/data_layer/data_layer_util.py

@@ -4,9 +4,11 @@ import dataclasses
 from dataclasses import dataclass, field
 from enum import Enum, IntEnum
 from hashlib import sha256
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, Union
 
 import aiosqlite
+from chia_rs.datalayer import ProofOfInclusion, ProofOfInclusionLayer
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint64
 from typing_extensions import final
@@ -48,6 +50,45 @@ def key_hash(key: bytes) -> bytes32:
     return bytes32(sha256(b"\1" + key).digest())
 
 
+# TODO: allow Optional[bytes32] for `node_hash` and resolve the filenames here
+def get_full_tree_filename(store_id: bytes32, node_hash: bytes32, generation: int, group_by_store: bool = False) -> str:
+    if group_by_store:
+        return f"{store_id}/{node_hash}-full-{generation}-v1.0.dat"
+    return f"{store_id}-{node_hash}-full-{generation}-v1.0.dat"
+
+
+def get_delta_filename(store_id: bytes32, node_hash: bytes32, generation: int, group_by_store: bool = False) -> str:
+    if group_by_store:
+        return f"{store_id}/{node_hash}-delta-{generation}-v1.0.dat"
+    return f"{store_id}-{node_hash}-delta-{generation}-v1.0.dat"
+
+
+def get_full_tree_filename_path(
+    foldername: Path,
+    store_id: bytes32,
+    node_hash: bytes32,
+    generation: int,
+    group_by_store: bool = False,
+) -> Path:
+    if group_by_store:
+        path = foldername.joinpath(f"{store_id}")
+        return path.joinpath(f"{node_hash}-full-{generation}-v1.0.dat")
+    return foldername.joinpath(f"{store_id}-{node_hash}-full-{generation}-v1.0.dat")
+
+
+def get_delta_filename_path(
+    foldername: Path,
+    store_id: bytes32,
+    node_hash: bytes32,
+    generation: int,
+    group_by_store: bool = False,
+) -> Path:
+    if group_by_store:
+        path = foldername.joinpath(f"{store_id}")
+        return path.joinpath(f"{node_hash}-delta-{generation}-v1.0.dat")
+    return foldername.joinpath(f"{store_id}-{node_hash}-delta-{generation}-v1.0.dat")
+
+
 @dataclasses.dataclass(frozen=True)
 class PaginationData:
     total_pages: int
@@ -94,7 +135,6 @@ async def _dot_dump(
     root_hash: bytes32,
 ) -> str:
     terminal_nodes = await data_store.get_keys_values(store_id=store_id, root_hash=root_hash)
-    internal_nodes = await data_store.get_internal_nodes(store_id=store_id, root_hash=root_hash)
 
     n = 8
 
@@ -108,16 +148,7 @@ async def _dot_dump(
         value = terminal_node.value.hex()
         dot_nodes.append(f"""node_{hash} [shape=box, label="{hash[:n]}\\nkey: {key}\\nvalue: {value}"];""")
 
-    for internal_node in internal_nodes:
-        hash = internal_node.hash.hex()
-        left = internal_node.left_hash.hex()
-        right = internal_node.right_hash.hex()
-        dot_nodes.append(f"""node_{hash} [label="{hash[:n]}"]""")
-        dot_connections.append(f"""node_{hash} -> node_{left} [label="L"];""")
-        dot_connections.append(f"""node_{hash} -> node_{right} [label="R"];""")
-        dot_pair_boxes.append(
-            f"node [shape = box]; {{rank = same; node_{left}->node_{right}[style=invis]; rankdir = LR}}"
-        )
+    # TODO: implement for internal nodes. currently this prints only terminal nodes
 
     lines = [
         "digraph {",
@@ -130,11 +161,6 @@ async def _dot_dump(
     return "\n".join(lines)
 
 
-def row_to_node(row: aiosqlite.Row) -> Node:
-    cls = node_type_to_class[row["node_type"]]
-    return cls.from_row(row=row)
-
-
 class Status(IntEnum):
     PENDING = 1
     COMMITTED = 2
@@ -147,9 +173,9 @@ class NodeType(IntEnum):
 
 
 @final
-class Side(IntEnum):
-    LEFT = 0
-    RIGHT = 1
+class Side(uint8, Enum):
+    LEFT = uint8(0)
+    RIGHT = uint8(1)
 
     def other(self) -> Side:
         if self == Side.LEFT:
@@ -208,78 +234,12 @@ class TerminalNode:
         )
 
 
-@final
-@dataclass(frozen=True)
-class ProofOfInclusionLayer:
-    other_hash_side: Side
-    other_hash: bytes32
-    combined_hash: bytes32
-
-    @classmethod
-    def from_internal_node(
-        cls,
-        internal_node: InternalNode,
-        traversal_child_hash: bytes32,
-    ) -> ProofOfInclusionLayer:
-        return ProofOfInclusionLayer(
-            other_hash_side=internal_node.other_child_side(hash=traversal_child_hash),
-            other_hash=internal_node.other_child_hash(hash=traversal_child_hash),
-            combined_hash=internal_node.hash,
-        )
-
-    @classmethod
-    def from_hashes(cls, primary_hash: bytes32, other_hash_side: Side, other_hash: bytes32) -> ProofOfInclusionLayer:
-        combined_hash = calculate_internal_hash(
-            hash=primary_hash,
-            other_hash_side=other_hash_side,
-            other_hash=other_hash,
-        )
-
-        return cls(other_hash_side=other_hash_side, other_hash=other_hash, combined_hash=combined_hash)
-
-
-other_side_to_bit = {Side.LEFT: 1, Side.RIGHT: 0}
-
-
-@dataclass(frozen=True)
-class ProofOfInclusion:
-    node_hash: bytes32
-    # children before parents
-    layers: list[ProofOfInclusionLayer]
-
-    @property
-    def root_hash(self) -> bytes32:
-        if len(self.layers) == 0:
-            return self.node_hash
-
-        return self.layers[-1].combined_hash
-
-    def sibling_sides_integer(self) -> int:
-        return sum(other_side_to_bit[layer.other_hash_side] << index for index, layer in enumerate(self.layers))
-
-    def sibling_hashes(self) -> list[bytes32]:
-        return [layer.other_hash for layer in self.layers]
-
-    def as_program(self) -> Program:
-        return Program.to([self.sibling_sides_integer(), self.sibling_hashes()])
-
-    def valid(self) -> bool:
-        existing_hash = self.node_hash
-
-        for layer in self.layers:
-            calculated_hash = calculate_internal_hash(
-                hash=existing_hash, other_hash_side=layer.other_hash_side, other_hash=layer.other_hash
-            )
-
-            if calculated_hash != layer.combined_hash:
-                return False
-
-            existing_hash = calculated_hash
+def calculate_sibling_sides_integer(proof: ProofOfInclusion) -> int:
+    return sum((1 << index if layer.other_hash_side == Side.LEFT else 0) for index, layer in enumerate(proof.layers))
 
-        if existing_hash != self.root_hash:
-            return False
 
-        return True
+def collect_sibling_hashes(proof: ProofOfInclusion) -> list[bytes32]:
+    return [layer.other_hash for layer in proof.layers]
 
 
 @final

chia/data_layer/data_layer_wallet.py

@@ -6,13 +6,14 @@ import time
 from typing import TYPE_CHECKING, Any, ClassVar, Optional, cast
 
 from chia_rs import BlockRecord, CoinSpend, CoinState, G1Element, G2Element
+from chia_rs.datalayer import ProofOfInclusion, ProofOfInclusionLayer
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint32, uint64, uint128
 from clvm.EvalError import EvalError
 from typing_extensions import Unpack, final
 
 from chia.data_layer.data_layer_errors import LauncherCoinNotFoundError, OfferIntegrityError
-from chia.data_layer.data_layer_util import OfferStore, ProofOfInclusion, ProofOfInclusionLayer, StoreProofs, leaf_hash
+from chia.data_layer.data_layer_util import OfferStore, StoreProofs, leaf_hash
 from chia.data_layer.singleton_record import SingletonRecord
 from chia.server.ws_connection import WSChiaConnection
 from chia.types.blockchain_format.coin import Coin
@@ -536,8 +537,9 @@ class DataLayerWallet:
 
         dl_tx = TransactionRecord(
             confirmed_at_height=uint32(0),
-            created_at_time=uint64(int(time.time())),
+            created_at_time=uint64(time.time()),
             to_puzzle_hash=new_puz_hash,
+            to_address=self.wallet_state_manager.encode_puzzle_hash(new_puz_hash),
             amount=uint64(singleton_record.lineage_proof.amount),
             fee_amount=fee,
             confirmed=False,
@@ -545,7 +547,7 @@ class DataLayerWallet:
             spend_bundle=spend_bundle,
             additions=spend_bundle.additions(),
             removals=spend_bundle.removals(),
-            memos=list(compute_memos(spend_bundle).items()),
+            memos=compute_memos(spend_bundle),
             wallet_id=self.id(),
             sent_to=[],
             trade_id=None,
@@ -589,15 +591,14 @@ class DataLayerWallet:
         if coins is None or len(coins) == 0:
             if launcher_id is None:
                 raise ValueError("Not enough info to know which DL coin to send")
+        elif len(coins) != 1:
+            raise ValueError("The wallet can only send one DL coin at a time")
         else:
-            if len(coins) != 1:
-                raise ValueError("The wallet can only send one DL coin at a time")
+            record = await self.wallet_state_manager.dl_store.get_singleton_record(next(iter(coins)).name())
+            if record is None:
+                raise ValueError("The specified coin is not a tracked DL")
             else:
-                record = await self.wallet_state_manager.dl_store.get_singleton_record(next(iter(coins)).name())
-                if record is None:
-                    raise ValueError("The specified coin is not a tracked DL")
-                else:
-                    launcher_id = record.launcher_id
+                launcher_id = record.launcher_id
 
         if len(amounts) != 1 or len(puzzle_hashes) != 1:
             raise ValueError("The wallet can only send one DL coin to one place at a time")
@@ -731,8 +732,9 @@ class DataLayerWallet:
             interface.side_effects.transactions.append(
                 TransactionRecord(
                     confirmed_at_height=uint32(0),
-                    created_at_time=uint64(int(time.time())),
+                    created_at_time=uint64(time.time()),
                     to_puzzle_hash=new_puzhash,
+                    to_address=self.wallet_state_manager.encode_puzzle_hash(new_puzhash),
                     amount=uint64(mirror_coin.amount),
                     fee_amount=fee,
                     confirmed=False,
@@ -740,7 +742,7 @@ class DataLayerWallet:
                     spend_bundle=mirror_bundle,
                     additions=mirror_bundle.additions(),
                     removals=mirror_bundle.removals(),
-                    memos=list(compute_memos(mirror_bundle).items()),
+                    memos=compute_memos(mirror_bundle),
                     wallet_id=self.id(),  # This is being called before the wallet is created so we're using a ID of 0
                     sent_to=[],
                     trade_id=None,
@@ -1088,12 +1090,13 @@ class DataLayerWallet:
         # Create all of the new solutions
         new_spends: list[CoinSpend] = []
         for spend in offer.coin_spends():
+            spend_to_add = spend
             solution = Program.from_serialized(spend.solution)
             if match_dl_singleton(spend.puzzle_reveal)[0]:
                 try:
                     graftroot: Program = solution.at("rrffrf")
                 except EvalError:
-                    new_spends.append(spend)
+                    new_spends.append(spend_to_add)
                     continue
                 mod, curried_args_prg = graftroot.uncurry()
                 if mod == GRAFTROOT_DL_OFFERS:
@@ -1136,9 +1139,9 @@ class DataLayerWallet:
                             ]
                         )
                     )
-                    new_spend: CoinSpend = spend.replace(solution=new_solution.to_serialized())
-                    spend = new_spend
-            new_spends.append(spend)
+                    spend_to_add = spend.replace(solution=new_solution.to_serialized())
+
+            new_spends.append(spend_to_add)
 
         return Offer({}, WalletSpendBundle(new_spends, offer.aggregated_signature()), offer.driver_dict)
 
@@ -1224,7 +1227,7 @@ def verify_offer(
                 raise OfferIntegrityError("maker: invalid proof of inclusion found")
 
         # TODO: verify each kv hash to the proof's node hash
-        roots = {proof.root_hash for proof in proofs}
+        roots = {proof.root_hash() for proof in proofs}
         if len(roots) > 1:
             raise OfferIntegrityError("maker: multiple roots referenced for a single store id")
         if len(roots) < 1: