chia-blockchain 2.1.3__tar.gz → 2.1.4rc1__tar.gz

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chia-blockchain
-Version: 2.1.3
+Version: 2.1.4rc1
 Summary: Chia blockchain full node, farmer, timelord, and wallet.
 Home-page: https://chia.net/
 Author: Mariano Sorgente

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/benchmarks/mempool-long-lived.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import asyncio
 from dataclasses import dataclass
 from time import monotonic
-from typing import Dict, Optional
+from typing import Collection, Dict, List, Optional
 
 from chia_rs import G2Element
 from clvm.casts import int_to_bytes
@@ -81,8 +81,13 @@ def fake_block_record(block_height: uint32, timestamp: uint64) -> BenchBlockReco
 async def run_mempool_benchmark() -> None:
     coin_records: Dict[bytes32, CoinRecord] = {}
 
-    async def get_coin_record(coin_id: bytes32) -> Optional[CoinRecord]:
-        return coin_records.get(coin_id)
+    async def get_coin_record(coin_ids: Collection[bytes32]) -> List[CoinRecord]:
+        ret: List[CoinRecord] = []
+        for name in coin_ids:
+            r = coin_records.get(name)
+            if r is not None:
+                ret.append(r)
+        return ret
 
     timestamp = uint64(1631794488)
 

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/benchmarks/mempool.py

@@ -6,7 +6,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass
 from subprocess import check_call
 from time import monotonic
-from typing import Dict, Iterator, List, Optional, Tuple
+from typing import Collection, Dict, Iterator, List, Optional, Tuple
 
 from chia.consensus.coinbase import create_farmer_coin, create_pool_coin
 from chia.consensus.default_constants import DEFAULT_CONSTANTS
@@ -78,8 +78,13 @@ def fake_block_record(block_height: uint32, timestamp: uint64) -> BenchBlockReco
 async def run_mempool_benchmark() -> None:
     all_coins: Dict[bytes32, CoinRecord] = {}
 
-    async def get_coin_record(coin_id: bytes32) -> Optional[CoinRecord]:
-        return all_coins.get(coin_id)
+    async def get_coin_records(coin_ids: Collection[bytes32]) -> List[CoinRecord]:
+        ret: List[CoinRecord] = []
+        for name in coin_ids:
+            r = all_coins.get(name)
+            if r is not None:
+                ret.append(r)
+        return ret
 
     wt = WalletTool(DEFAULT_CONSTANTS)
 
@@ -156,7 +161,7 @@ async def run_mempool_benchmark() -> None:
         else:
             print("\n== Multi-threaded")
 
-        mempool = MempoolManager(get_coin_record, DEFAULT_CONSTANTS, single_threaded=single_threaded)
+        mempool = MempoolManager(get_coin_records, DEFAULT_CONSTANTS, single_threaded=single_threaded)
 
         height = start_height
         rec = fake_block_record(height, timestamp)
@@ -186,7 +191,7 @@ async def run_mempool_benchmark() -> None:
         print(f"  time: {stop - start:0.4f}s")
         print(f"  per call: {(stop - start) / total_bundles * 1000:0.2f}ms")
 
-        mempool = MempoolManager(get_coin_record, DEFAULT_CONSTANTS, single_threaded=single_threaded)
+        mempool = MempoolManager(get_coin_records, DEFAULT_CONSTANTS, single_threaded=single_threaded)
 
         height = start_height
         rec = fake_block_record(height, timestamp)

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/clvm/spend_sim.py

@@ -156,7 +156,7 @@ class SpendSim:
         self.db_wrapper = await DBWrapper2.create(database=uri, uri=True, reader_count=1, db_version=2)
 
         self.coin_store = await CoinStore.create(self.db_wrapper)
-        self.mempool_manager = MempoolManager(self.coin_store.get_coin_record, defaults)
+        self.mempool_manager = MempoolManager(self.coin_store.get_coin_records, defaults)
         self.defaults = defaults
 
         # Load the next data if there is any

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/consensus/default_constants.py

@@ -43,8 +43,8 @@ DEFAULT_CONSTANTS = ConsensusConstants(
         "3d8765d3a597ec1d99663f6c9816d915b9f68613ac94009884c4addaefcce6af"
     ),
     MAX_VDF_WITNESS_SIZE=64,
-    # Size of mempool = 50x the size of block
-    MEMPOOL_BLOCK_BUFFER=50,
+    # Size of mempool = 10x the size of block
+    MEMPOOL_BLOCK_BUFFER=10,
     # Max coin amount, fits into 64 bits
     MAX_COIN_AMOUNT=uint64((1 << 64) - 1),
     # Max block cost in clvm cost units

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/full_node/coin_store.py

@@ -146,7 +146,7 @@ class CoinStore:
                     return CoinRecord(coin, row[0], row[1], row[2], row[6])
         return None
 
-    async def get_coin_records(self, names: List[bytes32]) -> List[CoinRecord]:
+    async def get_coin_records(self, names: Collection[bytes32]) -> List[CoinRecord]:
         if len(names) == 0:
             return []
 

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/full_node/full_node.py

@@ -276,7 +276,7 @@ class FullNode:
             )
 
             self._mempool_manager = MempoolManager(
-                get_coin_record=self.coin_store.get_coin_record,
+                get_coin_records=self.coin_store.get_coin_records,
                 consensus_constants=self.constants,
                 multiprocessing_context=self.multiprocessing_context,
                 single_threaded=single_threaded,

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/full_node/mempool.py

@@ -48,11 +48,16 @@ class Mempool:
     _block_height: uint32
     _timestamp: uint64
 
+    _total_fee: int
+    _total_cost: int
+
     def __init__(self, mempool_info: MempoolInfo, fee_estimator: FeeEstimatorInterface):
         self._db_conn = sqlite3.connect(":memory:")
         self._items = {}
         self._block_height = uint32(0)
         self._timestamp = uint64(0)
+        self._total_fee = 0
+        self._total_cost = 0
 
         with self._db_conn:
             # name means SpendBundle hash
@@ -75,8 +80,6 @@ class Mempool:
                 """
             )
             self._db_conn.execute("CREATE INDEX name_idx ON tx(name)")
-            self._db_conn.execute("CREATE INDEX fee_sum ON tx(fee)")
-            self._db_conn.execute("CREATE INDEX cost_sum ON tx(cost)")
             self._db_conn.execute("CREATE INDEX feerate ON tx(fee_per_cost)")
             self._db_conn.execute(
                 "CREATE INDEX assert_before ON tx(assert_before_height, assert_before_seconds) "
@@ -121,16 +124,10 @@ class Mempool:
         )
 
     def total_mempool_fees(self) -> int:
-        with self._db_conn:
-            cursor = self._db_conn.execute("SELECT SUM(fee) FROM tx")
-            val = cursor.fetchone()[0]
-            return uint64(0) if val is None else uint64(val)
+        return self._total_fee
 
     def total_mempool_cost(self) -> CLVMCost:
-        with self._db_conn:
-            cursor = self._db_conn.execute("SELECT SUM(cost) FROM tx")
-            val = cursor.fetchone()[0]
-            return CLVMCost(uint64(0) if val is None else uint64(val))
+        return CLVMCost(uint64(self._total_cost))
 
     def all_items(self) -> Iterator[MempoolItem]:
         with self._db_conn:
@@ -189,28 +186,28 @@ class Mempool:
         Gets the minimum fpc rate that a transaction with specified cost will need in order to get included.
         """
 
-        if self.at_full_capacity(cost):
-            # TODO: make MempoolItem.cost be CLVMCost
-            current_cost = int(self.total_mempool_cost())
-
-            # Iterates through all spends in increasing fee per cost
-            with self._db_conn:
-                cursor = self._db_conn.execute("SELECT cost,fee_per_cost FROM tx ORDER BY fee_per_cost ASC, seq DESC")
-
-                item_cost: int
-                fee_per_cost: float
-                for item_cost, fee_per_cost in cursor:
-                    current_cost -= item_cost
-                    # Removing one at a time, until our transaction of size cost fits
-                    if current_cost + cost <= self.mempool_info.max_size_in_cost:
-                        return fee_per_cost
-
-            raise ValueError(
-                f"Transaction with cost {cost} does not fit in mempool of max cost {self.mempool_info.max_size_in_cost}"
-            )
-        else:
+        if not self.at_full_capacity(cost):
             return 0
 
+        # TODO: make MempoolItem.cost be CLVMCost
+        current_cost = self._total_cost
+
+        # Iterates through all spends in increasing fee per cost
+        with self._db_conn:
+            cursor = self._db_conn.execute("SELECT cost,fee_per_cost FROM tx ORDER BY fee_per_cost ASC, seq DESC")
+
+            item_cost: int
+            fee_per_cost: float
+            for item_cost, fee_per_cost in cursor:
+                current_cost -= item_cost
+                # Removing one at a time, until our transaction of size cost fits
+                if current_cost + cost <= self.mempool_info.max_size_in_cost:
+                    return fee_per_cost
+
+        raise ValueError(
+            f"Transaction with cost {cost} does not fit in mempool of max cost {self.mempool_info.max_size_in_cost}"
+        )
+
     def new_tx_block(self, block_height: uint32, timestamp: uint64) -> None:
         """
         Remove all items that became invalid because of this new height and
@@ -255,9 +252,19 @@ class Mempool:
         for batch in to_batches(items, SQLITE_MAX_VARIABLE_NUMBER):
             args = ",".join(["?"] * len(batch.entries))
             with self._db_conn:
+                cursor = self._db_conn.execute(
+                    f"SELECT SUM(cost), SUM(fee) FROM tx WHERE name in ({args})", batch.entries
+                )
+                cost_to_remove, fee_to_remove = cursor.fetchone()
+
                 self._db_conn.execute(f"DELETE FROM tx WHERE name in ({args})", batch.entries)
                 self._db_conn.execute(f"DELETE FROM spends WHERE tx in ({args})", batch.entries)
 
+            self._total_cost -= cost_to_remove
+            self._total_fee -= fee_to_remove
+            assert self._total_cost >= 0
+            assert self._total_fee >= 0
+
         if reason != MempoolRemoveReason.BLOCK_INCLUSION:
             info = FeeMempoolInfo(
                 self.mempool_info, self.total_mempool_cost(), self.total_mempool_fees(), datetime.now()
@@ -310,14 +317,15 @@ class Mempool:
                     if fee_per_cost > item.fee_per_cost:
                         return Err.INVALID_FEE_LOW_FEE
                     to_remove.append(name)
+
                 self.remove_from_pool(to_remove, MempoolRemoveReason.EXPIRED)
+
                 # if we don't find any entries, it's OK to add this entry
 
-            total_cost = int(self.total_mempool_cost())
-            if total_cost + item.cost > self.mempool_info.max_size_in_cost:
+            if self._total_cost + item.cost > self.mempool_info.max_size_in_cost:
                 # pick the items with the lowest fee per cost to remove
                 cursor = self._db_conn.execute(
-                    """SELECT name FROM tx
+                    """SELECT name, cost, fee FROM tx
                     WHERE name NOT IN (
                         SELECT name FROM (
                             SELECT name,
@@ -328,6 +336,7 @@ class Mempool:
                     (self.mempool_info.max_size_in_cost - item.cost,),
                 )
                 to_remove = [bytes32(row[0]) for row in cursor]
+
                 self.remove_from_pool(to_remove, MempoolRemoveReason.POOL_FULL)
 
             # TODO: In the future, for the "fee_per_cost" field, opt for
@@ -354,6 +363,9 @@ class Mempool:
                 item.spend_bundle, item.npc_result, item.height_added_to_mempool, item.bundle_coin_spends
             )
 
+            self._total_cost += item.cost
+            self._total_fee += item.fee
+
         info = FeeMempoolInfo(self.mempool_info, self.total_mempool_cost(), self.total_mempool_fees(), datetime.now())
         self.fee_estimator.add_mempool_item(info, MempoolItemInfo(item.cost, item.fee, item.height_added_to_mempool))
         return None
@@ -363,7 +375,7 @@ class Mempool:
         Checks whether the mempool is at full capacity and cannot accept a transaction with size cost.
         """
 
-        return self.total_mempool_cost() + cost > self.mempool_info.max_size_in_cost
+        return self._total_cost + cost > self.mempool_info.max_size_in_cost
 
     def create_bundle_from_mempool_items(
         self, item_inclusion_filter: Callable[[bytes32], bool]

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/full_node/mempool_manager.py

@@ -7,7 +7,7 @@ from concurrent.futures import Executor
 from concurrent.futures.process import ProcessPoolExecutor
 from dataclasses import dataclass
 from multiprocessing.context import BaseContext
-from typing import Awaitable, Callable, Dict, List, Optional, Set, Tuple, TypeVar
+from typing import Awaitable, Callable, Collection, Dict, List, Optional, Set, Tuple, TypeVar
 
 from chia_rs import ELIGIBLE_FOR_DEDUP, GTElement
 from chiabip158 import PyBIP158
@@ -146,7 +146,7 @@ class MempoolManager:
     pool: Executor
     constants: ConsensusConstants
     seen_bundle_hashes: Dict[bytes32, bytes32]
-    get_coin_record: Callable[[bytes32], Awaitable[Optional[CoinRecord]]]
+    get_coin_records: Callable[[Collection[bytes32]], Awaitable[List[CoinRecord]]]
     nonzero_fee_minimum_fpc: int
     mempool_max_total_cost: int
     # a cache of MempoolItems that conflict with existing items in the pool
@@ -159,7 +159,7 @@ class MempoolManager:
 
     def __init__(
         self,
-        get_coin_record: Callable[[bytes32], Awaitable[Optional[CoinRecord]]],
+        get_coin_records: Callable[[Collection[bytes32]], Awaitable[List[CoinRecord]]],
         consensus_constants: ConsensusConstants,
         multiprocessing_context: Optional[BaseContext] = None,
         *,
@@ -170,7 +170,7 @@ class MempoolManager:
         # Keep track of seen spend_bundles
         self.seen_bundle_hashes: Dict[bytes32, bytes32] = {}
 
-        self.get_coin_record = get_coin_record
+        self.get_coin_records = get_coin_records
 
         # The fee per cost must be above this amount to consider the fee "nonzero", and thus able to kick out other
         # transactions. This prevents spam. This is equivalent to 0.055 XCH per block, or about 0.00005 XCH for two
@@ -303,7 +303,12 @@ class MempoolManager:
         return ret
 
     async def add_spend_bundle(
-        self, new_spend: SpendBundle, npc_result: NPCResult, spend_name: bytes32, first_added_height: uint32
+        self,
+        new_spend: SpendBundle,
+        npc_result: NPCResult,
+        spend_name: bytes32,
+        first_added_height: uint32,
+        get_coin_records: Optional[Callable[[Collection[bytes32]], Awaitable[List[CoinRecord]]]] = None,
     ) -> Tuple[Optional[uint64], MempoolInclusionStatus, Optional[Err]]:
         """
         Validates and adds to mempool a new_spend with the given NPCResult, and spend_name, and the current mempool.
@@ -327,8 +332,14 @@ class MempoolManager:
         if existing_item is not None:
             return existing_item.cost, MempoolInclusionStatus.SUCCESS, None
 
+        if get_coin_records is None:
+            get_coin_records = self.get_coin_records
         err, item, remove_items = await self.validate_spend_bundle(
-            new_spend, npc_result, spend_name, first_added_height
+            new_spend,
+            npc_result,
+            spend_name,
+            first_added_height,
+            get_coin_records,
         )
         if err is None:
             # No error, immediately add to mempool, after removing conflicting TXs.
@@ -358,6 +369,7 @@ class MempoolManager:
         npc_result: NPCResult,
         spend_name: bytes32,
         first_added_height: uint32,
+        get_coin_records: Callable[[Collection[bytes32]], Awaitable[List[CoinRecord]]],
     ) -> Tuple[Optional[Err], Optional[MempoolItem], List[bytes32]]:
         """
         Validates new_spend with the given NPCResult, and spend_name, and the current mempool. The mempool should
@@ -420,11 +432,14 @@ class MempoolManager:
 
         removal_record_dict: Dict[bytes32, CoinRecord] = {}
         removal_amount: int = 0
+        removal_records = await get_coin_records(removal_names)
+        for record in removal_records:
+            removal_record_dict[record.coin.name()] = record
+
         for name in removal_names:
-            removal_record = await self.get_coin_record(name)
-            if removal_record is None and name not in additions_dict:
+            if name not in removal_record_dict and name not in additions_dict:
                 return Err.UNKNOWN_UNSPENT, None, []
-            elif name in additions_dict:
+            if name in additions_dict:
                 removal_coin = additions_dict[name]
                 # The timestamp and block-height of this coin being spent needs
                 # to be consistent with what we use to check time-lock
@@ -440,10 +455,10 @@ class MempoolManager:
                     False,
                     self.peak.timestamp,
                 )
-
-            assert removal_record is not None
+                removal_record_dict[name] = removal_record
+            else:
+                removal_record = removal_record_dict[name]
             removal_amount = removal_amount + removal_record.coin.amount
-            removal_record_dict[name] = removal_record
 
         fees = uint64(removal_amount - addition_amount)
 
@@ -614,12 +629,44 @@ class MempoolManager:
                     spendbundle_ids_to_remove.add(item.name)
             self.mempool.remove_from_pool(list(spendbundle_ids_to_remove), MempoolRemoveReason.BLOCK_INCLUSION)
         else:
+            log.warning(
+                "updating the mempool using the slow-path. "
+                f"peak: {self.peak.header_hash} "
+                f"new-peak-prev: {new_peak.prev_transaction_block_hash} "
+                f"coins: {'not set' if spent_coins is None else 'set'}"
+            )
             old_pool = self.mempool
             self.mempool = Mempool(old_pool.mempool_info, old_pool.fee_estimator)
             self.seen_bundle_hashes = {}
+
+            # in order to make this a bit quicker, we look-up all the spends in
+            # a single query, rather than one at a time.
+            coin_records: Dict[bytes32, CoinRecord] = {}
+
+            removals: Set[bytes32] = set()
+            for item in old_pool.all_items():
+                for s in item.spend_bundle.coin_spends:
+                    removals.add(s.coin.name())
+
+            for record in await self.get_coin_records(removals):
+                name = record.coin.name()
+                coin_records[name] = record
+
+            async def local_get_coin_records(names: Collection[bytes32]) -> List[CoinRecord]:
+                ret: List[CoinRecord] = []
+                for name in names:
+                    r = coin_records.get(name)
+                    if r is not None:
+                        ret.append(r)
+                return ret
+
             for item in old_pool.all_items():
                 _, result, err = await self.add_spend_bundle(
-                    item.spend_bundle, item.npc_result, item.spend_bundle_name, item.height_added_to_mempool
+                    item.spend_bundle,
+                    item.npc_result,
+                    item.spend_bundle_name,
+                    item.height_added_to_mempool,
+                    local_get_coin_records,
                 )
                 # Only add to `seen` if inclusion worked, so it can be resubmitted in case of a reorg
                 if result == MempoolInclusionStatus.SUCCESS:
@@ -636,7 +683,11 @@ class MempoolManager:
         txs_added = []
         for item in potential_txs.values():
             cost, status, error = await self.add_spend_bundle(
-                item.spend_bundle, item.npc_result, item.spend_bundle_name, item.height_added_to_mempool
+                item.spend_bundle,
+                item.npc_result,
+                item.spend_bundle_name,
+                item.height_added_to_mempool,
+                self.get_coin_records,
             )
             if status == MempoolInclusionStatus.SUCCESS:
                 txs_added.append((item.spend_bundle, item.npc_result, item.spend_bundle_name))

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/server/server.py

@@ -682,15 +682,15 @@ class ChiaServer:
             return True
         inbound_count = len(self.get_connections(node_type, outbound=False))
         if node_type == NodeType.FULL_NODE:
-            return inbound_count < cast(int, self.config["target_peer_count"]) - cast(
-                int, self.config["target_outbound_peer_count"]
+            return inbound_count < cast(int, self.config.get("target_peer_count", 40)) - cast(
+                int, self.config.get("target_outbound_peer_count", 8)
             )
         if node_type == NodeType.WALLET:
-            return inbound_count < cast(int, self.config["max_inbound_wallet"])
+            return inbound_count < cast(int, self.config.get("max_inbound_wallet", 20))
         if node_type == NodeType.FARMER:
-            return inbound_count < cast(int, self.config["max_inbound_farmer"])
+            return inbound_count < cast(int, self.config.get("max_inbound_farmer", 10))
         if node_type == NodeType.TIMELORD:
-            return inbound_count < cast(int, self.config["max_inbound_timelord"])
+            return inbound_count < cast(int, self.config.get("max_inbound_timelord", 5))
         return True
 
     def is_trusted_peer(self, peer: WSChiaConnection, trusted_peers: Dict[str, Any]) -> bool:

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/server/start_full_node.py

@@ -90,7 +90,7 @@ def main() -> int:
 
     with maybe_manage_task_instrumentation(enable=os.environ.get("CHIA_INSTRUMENT_NODE") is not None):
         service_config = load_config_cli(DEFAULT_ROOT_PATH, "config.yaml", SERVICE_NAME)
-        target_peer_count = service_config.get("target_peer_count", 80) - service_config.get(
+        target_peer_count = service_config.get("target_peer_count", 40) - service_config.get(
             "target_outbound_peer_count", 8
         )
         if target_peer_count < 0:

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/util/db_synchronous.py

@@ -9,7 +9,7 @@ def db_synchronous_on(setting: str) -> str:
     if setting == "full":
         return "FULL"
 
-    # for now, default to synchronous=FULL mode. This can be made more
+    # for now, default to synchronous=NORMAL mode. This can be made more
     # sophisticated in the future. There are still material performance
     # improvements to be had in cases where the risks are low.
 
@@ -18,4 +18,4 @@ def db_synchronous_on(setting: str) -> str:
     # if type == DRIVE_FIXED or type == DRIVE_RAMDISK:
     #     return "OFF"
 
-    return "FULL"
+    return "NORMAL"

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia/util/initial-config.yaml

@@ -408,7 +408,7 @@ full_node:
   # How long to wait for a peer connection
   peer_connect_timeout: 30
   # Accept peers until this number of connections
-  target_peer_count: 80
+  target_peer_count: 40
   # Initiate outbound connections until this number is hit.
   target_outbound_peer_count: 8
   # IPv4/IPv6 network addresses and CIDR blocks allowed to connect even when target_peer_count has been hit.

{chia-blockchain-2.1.3 → chia-blockchain-2.1.4rc1}/chia_blockchain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chia-blockchain
-Version: 2.1.3
+Version: 2.1.4rc1
 Summary: Chia blockchain full node, farmer, timelord, and wallet.
 Home-page: https://chia.net/
 Author: Mariano Sorgente