chia-blockchain 2.5.6rc2__py3-none-any.whl → 2.5.7__py3-none-any.whl

chia/harvester/harvester_api.py

@@ -3,22 +3,24 @@ from __future__ import annotations
 import asyncio
 import logging
 import time
+from collections.abc import Awaitable, Sequence
 from pathlib import Path
 from typing import TYPE_CHECKING, ClassVar, Optional, cast
 
-from chia_rs import AugSchemeMPL, G1Element, G2Element, PlotSize, ProofOfSpace
+from chia_rs import AugSchemeMPL, G1Element, G2Element, ProofOfSpace
 from chia_rs.sized_bytes import bytes32
-from chia_rs.sized_ints import uint8, uint32, uint64
+from chia_rs.sized_ints import uint32, uint64
 
 from chia.consensus.pot_iterations import (
     calculate_iterations_quality,
     calculate_sp_interval_iters,
 )
 from chia.harvester.harvester import Harvester
+from chia.plotting.prover import PlotVersion
 from chia.plotting.util import PlotInfo, parse_plot_info
 from chia.protocols import harvester_protocol
 from chia.protocols.farmer_protocol import FarmingInfo
-from chia.protocols.harvester_protocol import Plot, PlotSyncResponse
+from chia.protocols.harvester_protocol import PartialProofsData, Plot, PlotSyncResponse
 from chia.protocols.outbound_message import Message, make_msg
 from chia.protocols.protocol_message_types import ProtocolMessageTypes
 from chia.server.api_protocol import ApiMetadata
@@ -26,8 +28,11 @@ from chia.server.ws_connection import WSChiaConnection
 from chia.types.blockchain_format.proof_of_space import (
     calculate_pos_challenge,
     calculate_prefix_bits,
+    calculate_required_plot_strength,
     generate_plot_public_key,
+    make_pos,
     passes_plot_filter,
+    quality_for_partial_proof,
 )
 from chia.wallet.derive_keys import master_sk_to_local_sk
 
@@ -49,6 +54,60 @@ class HarvesterAPI:
     def ready(self) -> bool:
         return True
 
+    def _plot_passes_filter(self, plot_info: PlotInfo, challenge: harvester_protocol.NewSignagePointHarvester2) -> bool:
+        filter_prefix_bits = calculate_prefix_bits(
+            self.harvester.constants,
+            challenge.peak_height,
+            plot_info.prover.get_size(),
+        )
+        return passes_plot_filter(
+            filter_prefix_bits,
+            plot_info.prover.get_id(),
+            challenge.challenge_hash,
+            challenge.sp_hash,
+        )
+
+    async def _handle_v1_responses(
+        self,
+        awaitables: Sequence[Awaitable[tuple[Path, list[harvester_protocol.NewProofOfSpace]]]],
+        start_time: float,
+        peer: WSChiaConnection,
+    ) -> int:
+        proofs_found = 0
+        for filename_sublist_awaitable in asyncio.as_completed(awaitables):
+            filename, sublist = await filename_sublist_awaitable
+            time_taken = time.monotonic() - start_time
+            if time_taken > 8:
+                self.harvester.log.warning(
+                    f"Looking up qualities on {filename} took: {time_taken}. This should be below 8 seconds"
+                    f" to minimize risk of losing rewards."
+                )
+            for response in sublist:
+                proofs_found += 1
+                msg = make_msg(ProtocolMessageTypes.new_proof_of_space, response)
+                await peer.send_message(msg)
+        return proofs_found
+
+    async def _handle_v2_responses(
+        self, v2_awaitables: Sequence[Awaitable[Optional[PartialProofsData]]], start_time: float, peer: WSChiaConnection
+    ) -> int:
+        partial_proofs_found = 0
+        for quality_awaitable in asyncio.as_completed(v2_awaitables):
+            partial_proofs_data = await quality_awaitable
+            if partial_proofs_data is None:
+                continue
+            time_taken = time.monotonic() - start_time
+            if time_taken > 8:
+                self.harvester.log.warning(
+                    f"Looking up partial proofs on {partial_proofs_data.plot_identifier}"
+                    f"took: {time_taken}. This should be below 8 seconds"
+                    f"to minimize risk of losing rewards."
+                )
+            partial_proofs_found += len(partial_proofs_data.partial_proofs)
+            msg = make_msg(ProtocolMessageTypes.partial_proofs, partial_proofs_data)
+            await peer.send_message(msg)
+        return partial_proofs_found
+
     @metadata.request(peer_required=True)
     async def harvester_handshake(
         self, harvester_handshake: harvester_protocol.HarvesterHandshake, peer: WSChiaConnection
@@ -94,10 +153,82 @@ class HarvesterAPI:
         start = time.monotonic()
         assert len(new_challenge.challenge_hash) == 32
 
+        required_plot_strength = calculate_required_plot_strength(
+            self.harvester.constants, new_challenge.last_tx_height
+        )
+
         loop = asyncio.get_running_loop()
 
+        def blocking_lookup_v2_partial_proofs(filename: Path, plot_info: PlotInfo) -> Optional[PartialProofsData]:
+            # Uses the V2 Prover object to lookup qualities only. No full proofs generated.
+            try:
+                plot_id = plot_info.prover.get_id()
+                sp_challenge_hash = calculate_pos_challenge(
+                    plot_id,
+                    new_challenge.challenge_hash,
+                    new_challenge.sp_hash,
+                )
+                partial_proofs = plot_info.prover.get_partial_proofs_for_challenge(
+                    sp_challenge_hash, required_plot_strength
+                )
+
+                # If no partial proofs are found, return None
+                if len(partial_proofs) == 0:
+                    return None
+
+                # Get the appropriate difficulty for this plot
+                difficulty = new_challenge.difficulty
+                sub_slot_iters = new_challenge.sub_slot_iters
+                if plot_info.pool_contract_puzzle_hash is not None:
+                    # Check for pool-specific difficulty
+                    for pool_difficulty in new_challenge.pool_difficulties:
+                        if pool_difficulty.pool_contract_puzzle_hash == plot_info.pool_contract_puzzle_hash:
+                            difficulty = pool_difficulty.difficulty
+                            sub_slot_iters = pool_difficulty.sub_slot_iters
+                            break
+
+                # Filter qualities that pass the required_iters check (same as V1 flow)
+                good_partial_proofs = []
+                sp_interval_iters = calculate_sp_interval_iters(self.harvester.constants, sub_slot_iters)
+
+                for partial_proof in partial_proofs:
+                    quality_str = quality_for_partial_proof(partial_proof, new_challenge.challenge_hash)
+                    required_iters: uint64 = calculate_iterations_quality(
+                        self.harvester.constants,
+                        quality_str,
+                        plot_info.prover.get_size(),
+                        difficulty,
+                        new_challenge.sp_hash,
+                        sub_slot_iters,
+                        new_challenge.last_tx_height,
+                    )
+
+                    if required_iters < sp_interval_iters:
+                        good_partial_proofs.append(partial_proof)
+
+                if len(good_partial_proofs) == 0:
+                    return None
+
+                size = plot_info.prover.get_size().size_v2
+                assert size is not None
+                return PartialProofsData(
+                    new_challenge.challenge_hash,
+                    new_challenge.sp_hash,
+                    good_partial_proofs[0].hex() + str(filename.resolve()),
+                    good_partial_proofs,
+                    new_challenge.signage_point_index,
+                    size,
+                    plot_info.pool_public_key,
+                    plot_info.pool_contract_puzzle_hash,
+                    plot_info.plot_public_key,
+                )
+                return None
+            except Exception:
+                self.harvester.log.exception("Failed V2 partial proof lookup")
+                return None
+
         def blocking_lookup(filename: Path, plot_info: PlotInfo) -> list[tuple[bytes32, ProofOfSpace]]:
-            # Uses the DiskProver object to lookup qualities. This is a blocking call,
+            # Uses the Prover object to lookup qualities. This is a blocking call,
             # so it should be run in a thread pool.
             try:
                 plot_id = plot_info.prover.get_id()
@@ -147,11 +278,10 @@ class HarvesterAPI:
 
                     # Found proofs of space (on average 1 is expected per plot)
                     for index, quality_str in enumerate(quality_strings):
-                        # TODO: todo_v2_plots
                         required_iters: uint64 = calculate_iterations_quality(
                             self.harvester.constants,
                             quality_str,
-                            PlotSize.make_v1(plot_info.prover.get_size()),
+                            plot_info.prover.get_size(),
                             difficulty,
                             new_challenge.sp_hash,
                             sub_slot_iters,
@@ -200,12 +330,12 @@ class HarvesterAPI:
                             responses.append(
                                 (
                                     quality_str,
-                                    ProofOfSpace(
+                                    make_pos(
                                         sp_challenge_hash,
                                         plot_info.pool_public_key,
                                         plot_info.pool_contract_puzzle_hash,
                                         plot_info.plot_public_key,
-                                        uint8(plot_info.prover.get_size()),
+                                        plot_info.prover.get_size(),
                                         proof_xs,
                                     ),
                                 )
@@ -218,7 +348,7 @@ class HarvesterAPI:
         async def lookup_challenge(
             filename: Path, plot_info: PlotInfo
         ) -> tuple[Path, list[harvester_protocol.NewProofOfSpace]]:
-            # Executes a DiskProverLookup in a thread pool, and returns responses
+            # Executes a ProverLookup in a thread pool, and returns responses
             all_responses: list[harvester_protocol.NewProofOfSpace] = []
             if self.harvester._shut_down:
                 return filename, []
@@ -241,6 +371,7 @@ class HarvesterAPI:
             return filename, all_responses
 
         awaitables = []
+        v2_awaitables = []
         passed = 0
         total = 0
         with self.harvester.plot_manager:
@@ -249,21 +380,19 @@ class HarvesterAPI:
                 # Passes the plot filter (does not check sp filter yet though, since we have not reached sp)
                 # This is being executed at the beginning of the slot
                 total += 1
-
-                # TODO: todo_v2_plots support v2 plots in PlotManager
-                filter_prefix_bits = uint8(
-                    calculate_prefix_bits(
-                        self.harvester.constants,
-                        new_challenge.peak_height,
-                        PlotSize.make_v1(try_plot_info.prover.get_size()),
+                if not self._plot_passes_filter(try_plot_info, new_challenge):
+                    continue
+                if try_plot_info.prover.get_version() == PlotVersion.V2:
+                    v2_awaitables.append(
+                        loop.run_in_executor(
+                            self.harvester.executor,
+                            blocking_lookup_v2_partial_proofs,
+                            try_plot_filename,
+                            try_plot_info,
+                        )
                     )
-                )
-                if passes_plot_filter(
-                    filter_prefix_bits,
-                    try_plot_info.prover.get_id(),
-                    new_challenge.challenge_hash,
-                    new_challenge.sp_hash,
-                ):
+                    passed += 1
+                else:
                     passed += 1
                     awaitables.append(lookup_challenge(try_plot_filename, try_plot_info))
             self.harvester.log.debug(f"new_signage_point_harvester {passed} plots passed the plot filter")
@@ -271,23 +400,26 @@ class HarvesterAPI:
         # Concurrently executes all lookups on disk, to take advantage of multiple disk parallelism
         time_taken = time.monotonic() - start
         total_proofs_found = 0
-        for filename_sublist_awaitable in asyncio.as_completed(awaitables):
-            filename, sublist = await filename_sublist_awaitable
-            time_taken = time.monotonic() - start
-            if time_taken > 8:
-                self.harvester.log.warning(
-                    f"Looking up qualities on {filename} took: {time_taken}. This should be below 8 seconds"
-                    f" to minimize risk of losing rewards."
-                )
-            else:
-                pass
-                # self.harvester.log.info(f"Looking up qualities on {filename} took: {time_taken}")
-            for response in sublist:
-                total_proofs_found += 1
-                msg = make_msg(ProtocolMessageTypes.new_proof_of_space, response)
-                await peer.send_message(msg)
-
-        now = uint64(int(time.time()))
+        total_v2_partial_proofs_found = 0
+
+        # run both concurrently
+        tasks = []
+        if awaitables:
+            tasks.append(self._handle_v1_responses(awaitables, start, peer))
+        if v2_awaitables:
+            tasks.append(self._handle_v2_responses(v2_awaitables, start, peer))
+
+        if tasks:
+            results = await asyncio.gather(*tasks)
+            if len(results) == 2:
+                total_proofs_found, total_v2_partial_proofs_found = results
+            elif len(results) == 1:
+                if awaitables:
+                    total_proofs_found = results[0]
+                else:
+                    total_v2_partial_proofs_found = results[0]
+
+        now = uint64(time.time())
 
         farming_info = FarmingInfo(
             new_challenge.challenge_hash,
@@ -302,9 +434,10 @@ class HarvesterAPI:
         await peer.send_message(pass_msg)
 
         self.harvester.log.info(
-            f"{len(awaitables)} plots were eligible for farming {new_challenge.challenge_hash.hex()[:10]}..."
-            f" Found {total_proofs_found} proofs. Time: {time_taken:.5f} s. "
-            f"Total {self.harvester.plot_manager.plot_count()} plots"
+            f"challenge_hash: {new_challenge.challenge_hash.hex()[:10]} ..."
+            f"{len(awaitables) + len(v2_awaitables)} plots were eligible for farming challenge"
+            f"Found {total_proofs_found} V1 proofs and {total_v2_partial_proofs_found} V2 qualities."
+            f" Time: {time_taken:.5f} s. Total {self.harvester.plot_manager.plot_count()} plots"
         )
         self.harvester.state_changed(
             "farming_info",
@@ -312,7 +445,8 @@ class HarvesterAPI:
                 "challenge_hash": new_challenge.challenge_hash.hex(),
                 "total_plots": self.harvester.plot_manager.plot_count(),
                 "found_proofs": total_proofs_found,
-                "eligible_plots": len(awaitables),
+                "found_v2_partial_proofs": total_v2_partial_proofs_found,
+                "eligible_plots": len(awaitables) + len(v2_awaitables),
                 "time": time_taken,
             },
         )

chia/harvester/harvester_service.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from chia.harvester.harvester import Harvester
+from chia.harvester.harvester_api import HarvesterAPI
+from chia.harvester.harvester_rpc_api import HarvesterRpcApi
+from chia.server.start_service import Service
+
+HarvesterService = Service[Harvester, HarvesterAPI, HarvesterRpcApi]

chia/{server → harvester}/start_harvester.py

@@ -13,8 +13,8 @@ from chia.consensus.default_constants import DEFAULT_CONSTANTS, update_testnet_o
 from chia.harvester.harvester import Harvester
 from chia.harvester.harvester_api import HarvesterAPI
 from chia.harvester.harvester_rpc_api import HarvesterRpcApi
+from chia.harvester.harvester_service import HarvesterService
 from chia.protocols.outbound_message import NodeType
-from chia.server.aliases import HarvesterService
 from chia.server.resolve_peer_info import get_unresolved_peer_infos
 from chia.server.signal_handlers import SignalHandlers
 from chia.server.start_service import RpcInfo, Service, async_run

chia/introducer/introducer_service.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from chia.full_node.full_node_rpc_api import FullNodeRpcApi
+from chia.introducer.introducer import Introducer
+from chia.introducer.introducer_api import IntroducerAPI
+from chia.server.start_service import Service
+
+IntroducerService = Service[Introducer, IntroducerAPI, FullNodeRpcApi]

chia/{server → introducer}/start_introducer.py

@@ -8,8 +8,8 @@ from typing import Any, Optional
 from chia.apis import ApiProtocolRegistry
 from chia.introducer.introducer import Introducer
 from chia.introducer.introducer_api import IntroducerAPI
+from chia.introducer.introducer_service import IntroducerService
 from chia.protocols.outbound_message import NodeType
-from chia.server.aliases import IntroducerService
 from chia.server.signal_handlers import SignalHandlers
 from chia.server.start_service import Service, async_run
 from chia.util.chia_logging import initialize_service_logging

chia/plot_sync/receiver.py

@@ -6,6 +6,7 @@ from collections.abc import Awaitable, Collection, Sequence
 from dataclasses import dataclass, field
 from typing import Any, Callable, Optional, Union
 
+from chia_rs import PlotSize
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import int16, uint32, uint64
 from typing_extensions import Protocol
@@ -349,7 +350,11 @@ class Receiver:
         self._duplicates = self._current_sync.delta.duplicates.additions.copy()
         self._total_plot_size = sum(plot.file_size for plot in self._plots.values())
         self._total_effective_plot_size = int(
-            sum(UI_ACTUAL_SPACE_CONSTANT_FACTOR * int(_expected_plot_size(plot.size)) for plot in self._plots.values())
+            # TODO: todo_v2_plots support v2 plots
+            sum(
+                UI_ACTUAL_SPACE_CONSTANT_FACTOR * int(_expected_plot_size(PlotSize.make_v1(plot.size)))
+                for plot in self._plots.values()
+            )
         )
         # Save current sync as last sync and create a new current sync
         self._last_sync = self._current_sync

chia/plot_sync/sender.py

@@ -37,16 +37,19 @@ log = logging.getLogger(__name__)
 def _convert_plot_info_list(plot_infos: list[PlotInfo]) -> list[Plot]:
     converted: list[Plot] = []
     for plot_info in plot_infos:
+        # TODO: todo_v2_plots support v2 plots
+        k = plot_info.prover.get_size().size_v1
+        assert k is not None
         converted.append(
             Plot(
                 filename=plot_info.prover.get_filename(),
-                size=plot_info.prover.get_size(),
+                size=k,
                 plot_id=plot_info.prover.get_id(),
                 pool_public_key=plot_info.pool_public_key,
                 pool_contract_puzzle_hash=plot_info.pool_contract_puzzle_hash,
                 plot_public_key=plot_info.plot_public_key,
                 file_size=uint64(plot_info.file_size),
-                time_modified=uint64(int(plot_info.time_modified)),
+                time_modified=uint64(plot_info.time_modified),
                 compression_level=plot_info.prover.get_compression_level(),
             )
         )
@@ -72,7 +75,7 @@ class MessageGenerator(Generic[T]):
     args: Iterable[object]
 
     def generate(self) -> tuple[PlotSyncIdentifier, T]:
-        identifier = PlotSyncIdentifier(uint64(int(time.time())), self.sync_id, self.message_id)
+        identifier = PlotSyncIdentifier(uint64(time.time()), self.sync_id, self.message_id)
         payload = self.payload_type(identifier, *self.args)
         return identifier, payload
 
@@ -277,7 +280,7 @@ class Sender:
             PlotSyncStart,
             initial,
             self._last_sync_id,
-            uint32(int(count)),
+            uint32(count),
             self._harvesting_mode,
         )
 

chia/plotting/cache.py

@@ -7,13 +7,16 @@ from collections.abc import ItemsView, KeysView, ValuesView
 from dataclasses import dataclass, field
 from math import ceil
 from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from chia.plotting.prover import ProverProtocol
 
 from chia_rs import G1Element
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint16, uint64
-from chiapos import DiskProver
 
+from chia.plotting.prover import get_prover_from_bytes
 from chia.plotting.util import parse_plot_info
 from chia.types.blockchain_format.proof_of_space import generate_plot_public_key
 from chia.util.streamable import Streamable, VersionedBlob, streamable
@@ -43,7 +46,7 @@ class CacheDataV1(Streamable):
 
 @dataclass
 class CacheEntry:
-    prover: DiskProver
+    prover: ProverProtocol
     farmer_public_key: G1Element
     pool_public_key: Optional[G1Element]
     pool_contract_puzzle_hash: Optional[bytes32]
@@ -51,7 +54,7 @@ class CacheEntry:
     last_use: float
 
     @classmethod
-    def from_disk_prover(cls, prover: DiskProver) -> CacheEntry:
+    def from_prover(cls, prover: ProverProtocol) -> CacheEntry:
         (
             pool_public_key_or_puzzle_hash,
             farmer_public_key,
@@ -113,7 +116,7 @@ class Cache:
                     cache_entry.pool_public_key,
                     cache_entry.pool_contract_puzzle_hash,
                     cache_entry.plot_public_key,
-                    uint64(int(cache_entry.last_use)),
+                    uint64(cache_entry.last_use),
                 )
                 for path, cache_entry in self.items()
             }
@@ -149,8 +152,9 @@ class Cache:
                     39: 44367,
                 }
                 for path, cache_entry in cache_data.entries:
+                    prover: ProverProtocol = get_prover_from_bytes(path, cache_entry.prover_data)
                     new_entry = CacheEntry(
-                        DiskProver.from_bytes(cache_entry.prover_data),
+                        prover,
                         cache_entry.farmer_public_key,
                         cache_entry.pool_public_key,
                         cache_entry.pool_contract_puzzle_hash,
@@ -161,28 +165,33 @@ class Cache:
                     #       it's here to filter invalid cache entries coming from bladebit RAM plotting.
                     #       Related: - https://github.com/Chia-Network/chia-blockchain/issues/13084
                     #                - https://github.com/Chia-Network/chiapos/pull/337
-                    k = new_entry.prover.get_size()
-                    if k not in estimated_c2_sizes:
-                        estimated_c2_sizes[k] = ceil(2**k / 100_000_000) * ceil(k / 8)
-                    memo_size = len(new_entry.prover.get_memo())
-                    prover_size = len(cache_entry.prover_data)
-                    # Estimated C2 size + memo size + 2000 (static data + path)
-                    # static data: version(2) + table pointers (<=96) + id(32) + k(1) => ~130
-                    # path: up to ~1870, all above will lead to false positive.
-                    # See https://github.com/Chia-Network/chiapos/blob/3ee062b86315823dd775453ad320b8be892c7df3/src/prover_disk.hpp#L282-L287  # noqa: E501
-
-                    # Use experimental measurements if more than estimates
-                    # https://github.com/Chia-Network/chia-blockchain/issues/16063
-                    check_size = estimated_c2_sizes[k] + memo_size + 2000
-                    if k in measured_sizes:
-                        check_size = max(check_size, measured_sizes[k])
-
-                    if prover_size > check_size:
-                        log.warning(
-                            "Suspicious cache entry dropped. Recommended: stop the harvester, remove "
-                            f"{self._path}, restart. Entry: size {prover_size}, path {path}"
-                        )
-                    else:
+                    ps = new_entry.prover.get_size()
+                    if ps.size_v1 is not None:
+                        k = ps.size_v1
+                        if k not in estimated_c2_sizes:
+                            estimated_c2_sizes[k] = ceil(2**k / 100_000_000) * ceil(k / 8)
+                        memo_size = len(new_entry.prover.get_memo())
+                        prover_size = len(cache_entry.prover_data)
+                        # Estimated C2 size + memo size + 2000 (static data + path)
+                        # static data: version(2) + table pointers (<=96) + id(32) + k(1) => ~130
+                        # path: up to ~1870, all above will lead to false positive.
+                        # See https://github.com/Chia-Network/chiapos/blob/3ee062b86315823dd775453ad320b8be892c7df3/src/prover_disk.hpp#L282-L287  # noqa: E501
+
+                        # Use experimental measurements if more than estimates
+                        # https://github.com/Chia-Network/chia-blockchain/issues/16063
+                        check_size = estimated_c2_sizes[k] + memo_size + 2000
+                        if k in measured_sizes:
+                            check_size = max(check_size, measured_sizes[k])
+
+                        if prover_size > check_size:
+                            log.warning(
+                                "Suspicious cache entry dropped. Recommended: stop the harvester, remove "
+                                f"{self._path}, restart. Entry: size {prover_size}, path {path}"
+                            )
+                        else:
+                            self._data[Path(path)] = new_entry
+                    elif ps.size_v2 is not None:
+                        # TODO: todo_v2_plots validate prover size
                         self._data[Path(path)] = new_entry
 
                 log.info(f"Parsed {len(self._data)} cache entries in {time.time() - start:.2f}s")