chia-blockchain 2.5.1rc1__py3-none-any.whl

chia/_tests/farmer_harvester/test_farmer_harvester.py

@@ -0,0 +1,292 @@
+from __future__ import annotations
+
+import asyncio
+from math import floor
+from pathlib import Path
+from typing import Any, Optional
+
+import pytest
+from chia_rs import G1Element
+
+from chia._tests.conftest import HarvesterFarmerEnvironment
+from chia._tests.util.split_managers import split_async_manager
+from chia._tests.util.time_out_assert import time_out_assert
+from chia.cmds.cmds_util import get_any_service_client
+from chia.farmer.farmer import Farmer
+from chia.plotting.util import PlotsRefreshParameter
+from chia.protocols import farmer_protocol, harvester_protocol
+from chia.protocols.protocol_message_types import ProtocolMessageTypes
+from chia.rpc.harvester_rpc_client import HarvesterRpcClient
+from chia.server.outbound_message import NodeType, make_msg
+from chia.simulator.block_tools import BlockTools
+from chia.types.aliases import FarmerService, HarvesterService
+from chia.types.blockchain_format.sized_bytes import bytes32
+from chia.types.peer_info import UnresolvedPeerInfo
+from chia.util.config import load_config
+from chia.util.hash import std_hash
+from chia.util.ints import uint8, uint32, uint64
+from chia.util.keychain import generate_mnemonic
+
+
+def farmer_is_started(farmer: Farmer) -> bool:
+    return farmer.started
+
+
+async def get_harvester_config(harvester_rpc_port: Optional[int], root_path: Path) -> dict[str, Any]:
+    async with get_any_service_client(HarvesterRpcClient, root_path, harvester_rpc_port) as (harvester_client, _):
+        return await harvester_client.get_harvester_config()
+
+
+async def update_harvester_config(harvester_rpc_port: Optional[int], root_path: Path, config: dict[str, Any]) -> bool:
+    async with get_any_service_client(HarvesterRpcClient, root_path, harvester_rpc_port) as (harvester_client, _):
+        return await harvester_client.update_harvester_config(config)
+
+
+@pytest.mark.anyio
+async def test_start_with_empty_keychain(
+    farmer_one_harvester_not_started: tuple[list[HarvesterService], FarmerService, BlockTools],
+) -> None:
+    _, farmer_service, bt = farmer_one_harvester_not_started
+    farmer: Farmer = farmer_service._node
+    farmer_service.reconnect_retry_seconds = 1
+    # First remove all keys from the keychain
+    assert bt.local_keychain is not None
+    bt.local_keychain.delete_all_keys()
+    # Make sure the farmer service is not initialized yet
+    assert not farmer.started
+    # Start it, wait 5 seconds and make sure it still isn't initialized (since the keychain is empty)
+    async with farmer_service.manage():
+        await asyncio.sleep(5)
+        assert not farmer.started
+        # Add a key to the keychain, this should lead to the start task passing
+        # `setup_keys` and set `Farmer.initialized`
+        bt.local_keychain.add_key(generate_mnemonic())
+        await time_out_assert(5, farmer_is_started, True, farmer)
+    assert not farmer.started
+
+
+@pytest.mark.anyio
+async def test_harvester_handshake(
+    farmer_one_harvester_not_started: tuple[list[HarvesterService], FarmerService, BlockTools],
+) -> None:
+    harvesters, farmer_service, bt = farmer_one_harvester_not_started
+    harvester_service = harvesters[0]
+    harvester = harvester_service._node
+    farmer = farmer_service._node
+
+    farmer_service.reconnect_retry_seconds = 1
+    harvester_service.reconnect_retry_seconds = 1
+
+    def farmer_has_connections() -> bool:
+        return len(farmer.server.get_connections()) > 0
+
+    def handshake_task_active() -> bool:
+        return farmer.harvester_handshake_task is not None
+
+    async def handshake_done() -> bool:
+        await asyncio.sleep(1)
+        return harvester.plot_manager._refresh_thread is not None and len(harvester.plot_manager.farmer_public_keys) > 0
+
+    # First remove all keys from the keychain
+    assert bt.local_keychain is not None
+    bt.local_keychain.delete_all_keys()
+    # Handshake task and plot manager thread should not be running yet
+    assert farmer.harvester_handshake_task is None
+    assert harvester.plot_manager._refresh_thread is None
+    async with split_async_manager(
+        manager=harvester_service.manage(), object=harvester_service
+    ) as split_harvester_manager:
+        # Start both services and wait a bit
+        async with farmer_service.manage():
+            async with harvester_service.manage():
+                harvester_service.add_peer(
+                    UnresolvedPeerInfo(str(farmer_service.self_hostname), farmer_service._server.get_port())
+                )
+                # Handshake task should be started but the handshake should not be done
+                await time_out_assert(5, handshake_task_active, True)
+                assert not await handshake_done()
+            # wait for the farmer to lose the connection
+            await time_out_assert(10, farmer_has_connections, False)
+            assert not await handshake_done()
+            # Handshake task should be stopped again
+            await time_out_assert(5, handshake_task_active, False)
+            await asyncio.sleep(1)
+            assert harvester.plot_manager._refresh_thread is None
+            assert len(harvester.plot_manager.farmer_public_keys) == 0
+            # Re-start the harvester and make sure the handshake task gets started but
+            # the handshake still doesn't go through
+            await split_harvester_manager.enter()
+            harvester_service.add_peer(
+                UnresolvedPeerInfo(str(farmer_service.self_hostname), farmer_service._server.get_port())
+            )
+            await time_out_assert(5, handshake_task_active, True)
+            assert not await handshake_done()
+        # make sure the handshake_task doesn't block the shutdown
+        await time_out_assert(5, handshake_task_active, False)
+        # Re-start the farmer and make sure the handshake task succeeds if a key get added to the keychain
+        async with farmer_service.manage():
+            await time_out_assert(5, handshake_task_active, True)
+            assert not await handshake_done()
+            bt.local_keychain.add_key(generate_mnemonic())
+            await time_out_assert(5, farmer_is_started, True, farmer)
+            await time_out_assert(5, handshake_task_active, False)
+            await time_out_assert(5, handshake_done, True)
+
+
+@pytest.mark.anyio
+async def test_farmer_respond_signatures(
+    caplog: pytest.LogCaptureFixture, harvester_farmer_environment: HarvesterFarmerEnvironment
+) -> None:
+    # This test ensures that the farmer correctly rejects invalid RespondSignatures
+    # messages from the harvester.
+    # In this test we're leveraging the fact that the farmer can handle RespondSignatures
+    # messages even though it didn't request them, to cover when the farmer doesn't know
+    # about an sp_hash, so it fails at the sp record check.
+
+    def log_is_ready() -> bool:
+        return len(caplog.text) > 0
+
+    _, _, harvester_service, _, _ = harvester_farmer_environment
+    # We won't have an sp record for this one
+    challenge_hash = bytes32(b"1" * 32)
+    sp_hash = bytes32(b"2" * 32)
+    response: harvester_protocol.RespondSignatures = harvester_protocol.RespondSignatures(
+        plot_identifier="test",
+        challenge_hash=challenge_hash,
+        sp_hash=sp_hash,
+        local_pk=G1Element(),
+        farmer_pk=G1Element(),
+        message_signatures=[],
+        include_source_signature_data=False,
+        farmer_reward_address_override=None,
+    )
+    msg = make_msg(ProtocolMessageTypes.respond_signatures, response)
+    await harvester_service._node.server.send_to_all([msg], NodeType.FARMER)
+    await time_out_assert(5, log_is_ready)
+    # We fail the sps record check
+    expected_error = f"Do not have challenge hash {challenge_hash}"
+    assert expected_error in caplog.text
+
+
+@pytest.mark.anyio
+async def test_harvester_config(farmer_one_harvester: tuple[list[HarvesterService], FarmerService, BlockTools]) -> None:
+    harvester_services, _farmer_service, bt = farmer_one_harvester
+    harvester_service = harvester_services[0]
+
+    assert harvester_service.rpc_server and harvester_service.rpc_server.webserver
+
+    harvester_rpc_port = harvester_service.rpc_server.webserver.listen_port
+    harvester_config = await get_harvester_config(harvester_rpc_port, bt.root_path)
+    assert harvester_config["success"] is True
+
+    def check_config_match(config1: dict[str, Any], config2: dict[str, Any]) -> None:
+        assert config1["harvester"]["use_gpu_harvesting"] == config2["use_gpu_harvesting"]
+        assert config1["harvester"]["gpu_index"] == config2["gpu_index"]
+        assert config1["harvester"]["enforce_gpu_index"] == config2["enforce_gpu_index"]
+        assert config1["harvester"]["disable_cpu_affinity"] == config2["disable_cpu_affinity"]
+        assert config1["harvester"]["parallel_decompressor_count"] == config2["parallel_decompressor_count"]
+        assert config1["harvester"]["decompressor_thread_count"] == config2["decompressor_thread_count"]
+        assert config1["harvester"]["recursive_plot_scan"] == config2["recursive_plot_scan"]
+        assert (
+            config2["refresh_parameter_interval_seconds"] == config1["harvester"]["refresh_parameter_interval_seconds"]
+            if "refresh_parameter_interval_seconds" in config1["harvester"]
+            else PlotsRefreshParameter().interval_seconds
+        )
+
+    check_config_match(bt.config, harvester_config)
+
+    harvester_config["use_gpu_harvesting"] = not harvester_config["use_gpu_harvesting"]
+    harvester_config["gpu_index"] += 1
+    harvester_config["enforce_gpu_index"] = not harvester_config["enforce_gpu_index"]
+    harvester_config["disable_cpu_affinity"] = not harvester_config["disable_cpu_affinity"]
+    harvester_config["parallel_decompressor_count"] += 1
+    harvester_config["decompressor_thread_count"] += 1
+    harvester_config["recursive_plot_scan"] = not harvester_config["recursive_plot_scan"]
+    harvester_config["refresh_parameter_interval_seconds"] += 1
+
+    res = await update_harvester_config(harvester_rpc_port, bt.root_path, harvester_config)
+    assert res is True
+    new_config = load_config(harvester_service.root_path, "config.yaml")
+    check_config_match(new_config, harvester_config)
+
+
+@pytest.mark.anyio
+async def test_missing_signage_point(
+    farmer_one_harvester: tuple[list[HarvesterService], FarmerService, BlockTools],
+) -> None:
+    _, farmer_service, _bt = farmer_one_harvester
+    farmer_api = farmer_service._api
+    farmer = farmer_api.farmer
+
+    def create_sp(index: int, challenge_hash: bytes32) -> tuple[uint64, farmer_protocol.NewSignagePoint]:
+        time = uint64(index + 1)
+        sp = farmer_protocol.NewSignagePoint(
+            challenge_hash, std_hash(b"2"), std_hash(b"3"), uint64(1), uint64(1000000), uint8(index), uint32(1)
+        )
+        return time, sp
+
+    # First sp. No missing sps
+    time0, sp0 = create_sp(index=0, challenge_hash=std_hash(b"1"))
+    assert farmer.prev_signage_point is None
+    ret = farmer.check_missing_signage_points(time0, sp0)
+    assert ret is None
+    assert farmer.prev_signage_point == (time0, sp0)
+
+    # 2nd sp. No missing sps
+    time1, sp1 = create_sp(index=1, challenge_hash=std_hash(b"1"))
+    ret = farmer.check_missing_signage_points(time1, sp1)
+    assert ret is None
+
+    # 3rd sp. 1 missing sp
+    time3, sp3 = create_sp(index=3, challenge_hash=std_hash(b"1"))
+    ret = farmer.check_missing_signage_points(time3, sp3)
+    assert ret == (time3, uint32(1))
+
+    # New challenge hash. Not counted as missing sp
+    _, sp_new_cc1 = create_sp(index=0, challenge_hash=std_hash(b"2"))
+    time_new_cc1 = time3
+    ret = farmer.check_missing_signage_points(time_new_cc1, sp_new_cc1)
+    assert ret is None
+
+    # Another new challenge hash. Calculating missing sps by timestamp
+    _, sp_new_cc2 = create_sp(index=0, challenge_hash=std_hash(b"3"))
+    # New sp is not in 9s but 12s is allowed
+    # since allowance multiplier is 1.6. (12 < 9 * 1.6)
+    time_new_cc2 = uint64(time_new_cc1 + 12)
+    ret = farmer.check_missing_signage_points(time_new_cc2, sp_new_cc2)
+    assert ret is None
+
+    # Another new challenge hash. Calculating missing sps by timestamp
+    _, sp_new_cc3 = create_sp(index=0, challenge_hash=std_hash(b"4"))
+    time_new_cc3 = uint64(time_new_cc2 + 601)  # roughly 10 minutes passed.
+    ret = farmer.check_missing_signage_points(time_new_cc3, sp_new_cc3)
+    assert ret is not None
+    ret_time, ret_skipped_sps = ret
+    assert ret_time == time_new_cc3
+    assert ret_skipped_sps == uint32(
+        floor(601 / (farmer.constants.SUB_SLOT_TIME_TARGET / farmer.constants.NUM_SPS_SUB_SLOT))
+    )
+
+    original_state_changed_callback = farmer.state_changed_callback
+    assert original_state_changed_callback is not None
+    number_of_missing_sps: uint32 = uint32(0)
+
+    def state_changed(change: str, data: dict[str, Any]) -> None:
+        nonlocal number_of_missing_sps
+        number_of_missing_sps = data["missing_signage_points"][1]
+        original_state_changed_callback(change, data)
+
+    farmer.state_changed_callback = state_changed  # type: ignore
+    _, sp_for_farmer_api = create_sp(index=2, challenge_hash=std_hash(b"4"))
+    await farmer_api.new_signage_point(sp_for_farmer_api)
+    assert number_of_missing_sps == uint32(1)
+
+
+@pytest.mark.anyio
+async def test_harvester_has_no_server(
+    farmer_one_harvester: tuple[list[FarmerService], HarvesterService, BlockTools],
+) -> None:
+    harvesters, _, _bt = farmer_one_harvester
+    harvester_server = harvesters[0]._server
+
+    assert harvester_server.webserver is None

chia/_tests/farmer_harvester/test_filter_prefix_bits.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from pathlib import Path
+from typing import Any, Optional
+
+import pytest
+
+from chia._tests.conftest import ConsensusMode
+from chia._tests.core.test_farmer_harvester_rpc import wait_for_plot_sync
+from chia._tests.util.setup_nodes import setup_farmer_multi_harvester
+from chia._tests.util.time_out_assert import time_out_assert
+from chia.farmer.farmer_api import FarmerAPI
+from chia.protocols import farmer_protocol
+from chia.rpc.farmer_rpc_client import FarmerRpcClient
+from chia.rpc.harvester_rpc_client import HarvesterRpcClient
+from chia.simulator.block_tools import create_block_tools_async, test_constants
+from chia.types.aliases import HarvesterService
+from chia.types.blockchain_format.proof_of_space import get_plot_id, passes_plot_filter
+from chia.types.blockchain_format.sized_bytes import bytes32
+from chia.types.full_block import FullBlock
+from chia.util.ints import uint8, uint32, uint64
+from chia.util.keychain import Keychain
+
+
+# these numbers are only valid for chains farmed with the fixed original plot
+# filter. The HARD_FORK_2_0 consensus mode uses a chain where blocks are farmed
+# with wider filters. i.e. some valid blocks may still not pass the filter in
+# this test
+@pytest.mark.limit_consensus_modes(allowed=[ConsensusMode.PLAIN])
+@pytest.mark.parametrize(
+    argnames=["filter_prefix_bits", "should_pass"], argvalues=[(9, 34), (8, 89), (7, 162), (6, 295), (5, 579)]
+)
+def test_filter_prefix_bits_on_blocks(
+    default_10000_blocks: list[FullBlock], filter_prefix_bits: uint8, should_pass: int
+) -> None:
+    passed = 0
+    for block in default_10000_blocks:
+        plot_id = get_plot_id(block.reward_chain_block.proof_of_space)
+        original_challenge_hash = block.reward_chain_block.pos_ss_cc_challenge_hash
+        if block.reward_chain_block.challenge_chain_sp_vdf is None:
+            assert block.reward_chain_block.signage_point_index == 0
+            signage_point = original_challenge_hash
+        else:
+            signage_point = block.reward_chain_block.challenge_chain_sp_vdf.output.get_hash()
+        if passes_plot_filter(filter_prefix_bits, plot_id, original_challenge_hash, signage_point):
+            passed += 1
+    assert passed == should_pass
+
+
+@pytest.fixture(scope="function")
+async def farmer_harvester_with_filter_size_9(
+    get_temp_keyring: Keychain, tmp_path: Path, self_hostname: str
+) -> AsyncIterator[tuple[HarvesterService, FarmerAPI]]:
+    async def have_connections() -> bool:
+        return len(await farmer_rpc_cl.get_connections()) > 0
+
+    local_b_tools = await create_block_tools_async(
+        constants=test_constants.replace(NUMBER_ZERO_BITS_PLOT_FILTER=uint8(9)), keychain=get_temp_keyring
+    )
+    new_config = local_b_tools._config
+    local_b_tools.change_config(new_config)
+    async with setup_farmer_multi_harvester(
+        local_b_tools, 1, tmp_path, local_b_tools.constants, start_services=True
+    ) as (harvesters, farmer_service, _):
+        harvester_service = harvesters[0]
+        assert farmer_service.rpc_server is not None
+        farmer_rpc_cl = await FarmerRpcClient.create(
+            self_hostname, farmer_service.rpc_server.listen_port, farmer_service.root_path, farmer_service.config
+        )
+        assert harvester_service.rpc_server is not None
+        harvester_rpc_cl = await HarvesterRpcClient.create(
+            self_hostname,
+            harvester_service.rpc_server.listen_port,
+            harvester_service.root_path,
+            harvester_service.config,
+        )
+        await time_out_assert(15, have_connections, True)
+        yield harvester_service, farmer_service._api
+
+    farmer_rpc_cl.close()
+    harvester_rpc_cl.close()
+    await farmer_rpc_cl.await_closed()
+    await harvester_rpc_cl.await_closed()
+
+
+@pytest.mark.parametrize(argnames=["peak_height", "eligible_plots"], argvalues=[(5495999, 0), (5496000, 1)])
+@pytest.mark.anyio
+async def test_filter_prefix_bits_with_farmer_harvester(
+    farmer_harvester_with_filter_size_9: tuple[HarvesterService, FarmerAPI],
+    peak_height: uint32,
+    eligible_plots: int,
+) -> None:
+    state_change = None
+    state_change_data = None
+
+    def state_changed_callback(change: str, change_data: Optional[dict[str, Any]]) -> None:
+        nonlocal state_change, state_change_data
+        state_change = change
+        state_change_data = change_data
+
+    def state_has_changed() -> bool:
+        return state_change is not None and state_change_data is not None
+
+    # We need a custom block tools with constants that set the initial filter
+    # size to 9 in order to test peak heights that cover sizes 9 and 8 respectively
+    harvester_service, farmer_api = farmer_harvester_with_filter_size_9
+    harvester_service._node.state_changed_callback = state_changed_callback
+    harvester_id = harvester_service._server.node_id
+    receiver = farmer_api.farmer.plot_sync_receivers[harvester_id]
+    if receiver.initial_sync():
+        await wait_for_plot_sync(receiver, receiver.last_sync().sync_id)
+    # This allows us to pass the plot filter with prefix bits 8 but not 9
+    challenge_hash = bytes32.from_hexstr("0x73490e166d0b88347c37d921660b216c27316aae9a3450933d3ff3b854e5831a")
+    sp_hash = bytes32.from_hexstr("0x7b3e23dbd438f9aceefa9827e2c5538898189987f49b06eceb7a43067e77b531")
+    sp = farmer_protocol.NewSignagePoint(
+        challenge_hash=challenge_hash,
+        challenge_chain_sp=sp_hash,
+        reward_chain_sp=bytes32(b"1" * 32),
+        difficulty=uint64(1),
+        sub_slot_iters=uint64(1000000),
+        signage_point_index=uint8(2),
+        peak_height=peak_height,
+    )
+    await farmer_api.new_signage_point(sp)
+    await time_out_assert(5, state_has_changed, True)
+    # We're intercepting the harvester's state changes as we're expecting
+    # a farming_info one.
+    assert state_change == "farming_info"
+    assert state_change_data is not None
+    assert state_change_data.get("eligible_plots") == eligible_plots