chia-blockchain 2.5.1rc1__py3-none-any.whl

chia/_tests/util/full_sync.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import cProfile
+import logging
+import shutil
+import tempfile
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Callable, Optional, cast
+
+import aiosqlite
+import zstd
+
+from chia._tests.util.constants import test_constants as TEST_CONSTANTS
+from chia.cmds.init_funcs import chia_init
+from chia.consensus.block_body_validation import ForkInfo
+from chia.consensus.constants import replace_str_to_bytes
+from chia.consensus.default_constants import DEFAULT_CONSTANTS
+from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty
+from chia.full_node.full_node import FullNode
+from chia.server.outbound_message import Message, NodeType
+from chia.server.server import ChiaServer
+from chia.server.ws_connection import ConnectionCallback, WSChiaConnection
+from chia.simulator.block_tools import make_unfinished_block
+from chia.types.blockchain_format.sized_bytes import bytes32
+from chia.types.full_block import FullBlock
+from chia.types.peer_info import PeerInfo
+from chia.types.validation_state import ValidationState
+from chia.util.config import load_config
+from chia.util.ints import uint16
+
+
+class ExitOnError(logging.Handler):
+    def __init__(self) -> None:
+        super().__init__()
+        self.exit_with_failure = False
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if record.levelno != logging.ERROR:
+            return
+        self.exit_with_failure = True
+
+
+@contextmanager
+def enable_profiler(profile: bool, counter: int) -> Iterator[None]:
+    if not profile:
+        yield
+        return
+
+    with cProfile.Profile() as pr:
+        receive_start_time = time.monotonic()
+        yield
+
+    if time.monotonic() - receive_start_time > 5:
+        pr.create_stats()
+        pr.dump_stats(f"slow-batch-{counter:05d}.profile")
+
+
+class FakeServer:
+    async def send_to_all(
+        self, messages: list[Message], node_type: NodeType, exclude: Optional[bytes32] = None
+    ) -> None:
+        pass
+
+    async def send_to_all_if(
+        self,
+        messages: list[Message],
+        node_type: NodeType,
+        predicate: Callable[[WSChiaConnection], bool],
+        exclude: Optional[bytes32] = None,
+    ) -> None:
+        pass
+
+    def set_received_message_callback(self, callback: ConnectionCallback) -> None:
+        pass
+
+    async def get_peer_info(self) -> Optional[PeerInfo]:
+        return None
+
+    def get_connections(
+        self, node_type: Optional[NodeType] = None, *, outbound: Optional[bool] = False
+    ) -> list[WSChiaConnection]:
+        return []
+
+    def is_duplicate_or_self_connection(self, target_node: PeerInfo) -> bool:
+        return False
+
+    async def start_client(
+        self,
+        target_node: PeerInfo,
+        on_connect: Optional[ConnectionCallback] = None,
+        auth: bool = False,
+        is_feeler: bool = False,
+    ) -> bool:
+        return False
+
+
+class FakePeer:
+    def get_peer_logging(self) -> PeerInfo:
+        return PeerInfo("0.0.0.0", uint16(0))
+
+    def __init__(self) -> None:
+        self.peer_node_id = bytes([0] * 32)
+
+    async def get_peer_info(self) -> Optional[PeerInfo]:
+        return None
+
+
+async def run_sync_test(
+    file: Path,
+    db_version: int,
+    profile: bool,
+    single_thread: bool,
+    test_constants: bool,
+    keep_up: bool,
+    db_sync: str,
+    node_profiler: bool,
+    start_at_checkpoint: Optional[str],
+) -> None:
+    logger = logging.getLogger()
+    logger.setLevel(logging.WARNING)
+    handler = logging.FileHandler("test-full-sync.log")
+    handler.setFormatter(
+        logging.Formatter(
+            "%(levelname)-8s %(message)s",
+            datefmt="%Y-%m-%dT%H:%M:%S",
+        )
+    )
+    logger.addHandler(handler)
+    check_log = ExitOnError()
+    logger.addHandler(check_log)
+
+    with tempfile.TemporaryDirectory() as root_dir:
+        root_path = Path(root_dir, "root")
+        if start_at_checkpoint is not None:
+            shutil.copytree(start_at_checkpoint, root_path)
+
+        chia_init(root_path, should_check_keys=False, v1_db=(db_version == 1))
+        config = load_config(root_path, "config.yaml")
+
+        if test_constants:
+            constants = TEST_CONSTANTS
+        else:
+            overrides = config["network_overrides"]["constants"][config["selected_network"]]
+            constants = replace_str_to_bytes(DEFAULT_CONSTANTS, **overrides)
+        if single_thread:
+            config["full_node"]["single_threaded"] = True
+        config["full_node"]["db_sync"] = db_sync
+        config["full_node"]["enable_profiler"] = node_profiler
+        full_node = await FullNode.create(
+            config["full_node"],
+            root_path=root_path,
+            consensus_constants=constants,
+        )
+
+        full_node.set_server(cast(ChiaServer, FakeServer()))
+        async with full_node.manage():
+            peak = full_node.blockchain.get_peak()
+            if peak is not None:
+                height = int(peak.height)
+            else:
+                height = 0
+
+            peer: WSChiaConnection = cast(WSChiaConnection, FakePeer())
+
+            print()
+            counter = 0
+            monotonic = height
+            prev_hash = None
+            async with aiosqlite.connect(file) as in_db:
+                await in_db.execute("pragma query_only")
+                rows = await in_db.execute(
+                    "SELECT header_hash, height, block FROM full_blocks "
+                    "WHERE height >= ? AND in_main_chain=1 ORDER BY height",
+                    (height,),
+                )
+
+                block_batch = []
+
+                start_time = time.monotonic()
+                logger.warning(f"starting test {start_time}")
+                worst_batch_height = None
+                worst_batch_time_per_block = None
+                peer_info = peer.get_peer_logging()
+                async for r in rows:
+                    batch_start_time = time.monotonic()
+                    with enable_profiler(profile, height):
+                        block = FullBlock.from_bytes(zstd.decompress(r[2]))
+                        block_batch.append(block)
+
+                        assert block.height == monotonic
+                        monotonic += 1
+                        assert prev_hash is None or block.prev_header_hash == prev_hash
+                        prev_hash = block.header_hash
+
+                        if len(block_batch) < 32:
+                            continue
+
+                        if keep_up:
+                            for b in block_batch:
+                                await full_node.add_unfinished_block(make_unfinished_block(b, constants), peer)
+                                await full_node.add_block(b, None, full_node._bls_cache)
+                        else:
+                            block_record = await full_node.blockchain.get_block_record_from_db(
+                                block_batch[0].prev_header_hash
+                            )
+                            ssi, diff = get_next_sub_slot_iters_and_difficulty(
+                                full_node.constants, True, block_record, full_node.blockchain
+                            )
+                            fork_height = block_batch[0].height - 1
+                            header_hash = block_batch[0].prev_header_hash
+                            success, summary = await full_node.add_block_batch(
+                                block_batch,
+                                peer_info,
+                                ForkInfo(fork_height, fork_height, header_hash),
+                                ValidationState(ssi, diff, None),
+                            )
+                            end_height = block_batch[-1].height
+                            full_node.blockchain.clean_block_record(end_height - full_node.constants.BLOCKS_CACHE_SIZE)
+
+                            if not success:
+                                raise RuntimeError("failed to ingest block batch")
+
+                            assert summary is not None
+
+                        time_per_block = (time.monotonic() - batch_start_time) / len(block_batch)
+                        if not worst_batch_height or worst_batch_time_per_block > time_per_block:
+                            worst_batch_height = height
+                            worst_batch_time_per_block = time_per_block
+
+                    counter += len(block_batch)
+                    height += len(block_batch)
+                    print(
+                        f"\rheight {height} {time_per_block:0.2f} s/block   ",
+                        end="",
+                    )
+                    block_batch = []
+                    if check_log.exit_with_failure:
+                        raise RuntimeError("error printed to log. exiting")
+
+                    if counter >= 100000:
+                        counter = 0
+                        print()
+                end_time = time.monotonic()
+                logger.warning(f"test completed at {end_time}")
+                logger.warning(f"duration: {end_time - start_time:0.2f} s")
+                logger.warning(f"worst time-per-block: {worst_batch_time_per_block:0.2f} s")
+                logger.warning(f"worst height: {worst_batch_height}")
+                logger.warning(f"end-height: {height}")
+            if node_profiler:
+                (root_path / "profile-node").rename("./profile-node")

chia/_tests/util/gen_ssl_certs.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import click
+from pytest import MonkeyPatch
+
+from chia.ssl.create_ssl import generate_ca_signed_cert, get_chia_ca_crt_key, make_ca_cert
+
+# NOTE: This is a standalone tool that can be used to generate a CA cert/key as well as node certs/keys.
+
+
+@click.command()
+@click.option(
+    "--suffix",
+    type=str,
+    default="",
+    help="Suffix to append to the generated cert/key symbols.",
+    required=True,
+)
+def gen_ssl(suffix: str = "") -> None:
+    captured_crt: Optional[bytes] = None
+    captured_key: Optional[bytes] = None
+    capture_cert_and_key = False
+
+    def patched_write_ssl_cert_and_key(cert_path: Path, cert_data: bytes, key_path: Path, key_data: bytes) -> None:
+        nonlocal capture_cert_and_key, captured_crt, captured_key
+
+        if capture_cert_and_key:
+            captured_crt = cert_data
+            captured_key = key_data
+
+        print(f"{cert_path} = b\"\"\"{cert_data.decode(encoding='utf8')}\"\"\"")
+        print()
+        print(f"{key_path} = b\"\"\"{key_data.decode(encoding='utf8')}\"\"\"")
+        print()
+
+    patch = MonkeyPatch()
+    patch.setattr("chia.ssl.create_ssl.write_ssl_cert_and_key", patched_write_ssl_cert_and_key)
+
+    private_ca_crt: Optional[bytes] = None
+    private_ca_key: Optional[bytes] = None
+    capture_cert_and_key = True
+
+    make_ca_cert(Path("SSL_TEST_PRIVATE_CA_CRT"), Path("SSL_TEST_PRIVATE_CA_KEY"))
+
+    capture_cert_and_key = False
+    private_ca_crt = captured_crt
+    private_ca_key = captured_key
+
+    node_certs_and_keys = {
+        "full_node": {
+            "private": {"crt": "SSL_TEST_FULLNODE_PRIVATE_CRT", "key": "SSL_TEST_FULLNODE_PRIVATE_KEY"},
+            "public": {"crt": "SSL_TEST_FULLNODE_PUBLIC_CRT", "key": "SSL_TEST_FULLNODE_PUBLIC_KEY"},
+        },
+        "wallet": {
+            "private": {"crt": "SSL_TEST_WALLET_PRIVATE_CRT", "key": "SSL_TEST_WALLET_PRIVATE_KEY"},
+            "public": {"crt": "SSL_TEST_WALLET_PUBLIC_CRT", "key": "SSL_TEST_WALLET_PUBLIC_KEY"},
+        },
+        "farmer": {
+            "private": {"crt": "SSL_TEST_FARMER_PRIVATE_CRT", "key": "SSL_TEST_FARMER_PRIVATE_KEY"},
+            "public": {"crt": "SSL_TEST_FARMER_PUBLIC_CRT", "key": "SSL_TEST_FARMER_PUBLIC_KEY"},
+        },
+        "harvester": {"private": {"crt": "SSL_TEST_HARVESTER_PRIVATE_CRT", "key": "SSL_TEST_HARVESTER_PRIVATE_KEY"}},
+        "timelord": {
+            "private": {"crt": "SSL_TEST_TIMELORD_PRIVATE_CRT", "key": "SSL_TEST_TIMELORD_PRIVATE_KEY"},
+            "public": {"crt": "SSL_TEST_TIMELORD_PUBLIC_CRT", "key": "SSL_TEST_TIMELORD_PUBLIC_KEY"},
+        },
+        "crawler": {"private": {"crt": "SSL_TEST_CRAWLER_PRIVATE_CRT", "key": "SSL_TEST_CRAWLER_PRIVATE_KEY"}},
+        "daemon": {"private": {"crt": "SSL_TEST_DAEMON_PRIVATE_CRT", "key": "SSL_TEST_DAEMON_PRIVATE_KEY"}},
+        "introducer": {
+            "public": {"crt": "SSL_TEST_INTRODUCER_PUBLIC_CRT", "key": "SSL_TEST_INTRODUCER_PUBLIC_KEY"},
+        },
+    }
+
+    chia_ca_crt, chia_ca_key = get_chia_ca_crt_key()
+
+    for node_name, cert_type_dict in node_certs_and_keys.items():
+        for cert_type, cert_dict in cert_type_dict.items():
+            crt = cert_dict["crt"]
+            key = cert_dict["key"]
+            ca_crt = chia_ca_crt if cert_type == "public" else private_ca_crt
+            ca_key = chia_ca_key if cert_type == "public" else private_ca_key
+
+            assert ca_crt is not None
+            assert ca_key is not None
+            generate_ca_signed_cert(ca_crt, ca_key, Path(crt), Path(key))
+
+    patch.undo()
+
+    append_str = "" if suffix == "" else f"_{suffix}"
+    print(
+        f"SSL_TEST_PRIVATE_CA_CERT_AND_KEY{append_str}: tuple[bytes, bytes] = "
+        "(SSL_TEST_PRIVATE_CA_CRT, SSL_TEST_PRIVATE_CA_KEY)"
+    )
+    print()
+    print(f"SSL_TEST_NODE_CERTS_AND_KEYS{append_str}: dict[str, dict[str, dict[str, bytes]]] = {{")
+    for node_name, cert_type_dict in node_certs_and_keys.items():
+        print(f'    "{node_name}": {{')
+        for cert_type, cert_dict in cert_type_dict.items():
+            crt = cert_dict["crt"]
+            key = cert_dict["key"]
+            print(f'        "{cert_type}": {{"crt": {crt}, "key": {key}}},')
+        print("    },")
+    print("}")
+
+
+def main() -> None:
+    gen_ssl()
+
+
+if __name__ == "__main__":
+    main()

chia/_tests/util/generator_tools_testing.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from chia._tests.util.get_name_puzzle_conditions import get_name_puzzle_conditions
+from chia.consensus.constants import ConsensusConstants
+from chia.consensus.default_constants import DEFAULT_CONSTANTS
+from chia.types.blockchain_format.coin import Coin
+from chia.types.blockchain_format.sized_bytes import bytes32
+from chia.types.full_block import FullBlock
+from chia.types.generator_types import BlockGenerator
+from chia.util.generator_tools import tx_removals_and_additions
+from chia.util.ints import uint32
+
+
+def run_and_get_removals_and_additions(
+    block: FullBlock,
+    max_cost: int,
+    *,
+    height: uint32,
+    constants: ConsensusConstants = DEFAULT_CONSTANTS,
+    mempool_mode: bool = False,
+) -> tuple[list[bytes32], list[Coin]]:
+    removals: list[bytes32] = []
+    additions: list[Coin] = []
+
+    assert len(block.transactions_generator_ref_list) == 0
+    if not block.is_transaction_block():
+        return [], []
+
+    if block.transactions_generator is not None:
+        npc_result = get_name_puzzle_conditions(
+            BlockGenerator(block.transactions_generator, []),
+            max_cost,
+            mempool_mode=mempool_mode,
+            height=height,
+            constants=constants,
+        )
+        assert npc_result.error is None
+        rem, add = tx_removals_and_additions(npc_result.conds)
+        # build removals list
+        removals.extend(rem)
+        additions.extend(add)
+
+    rewards = block.get_included_reward_coins()
+    additions.extend(rewards)
+    return removals, additions

chia/_tests/util/get_name_puzzle_conditions.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import logging
+
+from chia_rs import (
+    DONT_VALIDATE_SIGNATURE,
+    MEMPOOL_MODE,
+    G2Element,
+    get_flags_for_height_and_constants,
+    run_block_generator,
+    run_block_generator2,
+)
+
+from chia.consensus.constants import ConsensusConstants
+from chia.consensus.cost_calculator import NPCResult
+from chia.types.generator_types import BlockGenerator
+from chia.util.errors import Err
+from chia.util.ints import uint16, uint32
+
+log = logging.getLogger(__name__)
+
+
+def get_name_puzzle_conditions(
+    generator: BlockGenerator,
+    max_cost: int,
+    *,
+    mempool_mode: bool,
+    height: uint32,
+    constants: ConsensusConstants,
+) -> NPCResult:
+    flags = get_flags_for_height_and_constants(height, constants) | DONT_VALIDATE_SIGNATURE
+
+    if mempool_mode:
+        flags |= MEMPOOL_MODE
+
+    if height >= constants.HARD_FORK_HEIGHT:
+        run_block = run_block_generator2
+    else:
+        run_block = run_block_generator
+
+    try:
+        block_args = generator.generator_refs
+        err, result = run_block(bytes(generator.program), block_args, max_cost, flags, G2Element(), None, constants)
+        assert (err is None) != (result is None)
+        if err is not None:
+            return NPCResult(uint16(err), None)
+        else:
+            assert result is not None
+            return NPCResult(None, result)
+    except BaseException:
+        log.exception("get_name_puzzle_condition failed")
+        return NPCResult(uint16(Err.GENERATOR_RUNTIME_ERROR.value), None)

chia/_tests/util/key_tool.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from chia_rs import AugSchemeMPL, G1Element, G2Element, PrivateKey
+
+from chia._tests.core.make_block_generator import GROUP_ORDER, int_to_public_key
+from chia.simulator.block_tools import test_constants
+from chia.types.coin_spend import CoinSpend
+from chia.util.condition_tools import conditions_dict_for_solution, pkm_pairs_for_conditions_dict
+
+
+@dataclass
+class KeyTool:
+    dict: dict[G1Element, int] = field(default_factory=dict)
+
+    def add_secret_exponents(self, secret_exponents: list[int]) -> None:
+        for _ in secret_exponents:
+            self.dict[int_to_public_key(_)] = _ % GROUP_ORDER
+
+    def sign(self, public_key: G1Element, message: bytes) -> G2Element:
+        secret_exponent = self.dict.get(public_key)
+        if not secret_exponent:
+            raise ValueError(f"unknown pubkey {bytes(public_key).hex()}")
+        bls_private_key = PrivateKey.from_bytes(secret_exponent.to_bytes(32, "big"))
+        return AugSchemeMPL.sign(bls_private_key, message)
+
+    def signature_for_solution(self, coin_spend: CoinSpend, additional_data: bytes) -> G2Element:
+        signatures = []
+        conditions_dict = conditions_dict_for_solution(
+            coin_spend.puzzle_reveal, coin_spend.solution, test_constants.MAX_BLOCK_COST_CLVM
+        )
+        for public_key, message in pkm_pairs_for_conditions_dict(conditions_dict, coin_spend.coin, additional_data):
+            signature = self.sign(public_key, message)
+            signatures.append(signature)
+        return AugSchemeMPL.aggregate(signatures)