chia-blockchain 2.5.1rc1__py3-none-any.whl

chia/_tests/blockchain/test_build_chains.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import pytest
+
+from chia.types.full_block import FullBlock
+
+# These test targets are used to trigger a build of the test chains.
+# On CI we clone the test-cache repository to load the chains from, so they
+# don't need to be re-generated.
+
+# When running tests in parallel (with pytest-xdist) it's faster to first
+# generate all the chains, so the same chains aren't being created in parallel.
+
+# The cached test chains are stored in ~/.chia/blocks
+
+# To generate the chains, run:
+
+# pytest -m build_test_chains
+
+
+@pytest.mark.build_test_chains
+def test_trigger_default_400(default_400_blocks: list[FullBlock]) -> None:
+    pass
+
+
+@pytest.mark.build_test_chains
+def test_trigger_default_1000(default_1000_blocks: list[FullBlock]) -> None:
+    pass
+
+
+@pytest.mark.build_test_chains
+def test_trigger_pre_genesis_empty_1000(pre_genesis_empty_slots_1000_blocks: list[FullBlock]) -> None:
+    pass
+
+
+@pytest.mark.build_test_chains
+def test_trigger_default_1500(default_1500_blocks: list[FullBlock]) -> None:
+    pass
+
+
+@pytest.mark.build_test_chains
+def test_trigger_default_10000(
+    default_10000_blocks: list[FullBlock],
+    test_long_reorg_blocks: list[FullBlock],
+    test_long_reorg_blocks_light: list[FullBlock],
+    test_long_reorg_1500_blocks: list[FullBlock],
+    test_long_reorg_1500_blocks_light: list[FullBlock],
+) -> None:
+    pass
+
+
+@pytest.mark.build_test_chains
+def test_trigger_default_2000_compact(default_2000_blocks_compact: list[FullBlock]) -> None:
+    pass
+
+
+@pytest.mark.build_test_chains
+def test_trigger_default_10000_compact(default_10000_blocks_compact: list[FullBlock]) -> None:
+    pass

chia/_tests/blockchain/test_get_block_generator.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+import pytest
+from clvm.casts import int_to_bytes
+
+from chia.consensus.get_block_generator import get_block_generator
+from chia.types.blockchain_format.serialized_program import SerializedProgram
+from chia.types.blockchain_format.sized_bytes import bytes32
+from chia.types.generator_types import BlockGenerator
+from chia.util.ints import uint32
+
+
+@dataclass(frozen=True)
+class BR:
+    prev_header_hash: bytes32
+    transactions_generator: Optional[SerializedProgram]
+    transactions_generator_ref_list: list[uint32]
+
+
+@dataclass(frozen=True)
+class FB:
+    prev_header_hash: bytes32
+    transactions_generator: Optional[SerializedProgram]
+    height: uint32
+
+
+def blockhash(i: int) -> bytes32:
+    return bytes32([i] * 32)
+
+
+def program(i: int) -> SerializedProgram:
+    return SerializedProgram.from_bytes(int_to_bytes(i))
+
+
+async def zero_hits(hh: bytes32, refs: set[uint32]) -> dict[uint32, bytes]:
+    return {}
+
+
+async def never_called(hh: bytes32, refs: set[uint32]) -> dict[uint32, bytes]:
+    assert False  # pragma: no cover
+
+
+async def only_lookup_5(hh: bytes32, refs: set[uint32]) -> dict[uint32, bytes]:
+    assert refs == {uint32(5)}
+    return {uint32(5): bytes(program(5))}
+
+
+DUMMY_PROGRAM = SerializedProgram.from_bytes(b"\x80")
+
+
+@pytest.mark.anyio
+async def test_failing_lookup() -> None:
+    br = BR(bytes32.zeros, DUMMY_PROGRAM, [uint32(1)])
+    with pytest.raises(KeyError):
+        await get_block_generator(zero_hits, br)
+
+
+@pytest.mark.anyio
+async def test_no_generator() -> None:
+    br = BR(bytes32.zeros, None, [uint32(1)])
+    with pytest.raises(AssertionError):
+        await get_block_generator(zero_hits, br)
+
+
+@pytest.mark.anyio
+async def test_no_refs() -> None:
+    br = BR(bytes32.zeros, DUMMY_PROGRAM, [])
+    bg = await get_block_generator(never_called, br)
+    assert bg == BlockGenerator(DUMMY_PROGRAM, [])

chia/_tests/blockchain/test_lookup_fork_chain.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+
+from chia._tests.util.benchmarks import rand_hash
+from chia.consensus.block_record import BlockRecord
+from chia.consensus.blockchain_interface import BlockRecordsProtocol
+from chia.consensus.find_fork_point import find_fork_point_in_chain, lookup_fork_chain
+from chia.simulator.block_tools import test_constants
+from chia.types.blockchain_format.sized_bytes import bytes32
+from chia.util.ints import uint32
+
+
+class DummyChain:
+    _chain: dict[bytes32, bytes32]
+
+    def __init__(self) -> None:
+        self._chain = {}
+
+    def add_block(self, h: bytes32, prev: bytes32) -> None:
+        self._chain[h] = prev
+
+    async def prev_block_hash(self, header_hashes: list[bytes32]) -> list[bytes32]:
+        ret: list[bytes32] = []
+        for h in header_hashes:
+            ret.append(self._chain[h])
+        return ret
+
+
+A = rand_hash()
+B = rand_hash()
+C = rand_hash()
+D = rand_hash()
+E = rand_hash()
+F = rand_hash()
+G = rand_hash()
+
+dummy_chain = DummyChain()
+dummy_chain.add_block(G, test_constants.GENESIS_CHALLENGE)
+dummy_chain.add_block(D, G)
+dummy_chain.add_block(C, D)
+dummy_chain.add_block(B, C)
+dummy_chain.add_block(A, B)
+dummy_chain.add_block(E, D)
+dummy_chain.add_block(F, E)
+
+test_chain: BlockRecordsProtocol = dummy_chain  # type: ignore[assignment]
+
+#    A
+#    |
+#    v
+#    B     F
+#    |     |
+#    v     v
+#    C     E
+#     \   /
+#      v v
+#       D
+#       |
+#       v
+#       G
+
+
+@dataclass
+class FakeBlockRecord:
+    height: uint32
+    header_hash: bytes32
+    prev_hash: bytes32
+
+
+def BR(height: int, header_hash: bytes32, prev_hash: bytes32) -> BlockRecord:
+    ret = FakeBlockRecord(uint32(height), header_hash, prev_hash)
+    return ret  # type: ignore[return-value]
+
+
+@pytest.mark.anyio
+async def test_no_fork() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (42, A), (42, A), test_constants)
+    assert chain == {}
+    assert fork_hash == A
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(42, A, B), BR(42, A, B))
+    assert fork_height == 42
+
+
+@pytest.mark.anyio
+async def test_fork_left() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (42, A), (41, F), test_constants)
+    assert chain == {40: E, 41: F}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(42, A, B), BR(41, F, E))
+    assert fork_height == 39
+
+
+@pytest.mark.anyio
+async def test_fork_left_short() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (41, B), (41, F), test_constants)
+    assert chain == {40: E, 41: F}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(41, B, C), BR(41, F, E))
+    assert fork_height == 39
+
+
+@pytest.mark.anyio
+async def test_fork_right() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (41, F), (42, A), test_constants)
+    assert chain == {40: C, 41: B, 42: A}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(41, F, E), BR(42, A, B))
+    assert fork_height == 39
+
+
+@pytest.mark.anyio
+async def test_fork_right_short() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (41, F), (41, B), test_constants)
+    assert chain == {40: C, 41: B}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(41, F, E), BR(41, B, C))
+    assert fork_height == 39
+
+
+@pytest.mark.anyio
+async def test_linear_long() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (39, D), (42, A), test_constants)
+    assert chain == {40: C, 41: B, 42: A}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(39, D, G), BR(42, A, B))
+    assert fork_height == 39
+
+
+@pytest.mark.anyio
+async def test_linear_short() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (42, A), (39, D), test_constants)
+    assert chain == {}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(42, A, B), BR(39, D, G))
+    assert fork_height == 39
+
+
+@pytest.mark.anyio
+async def test_no_shared_left() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (1, F), (1, B), test_constants)
+    assert chain == {0: C, 1: B}
+    assert fork_hash == test_constants.GENESIS_CHALLENGE
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(1, F, E), BR(1, B, C))
+    assert fork_height == -1
+
+
+@pytest.mark.anyio
+async def test_no_shared_right() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (1, B), (1, F), test_constants)
+    assert chain == {0: E, 1: F}
+    assert fork_hash == test_constants.GENESIS_CHALLENGE
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(1, B, C), BR(1, F, E))
+    assert fork_height == -1
+
+
+@pytest.mark.anyio
+async def test_root_shared_left() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (2, F), (2, B), test_constants)
+    assert chain == {1: C, 2: B}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(2, F, E), BR(2, B, C))
+    assert fork_height == 0
+
+
+@pytest.mark.anyio
+async def test_root_shared_right() -> None:
+    chain, fork_hash = await lookup_fork_chain(test_chain, (2, B), (2, F), test_constants)
+    assert chain == {1: E, 2: F}
+    assert fork_hash == D
+
+    fork_height = await find_fork_point_in_chain(test_chain, BR(2, B, C), BR(2, F, E))
+    assert fork_height == 0
+
+
+@pytest.mark.anyio
+async def test_no_left_chain() -> None:
+    chain, fork_hash = await lookup_fork_chain(
+        test_chain, (-1, test_constants.GENESIS_CHALLENGE), (3, F), test_constants
+    )
+    assert chain == {0: G, 1: D, 2: E, 3: F}
+    assert fork_hash == test_constants.GENESIS_CHALLENGE

chia/_tests/build-init-files.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+# Create missing `__init__.py` files in the source code folders (in "chia/").
+#
+# They are required by the python interpreter to properly identify modules/packages so that tools like `mypy` or an IDE
+# can work with their full capabilities.
+#
+# See https://docs.python.org/3/tutorial/modules.html#packages.
+#
+# Note: This script is run in a `pre-commit` hook (which runs on CI) to make sure we don't miss out any folder.
+
+from __future__ import annotations
+
+import logging
+import pathlib
+
+import click
+
+log_levels = {
+    0: logging.ERROR,
+    1: logging.WARNING,
+    2: logging.INFO,
+}
+
+
+ignores = {"__pycache__", ".pytest_cache"}
+
+
+def traverse_directory(path: pathlib.Path) -> list[pathlib.Path]:
+    of_interest: list[pathlib.Path] = []
+
+    file_found = False
+
+    for member in path.iterdir():
+        if not member.is_dir():
+            file_found = True
+            continue
+
+        if member.name in ignores:
+            continue
+
+        found = traverse_directory(path=member)
+        of_interest.extend(found)
+
+        if len(found) > 0:
+            of_interest.append(member)
+
+    if len(of_interest) > 0 or file_found:
+        of_interest.append(path)
+
+    return of_interest
+
+
+@click.command()
+@click.option(
+    "-r", "--root", "root_str", type=click.Path(dir_okay=True, file_okay=False, resolve_path=True), default="."
+)
+@click.option("-v", "--verbose", count=True, help=f"Increase verbosity up to {len(log_levels) - 1} times")
+def command(verbose, root_str):
+    logger = logging.getLogger()
+    log_level = log_levels.get(verbose, min(log_levels.values()))
+    logger.setLevel(log_level)
+    stream_handler = logging.StreamHandler()
+    logger.addHandler(stream_handler)
+
+    tree_roots = ["benchmarks", "build_scripts", "chia", "tools"]
+    failed = False
+    root = pathlib.Path(root_str).resolve()
+    directories = [
+        directory for tree_root in tree_roots for directory in traverse_directory(path=root.joinpath(tree_root))
+    ]
+
+    for path in directories:
+        init_path = path.joinpath("__init__.py")
+        # This has plenty of race hazards. If it messes up,
+        # it will likely get caught the next time.
+        if init_path.is_file() and not init_path.is_symlink():
+            logger.info(f"Found   : {init_path}")
+            continue
+        elif not init_path.exists():
+            failed = True
+            init_path.touch()
+            logger.warning(f"Created : {init_path}")
+        else:
+            failed = True
+            logger.error(f"Fail    : present but not a regular file: {init_path}")
+
+    if failed:
+        raise click.ClickException("At least one __init__.py created or not a regular file")
+
+
+command()

chia/_tests/build-job-matrix.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import json
+import logging
+import types
+from pathlib import Path
+from typing import Any
+
+import testconfig
+
+root_path = Path(__file__).parent.absolute()
+project_root_path = root_path.parent.parent
+
+
+def skip(path: Path) -> bool:
+    return any(part.startswith(("_", ".")) and part != "_tests" for part in path.parts)
+
+
+def subdirs(per: str) -> list[Path]:
+    if per == "directory":
+        glob_pattern = "**/"
+    elif per == "file":
+        glob_pattern = "**/test_*.py"
+    else:
+        raise Exception(f"Unrecognized per: {per!r}")
+
+    paths = [path for path in root_path.rglob(glob_pattern) if not skip(path=path)]
+
+    if per == "directory":
+        filtered_paths = []
+        for path in paths:
+            relative_path = path.relative_to(root_path)
+            logging.info(f"Considering: {relative_path}")
+            if len([f for f in path.glob("test_*.py")]) == 0:
+                logging.info(f"Skipping {relative_path}: no tests collected")
+                continue
+
+            filtered_paths.append(path)
+
+        paths = filtered_paths
+
+    return sorted(paths)
+
+
+def module_dict(module: types.ModuleType) -> dict[str, Any]:
+    return {k: v for k, v in module.__dict__.items() if not k.startswith("_") and k != "annotations"}
+
+
+def dir_config(dir: Path) -> dict[str, Any]:
+    import importlib
+
+    module_name = ".".join([*dir.relative_to(root_path).parts, "config"])
+    try:
+        return module_dict(importlib.import_module(module_name))
+    except ModuleNotFoundError:
+        return {}
+
+
+@dataclasses.dataclass
+class SpecifiedDefaultsError(Exception):
+    overlap: dict[str, Any]
+
+    def __post_init__(self) -> None:
+        super().__init__()
+
+
+# Overwrite with directory specific values
+def update_config(parent: dict[str, Any], child: dict[str, Any]) -> dict[str, Any]:
+    if child is None:
+        return parent
+    conf = child
+
+    # avoid manual configuration set to default values
+    common_keys = set(parent.keys()).intersection(child.keys())
+    specified_defaulted_values = {k: parent[k] for k in common_keys if parent[k] == child[k]}
+    if len(specified_defaulted_values) > 0:
+        raise SpecifiedDefaultsError(overlap=specified_defaulted_values)
+
+    for k, v in parent.items():
+        if k not in child:
+            conf[k] = v
+
+    return conf
+
+
+# args
+arg_parser = argparse.ArgumentParser(description="Generate GitHub test matrix configuration")
+arg_parser.add_argument("--per", type=str, choices=["directory", "file"], required=True)
+arg_parser.add_argument("--verbose", "-v", action="store_true")
+arg_parser.add_argument("--only", action="append", default=[])
+arg_parser.add_argument("--duplicates", type=int, default=1)
+arg_parser.add_argument("--timeout-multiplier", type=float, default=1)
+args = arg_parser.parse_args()
+
+if args.verbose:
+    logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG)
+
+# main
+if len(args.only) == 0:
+    test_paths = subdirs(per=args.per)
+else:
+    test_paths = [root_path.joinpath(path) for path in args.only]
+
+test_paths_with_index = [(path, index + 1) for path in test_paths for index in range(args.duplicates)]
+
+configuration = []
+
+specified_defaults: dict[Path, dict[str, Any]] = {}
+pytest_monitor_enabling_paths: list[Path] = []
+
+for path, index in test_paths_with_index:
+    if path.is_dir():
+        test_files = sorted(path.glob("test_*.py"))
+        paths_for_cli_list = [file.relative_to(project_root_path) for file in test_files]
+        config_path = path
+    else:
+        paths_for_cli_list = [path.relative_to(project_root_path)]
+        config_path = path.parent
+
+    def mung_path(path: Path) -> str:
+        # TODO: shell escaping, but that's per platform...
+        return ".".join(path.with_suffix("").parts)
+
+    paths_for_cli = " ".join(mung_path(path) for path in paths_for_cli_list)
+    paths_for_cli = f"--pyargs {paths_for_cli}"
+
+    try:
+        conf = update_config(module_dict(testconfig), dir_config(config_path))
+    except SpecifiedDefaultsError as e:
+        specified_defaults[root_path.joinpath(config_path, "config.py")] = e.overlap
+        continue
+
+    # TODO: design a configurable system for this
+    process_count = {
+        "macos": {False: 0, True: 4}.get(conf["parallel"], conf["parallel"]),
+        "ubuntu": {False: 0, True: 6}.get(conf["parallel"], conf["parallel"]),
+        "windows": {False: 0, True: 4}.get(conf["parallel"], conf["parallel"]),
+    }
+    pytest_parallel_args = {os: f" -n {count}" for os, count in process_count.items()}
+
+    enable_pytest_monitor = conf["check_resource_usage"]
+
+    if enable_pytest_monitor:
+        # NOTE: do not use until the hangs are fixed
+        #       https://github.com/CFMTech/pytest-monitor/issues/53
+        #       https://github.com/pythonprofilers/memory_profiler/issues/342
+
+        pytest_monitor_enabling_paths.append(path)
+
+    max_index_characters = len(str(args.duplicates))
+    index_string = f"{index:0{max_index_characters}d}"
+    module_import_path = ".".join(path.relative_to(root_path).with_suffix("").parts)
+    if args.duplicates == 1:
+        name = module_import_path
+        file_name_index = ""
+    else:
+        name = f"{module_import_path} #{index_string}"
+        file_name_index = f"_{index_string}"
+
+    for_matrix = {
+        "check_resource_usage": conf["check_resource_usage"],
+        "enable_pytest_monitor": "-p monitor" if enable_pytest_monitor else "",
+        "job_timeout": round(conf["job_timeout"] * args.timeout_multiplier),
+        "pytest_parallel_args": pytest_parallel_args,
+        "checkout_blocks_and_plots": conf["checkout_blocks_and_plots"],
+        "install_timelord": conf["install_timelord"],
+        "test_files": paths_for_cli,
+        "name": name,
+        "legacy_keyring_required": conf.get("legacy_keyring_required", False),
+        "index": index,
+        "index_string": index_string,
+        "module_import_path": module_import_path,
+        "file_name_index": file_name_index,
+    }
+    for_matrix = dict(sorted(for_matrix.items()))
+    configuration.append(for_matrix)
+
+messages: list[str] = []
+
+if len(specified_defaults) > 0:
+    message = f"Found {len(specified_defaults)} directories with specified defaults"
+    messages.append(message)
+    logging.error(f"{message}:")
+    for path, overlap in sorted(specified_defaults.items()):
+        logging.error(f" {path} : {overlap}")
+
+if len(pytest_monitor_enabling_paths) > 0:
+    message = f"Found {len(pytest_monitor_enabling_paths)} directories with pytest-monitor enabled"
+    messages.append(message)
+    logging.error(f"{message}:")
+    for path in sorted(pytest_monitor_enabling_paths):
+        logging.error(f" {path}")
+
+if len(messages) > 0:
+    raise Exception("\n".join(messages))
+
+configuration_json = json.dumps(configuration)
+
+for line in json.dumps(configuration, indent=2).splitlines():
+    logging.info(line)
+
+print(f"{configuration_json}")

chia/_tests/check_pytest_monitor_output.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import sys
+
+ret = 0
+
+# example input line
+# test_non_tx_aggregate_limits 0.997759588095738 1.45325589179993 554.45703125
+for ln in sys.stdin:
+    line = ln.strip().split()
+
+    print(f"{float(line[1]) * 100.0: 8.1f}% CPU {float(line[2]):7.1f}s {float(line[3]): 8.2f} MB RAM  {line[0]}")
+    limit = 800
+
+    # until this can be optimized, use higher limits
+    if "test_duplicate_coin_announces" in line[0]:
+        limit = 2200
+    elif (
+        "test_duplicate_large_integer_substr" in line[0]
+        or "test_duplicate_reserve_fee" in line[0]
+        or "test_duplicate_large_integer_negative" in line[0]
+        or "test_duplicate_large_integer" in line[0]
+    ):
+        limit = 1100
+
+    if float(line[3]) > limit:
+        print("   ERROR: ^^ exceeded RAM limit ^^ \n")
+        ret += 1
+
+if ret > 0:
+    print("some tests used too much RAM")
+
+sys.exit(ret)