chia-blockchain 2.5.1rc1__py3-none-any.whl

chia/server/start_service.py

@@ -0,0 +1,316 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import logging.config
+import os
+import signal
+from collections.abc import AsyncIterator, Awaitable, Coroutine
+from pathlib import Path
+from types import FrameType
+from typing import Any, Callable, Generic, Optional, TypeVar, cast
+
+from chia.daemon.server import service_launch_lock_path
+from chia.protocols.shared_protocol import default_capabilities
+from chia.rpc.rpc_server import RpcApiProtocol, RpcServer, RpcServiceProtocol, start_rpc_server
+from chia.server.api_protocol import ApiProtocol
+from chia.server.chia_policy import set_chia_policy
+from chia.server.outbound_message import NodeType
+from chia.server.server import ChiaServer
+from chia.server.signal_handlers import SignalHandlers
+from chia.server.ssl_context import chia_ssl_ca_paths, private_ssl_ca_paths
+from chia.server.upnp import UPnP
+from chia.server.ws_connection import WSChiaConnection
+from chia.types.peer_info import PeerInfo, UnresolvedPeerInfo
+from chia.util.chia_version import chia_short_version
+from chia.util.ints import uint16
+from chia.util.lock import Lockfile, LockfileError
+from chia.util.log_exceptions import log_exceptions
+from chia.util.network import resolve
+from chia.util.setproctitle import setproctitle
+from chia.util.task_referencer import create_referenced_task
+
+# this is used to detect whether we are running in the main process or not, in
+# signal handlers. We need to ignore signals in the sub processes.
+main_pid: Optional[int] = None
+
+T = TypeVar("T")
+_T_RpcServiceProtocol = TypeVar("_T_RpcServiceProtocol", bound=RpcServiceProtocol)
+_T_ApiProtocol = TypeVar("_T_ApiProtocol", bound=ApiProtocol)
+_T_RpcApiProtocol = TypeVar("_T_RpcApiProtocol", bound=RpcApiProtocol)
+
+RpcInfo = tuple[type[_T_RpcApiProtocol], int]
+
+log = logging.getLogger(__name__)
+
+
+class ServiceException(Exception):
+    pass
+
+
+class Service(Generic[_T_RpcServiceProtocol, _T_ApiProtocol, _T_RpcApiProtocol]):
+    def __init__(
+        self,
+        root_path: Path,
+        node: _T_RpcServiceProtocol,
+        peer_api: _T_ApiProtocol,
+        node_type: NodeType,
+        advertised_port: Optional[int],
+        service_name: str,
+        network_id: str,
+        *,
+        config: dict[str, Any],
+        class_for_type: dict[NodeType, type[ApiProtocol]],
+        upnp_ports: Optional[list[int]] = None,
+        connect_peers: Optional[set[UnresolvedPeerInfo]] = None,
+        on_connect_callback: Optional[Callable[[WSChiaConnection], Awaitable[None]]] = None,
+        rpc_info: Optional[RpcInfo[_T_RpcApiProtocol]] = None,
+        connect_to_daemon: bool = True,
+        max_request_body_size: Optional[int] = None,
+        override_capabilities: Optional[list[tuple[uint16, str]]] = None,
+    ) -> None:
+        if upnp_ports is None:
+            upnp_ports = []
+
+        if connect_peers is None:
+            connect_peers = set()
+
+        self.root_path = root_path
+        self.config = config
+        ping_interval = self.config.get("ping_interval")
+        self.self_hostname = cast(str, self.config.get("self_hostname"))
+        self.daemon_port = self.config.get("daemon_port")
+        assert ping_interval is not None
+        self._connect_to_daemon = connect_to_daemon
+        self._node_type = node_type
+        self._service_name = service_name
+        self.rpc_server: Optional[RpcServer[_T_RpcApiProtocol]] = None
+        self._network_id: str = network_id
+        self.max_request_body_size = max_request_body_size
+        self.reconnect_retry_seconds: int = 3
+
+        self._log = logging.getLogger(service_name)
+        self._log.info(f"Starting service {self._service_name} ...")
+        self._log.info(f"chia-blockchain version: {chia_short_version()}")
+
+        self.service_config = self.config[service_name]
+
+        self._rpc_info = rpc_info
+        private_ca_crt, private_ca_key = private_ssl_ca_paths(root_path, self.config)
+        chia_ca_crt, chia_ca_key = chia_ssl_ca_paths(root_path, self.config)
+        inbound_rlp = self.config.get("inbound_rate_limit_percent")
+        outbound_rlp = self.config.get("outbound_rate_limit_percent")
+        if node_type == NodeType.WALLET:
+            inbound_rlp = self.service_config.get("inbound_rate_limit_percent", inbound_rlp)
+            outbound_rlp = 60
+        capabilities_to_use: list[tuple[uint16, str]] = default_capabilities[node_type]
+        if override_capabilities is not None:
+            capabilities_to_use = override_capabilities
+
+        assert inbound_rlp and outbound_rlp
+        self._server = ChiaServer.create(
+            advertised_port,
+            node,
+            peer_api,
+            node_type,
+            ping_interval,
+            network_id,
+            inbound_rlp,
+            outbound_rlp,
+            capabilities_to_use,
+            root_path,
+            self.service_config,
+            (private_ca_crt, private_ca_key),
+            (chia_ca_crt, chia_ca_key),
+            class_for_type=class_for_type,
+            name=f"{service_name}_server",
+        )
+        f = getattr(node, "set_server", None)
+        if f:
+            f(self._server)
+        else:
+            self._log.warning(f"No set_server method for {service_name}")
+
+        self._upnp_ports = upnp_ports
+
+        self._api = peer_api
+        self._node = node
+        self._stopped_by_rpc = False
+
+        self._on_connect_callback = on_connect_callback
+        self._advertised_port = advertised_port
+        self._connect_peers = connect_peers
+        self._connect_peers_task: Optional[asyncio.Task[None]] = None
+        self.upnp: UPnP = UPnP()
+        self.stop_requested = asyncio.Event()
+
+    async def _connect_peers_task_handler(self) -> None:
+        resolved_peers: dict[UnresolvedPeerInfo, PeerInfo] = {}
+        prefer_ipv6 = self.config.get("prefer_ipv6", False)
+        while True:
+            for unresolved in self._connect_peers:
+                resolved = resolved_peers.get(unresolved)
+                if resolved is None:
+                    try:
+                        resolved = PeerInfo(await resolve(unresolved.host, prefer_ipv6=prefer_ipv6), unresolved.port)
+                    except Exception as e:
+                        self._log.warning(f"Failed to resolve {unresolved.host}: {e}")
+                        continue
+                    self._log.info(f"Add resolved {resolved}")
+                    resolved_peers[unresolved] = resolved
+
+                if any(connection.peer_info == resolved for connection in self._server.all_connections.values()):
+                    continue
+                if any(
+                    connection.peer_info.host == resolved.host and connection.peer_server_port == resolved.port
+                    for connection in self._server.all_connections.values()
+                ):
+                    continue
+
+                if not await self._server.start_client(resolved, None):
+                    self._log.info(f"Failed to connect to {resolved}")
+                    # Re-resolve to make sure the IP didn't change, this helps for example to keep dyndns hostnames
+                    # up to date.
+                    try:
+                        resolved_new = PeerInfo(
+                            await resolve(unresolved.host, prefer_ipv6=prefer_ipv6), unresolved.port
+                        )
+                    except Exception as e:
+                        self._log.warning(f"Failed to resolve after connection failure {unresolved.host}: {e}")
+                        continue
+                    if resolved_new != resolved:
+                        self._log.info(f"Host {unresolved.host} changed from {resolved} to {resolved_new}")
+                        resolved_peers[unresolved] = resolved_new
+            await asyncio.sleep(self.reconnect_retry_seconds)
+
+    async def run(self) -> None:
+        try:
+            with Lockfile.create(service_launch_lock_path(self.root_path, self._service_name), timeout=1):
+                async with self.manage():
+                    await self.stop_requested.wait()
+        except LockfileError as e:
+            self._log.error(f"{self._service_name}: already running")
+            raise ValueError(f"{self._service_name}: already running") from e
+
+    @contextlib.asynccontextmanager
+    async def manage(self, *, start: bool = True) -> AsyncIterator[None]:
+        # NOTE: avoid start=False, this is presently used for corner case setup type tests
+        async with contextlib.AsyncExitStack() as async_exit_stack:
+            try:
+                if start:
+                    self.stop_requested = asyncio.Event()
+
+                    assert self.self_hostname is not None
+                    assert self.daemon_port is not None
+
+                    await async_exit_stack.enter_async_context(self._node.manage())
+                    self._node._shut_down = False
+
+                    if len(self._upnp_ports) > 0:
+                        async_exit_stack.enter_context(self.upnp.manage(self._upnp_ports))
+
+                    await self._server.start(
+                        prefer_ipv6=self.config.get("prefer_ipv6", False),
+                        on_connect=self._on_connect_callback,
+                    )
+                    try:
+                        self._advertised_port = self._server.get_port()
+                    except ValueError:
+                        pass
+
+                    self._connect_peers_task = create_referenced_task(self._connect_peers_task_handler())
+
+                    self._log.info(
+                        f"Started {self._service_name} service on network_id: {self._network_id} "
+                        f"at port {self._advertised_port}"
+                    )
+
+                    if self._rpc_info:
+                        rpc_api, rpc_port = self._rpc_info
+                        self.rpc_server = await start_rpc_server(
+                            rpc_api(self._node),
+                            self.self_hostname,
+                            self.daemon_port,
+                            uint16(rpc_port),
+                            self.stop_requested.set,
+                            self.root_path,
+                            self.config,
+                            service_config=self.service_config,
+                            connect_to_daemon=self._connect_to_daemon,
+                            max_request_body_size=self.max_request_body_size,
+                        )
+                yield
+            finally:
+                self._log.info(f"Stopping service {self._service_name} at port {self._advertised_port} ...")
+
+                # start with UPnP, since this can take a while, we want it to happen
+                # in the background while shutting down everything else
+                for port in self._upnp_ports:
+                    self.upnp.release(port)
+
+                self._log.info("Cancelling reconnect task")
+                if self._connect_peers_task is not None:
+                    self._connect_peers_task.cancel()
+                self._log.info("Closing connections")
+                self._server.close_all()
+
+                if self.rpc_server is not None:
+                    self._log.info("Closing RPC server")
+                    self.rpc_server.close()
+
+                self._log.info("Waiting for socket to be closed (if opened)")
+
+                self._log.info("Waiting for ChiaServer to be closed")
+                await self._server.await_closed()
+
+                if self.rpc_server:
+                    self._log.info("Waiting for RPC server")
+                    await self.rpc_server.await_closed()
+                    self._log.info("Closed RPC server")
+
+                self._log.info(f"Service {self._service_name} at port {self._advertised_port} fully stopped")
+
+    def add_peer(self, peer: UnresolvedPeerInfo) -> None:
+        self._connect_peers.add(peer)
+
+    async def setup_process_global_state(self, signal_handlers: SignalHandlers) -> None:
+        # Being async forces this to be run from within an active event loop as is
+        # needed for the signal handler setup.
+        proctitle_name = f"chia_{self._service_name}"
+        setproctitle(proctitle_name)
+
+        global main_pid
+        main_pid = os.getpid()
+        signal_handlers.setup_sync_signal_handler(handler=self._accept_signal)
+
+    def _accept_signal(
+        self,
+        signal_: signal.Signals,
+        stack_frame: Optional[FrameType],
+        loop: asyncio.AbstractEventLoop,
+    ) -> None:
+        # we only handle signals in the main process. In the ProcessPoolExecutor
+        # processes, we have to ignore them. We'll shut them down gracefully
+        # from the main process
+        ignore = os.getpid() != main_pid
+
+        # TODO: if we remove this conditional behavior, consider moving logging to common signal handling
+        if ignore:
+            message = "ignoring in worker process"
+        else:
+            message = "shutting down"
+
+        self._log.info("Received signal %s (%s), %s.", signal_.name, signal_.value, message)
+
+        if ignore:
+            return
+
+        self.stop_requested.set()
+
+
+def async_run(coro: Coroutine[object, object, T], connection_limit: Optional[int] = None) -> T:
+    with log_exceptions(log=log, message="fatal uncaught exception"):
+        if connection_limit is not None:
+            set_chia_policy(connection_limit)
+        return asyncio.run(coro)

chia/server/start_timelord.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+from typing import Any, Optional
+
+from chia.apis import ApiProtocolRegistry
+from chia.consensus.constants import ConsensusConstants, replace_str_to_bytes
+from chia.consensus.default_constants import DEFAULT_CONSTANTS
+from chia.rpc.timelord_rpc_api import TimelordRpcApi
+from chia.server.outbound_message import NodeType
+from chia.server.signal_handlers import SignalHandlers
+from chia.server.start_service import RpcInfo, Service, async_run
+from chia.timelord.timelord import Timelord
+from chia.timelord.timelord_api import TimelordAPI
+from chia.types.aliases import TimelordService
+from chia.util.chia_logging import initialize_service_logging
+from chia.util.config import get_unresolved_peer_infos, load_config, load_config_cli
+from chia.util.default_root import resolve_root_path
+from chia.util.task_timing import maybe_manage_task_instrumentation
+
+# See: https://bugs.python.org/issue29288
+"".encode("idna")
+
+SERVICE_NAME = "timelord"
+
+
+def create_timelord_service(
+    root_path: pathlib.Path,
+    config: dict[str, Any],
+    consensus_constants: ConsensusConstants,
+    connect_to_daemon: bool = True,
+) -> TimelordService:
+    service_config = config[SERVICE_NAME]
+
+    network_id = service_config["selected_network"]
+    overrides = service_config["network_overrides"]["constants"][network_id]
+    updated_constants = replace_str_to_bytes(consensus_constants, **overrides)
+
+    node = Timelord(root_path, service_config, updated_constants)
+    peer_api = TimelordAPI(node)
+
+    rpc_info: Optional[RpcInfo[TimelordRpcApi]] = None
+    if service_config.get("start_rpc_server", True):
+        rpc_info = (TimelordRpcApi, service_config.get("rpc_port", 8557))
+
+    return Service(
+        root_path=root_path,
+        config=config,
+        node=node,
+        peer_api=peer_api,
+        node_type=NodeType.TIMELORD,
+        advertised_port=None,
+        service_name=SERVICE_NAME,
+        connect_peers=get_unresolved_peer_infos(service_config, NodeType.FULL_NODE),
+        network_id=network_id,
+        rpc_info=rpc_info,
+        connect_to_daemon=connect_to_daemon,
+        class_for_type=ApiProtocolRegistry,
+    )
+
+
+async def async_main(root_path: pathlib.Path) -> int:
+    # TODO: refactor to avoid the double load
+    config = load_config(root_path, "config.yaml")
+    service_config = load_config_cli(root_path, "config.yaml", SERVICE_NAME)
+    config[SERVICE_NAME] = service_config
+    initialize_service_logging(service_name=SERVICE_NAME, config=config, root_path=root_path)
+
+    service = create_timelord_service(root_path, config, DEFAULT_CONSTANTS)
+    async with SignalHandlers.manage() as signal_handlers:
+        await service.setup_process_global_state(signal_handlers=signal_handlers)
+        await service.run()
+
+    return 0
+
+
+def main() -> int:
+    root_path = resolve_root_path(override=None)
+
+    with maybe_manage_task_instrumentation(
+        enable=os.environ.get(f"CHIA_INSTRUMENT_{SERVICE_NAME.upper()}") is not None
+    ):
+        return async_run(coro=async_main(root_path=root_path))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

chia/server/start_wallet.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+from multiprocessing import freeze_support
+from typing import Any, Optional
+
+from chia.apis import ApiProtocolRegistry
+from chia.consensus.constants import ConsensusConstants, replace_str_to_bytes
+from chia.consensus.default_constants import DEFAULT_CONSTANTS
+from chia.rpc.wallet_rpc_api import WalletRpcApi
+from chia.server.outbound_message import NodeType
+from chia.server.signal_handlers import SignalHandlers
+from chia.server.start_service import RpcInfo, Service, async_run
+from chia.types.aliases import WalletService
+from chia.util.chia_logging import initialize_service_logging
+from chia.util.config import get_unresolved_peer_infos, load_config, load_config_cli
+from chia.util.default_root import resolve_root_path
+from chia.util.keychain import Keychain
+from chia.util.task_timing import maybe_manage_task_instrumentation
+from chia.wallet.wallet_node import WalletNode
+
+# See: https://bugs.python.org/issue29288
+from chia.wallet.wallet_node_api import WalletNodeAPI
+
+"".encode("idna")
+
+SERVICE_NAME = "wallet"
+
+
+def create_wallet_service(
+    root_path: pathlib.Path,
+    config: dict[str, Any],
+    consensus_constants: ConsensusConstants,
+    keychain: Optional[Keychain] = None,
+    connect_to_daemon: bool = True,
+) -> WalletService:
+    service_config = config[SERVICE_NAME]
+
+    network_id = service_config["selected_network"]
+    overrides = service_config["network_overrides"]["constants"][network_id]
+    updated_constants = replace_str_to_bytes(consensus_constants, **overrides)
+    service_config.setdefault("short_sync_blocks_behind_threshold", 20)
+
+    node = WalletNode(
+        service_config,
+        root_path,
+        constants=updated_constants,
+        local_keychain=keychain,
+    )
+    peer_api = WalletNodeAPI(node)
+
+    rpc_info: Optional[RpcInfo[WalletRpcApi]] = None
+    if service_config.get("start_rpc_server", True):
+        rpc_info = (WalletRpcApi, service_config["rpc_port"])
+
+    return Service(
+        root_path=root_path,
+        config=config,
+        node=node,
+        peer_api=peer_api,
+        node_type=NodeType.WALLET,
+        advertised_port=None,
+        service_name=SERVICE_NAME,
+        connect_peers=get_unresolved_peer_infos(service_config, NodeType.FULL_NODE),
+        on_connect_callback=node.on_connect,
+        network_id=network_id,
+        rpc_info=rpc_info,
+        connect_to_daemon=connect_to_daemon,
+        class_for_type=ApiProtocolRegistry,
+    )
+
+
+async def async_main(root_path: pathlib.Path) -> int:
+    # TODO: refactor to avoid the double load
+    config = load_config(root_path, "config.yaml")
+    service_config = load_config_cli(root_path, "config.yaml", SERVICE_NAME)
+    config[SERVICE_NAME] = service_config
+
+    # This is simulator
+    local_test = service_config.get("testing", False)
+    if local_test is True:
+        from chia.simulator.block_tools import test_constants
+
+        constants = test_constants
+        current = service_config["database_path"]
+        service_config["database_path"] = f"{current}_simulation"
+        service_config["selected_network"] = "testnet0"
+    else:
+        constants = DEFAULT_CONSTANTS
+    initialize_service_logging(service_name=SERVICE_NAME, config=config, root_path=root_path)
+
+    service = create_wallet_service(root_path, config, constants)
+    async with SignalHandlers.manage() as signal_handlers:
+        await service.setup_process_global_state(signal_handlers=signal_handlers)
+        await service.run()
+
+    return 0
+
+
+def main() -> int:
+    freeze_support()
+    root_path = resolve_root_path(override=None)
+
+    with maybe_manage_task_instrumentation(
+        enable=os.environ.get(f"CHIA_INSTRUMENT_{SERVICE_NAME.upper()}") is not None
+    ):
+        return async_run(coro=async_main(root_path=root_path))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

chia/server/upnp.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import contextlib
+import logging
+import threading
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from queue import Queue
+from typing import Optional, Union
+
+from typing_extensions import Literal
+
+log = logging.getLogger(__name__)
+
+try:
+    import miniupnpc
+except ImportError:
+    log.info(
+        "importing miniupnpc failed."
+        " This is not required to run chia, it allows incoming connections from other peers."
+    )
+    miniupnpc = None
+
+
+@dataclass
+class UPnP:
+    _thread: Optional[threading.Thread] = None
+    _queue: Queue[Union[tuple[Literal["remap", "release"], int], tuple[Literal["shutdown"]]]] = field(
+        default_factory=Queue,
+    )
+    _upnp: Optional[miniupnpc.UPnP] = None
+
+    @contextlib.contextmanager
+    def manage(self, ports: list[int]) -> Iterator[None]:
+        self.setup()
+        try:
+            for port in ports:
+                self.remap(port)
+            yield
+        finally:
+            for port in ports:
+                self.release(port)
+            self.shutdown()
+
+    def setup(self) -> None:
+        if miniupnpc is None:
+            return
+
+        if self._thread is not None:
+            raise Exception(f"already started, {type(self).__name__} instances are not reusable")
+
+        self._thread = threading.Thread(target=self._run)
+        self._thread.start()
+
+    def _is_alive(self) -> bool:
+        if self._thread is None:
+            return False
+
+        return self._thread.is_alive()
+
+    def _run(self) -> None:
+        try:
+            self._upnp = miniupnpc.UPnP()
+            self._upnp.discoverdelay = 30
+            self._upnp.discover()
+            self._upnp.selectigd()
+            keep_going = True
+            while keep_going:
+                msg = self._queue.get()
+                if msg[0] == "remap":
+                    port = msg[1]
+                    log.info(f"Attempting to enable UPnP (open up port {port})")
+                    try:
+                        self._upnp.deleteportmapping(port, "TCP")
+                    except Exception as e:
+                        log.info(f"Removal of previous portmapping failed. This does not indicate an error: {e}")
+                    self._upnp.addportmapping(port, "TCP", self._upnp.lanaddr, port, "chia", "")
+                    log.info(
+                        f"Port {port} opened with UPnP. lanaddr {self._upnp.lanaddr} "
+                        f"external: {self._upnp.externalipaddress()}"
+                    )
+                elif msg[0] == "release":
+                    port = msg[1]
+                    log.info(f"UPnP, releasing port {port}")
+                    self._upnp.deleteportmapping(port, "TCP")
+                    log.info(f"UPnP, Port {port} closed")
+                elif msg[0] == "shutdown":
+                    keep_going = False
+        except Exception as e:
+            log.info("UPnP failed. This is not required to run chia, it allows incoming connections from other peers.")
+            log.info(e)
+
+    def remap(self, port: int) -> None:
+        if not self._is_alive():
+            return
+
+        self._queue.put(("remap", port))
+
+    def release(self, port: int) -> None:
+        if not self._is_alive():
+            return
+
+        self._queue.put(("release", port))
+
+    def shutdown(self) -> None:
+        if self._thread is None:
+            return
+
+        if self._is_alive():
+            self._queue.put(("shutdown",))
+            log.info("UPnP, shutting down thread")
+
+        self._thread.join(5)
+
+    # this is here just in case the UPnP object is destroyed non-gracefully,
+    # e.g. via an exception before the main thread can call shutdown()
+    def __del__(self) -> None:
+        self.shutdown()