remote-exec-grpc 0.2.1__py3-none-any.whl

remote_exec/cmdlog.py

@@ -0,0 +1,159 @@
+"""Command log: every executed command together with its output.
+
+One text file shared by all executions (each line carries the execution id,
+so concurrent commands can be told apart)::
+
+    2026-10-04 10:00:00.120 [3f2a9c1d0b7e] START client=ipv4:10.0.0.7:51234 cwd="/proj" timeout=0 env_keys="" command="make"
+    2026-10-04 10:00:00.250 [3f2a9c1d0b7e] OUT| gcc -c main.c
+    2026-10-04 10:00:00.260 [3f2a9c1d0b7e] ERR| warning: unused variable
+    2026-10-04 10:00:01.900 [3f2a9c1d0b7e] END status=exited exit_code=0 duration=1.780s
+
+Output is split into lines per stream (undecodable bytes are replaced).
+When the file would exceed ``max_bytes`` it is renamed to
+``<name>.<timestamp><suffix>`` and a new file is started; ``keep`` limits how
+many rotated files are retained (0 = keep all). Files are created with mode
+600 because command output may contain sensitive data. Environment variable
+values are never written.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+DEFAULT_COMMAND_LOG = "remote-exec-commands.log"
+DEFAULT_MAX_MB = 10.0
+MAX_LINE_BYTES = 64 * 1024  # longer output without newline is written in pieces
+
+
+def _q(value: str) -> str:
+    return json.dumps(value, ensure_ascii=False)
+
+
+class CommandLog:
+    def __init__(self, path: str | os.PathLike, max_bytes: int = int(DEFAULT_MAX_MB * 1024 * 1024), keep: int = 0):
+        if max_bytes < 1024:
+            raise ValueError("command log size limit must be at least 1 KiB")
+        if keep < 0:
+            raise ValueError("number of rotated command logs to keep must be >= 0")
+        self.path = Path(path).expanduser().absolute()
+        self.max_bytes = max_bytes
+        self.keep = keep
+        self._pending: dict[tuple[str, str], bytearray] = {}
+        self._error_reported = False
+        self._file = None
+        self._open()
+
+    # ------------------------------------------------------------ public API
+
+    def start(self, execution_id: str, client: str, cwd: str, command: str, timeout: float, env_keys) -> None:
+        self._write(self._line(
+            execution_id,
+            f"START client={client} cwd={_q(cwd)} timeout={timeout:g} "
+            f"env_keys={_q(','.join(sorted(env_keys)))} command={_q(command)}",
+        ))
+
+    def output(self, execution_id: str, stream: str, data: bytes) -> None:
+        """Record a chunk of ``stream`` ("stdout"/"stderr"); complete lines are written."""
+        key = (execution_id, stream)
+        buf = self._pending.setdefault(key, bytearray())
+        buf.extend(data)
+        lines = []
+        while (nl := buf.find(b"\n")) >= 0:
+            lines.append(bytes(buf[:nl]))
+            del buf[:nl + 1]
+        while len(buf) > MAX_LINE_BYTES:
+            lines.append(bytes(buf[:MAX_LINE_BYTES]))
+            del buf[:MAX_LINE_BYTES]
+        if lines:
+            self._write("".join(self._output_line(execution_id, stream, line) for line in lines))
+
+    def end(self, execution_id: str, status: str, exit_code, duration: float) -> None:
+        text = []
+        for stream in ("stdout", "stderr"):
+            rest = self._pending.pop((execution_id, stream), None)
+            if rest:
+                text.append(self._output_line(execution_id, stream, bytes(rest)))
+        text.append(self._line(execution_id, f"END status={status} exit_code={exit_code} duration={duration:.3f}s"))
+        self._write("".join(text))
+
+    def rejected(self, client: str, command: str, reason: str) -> None:
+        self._write(self._line("-" * 12, f"REJECTED client={client} reason={_q(reason)} command={_q(command)}"))
+
+    def close(self) -> None:
+        if self._file:
+            self._file.close()
+            self._file = None
+
+    # ------------------------------------------------------------ internals
+
+    @staticmethod
+    def _line(execution_id: str, message: str) -> str:
+        ts = datetime.now().isoformat(sep=" ", timespec="milliseconds")
+        return f"{ts} [{execution_id}] {message}\n"
+
+    def _output_line(self, execution_id: str, stream: str, raw: bytes) -> str:
+        text = raw.decode("utf-8", errors="replace").rstrip("\r")
+        return self._line(execution_id, f"{'OUT' if stream == 'stdout' else 'ERR'}| {text}")
+
+    def _open(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+        self._file = os.fdopen(fd, "a", encoding="utf-8")
+        self._size = os.fstat(fd).st_size
+
+    def _write(self, text: str) -> None:
+        if self._file is None:
+            return
+        data_len = len(text.encode("utf-8"))
+        try:
+            if self._size > 0 and self._size + data_len > self.max_bytes:
+                self._rotate()
+            self._file.write(text)
+            self._file.flush()
+            self._size += data_len
+        except OSError as exc:
+            # Never let logging problems (e.g. disk full) break command execution.
+            if not self._error_reported:
+                log.error("cannot write command log %s: %s (further errors suppressed)", self.path, exc)
+                self._error_reported = True
+
+    def _rotated_name(self) -> Path:
+        stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+        candidate = self.path.with_name(f"{self.path.stem}.{stamp}{self.path.suffix}")
+        n = 1
+        while candidate.exists():
+            candidate = self.path.with_name(f"{self.path.stem}.{stamp}-{n}{self.path.suffix}")
+            n += 1
+        return candidate
+
+    def rotated_files(self) -> list[Path]:
+        """Rotated files, oldest first (by last write time, then name)."""
+        pattern = f"{self.path.stem}.*{self.path.suffix}"
+        files = []
+        for p in self.path.parent.glob(pattern):
+            if p != self.path:
+                try:
+                    files.append((p.stat().st_mtime, p.name, p))
+                except OSError:
+                    pass
+        return [p for _, _, p in sorted(files)]
+
+    def _rotate(self) -> None:
+        self._file.close()
+        self._file = None
+        try:
+            os.replace(self.path, self._rotated_name())
+            if self.keep:
+                for old in self.rotated_files()[:-self.keep]:
+                    try:
+                        old.unlink()
+                    except OSError:
+                        pass
+        finally:
+            self._open()  # if the rename failed, keep appending to the current file

remote_exec/config.py

@@ -0,0 +1,202 @@
+"""Configuration loading.
+
+Client config (YAML), default location ``$REMOTE_EXEC_CONFIG`` or
+``${XDG_CONFIG_HOME:-~/.config}/remote-exec/config.yaml``::
+
+    defaults:              # optional, merged into every server entry
+      token_file: ~/.config/remote-exec/token
+    servers:
+      fpga01:
+        host: fpga01.example.local
+        port: 50052        # optional, defaults to 50051
+        description: FPGA builds (Vivado, Questa)
+
+Server config (YAML, only with ``remote-exec-server --config``) uses the long
+option names of the server CLI as keys, e.g. ``host``, ``port``, ``token_file``.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+DEFAULT_PORT = 50051
+DEFAULT_HOST = "127.0.0.1"
+CONFIG_ENV_VAR = "REMOTE_EXEC_CONFIG"
+TOKEN_ENV_VAR = "REMOTE_EXEC_TOKEN"
+
+
+class ConfigError(Exception):
+    pass
+
+
+def default_client_config_path() -> Path:
+    if env := os.environ.get(CONFIG_ENV_VAR):
+        return Path(env).expanduser()
+    base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
+    return Path(base) / "remote-exec" / "config.yaml"
+
+
+def validate_port(value: Any, where: str = "port") -> int:
+    if isinstance(value, bool):
+        raise ConfigError(f"{where}: invalid port {value!r}")
+    try:
+        port = int(value)
+    except (TypeError, ValueError):
+        raise ConfigError(f"{where}: invalid port {value!r}") from None
+    if not 1 <= port <= 65535:
+        raise ConfigError(f"{where}: port {port} out of range 1-65535")
+    return port
+
+
+def split_host_port(value: str, default_port: int = DEFAULT_PORT) -> tuple[str, int]:
+    """Parse ``host``, ``host:port``, ``[v6]`` or ``[v6]:port``."""
+    value = value.strip()
+    if value.startswith("["):
+        host, sep, rest = value[1:].partition("]")
+        if not sep:
+            raise ConfigError(f"invalid address {value!r}")
+        if not rest:
+            return host, default_port
+        if not rest.startswith(":"):
+            raise ConfigError(f"invalid address {value!r}")
+        return host, validate_port(rest[1:], value)
+    if value.count(":") == 1:
+        host, port = value.split(":")
+        return host, validate_port(port, value)
+    return value, default_port  # plain host or bare IPv6
+
+
+def format_address(host: str, port: int) -> str:
+    return f"[{host}]:{port}" if ":" in host else f"{host}:{port}"
+
+
+def _load_yaml(path: Path) -> dict:
+    try:
+        data = yaml.safe_load(path.read_text()) or {}
+    except OSError as exc:
+        raise ConfigError(f"cannot read config {path}: {exc.strerror or exc}") from exc
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"invalid YAML in {path}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ConfigError(f"{path}: top level must be a mapping")
+    return data
+
+
+# ---------------------------------------------------------------- client
+
+
+@dataclass
+class ServerEndpoint:
+    name: str
+    host: str
+    port: int = DEFAULT_PORT
+    description: str = ""
+    token: str | None = None
+    token_file: str | None = None
+    tls: bool = False
+    ca_cert: str | None = None
+    client_cert: str | None = None
+    client_key: str | None = None
+    tls_server_name: str | None = None
+    connect_timeout: float | None = None
+
+    @property
+    def address(self) -> str:
+        return format_address(self.host, self.port)
+
+    @property
+    def uses_tls(self) -> bool:
+        return self.tls or bool(self.ca_cert or self.client_cert)
+
+
+_ENDPOINT_KEYS = {f.name for f in fields(ServerEndpoint)} - {"name"}
+
+
+def _make_endpoint(name: str, raw: Any, defaults: dict, where: str) -> ServerEndpoint:
+    if not isinstance(raw, dict):
+        raise ConfigError(f"{where}: server entry must be a mapping")
+    if "token" in raw or "token_file" in raw:
+        # A server's own credential replaces the default one entirely; otherwise a
+        # default `token` would win over the server's `token_file`.
+        defaults = {k: v for k, v in defaults.items() if k not in ("token", "token_file")}
+    merged = {**defaults, **raw}
+    unknown = set(merged) - _ENDPOINT_KEYS
+    if unknown:
+        raise ConfigError(f"{where}: unknown key(s): {', '.join(sorted(unknown))}")
+    if not merged.get("host"):
+        raise ConfigError(f"{where}: 'host' is required")
+    merged["host"] = str(merged["host"])
+    merged["port"] = validate_port(merged.get("port", DEFAULT_PORT), f"{where}.port")
+    for key in ("token_file", "ca_cert", "client_cert", "client_key"):
+        if merged.get(key):
+            merged[key] = os.path.expanduser(str(merged[key]))
+    return ServerEndpoint(name=name, **merged)
+
+
+@dataclass
+class ClientConfig:
+    servers: dict[str, ServerEndpoint] = field(default_factory=dict)
+    defaults: dict = field(default_factory=dict)
+    path: Path | None = None
+
+    def resolve(self, target: str) -> ServerEndpoint:
+        """Look ``target`` up by name; otherwise treat it as ``host[:port]``."""
+        if target in self.servers:
+            return self.servers[target]
+        if not target:
+            raise ConfigError("empty server name")
+        host, port = split_host_port(target)
+        defaults = {k: v for k, v in self.defaults.items() if k != "port"}
+        return _make_endpoint(target, {**defaults, "host": host, "port": port}, {}, f"server {target!r}")
+
+
+def load_client_config(path: str | os.PathLike | None = None) -> ClientConfig:
+    """Load the client config. A missing *default* config is not an error."""
+    explicit = path is not None
+    cfg_path = Path(path).expanduser() if explicit else default_client_config_path()
+    if not cfg_path.exists():
+        if explicit or os.environ.get(CONFIG_ENV_VAR):
+            raise ConfigError(f"config file not found: {cfg_path}")
+        return ClientConfig(path=None)
+
+    data = _load_yaml(cfg_path)
+    unknown = set(data) - {"servers", "defaults"}
+    if unknown:
+        raise ConfigError(f"{cfg_path}: unknown top-level key(s): {', '.join(sorted(unknown))}")
+    defaults = data.get("defaults") or {}
+    servers_raw = data.get("servers") or {}
+    if not isinstance(defaults, dict) or not isinstance(servers_raw, dict):
+        raise ConfigError(f"{cfg_path}: 'defaults' and 'servers' must be mappings")
+    servers = {
+        str(name): _make_endpoint(str(name), raw, defaults, f"{cfg_path}: servers.{name}")
+        for name, raw in servers_raw.items()
+    }
+    return ClientConfig(servers=servers, defaults=defaults, path=cfg_path)
+
+
+# ---------------------------------------------------------------- server
+
+
+SERVER_CONFIG_KEYS = {
+    "host", "port", "listen", "token_file", "tls_cert", "tls_key", "tls_client_ca",
+    "max_concurrent", "max_timeout", "shell", "kill_grace", "log_level", "log_file",
+    "insecure_no_auth", "allow_root", "tools",
+    "command_log", "command_log_max_mb", "command_log_keep", "no_command_log",
+}
+
+
+def load_server_config(path: str | os.PathLike) -> dict[str, Any]:
+    cfg_path = Path(path).expanduser()
+    data = _load_yaml(cfg_path)
+    data = {k.replace("-", "_"): v for k, v in data.items()}
+    unknown = set(data) - SERVER_CONFIG_KEYS
+    if unknown:
+        raise ConfigError(f"{cfg_path}: unknown key(s): {', '.join(sorted(unknown))}")
+    if "port" in data:
+        data["port"] = validate_port(data["port"], f"{cfg_path}: port")
+    return data

remote_exec/executor.py

@@ -0,0 +1,287 @@
+"""Subprocess execution with streaming output and process-group cleanup.
+
+This module knows nothing about gRPC. It turns a :class:`CommandSpec` into an
+async stream of :class:`ExecEvent` objects, which keeps it reusable for future
+APIs (e.g. Submit/GetStatus/GetResult backed by a task registry).
+
+Process model
+-------------
+Every command runs as ``<shell> -c <command>`` in a *new session*
+(``start_new_session=True``), so the shell and everything it spawns share a
+process group whose id equals the shell's pid. Cleanup always targets the whole
+group (SIGTERM, then SIGKILL after a grace period) so that tools like Vivado do
+not leave orphaned children behind. This happens when:
+
+* the command times out,
+* the consumer stops iterating (client disconnect / cancellation),
+* the main process exits but left background processes in its group.
+
+A process that deliberately escapes the group (``setsid``/double fork) cannot be
+tracked this way; under systemd the unit's cgroup still covers it on stop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import enum
+import logging
+import os
+import signal
+import uuid
+from collections.abc import AsyncIterator, Mapping
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+log = logging.getLogger(__name__)
+
+READ_CHUNK_SIZE = 64 * 1024
+# Bounded so a slow consumer applies back-pressure all the way to the process
+# (its pipe fills up and it blocks), instead of buffering unbounded output.
+QUEUE_MAX_CHUNKS = 64
+DEFAULT_KILL_GRACE = 5.0
+# After the main process exited (and its group was cleaned up) wait at most
+# this long for the pipes to reach EOF. Only matters if an escaped process
+# still holds the pipes open.
+PIPE_DRAIN_TIMEOUT = 5.0
+
+
+class Stream(enum.Enum):
+    STDOUT = "stdout"
+    STDERR = "stderr"
+
+
+class Status(enum.Enum):
+    EXITED = "exited"
+    TIMEOUT = "timeout"
+    ERROR = "error"
+
+
+@dataclass(frozen=True)
+class CommandSpec:
+    command: str
+    cwd: str | None = None
+    env: Mapping[str, str] = field(default_factory=dict)
+    timeout: float | None = None
+
+
+@dataclass(frozen=True)
+class Started:
+    execution_id: str
+    pid: int
+    start_time: datetime
+
+
+@dataclass(frozen=True)
+class Output:
+    stream: Stream
+    data: bytes
+
+
+@dataclass(frozen=True)
+class Finished:
+    execution_id: str
+    status: Status
+    exit_code: int
+    signal: int
+    start_time: datetime
+    end_time: datetime
+    message: str = ""
+
+
+ExecEvent = Started | Output | Finished
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _signal_process_group(pgid: int, sig: int) -> bool:
+    """Send ``sig`` to a process group. Returns False if the group is gone."""
+    try:
+        os.killpg(pgid, sig)
+        return True
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        # Should not happen for our own children; treat as gone rather than crash.
+        log.warning("permission denied signalling process group %d", pgid)
+        return False
+
+
+async def terminate_process_group(pgid: int, grace: float = DEFAULT_KILL_GRACE) -> None:
+    """SIGTERM the group, wait up to ``grace`` seconds, then SIGKILL it."""
+    if not _signal_process_group(pgid, signal.SIGTERM):
+        return
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + grace
+    while loop.time() < deadline:
+        await asyncio.sleep(0.05)
+        if not _signal_process_group(pgid, 0):
+            return
+    log.info("process group %d survived SIGTERM for %.1fs, sending SIGKILL", pgid, grace)
+    _signal_process_group(pgid, signal.SIGKILL)
+
+
+class Executor:
+    """Runs commands; tracks background cleanup tasks so shutdown can await them."""
+
+    def __init__(self, shell: str = "/bin/sh", kill_grace: float = DEFAULT_KILL_GRACE):
+        self.shell = shell
+        self.kill_grace = kill_grace
+        self._cleanup_tasks: set[asyncio.Task] = set()
+        self._running_pgids: set[int] = set()
+
+    @property
+    def running_count(self) -> int:
+        return len(self._running_pgids)
+
+    def build_env(self, extra: Mapping[str, str]) -> dict[str, str]:
+        env = dict(os.environ)
+        env.update(extra)
+        return env
+
+    async def run(self, spec: CommandSpec, execution_id: str | None = None) -> AsyncIterator[ExecEvent]:
+        """Run ``spec`` and yield events. Always ends with exactly one Finished.
+
+        If the consumer stops early (``aclose()`` or task cancellation) the
+        process group is terminated in the background.
+        """
+        execution_id = execution_id or uuid.uuid4().hex
+        loop = asyncio.get_running_loop()
+        start_time = utcnow()
+
+        try:
+            proc = await asyncio.create_subprocess_exec(
+                self.shell,
+                "-c",
+                spec.command,
+                stdin=asyncio.subprocess.DEVNULL,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                cwd=spec.cwd or None,
+                env=self.build_env(spec.env),
+                start_new_session=True,
+            )
+        except OSError as exc:
+            where = f" (cwd={spec.cwd})" if spec.cwd else ""
+            yield Finished(
+                execution_id, Status.ERROR, -1, 0, start_time, utcnow(),
+                f"failed to start command{where}: {exc.strerror or exc}",
+            )
+            return
+
+        pgid = proc.pid  # session leader => pgid == pid
+        self._running_pgids.add(pgid)
+        queue: asyncio.Queue[tuple[Stream, bytes | None]] = asyncio.Queue(QUEUE_MAX_CHUNKS)
+
+        async def pump(reader: asyncio.StreamReader, stream: Stream) -> None:
+            try:
+                while chunk := await reader.read(READ_CHUNK_SIZE):
+                    await queue.put((stream, chunk))
+            finally:
+                # put_nowait could fail on a full queue; the consumer is gone
+                # in that case anyway (we are being cancelled).
+                try:
+                    queue.put_nowait((stream, None))
+                except asyncio.QueueFull:
+                    pass
+
+        async def wait_and_reap() -> int:
+            rc = await proc.wait()
+            # Kill whatever the command left behind in its process group.
+            await terminate_process_group(pgid, self.kill_grace)
+            return rc
+
+        timed_out = False
+        killer: asyncio.Task | None = None
+
+        def on_timeout() -> None:
+            nonlocal timed_out, killer
+            timed_out = True
+            log.info("execution %s timed out after %ss, terminating pgid %d", execution_id, spec.timeout, pgid)
+            killer = asyncio.ensure_future(terminate_process_group(pgid, self.kill_grace))
+
+        pumps = [
+            asyncio.ensure_future(pump(proc.stdout, Stream.STDOUT)),
+            asyncio.ensure_future(pump(proc.stderr, Stream.STDERR)),
+        ]
+        waiter = asyncio.ensure_future(wait_and_reap())
+        timer = loop.call_later(spec.timeout, on_timeout) if spec.timeout else None
+        get_task: asyncio.Future | None = None
+        finished = False
+
+        try:
+            yield Started(execution_id, proc.pid, start_time)
+
+            open_streams = 2
+            drain_deadline: float | None = None
+            while open_streams:
+                wait_timeout = None
+                if waiter.done():
+                    if drain_deadline is None:
+                        drain_deadline = loop.time() + PIPE_DRAIN_TIMEOUT
+                    wait_timeout = drain_deadline - loop.time()
+                    if wait_timeout <= 0:
+                        log.warning("execution %s: pipes still open after process exit; giving up", execution_id)
+                        break
+                if get_task is None:
+                    get_task = asyncio.ensure_future(queue.get())
+                waitables = {get_task} if waiter.done() else {get_task, waiter}
+                done, _ = await asyncio.wait(waitables, timeout=wait_timeout, return_when=asyncio.FIRST_COMPLETED)
+                if get_task in done:
+                    stream, data = get_task.result()
+                    get_task = None
+                    if data is None:
+                        open_streams -= 1
+                    else:
+                        yield Output(stream, data)
+
+            returncode = await waiter
+            if timer:
+                timer.cancel()
+            if killer:
+                await killer
+            sig = -returncode if returncode < 0 else 0
+            exit_code = 128 + sig if sig else returncode
+            if timed_out:
+                status, message = Status.TIMEOUT, f"command timed out after {spec.timeout:g}s"
+            else:
+                status, message = Status.EXITED, ""
+            finished = True
+            yield Finished(execution_id, status, exit_code, sig, start_time, utcnow(), message)
+        finally:
+            if timer:
+                timer.cancel()
+            if get_task is not None:
+                get_task.cancel()
+            for task in pumps:
+                task.cancel()
+            if not finished:
+                # Consumer went away (client disconnect / cancel). Must not
+                # await here: we may be running inside a cancelled task.
+                log.info("execution %s aborted, terminating pgid %d", execution_id, pgid)
+                waiter.cancel()
+                _signal_process_group(pgid, signal.SIGTERM)
+                self._spawn_cleanup(proc, pgid)
+            self._running_pgids.discard(pgid)
+
+    def _spawn_cleanup(self, proc: asyncio.subprocess.Process, pgid: int) -> None:
+        async def cleanup() -> None:
+            await terminate_process_group(pgid, self.kill_grace)
+            try:
+                await asyncio.wait_for(proc.wait(), self.kill_grace)
+            except asyncio.TimeoutError:
+                log.warning("process %d did not exit after SIGKILL", proc.pid)
+
+        task = asyncio.ensure_future(cleanup())
+        self._cleanup_tasks.add(task)
+        task.add_done_callback(self._cleanup_tasks.discard)
+
+    def kill_all(self) -> None:
+        """Synchronously SIGKILL all running process groups (last resort)."""
+        for pgid in list(self._running_pgids):
+            _signal_process_group(pgid, signal.SIGKILL)
+
+    async def wait_cleanups(self, timeout: float | None = None) -> None:
+        if self._cleanup_tasks:
+            await asyncio.wait(set(self._cleanup_tasks), timeout=timeout)