optio-host 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {optio_host-0.3.0 → optio_host-0.3.2}/PKG-INFO +1 -1
- {optio_host-0.3.0 → optio_host-0.3.2}/pyproject.toml +1 -1
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host/archive.py +10 -4
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host/host.py +323 -27
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host.egg-info/PKG-INFO +1 -1
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host.egg-info/SOURCES.txt +6 -1
- optio_host-0.3.2/tests/test_archive_progress_format.py +94 -0
- optio_host-0.3.2/tests/test_local_archive_workdir.py +336 -0
- optio_host-0.3.2/tests/test_remote_archive_workdir.py +391 -0
- optio_host-0.3.2/tests/test_remote_host_connect.py +37 -0
- optio_host-0.3.2/tests/test_run_command_unwind_trace.py +96 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/README.md +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/setup.cfg +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host/__init__.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host/download.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host/paths.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host/testing.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host/types.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host.egg-info/dependency_links.txt +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host.egg-info/requires.txt +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/src/optio_host.egg-info/top_level.txt +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_archive.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_download.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_env_scrub.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_local_host.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_local_host_large_lines.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_local_host_process_group.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_local_host_stdin.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_paths.py +0 -0
- {optio_host-0.3.0 → optio_host-0.3.2}/tests/test_remote_host_stdin.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: optio-host
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Local-or-remote host abstraction for optio task types (run commands local or remote, transfer files, tunnels).
|
|
5
5
|
Author-email: Kristof Csillag <kristof.csillag@deai-labs.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "optio-host"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "Local-or-remote host abstraction for optio task types (run commands local or remote, transfer files, tunnels)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -6,8 +6,10 @@ a stream, wiping the destination first and extracting in. Both run their
|
|
|
6
6
|
synchronous tarfile work in a thread executor so the event loop stays
|
|
7
7
|
responsive.
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
`
|
|
9
|
+
`consume_workdir_archive` backs `LocalHost.restore_workdir`.
|
|
10
|
+
`yield_workdir_archive` is only `LocalHost.archive_workdir`'s fallback for a
|
|
11
|
+
host without `tar` or `bash`; normally both hosts stream `tar | pigz -1` (or
|
|
12
|
+
`gzip -1`), RemoteHost over SSH and LocalHost as a local subprocess.
|
|
11
13
|
"""
|
|
12
14
|
|
|
13
15
|
from __future__ import annotations
|
|
@@ -51,9 +53,13 @@ def _excluded(relpath: str, patterns: list[str]) -> bool:
|
|
|
51
53
|
|
|
52
54
|
|
|
53
55
|
def _build_archive_bytes(root: str, patterns: list[str]) -> bytes:
|
|
54
|
-
"""Build the entire tar.gz in memory and return it as bytes.
|
|
56
|
+
"""Build the entire tar.gz in memory and return it as bytes.
|
|
57
|
+
|
|
58
|
+
Level 1, not tarfile's default 9: level 9 is ~7x slower for ~10% smaller
|
|
59
|
+
output, and a snapshot capture runs against the cancel grace.
|
|
60
|
+
"""
|
|
55
61
|
buf = io.BytesIO()
|
|
56
|
-
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
|
62
|
+
with tarfile.open(fileobj=buf, mode="w:gz", compresslevel=1) as tar:
|
|
57
63
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
58
64
|
rel_dir = os.path.relpath(dirpath, root)
|
|
59
65
|
dirnames[:] = [
|
|
@@ -16,7 +16,7 @@ etc.) live in ``optio_opencode.host_actions`` as free functions taking
|
|
|
16
16
|
from __future__ import annotations
|
|
17
17
|
|
|
18
18
|
from dataclasses import dataclass
|
|
19
|
-
from typing import AsyncIterator, Protocol
|
|
19
|
+
from typing import AsyncIterator, Callable, Protocol
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
@dataclass
|
|
@@ -230,6 +230,8 @@ class Host(Protocol):
|
|
|
230
230
|
# --- implementation -----------------------------------------------------
|
|
231
231
|
|
|
232
232
|
import asyncio
|
|
233
|
+
import contextlib
|
|
234
|
+
import contextvars
|
|
233
235
|
import fnmatch
|
|
234
236
|
import hashlib
|
|
235
237
|
import logging as _logging
|
|
@@ -242,6 +244,41 @@ import time as _time
|
|
|
242
244
|
_trace_logger = _logging.getLogger("optio_core.cancel_trace")
|
|
243
245
|
_CANCEL_TRACE = os.environ.get("OPTIO_CANCEL_TRACE", "0").lower() in ("1", "true", "yes")
|
|
244
246
|
|
|
247
|
+
# "Is a task-cancel/snapshot unwind in progress right now?" — set only around
|
|
248
|
+
# the finally-block brackets in optio_claudecode.session and
|
|
249
|
+
# optio_agents.protocol.session via unwind_tracing() below. RemoteHost.
|
|
250
|
+
# run_command has ~250 call sites, several of them polled once per second for
|
|
251
|
+
# a session's entire lifetime (e.g. tmux_session_alive); tracing every one of
|
|
252
|
+
# those unconditionally under OPTIO_CANCEL_TRACE=1 would flood the log with
|
|
253
|
+
# heartbeat noise the engine's existing filter does not catch (none of these
|
|
254
|
+
# lines contain "heartbeat"). Gating on this var, in addition to
|
|
255
|
+
# OPTIO_CANCEL_TRACE, confines run_command's trace to the unwind itself.
|
|
256
|
+
#
|
|
257
|
+
# A ContextVar set inside a coroutine is visible to whatever that coroutine
|
|
258
|
+
# awaits in-line, but NOT to a task created (asyncio.create_task) before the
|
|
259
|
+
# var was set — such a task captured its own context snapshot at creation
|
|
260
|
+
# time. Every remote call made during the two unwind brackets today is
|
|
261
|
+
# awaited in-line (verified: neither finally block spawns a new task), so
|
|
262
|
+
# wrapping each bracket's body in one `with unwind_tracing():` is sufficient;
|
|
263
|
+
# if a future unwind step moves work into a freshly created task, that task
|
|
264
|
+
# must enter its own unwind_tracing() block.
|
|
265
|
+
_unwind_tracing: "contextvars.ContextVar[bool]" = contextvars.ContextVar(
|
|
266
|
+
"optio_host_unwind_tracing", default=False,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
@contextlib.contextmanager
|
|
271
|
+
def unwind_tracing():
|
|
272
|
+
"""Mark "a task unwind is in progress" for this block (see ``_unwind_tracing``
|
|
273
|
+
above). Sync context manager — safe to wrap an ``async with``/``await``-heavy
|
|
274
|
+
block with plain ``with``; resets the var on exit, including when the
|
|
275
|
+
block raises."""
|
|
276
|
+
token = _unwind_tracing.set(True)
|
|
277
|
+
try:
|
|
278
|
+
yield
|
|
279
|
+
finally:
|
|
280
|
+
_unwind_tracing.reset(token)
|
|
281
|
+
|
|
245
282
|
# asyncio.StreamReader defaults its buffer cap to 64 KiB; readline() raises
|
|
246
283
|
# LimitOverrunError on any single line that exceeds it. Subprocess output we
|
|
247
284
|
# consume here (recipe-dsl NDJSON events, singer messages, tailed log lines)
|
|
@@ -256,6 +293,98 @@ def _trace(fmt: str, *args: object) -> None:
|
|
|
256
293
|
_trace_logger.warning("[%.3f] optio-host " + fmt, _time.monotonic(), *args)
|
|
257
294
|
|
|
258
295
|
|
|
296
|
+
class _traced:
|
|
297
|
+
"""Async context manager: START/DONE/RAISED span for one unwind step.
|
|
298
|
+
|
|
299
|
+
Sibling of the identically-shaped helper in optio_claudecode.session —
|
|
300
|
+
same logger (``optio_core.cancel_trace``), same OPTIO_CANCEL_TRACE gate,
|
|
301
|
+
same wording convention, just prefixed "optio-host" by ``_trace``.
|
|
302
|
+
``.start``/``.elapsed`` (monotonic seconds) stay populated regardless of
|
|
303
|
+
the trace gate. Diagnostic only: never swallows an exception raised
|
|
304
|
+
inside the ``async with`` block — logs RAISED (with the exception type)
|
|
305
|
+
and re-raises unchanged.
|
|
306
|
+
"""
|
|
307
|
+
|
|
308
|
+
def __init__(self, label: str, *, clock: "Callable[[], float]" = _time.monotonic) -> None:
|
|
309
|
+
self.label = label
|
|
310
|
+
self._clock = clock
|
|
311
|
+
self.start = 0.0
|
|
312
|
+
self.elapsed = 0.0
|
|
313
|
+
|
|
314
|
+
async def __aenter__(self) -> "_traced":
|
|
315
|
+
self.start = self._clock()
|
|
316
|
+
_trace("%s START", self.label)
|
|
317
|
+
return self
|
|
318
|
+
|
|
319
|
+
async def __aexit__(self, exc_type, exc, tb) -> bool:
|
|
320
|
+
self.elapsed = self._clock() - self.start
|
|
321
|
+
if exc_type is not None:
|
|
322
|
+
_trace("%s RAISED %s (%.1fs)", self.label, exc_type.__name__, self.elapsed)
|
|
323
|
+
else:
|
|
324
|
+
_trace("%s DONE (%.1fs)", self.label, self.elapsed)
|
|
325
|
+
return False
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
# Pure formatting helpers for the archive_workdir progress/DONE trace, kept
|
|
329
|
+
# free of asyncssh/SSH state so they can be unit-tested without a live
|
|
330
|
+
# connection (see tests/test_archive_progress_format.py).
|
|
331
|
+
def _mib(num_bytes: int) -> float:
|
|
332
|
+
"""Bytes to MiB, for trace/progress formatting."""
|
|
333
|
+
return num_bytes / (1024 * 1024)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _mib_per_s(num_bytes: int, elapsed_s: float) -> float:
|
|
337
|
+
"""MiB/s rate, for trace/progress formatting.
|
|
338
|
+
|
|
339
|
+
Returns 0.0 for a non-positive ``elapsed_s`` (the very first sample, or
|
|
340
|
+
a clock that hasn't advanced) instead of raising a division error.
|
|
341
|
+
"""
|
|
342
|
+
return _mib(num_bytes) / elapsed_s if elapsed_s > 0 else 0.0
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# How often archive_workdir's generator emits a bytes-so-far progress trace
|
|
346
|
+
# while streaming the tar.
|
|
347
|
+
_ARCHIVE_TRACE_PROGRESS_INTERVAL_S = 10.0
|
|
348
|
+
|
|
349
|
+
# archive_workdir reads the archive in blocks of this size. asyncssh's
|
|
350
|
+
# `async for` over stdout is a readline() loop, which cuts a gzip stream into
|
|
351
|
+
# ~300-byte pieces, each one a GridFS write in the snapshot consumer (~1 MiB/s
|
|
352
|
+
# overall). asyncssh returns at most ~200 KiB per read() however much is asked
|
|
353
|
+
# for, so anything >= 256 KiB behaves the same. See
|
|
354
|
+
# docs/2026-09-13-snapshot-archive-throughput-design.md. LocalHost uses it as
|
|
355
|
+
# its pipe reader's buffer limit too, so the archive is never buffered ahead
|
|
356
|
+
# of the consumer by more than a couple of blocks.
|
|
357
|
+
_ARCHIVE_READ_BLOCK = 256 * 1024
|
|
358
|
+
|
|
359
|
+
# How much of the archive pipeline's stderr archive_workdir keeps for its
|
|
360
|
+
# error message.
|
|
361
|
+
_ARCHIVE_STDERR_TAIL_BYTES = 4096
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
async def _read_tail(reader, limit: int) -> bytes:
|
|
365
|
+
"""Read ``reader`` to EOF, keeping only its last ``limit`` bytes."""
|
|
366
|
+
tail = b""
|
|
367
|
+
while chunk := await reader.read(_ARCHIVE_READ_BLOCK):
|
|
368
|
+
tail = (tail + chunk)[-limit:]
|
|
369
|
+
return tail
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _archive_command(workdir: str, patterns: list[str], compressor: str) -> str:
|
|
373
|
+
"""Shell command that writes a gzip tar of ``workdir`` to stdout.
|
|
374
|
+
|
|
375
|
+
``compressor`` is ``"pigz -1"`` or ``"gzip -1"``; both emit gzip, so
|
|
376
|
+
either host's ``restore_workdir`` reads either. Runs under ``bash``
|
|
377
|
+
explicitly (the remote login shell, or the local ``/bin/sh``, may be dash)
|
|
378
|
+
so ``pipefail`` carries a tar failure through the pipe into the exit status.
|
|
379
|
+
"""
|
|
380
|
+
excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in patterns)
|
|
381
|
+
script = (
|
|
382
|
+
f"set -o pipefail; cd {shlex.quote(workdir)} && "
|
|
383
|
+
f"tar cf - {excludes} . | {compressor}"
|
|
384
|
+
)
|
|
385
|
+
return f"bash -c {shlex.quote(script)}"
|
|
386
|
+
|
|
387
|
+
|
|
259
388
|
def _scrub_env(proc_env: dict, patterns: "list[str] | None") -> None:
|
|
260
389
|
"""Remove every env var whose NAME matches any glob pattern (in place).
|
|
261
390
|
|
|
@@ -496,8 +625,76 @@ class LocalHost:
|
|
|
496
625
|
def archive_workdir(
|
|
497
626
|
self, exclude: list[str] | None,
|
|
498
627
|
) -> "AsyncIterator[bytes]":
|
|
499
|
-
|
|
500
|
-
|
|
628
|
+
"""Stream a gzip tar of the workdir through the same ``tar | pigz -1``
|
|
629
|
+
(or ``gzip -1``) pipeline RemoteHost runs over SSH. Python ``tarfile``
|
|
630
|
+
at level 9, buffered whole in memory, took 62 s for a 743 MiB workdir
|
|
631
|
+
against a 30 s cancel grace (docs/2026-10-03-local-archive-throughput-design.md);
|
|
632
|
+
it remains only as the fallback for hosts without ``tar`` or ``bash``.
|
|
633
|
+
"""
|
|
634
|
+
from optio_host.archive import DEFAULT_WORKDIR_EXCLUDES, yield_workdir_archive
|
|
635
|
+
if shutil.which("tar") is None or shutil.which("bash") is None:
|
|
636
|
+
return yield_workdir_archive(self.workdir, exclude=exclude)
|
|
637
|
+
patterns = list(DEFAULT_WORKDIR_EXCLUDES) if exclude is None else list(exclude)
|
|
638
|
+
|
|
639
|
+
async def _gen() -> "AsyncIterator[bytes]":
|
|
640
|
+
compressor = "pigz -1" if shutil.which("pigz") else "gzip -1"
|
|
641
|
+
cmd = _archive_command(self.workdir, patterns, compressor)
|
|
642
|
+
_trace(
|
|
643
|
+
"LocalHost.archive_workdir START workdir=%s compressor=%s excludes=%r",
|
|
644
|
+
self.workdir, compressor, patterns,
|
|
645
|
+
)
|
|
646
|
+
t0 = _time.monotonic()
|
|
647
|
+
total = 0
|
|
648
|
+
last_report = t0
|
|
649
|
+
# Own process group, so an abandoned capture can kill tar and the
|
|
650
|
+
# compressor along with the bash wrapper.
|
|
651
|
+
proc = await asyncio.create_subprocess_exec(
|
|
652
|
+
"/bin/sh", "-c", cmd,
|
|
653
|
+
stdin=asyncio.subprocess.DEVNULL,
|
|
654
|
+
stdout=asyncio.subprocess.PIPE,
|
|
655
|
+
stderr=asyncio.subprocess.PIPE,
|
|
656
|
+
limit=_ARCHIVE_READ_BLOCK,
|
|
657
|
+
start_new_session=True,
|
|
658
|
+
)
|
|
659
|
+
stderr_task = asyncio.create_task(
|
|
660
|
+
_read_tail(proc.stderr, _ARCHIVE_STDERR_TAIL_BYTES),
|
|
661
|
+
)
|
|
662
|
+
completed = False
|
|
663
|
+
try:
|
|
664
|
+
while chunk := await proc.stdout.read(_ARCHIVE_READ_BLOCK):
|
|
665
|
+
total += len(chunk)
|
|
666
|
+
yield chunk
|
|
667
|
+
now = _time.monotonic()
|
|
668
|
+
if now - last_report >= _ARCHIVE_TRACE_PROGRESS_INTERVAL_S:
|
|
669
|
+
last_report = now
|
|
670
|
+
_trace(
|
|
671
|
+
"LocalHost.archive_workdir progress bytes=%d rate=%.1fMiB/s",
|
|
672
|
+
total, _mib_per_s(total, now - t0),
|
|
673
|
+
)
|
|
674
|
+
stderr_tail = await stderr_task
|
|
675
|
+
await proc.wait()
|
|
676
|
+
completed = True
|
|
677
|
+
finally:
|
|
678
|
+
if not completed:
|
|
679
|
+
# Abandoned mid-stream (consumer closed or cancelled): the
|
|
680
|
+
# pipeline would otherwise block on a full pipe for good.
|
|
681
|
+
self._killpg(proc.pid, _signal.SIGKILL)
|
|
682
|
+
stderr_task.cancel()
|
|
683
|
+
await proc.wait()
|
|
684
|
+
elapsed = _time.monotonic() - t0
|
|
685
|
+
_trace(
|
|
686
|
+
"LocalHost.archive_workdir DONE bytes=%d elapsed=%.1fs "
|
|
687
|
+
"rate=%.1fMiB/s exit=%s",
|
|
688
|
+
total, elapsed, _mib_per_s(total, elapsed), proc.returncode,
|
|
689
|
+
)
|
|
690
|
+
if proc.returncode != 0:
|
|
691
|
+
detail = stderr_tail.decode("utf-8", errors="replace").strip()
|
|
692
|
+
raise RuntimeError(
|
|
693
|
+
f"local workdir archive failed (exit {proc.returncode})"
|
|
694
|
+
+ (f": {detail}" if detail else "")
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
return _gen()
|
|
501
698
|
|
|
502
699
|
async def restore_workdir(
|
|
503
700
|
self, stream: "AsyncIterator[bytes]",
|
|
@@ -646,6 +843,15 @@ import asyncssh
|
|
|
646
843
|
|
|
647
844
|
from optio_host.types import SSHConfig
|
|
648
845
|
|
|
846
|
+
# SSH-level keepalive for every RemoteHost connection. asyncssh leaves it off
|
|
847
|
+
# by default (keepalive_interval=0), and our connections can live for a whole
|
|
848
|
+
# sync or agent session with long idle gaps: a NAT or firewall idle timeout
|
|
849
|
+
# would silently drop them, and a dead peer would go unnoticed until the next
|
|
850
|
+
# write. Probing every 30 s keeps middlebox state alive and closes the
|
|
851
|
+
# connection after 3 unanswered probes (about 90 s).
|
|
852
|
+
SSH_KEEPALIVE_INTERVAL = 30
|
|
853
|
+
SSH_KEEPALIVE_COUNT_MAX = 3
|
|
854
|
+
|
|
649
855
|
|
|
650
856
|
class RemoteHost:
|
|
651
857
|
"""Host implementation backed by a single asyncssh connection.
|
|
@@ -670,14 +876,18 @@ class RemoteHost:
|
|
|
670
876
|
self._tail_proc: asyncssh.SSHClientProcess | None = None
|
|
671
877
|
self._forward: asyncssh.SSHListener | None = None
|
|
672
878
|
self._host_home_cache: str | None = None
|
|
879
|
+
self._compressor: str | None = None
|
|
673
880
|
|
|
674
881
|
async def connect(self) -> None:
|
|
882
|
+
self._compressor = None
|
|
675
883
|
self._conn = await asyncssh.connect(
|
|
676
884
|
host=self._ssh.host,
|
|
677
885
|
username=self._ssh.user,
|
|
678
886
|
port=self._ssh.port,
|
|
679
887
|
client_keys=[self._ssh.key_path],
|
|
680
888
|
known_hosts=None, # Spec: known-hosts verification disabled in MVP.
|
|
889
|
+
keepalive_interval=SSH_KEEPALIVE_INTERVAL,
|
|
890
|
+
keepalive_count_max=SSH_KEEPALIVE_COUNT_MAX,
|
|
681
891
|
)
|
|
682
892
|
self._sftp = await self._conn.start_sftp_client()
|
|
683
893
|
|
|
@@ -840,24 +1050,86 @@ class RemoteHost:
|
|
|
840
1050
|
proc.kill()
|
|
841
1051
|
await proc.wait()
|
|
842
1052
|
|
|
1053
|
+
async def _archive_compressor(self) -> str:
|
|
1054
|
+
"""``pigz -1`` when the host has pigz, else ``gzip -1``. Probed once
|
|
1055
|
+
per connection."""
|
|
1056
|
+
if self._compressor is None:
|
|
1057
|
+
assert self._conn is not None
|
|
1058
|
+
probe = await self._conn.run("command -v pigz", check=False)
|
|
1059
|
+
self._compressor = "pigz -1" if probe.exit_status == 0 else "gzip -1"
|
|
1060
|
+
return self._compressor
|
|
1061
|
+
|
|
843
1062
|
def archive_workdir(
|
|
844
1063
|
self, exclude: list[str] | None,
|
|
845
1064
|
) -> "AsyncIterator[bytes]":
|
|
846
1065
|
from optio_host.archive import DEFAULT_WORKDIR_EXCLUDES
|
|
847
1066
|
assert self._conn is not None
|
|
848
1067
|
patterns = list(DEFAULT_WORKDIR_EXCLUDES) if exclude is None else list(exclude)
|
|
849
|
-
excludes = " ".join(f"--exclude={shlex.quote(p)}" for p in patterns)
|
|
850
|
-
cmd = f"cd {shlex.quote(self.workdir)} && tar czf - {excludes} ."
|
|
851
1068
|
|
|
852
1069
|
async def _gen() -> "AsyncIterator[bytes]":
|
|
853
1070
|
assert self._conn is not None
|
|
1071
|
+
compressor = await self._archive_compressor()
|
|
1072
|
+
cmd = _archive_command(self.workdir, patterns, compressor)
|
|
1073
|
+
_trace(
|
|
1074
|
+
"archive_workdir START workdir=%s compressor=%s excludes=%r",
|
|
1075
|
+
self.workdir, compressor, patterns,
|
|
1076
|
+
)
|
|
1077
|
+
t0 = _time.monotonic()
|
|
1078
|
+
total = 0
|
|
1079
|
+
chunk_count = 0
|
|
1080
|
+
last_report = t0
|
|
854
1081
|
proc = await self._conn.create_process(cmd, encoding=None)
|
|
855
|
-
|
|
856
|
-
|
|
1082
|
+
# Read stderr alongside stdout: asyncssh stops delivering stdout
|
|
1083
|
+
# once unread stderr fills the channel window.
|
|
1084
|
+
stderr_task = asyncio.create_task(
|
|
1085
|
+
_read_tail(proc.stderr, _ARCHIVE_STDERR_TAIL_BYTES),
|
|
1086
|
+
)
|
|
1087
|
+
completed = False
|
|
1088
|
+
try:
|
|
1089
|
+
while chunk := await proc.stdout.read(_ARCHIVE_READ_BLOCK):
|
|
1090
|
+
total += len(chunk)
|
|
1091
|
+
chunk_count += 1
|
|
857
1092
|
yield chunk
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
1093
|
+
now = _time.monotonic()
|
|
1094
|
+
if now - last_report >= _ARCHIVE_TRACE_PROGRESS_INTERVAL_S:
|
|
1095
|
+
last_report = now
|
|
1096
|
+
_trace(
|
|
1097
|
+
"archive_workdir progress bytes=%d rate=%.1fMiB/s",
|
|
1098
|
+
total, _mib_per_s(total, now - t0),
|
|
1099
|
+
)
|
|
1100
|
+
# Collect stderr before wait(): asyncssh's wait() discards
|
|
1101
|
+
# output still unread when the channel closes.
|
|
1102
|
+
stderr_tail = await stderr_task
|
|
1103
|
+
await proc.wait()
|
|
1104
|
+
completed = True
|
|
1105
|
+
finally:
|
|
1106
|
+
if not completed:
|
|
1107
|
+
# Abandoned mid-stream (consumer closed or cancelled):
|
|
1108
|
+
# closing the channel makes the remote pipeline die of
|
|
1109
|
+
# SIGPIPE instead of blocking on a full SSH window, and
|
|
1110
|
+
# frees whatever it still buffers. Idempotent.
|
|
1111
|
+
proc.close()
|
|
1112
|
+
stderr_task.cancel()
|
|
1113
|
+
elapsed = _time.monotonic() - t0
|
|
1114
|
+
_trace(
|
|
1115
|
+
"archive_workdir DONE bytes=%d chunks=%d elapsed=%.1fs "
|
|
1116
|
+
"rate=%.1fMiB/s exit=%s",
|
|
1117
|
+
total, chunk_count, elapsed, _mib_per_s(total, elapsed),
|
|
1118
|
+
proc.exit_status,
|
|
1119
|
+
)
|
|
1120
|
+
# None means the channel closed without an exit status: the
|
|
1121
|
+
# archive may be truncated, so it is a failure too.
|
|
1122
|
+
if proc.exit_status != 0:
|
|
1123
|
+
status = (
|
|
1124
|
+
"no exit status" if proc.exit_status is None
|
|
1125
|
+
else f"exit {proc.exit_status}"
|
|
1126
|
+
)
|
|
1127
|
+
_trace("archive_workdir RAISED RuntimeError (%s)", status)
|
|
1128
|
+
detail = stderr_tail.decode("utf-8", errors="replace").strip()
|
|
1129
|
+
raise RuntimeError(
|
|
1130
|
+
f"remote workdir archive failed ({status})"
|
|
1131
|
+
+ (f": {detail}" if detail else "")
|
|
1132
|
+
)
|
|
861
1133
|
|
|
862
1134
|
return _gen()
|
|
863
1135
|
|
|
@@ -892,22 +1164,27 @@ class RemoteHost:
|
|
|
892
1164
|
progress_cb=None,
|
|
893
1165
|
) -> bytes:
|
|
894
1166
|
assert self._conn is not None
|
|
895
|
-
|
|
896
|
-
|
|
1167
|
+
async with _traced(f"fetch_bytes_from_host path={absolute_path}") as t:
|
|
1168
|
+
sftp = await self._conn.start_sftp_client()
|
|
897
1169
|
try:
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
progress_cb
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
1170
|
+
try:
|
|
1171
|
+
async with sftp.open(absolute_path, "rb") as fh:
|
|
1172
|
+
data = await fh.read()
|
|
1173
|
+
except (asyncssh.SFTPError, asyncssh.SFTPNoSuchFile) as exc:
|
|
1174
|
+
# asyncssh maps "no such file" to a generic SFTPError in some
|
|
1175
|
+
# versions; check the message.
|
|
1176
|
+
if "No such file" in str(exc):
|
|
1177
|
+
raise FileNotFoundError(absolute_path) from exc
|
|
1178
|
+
raise
|
|
1179
|
+
if progress_cb is not None:
|
|
1180
|
+
progress_cb(100.0, None)
|
|
1181
|
+
finally:
|
|
1182
|
+
sftp.exit()
|
|
1183
|
+
_trace(
|
|
1184
|
+
"fetch_bytes_from_host path=%s bytes=%d (%.1fs)",
|
|
1185
|
+
absolute_path, len(data), t.elapsed,
|
|
1186
|
+
)
|
|
1187
|
+
return data
|
|
911
1188
|
|
|
912
1189
|
async def glob(self, pattern: str) -> list[str]:
|
|
913
1190
|
assert self._conn is not None
|
|
@@ -956,12 +1233,31 @@ class RemoteHost:
|
|
|
956
1233
|
f"export {k}={shlex.quote(v)};" for k, v in env.items()
|
|
957
1234
|
) + " "
|
|
958
1235
|
full_command = f"cd {shlex.quote(run_cwd)} && {exports}{command}"
|
|
1236
|
+
# Trace only while a task-cancel/snapshot unwind is in progress (see
|
|
1237
|
+
# unwind_tracing() above) — run_command's ~250 call sites include
|
|
1238
|
+
# once-per-second liveness polls for a session's entire lifetime, and
|
|
1239
|
+
# tracing every one of those would flood the log outside an unwind.
|
|
1240
|
+
trace_this = _CANCEL_TRACE and _unwind_tracing.get()
|
|
1241
|
+
if trace_this:
|
|
1242
|
+
_trace("run_command START %r", command[:160])
|
|
1243
|
+
t0 = _time.monotonic() if trace_this else 0.0
|
|
959
1244
|
result = await self._conn.run(full_command, check=False)
|
|
960
1245
|
stdout = result.stdout or ""
|
|
961
1246
|
stderr = result.stderr or ""
|
|
1247
|
+
stdout = stdout if isinstance(stdout, str) else stdout.decode("utf-8", errors="replace")
|
|
1248
|
+
stderr = stderr if isinstance(stderr, str) else stderr.decode("utf-8", errors="replace")
|
|
1249
|
+
if trace_this:
|
|
1250
|
+
# Byte-size computation is real work (an extra utf-8 encode over
|
|
1251
|
+
# possibly-large output) — only pay for it when the trace line
|
|
1252
|
+
# will actually be emitted.
|
|
1253
|
+
_trace(
|
|
1254
|
+
"run_command DONE %r exit=%s elapsed=%.1fs stdout=%dB stderr=%dB",
|
|
1255
|
+
command[:160], result.exit_status, _time.monotonic() - t0,
|
|
1256
|
+
len(stdout.encode("utf-8")), len(stderr.encode("utf-8")),
|
|
1257
|
+
)
|
|
962
1258
|
return RunResult(
|
|
963
|
-
stdout=stdout
|
|
964
|
-
stderr=stderr
|
|
1259
|
+
stdout=stdout,
|
|
1260
|
+
stderr=stderr,
|
|
965
1261
|
exit_code=result.exit_status if result.exit_status is not None else -1,
|
|
966
1262
|
)
|
|
967
1263
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: optio-host
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Local-or-remote host abstraction for optio task types (run commands local or remote, transfer files, tunnels).
|
|
5
5
|
Author-email: Kristof Csillag <kristof.csillag@deai-labs.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -13,11 +13,16 @@ src/optio_host.egg-info/dependency_links.txt
|
|
|
13
13
|
src/optio_host.egg-info/requires.txt
|
|
14
14
|
src/optio_host.egg-info/top_level.txt
|
|
15
15
|
tests/test_archive.py
|
|
16
|
+
tests/test_archive_progress_format.py
|
|
16
17
|
tests/test_download.py
|
|
17
18
|
tests/test_env_scrub.py
|
|
19
|
+
tests/test_local_archive_workdir.py
|
|
18
20
|
tests/test_local_host.py
|
|
19
21
|
tests/test_local_host_large_lines.py
|
|
20
22
|
tests/test_local_host_process_group.py
|
|
21
23
|
tests/test_local_host_stdin.py
|
|
22
24
|
tests/test_paths.py
|
|
23
|
-
tests/
|
|
25
|
+
tests/test_remote_archive_workdir.py
|
|
26
|
+
tests/test_remote_host_connect.py
|
|
27
|
+
tests/test_remote_host_stdin.py
|
|
28
|
+
tests/test_run_command_unwind_trace.py
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Tests for the OPTIO_CANCEL_TRACE diagnostic helpers in optio_host.host
|
|
2
|
+
that can be exercised without a live SSH connection:
|
|
3
|
+
|
|
4
|
+
* ``_mib`` / ``_mib_per_s`` — the pure byte-counting/rate-formatting helpers
|
|
5
|
+
used by ``RemoteHost.archive_workdir``'s progress/DONE trace lines.
|
|
6
|
+
* ``_traced`` — the async START/DONE/RAISED span helper shared by
|
|
7
|
+
``fetch_bytes_from_host`` (sibling of the identically-shaped helper in
|
|
8
|
+
optio_claudecode.session).
|
|
9
|
+
|
|
10
|
+
``RemoteHost.archive_workdir``/``run_command``/``fetch_bytes_from_host``
|
|
11
|
+
themselves need a real (or SSH-container) connection and are covered
|
|
12
|
+
elsewhere (test_remote_host_connect.py); this file only covers the
|
|
13
|
+
connection-free pieces of the new tracing.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
|
|
18
|
+
import pytest
|
|
19
|
+
|
|
20
|
+
from optio_host import host as host_mod
|
|
21
|
+
from optio_host.host import (
|
|
22
|
+
_ARCHIVE_TRACE_PROGRESS_INTERVAL_S,
|
|
23
|
+
_mib,
|
|
24
|
+
_mib_per_s,
|
|
25
|
+
_traced,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# --- _mib / _mib_per_s -----------------------------------------------------
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_mib_converts_bytes():
|
|
33
|
+
assert _mib(1024 * 1024) == pytest.approx(1.0)
|
|
34
|
+
assert _mib(0) == 0.0
|
|
35
|
+
assert _mib(5 * 1024 * 1024) == pytest.approx(5.0)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_mib_per_s_computes_rate():
|
|
39
|
+
assert _mib_per_s(10 * 1024 * 1024, 2.0) == pytest.approx(5.0)
|
|
40
|
+
assert _mib_per_s(1024 * 1024, 1.0) == pytest.approx(1.0)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_mib_per_s_zero_or_negative_elapsed_is_zero_not_a_division_error():
|
|
44
|
+
assert _mib_per_s(1024 * 1024, 0.0) == 0.0
|
|
45
|
+
assert _mib_per_s(1024 * 1024, -1.0) == 0.0
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_progress_interval_matches_spec():
|
|
49
|
+
assert _ARCHIVE_TRACE_PROGRESS_INTERVAL_S == 10.0
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# --- _traced ---------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class _FakeClock:
|
|
56
|
+
def __init__(self, t: float = 0.0) -> None:
|
|
57
|
+
self.t = t
|
|
58
|
+
|
|
59
|
+
def __call__(self) -> float:
|
|
60
|
+
return self.t
|
|
61
|
+
|
|
62
|
+
def tick(self, dt: float) -> None:
|
|
63
|
+
self.t += dt
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
async def test_traced_emits_nothing_when_trace_disabled(monkeypatch, caplog):
|
|
67
|
+
monkeypatch.setattr(host_mod, "_CANCEL_TRACE", False)
|
|
68
|
+
clock = _FakeClock()
|
|
69
|
+
with caplog.at_level(logging.WARNING, logger="optio_core.cancel_trace"):
|
|
70
|
+
async with _traced("fetch_bytes_from_host path=/x", clock=clock) as t:
|
|
71
|
+
clock.tick(0.7)
|
|
72
|
+
assert caplog.records == []
|
|
73
|
+
assert t.elapsed == pytest.approx(0.7)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
async def test_traced_emits_start_done_with_elapsed_when_enabled(monkeypatch, caplog):
|
|
77
|
+
monkeypatch.setattr(host_mod, "_CANCEL_TRACE", True)
|
|
78
|
+
clock = _FakeClock()
|
|
79
|
+
with caplog.at_level(logging.WARNING, logger="optio_core.cancel_trace"):
|
|
80
|
+
async with _traced("fetch_bytes_from_host path=/x", clock=clock):
|
|
81
|
+
clock.tick(1.2)
|
|
82
|
+
messages = [r.getMessage() for r in caplog.records]
|
|
83
|
+
assert any("fetch_bytes_from_host path=/x START" in m for m in messages)
|
|
84
|
+
done = [m for m in messages if "fetch_bytes_from_host path=/x DONE" in m]
|
|
85
|
+
assert len(done) == 1
|
|
86
|
+
assert "1.2s" in done[0]
|
|
87
|
+
assert all("optio-host" in m for m in messages)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
async def test_traced_raised_propagates(monkeypatch):
|
|
91
|
+
monkeypatch.setattr(host_mod, "_CANCEL_TRACE", False)
|
|
92
|
+
with pytest.raises(ValueError, match="nope"):
|
|
93
|
+
async with _traced("x"):
|
|
94
|
+
raise ValueError("nope")
|