quapp-hpc 0.0.1.dev7__tar.gz → 0.0.1.dev11__tar.gz

{quapp_hpc-0.0.1.dev7/quapp_hpc.egg-info → quapp_hpc-0.0.1.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev7
+Version: 0.0.1.dev11
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quapp-hpc"
-version = "0.0.1.dev7"
+version = "0.0.1.dev11"
 description = "Quapp HPC library — Slurm integration for Quapp Platform"
 readme = "README.md"
 authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev11/quapp_hpc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev7
+Version: 0.0.1.dev11
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev11}/quapp_hpc.egg-info/SOURCES.txt

@@ -24,4 +24,9 @@ quapp_hpc/model/device/slurm_device.py
 quapp_hpc/model/provider/__init__.py
 quapp_hpc/model/provider/slurm_provider.py
 quapp_slurm_hpc/__init__.py
-quapp_slurm_hpc/runner.py
+quapp_slurm_hpc/host_wrapper.py
+quapp_slurm_hpc/log_writer.py
+quapp_slurm_hpc/runner.py
+quapp_slurm_hpc/stream_exec.py
+tests/test_log_writer.py
+tests/test_stream_exec.py

quapp_hpc-0.0.1.dev11/quapp_slurm_hpc/host_wrapper.py

@@ -0,0 +1,84 @@
+"""Host-side wrapper that streams a command's output live to the backend's log pipeline
+(PQF-23860), the same way `runner.py` streams the function handler's own output — but for the
+*outer* `apptainer exec` invocation itself, run on the bare Slurm compute node, outside any
+container. That outer phase (image pull, OCI→SIF conversion, Apptainer's own warnings like
+"Could not find any nv files on this host!") happens entirely before `runner.py` (tier 1) ever
+starts, so without this wrapper it's invisible anywhere except a manual SSH read of the raw
+`slurm-<jobid>.out` file.
+
+Deployed standalone to Slurm compute node hosts — see the `apptainer` Ansible role
+(quapp-infrastructure, cts/slurm/ansible) — independent of the full `quapp-hpc` PyPI package:
+only this file plus `log_writer.py` and `stream_exec.py` side by side, relying solely on
+`requests` + the standard library (already present on these hosts; no pip/package install
+needed). Keep the three files identical to their `quapp_slurm_hpc/` counterparts in this repo.
+
+Usage:
+    python3 host_wrapper.py --input-file <job's input.json> -- <command...>
+
+Exits with the wrapped command's exit code. Best-effort like the rest of this feature: any
+failure to read `--input-file`, build a writer, or push log chunks only logs a warning to
+stderr — it never changes the wrapped command's exit code or stops it from running.
+"""
+import json
+import sys
+
+import log_writer
+import stream_exec
+
+
+def main(argv) -> int:
+    if "--" not in argv:
+        print(
+            "usage: host_wrapper.py --input-file <path> -- <command...>",
+            file=sys.stderr, flush=True,
+        )
+        return 2
+    sep = argv.index("--")
+    command = argv[sep + 1:]
+    if not command:
+        print("host_wrapper: no command given after --", file=sys.stderr, flush=True)
+        return 2
+
+    input_file = None
+    own_args = argv[:sep]
+    for i, arg in enumerate(own_args):
+        if arg == "--input-file" and i + 1 < len(own_args):
+            input_file = own_args[i + 1]
+    if not input_file:
+        print("host_wrapper: --input-file is required", file=sys.stderr, flush=True)
+        return 2
+
+    writer = None
+    try:
+        with open(input_file, encoding="utf-8") as f:
+            invocation_input = json.load(f)
+        writer = log_writer.from_input(invocation_input)
+        if writer is not None:
+            writer.open()
+    except Exception as exc:  # best-effort: a broken input.json must not block the real command
+        print(f"host_wrapper: could not set up log streaming: {exc}", file=sys.stderr, flush=True)
+        writer = None
+
+    # tee_to writes each line to our own stdout as it's read — not after the command exits — so
+    # a killed command (walltime, scancel, OOM) still leaves whatever it already printed in
+    # slurm-<jobid>.out, exactly as before this wrapper existed. keep_output=False: nothing here
+    # needs the content back (it's already teed), so don't also hold the whole thing in memory.
+    exit_code, _, _ = stream_exec.run_streamed(
+        command, writer, merge_stderr=True, flush_interval_fn=log_writer.flush_interval,
+        tee_to=sys.stdout.buffer, keep_output=False,
+    )
+
+    if writer is not None:
+        # The raw (possibly negative, signal-killed) code — JobLogCloseRequest.exitCode's own
+        # contract documents negative-for-a-signal, matching Popen.returncode directly.
+        writer.close(exit_code)
+
+    # This process's own exit status, by contrast, must follow the shell/Slurm convention a
+    # negative Popen.returncode doesn't: sys.exit() truncates a negative value to a nonsense
+    # positive byte (e.g. -15 -> 241) instead of the 128+signum a killed process is supposed to
+    # report (143 for SIGTERM).
+    return 128 + (-exit_code) if exit_code < 0 else exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

quapp_hpc-0.0.1.dev11/quapp_slurm_hpc/log_writer.py

@@ -0,0 +1,215 @@
+"""Pushes a job's stdout/stderr to the backend's live log pipeline (PQF-23860) as it runs,
+instead of only via the final output.json snapshot uploaded at the end.
+
+No client for this protocol exists anywhere yet (checked qapp-lib / quapp-libs) — this is the
+first one, reverse-engineered directly from the backend's ingest code:
+
+    JobLogIngestControllerV1  -- POST {base_url}/writers/{writer_id}/open?attempt=N
+                              -- POST {base_url}/writers/{writer_id}/chunks?attempt=N
+                              -- POST {base_url}/writers/{writer_id}/close?attempt=N
+    JobLogWriteResult         -- the response body, wrapped by the backend's global envelope
+                                 as {"data": {"outcome": ..., "endOffset": ..., "expectedSeq": ...}}
+
+Every call carries the token the backend issued for this job attempt in the `logIngest` block
+of the invocation input (see JobLogIngestPayloadProvider on the backend). Best-effort only: a
+failure here must never raise into the handler's own execution — the job still has to run to
+completion and still produce output.json exactly as before.
+"""
+import sys
+import time
+import uuid
+from typing import Optional
+
+import requests
+
+_TOKEN_HEADER = "X-Quapp-Log-Token"
+_SEQ_HEADER = "X-Log-Seq"
+_DROPPED_HEADER = "X-Log-Dropped"
+_REQUEST_TIMEOUT_SECONDS = 5
+_STOP_OUTCOMES = {"SEALED", "TOO_LATE", "TRUNCATED"}
+
+# Stay under the backend's `invocation.logging.max-chunk-size` (256KB) with margin — a chunk
+# over that limit gets a 400 (JOB_LOG_CHUNK_TOO_LARGE), and since that isn't one of the retryable
+# statuses below, a single oversized flush would otherwise stop the stream for the rest of the job.
+_MAX_CHUNK_BYTES = 250 * 1024
+
+# Transient: worth a few retries (same seq — the backend's DUPLICATE outcome makes retrying a
+# chunk that actually was stored safe). Any other status (400, 401, 403, 404, 409…) is treated as
+# non-retryable: retrying the same request wouldn't change the outcome. 401 in particular (token
+# expired after its ~48h TTL) is expected to stop the stream, not retry.
+_TRANSIENT_STATUSES = {429, 500, 502, 503, 504}
+_MAX_ATTEMPTS = 4
+_RETRY_BACKOFF_SECONDS = (1, 2, 5)
+
+# How often to flush: often at first for near-real-time visibility on typical short jobs, backing
+# off as the job runs longer so a multi-hour job doesn't exhaust the backend's 2000-chunks/10MB
+# per-attempt cap (at a flat 2s cadence that cap hits at ~66 minutes regardless of chunk size,
+# since every flush spends one chunk even when its content is tiny).
+FLUSH_SCHEDULE = (
+    (300.0, 2.0),  # first 5 minutes: every 2s
+    (1800.0, 10.0),  # next 25 minutes: every 10s
+    (float("inf"), 30.0),  # beyond 30 minutes: every 30s
+)
+
+
+def flush_interval(elapsed_seconds: float) -> float:
+    """The flush cadence to use at `elapsed_seconds` into the job, per FLUSH_SCHEDULE."""
+    for threshold, interval in FLUSH_SCHEDULE:
+        if elapsed_seconds < threshold:
+            return interval
+    return FLUSH_SCHEDULE[-1][1]
+
+
+def split_chunks(data: bytes, limit: int = _MAX_CHUNK_BYTES):
+    """Splits `data` into pieces of at most `limit` bytes each, cutting at the last newline
+    inside the window when there is one (so a line isn't split mid-line unless a single line
+    alone already exceeds `limit`, e.g. a `\\r`-driven progress bar — then it must be)."""
+    pieces = []
+    start = 0
+    n = len(data)
+    while start < n:
+        end = min(start + limit, n)
+        if end < n:
+            cut = data.rfind(b"\n", start, end)
+            if cut != -1 and cut + 1 > start:
+                end = cut + 1
+        pieces.append(data[start:end])
+        start = end
+    return pieces
+
+
+class JobLogWriter:
+    """A single log writer for one job attempt. Not thread-safe — call from one thread only."""
+
+    def __init__(self, base_url: str, token: str, attempt: int, headers: Optional[dict] = None):
+        self._base_url = base_url.rstrip("/")
+        self._token = token
+        self._attempt = attempt
+        self._extra_headers = headers or {}
+        self._writer_id = str(uuid.uuid4())
+        self._seq = 0
+        self._dropped_bytes = 0
+        self._stopped = False
+
+    @property
+    def stopped(self) -> bool:
+        """True once the backend told us to stop (sealed/too-late/truncated) or a request
+        failed after retries — callers should skip further append()/close() calls once this is
+        set, though each method is itself a safe no-op either way."""
+        return self._stopped
+
+    def open(self) -> None:
+        if self._stopped:
+            return
+        self._post("open")
+
+    def append(self, data: bytes) -> None:
+        """Sends new output, split into chunks the backend will actually accept. Silently drops
+        it (and counts it as dropped) once streaming has stopped — the handler's own run must
+        never block or fail on this."""
+        if not data:
+            return
+        # Pass the limit explicitly (not relying on split_chunks's own default) so it is read
+        # fresh from the module each call — needed for tests to override it via monkeypatching.
+        for piece in split_chunks(data, limit=_MAX_CHUNK_BYTES):
+            if self._stopped:
+                self._dropped_bytes += len(piece)
+                continue
+            self._append_one(piece)
+
+    def _append_one(self, piece: bytes) -> None:
+        outcome, expected_seq = self._post(
+            "chunks",
+            data=piece,
+            extra_headers={
+                _SEQ_HEADER: str(self._seq),
+                _DROPPED_HEADER: str(self._dropped_bytes),
+                # The backend's /chunks route is declared `consumes = APPLICATION_OCTET_STREAM`
+                # and rejects anything else with 415 — requests does not set this on its own
+                # for a raw bytes body.
+                "Content-Type": "application/octet-stream",
+            },
+        )
+        if outcome in ("APPENDED", "DUPLICATE"):
+            self._seq += 1
+            self._dropped_bytes = 0
+        elif outcome == "SEQ_GAP" and expected_seq is not None:
+            # The backend tells us exactly what it's missing — resync instead of retrying the
+            # same seq forever (which would never succeed and would spin indefinitely).
+            self._seq = int(expected_seq)
+            self._dropped_bytes += len(piece)
+
+    def close(self, exit_code: int, abandoned: bool = False, discarded: bool = False) -> None:
+        """Closes the writer. Safe to call even if streaming already stopped earlier (e.g. the
+        backend sealed it, or a request failed), so a crash can still attempt a clean close
+        from a `finally` block without extra branching."""
+        self._post(
+            "close",
+            json={
+                "lastSeq": self._seq - 1,
+                "droppedBytes": self._dropped_bytes,
+                "abandoned": abandoned,
+                "discarded": discarded,
+                "exitCode": exit_code,
+            },
+        )
+        self._stopped = True
+
+    def _post(self, segment: str, *, json=None, data=None, extra_headers: Optional[dict] = None):
+        url = f"{self._base_url}/writers/{self._writer_id}/{segment}?attempt={self._attempt}"
+        headers = {_TOKEN_HEADER: self._token, **self._extra_headers, **(extra_headers or {})}
+        for attempt in range(_MAX_ATTEMPTS):
+            last_attempt = attempt == _MAX_ATTEMPTS - 1
+            try:
+                response = requests.post(
+                    url, json=json, data=data, headers=headers, timeout=_REQUEST_TIMEOUT_SECONDS
+                )
+            except requests.exceptions.RequestException as exc:
+                if not last_attempt:
+                    time.sleep(_RETRY_BACKOFF_SECONDS[attempt])
+                    continue
+                print(
+                    f"[log_writer] stopping log stream, request failed after "
+                    f"{attempt + 1} attempts: {exc}",
+                    file=sys.stderr, flush=True,
+                )
+                self._stopped = True
+                return None, None
+
+            if response.status_code in _TRANSIENT_STATUSES and not last_attempt:
+                time.sleep(_RETRY_BACKOFF_SECONDS[attempt])
+                continue
+
+            try:
+                response.raise_for_status()
+            except requests.exceptions.HTTPError as exc:
+                # Non-transient (or retries exhausted): includes 401 once the ~48h token TTL
+                # expires, which is expected to stop the stream rather than retry.
+                print(f"[log_writer] stopping log stream, request failed: {exc}",
+                      file=sys.stderr, flush=True)
+                self._stopped = True
+                return None, None
+
+            payload = (response.json() or {}).get("data") or {}
+            outcome = payload.get("outcome")
+            if outcome in _STOP_OUTCOMES:
+                self._stopped = True
+            return outcome, payload.get("expectedSeq")
+
+        self._stopped = True
+        return None, None
+
+
+def from_input(invocation_input) -> Optional[JobLogWriter]:
+    """Builds a writer from the `logIngest` block the backend injects into the job input, or
+    None when push logging is disabled or the job predates this feature. Callers must treat
+    None as "don't stream this run" — not as an error."""
+    log_ingest = invocation_input.get("logIngest") if isinstance(invocation_input, dict) else None
+    if not isinstance(log_ingest, dict):
+        return None
+    base_url = log_ingest.get("baseUrl")
+    token = log_ingest.get("token")
+    attempt = log_ingest.get("attempt")
+    if not base_url or not token or attempt is None:
+        return None
+    return JobLogWriter(base_url, token, int(attempt), log_ingest.get("headers"))

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev11}/quapp_slurm_hpc/runner.py

@@ -11,7 +11,8 @@ This module:
   1. Reads the invocation input from --input-file (JSON written by the sbatch script).
   2. Loads handler.py from --handler-dir.
   3. Calls handler.processing(invocation_input) → bash script string.
-  4. Executes the bash script, captures stdout / stderr / exit_code.
+  4. Executes the bash script, streaming stdout/stderr live to the backend as it runs (when the
+     input carries a `logIngest` block) while also capturing it in full for the final result.
   5. Calls handler.post_processing(raw_result) if it exists.
   6. Writes the final result dict to --output-file.
   7. Uploads output.json to S3 (key: {QUAPP_JOB_ID}/output.json) when SLURM_S3_BUCKET is set.
@@ -20,13 +21,14 @@ import argparse
 import importlib.util
 import json
 import os
-import subprocess
 import sys
 import tempfile
 import traceback
 from pathlib import Path
 from types import ModuleType
 
+from quapp_slurm_hpc import log_writer, stream_exec
+
 
 def _load_handler(handler_dir: str) -> ModuleType:
     handler_path = Path(handler_dir) / "handler.py"
@@ -43,23 +45,27 @@ def _load_handler(handler_dir: str) -> ModuleType:
     return module
 
 
-def _run_script(script: str) -> dict:
-    """Write script to a temp file and execute it with bash, capturing output."""
+def _run_script(script: str, writer: "log_writer.JobLogWriter | None" = None) -> dict:
+    """Write script to a temp file and execute it with bash, capturing output.
+
+    Reads stdout/stderr incrementally as the script runs (instead of blocking until it exits)
+    so `writer`, when given, can push new output to the backend on the cadence
+    `log_writer.flush_interval()` defines (frequent at first, backing off on a long-running job
+    so it doesn't exhaust the backend's per-attempt chunk/size cap) — the final returned dict is
+    unchanged either way; the full stdout/stderr are still captured in full for `output.json`,
+    same as before.
+    """
     with tempfile.NamedTemporaryFile(
         mode="w", suffix=".sh", delete=False, encoding="utf-8"
     ) as f:
         f.write(script)
         tmp_path = f.name
     try:
-        result = subprocess.run(
-            ["bash", tmp_path],
-            capture_output=True,
-            text=True,
-        )
+        exit_code, stdout_bytes, stderr_bytes = stream_exec.run_streamed(["bash", tmp_path], writer)
         return {
-            "exit_code": result.returncode,
-            "stdout": result.stdout,
-            "stderr": result.stderr,
+            "exit_code": exit_code,
+            "stdout": stdout_bytes.decode("utf-8", errors="replace"),
+            "stderr": stderr_bytes.decode("utf-8", errors="replace"),
         }
     finally:
         Path(tmp_path).unlink(missing_ok=True)
@@ -75,10 +81,18 @@ def main() -> None:
     args = parser.parse_args()
 
     output: dict = {}
+    writer = None
+    exit_code_for_close = 1
     try:
         with open(args.input_file, encoding="utf-8") as f:
             invocation_input = json.load(f)
 
+        # Absent when log push is disabled or the job predates this feature — _run_script and
+        # writer.append/close below all treat `writer is None` as "don't stream", not an error.
+        writer = log_writer.from_input(invocation_input)
+        if writer is not None:
+            writer.open()
+
         handler = _load_handler(args.handler_dir)
 
         if not hasattr(handler, "processing"):
@@ -92,7 +106,8 @@ def main() -> None:
                 f"handler.processing() must return a str (bash script), got {type(script).__name__}"
             )
 
-        raw_result = _run_script(script)
+        raw_result = _run_script(script, writer)
+        exit_code_for_close = raw_result.get("exit_code", 1)
 
         if hasattr(handler, "post_processing"):
             output = handler.post_processing(raw_result)
@@ -105,6 +120,12 @@ def main() -> None:
             "stdout": "",
             "stderr": traceback.format_exc(),
         }
+        exit_code_for_close = output["exit_code"]
+    finally:
+        # Close even when the script never ran (e.g. handler.py failed to load) so a writer
+        # that was opened above is never left dangling on the backend.
+        if writer is not None:
+            writer.close(exit_code_for_close)
 
     with open(args.output_file, "w", encoding="utf-8") as f:
         json.dump(output, f)

quapp_hpc-0.0.1.dev11/quapp_slurm_hpc/stream_exec.py

@@ -0,0 +1,108 @@
+"""Runs a subprocess while streaming its output incrementally to a `log_writer.JobLogWriter`,
+instead of blocking until it exits. Shared by:
+
+  - `runner.py`'s `_run_script()` — the function handler's own bash script (tier 1 of the HPC
+    log-streaming feature), stdout/stderr captured separately for the final output.json.
+  - `host_wrapper.py` — wraps the *outer* `apptainer exec` invocation itself (tier 2), so the
+    image-pull/SIF-build messages Apptainer prints before the handler ever starts are visible
+    too. stdout/stderr merged there, since nothing downstream needs them separate.
+
+No import of `log_writer` at module load time: `host_wrapper.py` is deployed standalone to bare
+Slurm compute node hosts (no `quapp_slurm_hpc` package install there, just this file + log_writer.py
+side by side — see the `apptainer` Ansible role), so this module only takes an already-built
+writer and never assumes how it was constructed.
+"""
+import subprocess
+import threading
+import time
+from typing import List, Optional, Tuple
+
+
+def run_streamed(
+    cmd: List[str],
+    writer=None,
+    *,
+    merge_stderr: bool = False,
+    flush_interval_fn=None,
+    tee_to=None,
+    keep_output: bool = True,
+) -> Tuple[int, bytes, bytes]:
+    """Runs `cmd`, flushing new output to `writer.append()` (if given) on the cadence
+    `flush_interval_fn(elapsed_seconds)` returns (defaults to `log_writer.flush_interval`).
+
+    `tee_to`, when given a writable binary stream (e.g. `sys.stdout.buffer`), gets each line
+    written (and flushed) to it as soon as it's read — not after the process exits — so a killed
+    process (walltime, `scancel`, OOM) still leaves whatever it already printed somewhere durable
+    (`host_wrapper.py` uses this so Apptainer's output still lands in `slurm-<jobid>.out` exactly
+    as before this wrapper existed, line by line, instead of only on a clean exit).
+
+    `keep_output` (default True) buffers everything read into the returned bytes, same as
+    before. Set it False when the caller only needs the exit code and already has the content
+    some other way (`tee_to`, or doesn't need it) — otherwise a long, chatty command holds its
+    entire output in memory for no reason (`host_wrapper.py`'s use case: it tees to its own
+    stdout and never looks at the returned bytes).
+
+    Returns `(exit_code, stdout_bytes, stderr_bytes)` (both empty when `keep_output` is False).
+    When `merge_stderr` is True, stderr is redirected into the same stream as stdout, so
+    `stdout_bytes` holds everything combined and `stderr_bytes` is always `b""`.
+    """
+    if flush_interval_fn is None:
+        from quapp_slurm_hpc.log_writer import flush_interval as flush_interval_fn
+
+    process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT if merge_stderr else subprocess.PIPE,
+    )
+
+    stdout_parts: List[bytes] = []
+    stderr_parts: List[bytes] = []
+    pending_parts: List[bytes] = []
+    lock = threading.Lock()
+
+    def _drain(stream, sink: List[bytes]) -> None:
+        for line in iter(stream.readline, b""):
+            if tee_to is not None:
+                tee_to.write(line)
+                tee_to.flush()
+            with lock:
+                if keep_output:
+                    sink.append(line)
+                pending_parts.append(line)
+        stream.close()
+
+    readers = [threading.Thread(target=_drain, args=(process.stdout, stdout_parts), daemon=True)]
+    if not merge_stderr:
+        readers.append(
+            threading.Thread(target=_drain, args=(process.stderr, stderr_parts), daemon=True)
+        )
+    for reader in readers:
+        reader.start()
+
+    def _flush() -> None:
+        if writer is None:
+            return
+        with lock:
+            if not pending_parts:
+                return
+            chunk = b"".join(pending_parts)
+            pending_parts.clear()
+        writer.append(chunk)
+
+    start_time = time.monotonic()
+    last_flush = start_time
+    while True:
+        finished = process.poll() is not None and not any(r.is_alive() for r in readers)
+        now = time.monotonic()
+        if finished or now - last_flush >= flush_interval_fn(now - start_time):
+            _flush()
+            last_flush = now
+        if finished:
+            break
+        time.sleep(0.2)
+
+    for reader in readers:
+        reader.join()
+    exit_code = process.wait()
+
+    return exit_code, b"".join(stdout_parts), b"".join(stderr_parts)

quapp_hpc-0.0.1.dev11/tests/test_log_writer.py

@@ -0,0 +1,268 @@
+"""Tests for quapp_slurm_hpc.log_writer — the first client of the backend's PQF-23860 ingest
+protocol. No test infra existed in this repo before this file.
+"""
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+
+from quapp_slurm_hpc import log_writer
+from quapp_slurm_hpc.log_writer import JobLogWriter, flush_interval, split_chunks
+
+
+def _response(status_code=200, outcome=None, expected_seq=None):
+    resp = MagicMock()
+    resp.status_code = status_code
+    if status_code >= 400:
+        resp.raise_for_status.side_effect = requests.exceptions.HTTPError(response=resp)
+    else:
+        resp.raise_for_status.side_effect = None
+    resp.json.return_value = {
+        "data": {"outcome": outcome, "endOffset": 0, "expectedSeq": expected_seq}
+    }
+    return resp
+
+
+@pytest.fixture(autouse=True)
+def no_real_sleep():
+    # Retry backoff would otherwise make the failure-path tests slow for no reason.
+    with patch.object(log_writer.time, "sleep"):
+        yield
+
+
+# ── split_chunks ──────────────────────────────────────────────────────────────
+
+
+def test_split_chunks_under_limit_is_a_single_piece():
+    data = b"line 1\nline 2\n"
+    assert split_chunks(data, limit=1024) == [data]
+
+
+def test_split_chunks_cuts_at_the_last_newline_in_the_window():
+    data = b"a" * 10 + b"\n" + b"b" * 10 + b"\n" + b"c" * 10
+    pieces = split_chunks(data, limit=15)
+    assert b"".join(pieces) == data
+    assert all(len(p) <= 15 for p in pieces)
+    # First piece ends right after the first line's newline, not mid-line.
+    assert pieces[0] == b"a" * 10 + b"\n"
+
+
+def test_split_chunks_splits_mid_line_when_a_single_line_exceeds_the_limit():
+    # A \r-driven progress bar with no newline for a long stretch must still be split, even
+    # though that necessarily cuts inside what is semantically "one line".
+    data = b"x" * 50
+    pieces = split_chunks(data, limit=20)
+    assert b"".join(pieces) == data
+    assert all(len(p) <= 20 for p in pieces)
+    assert len(pieces) == 3
+
+
+def test_split_chunks_empty_input_yields_no_pieces():
+    assert split_chunks(b"", limit=10) == []
+
+
+# ── flush_interval ────────────────────────────────────────────────────────────
+
+
+def test_flush_interval_schedule():
+    assert flush_interval(0) == 2.0
+    assert flush_interval(299) == 2.0
+    assert flush_interval(300) == 10.0
+    assert flush_interval(1799) == 10.0
+    assert flush_interval(1800) == 30.0
+    assert flush_interval(999_999) == 30.0
+
+
+# ── JobLogWriter.append ───────────────────────────────────────────────────────
+
+
+def test_append_splits_oversized_data_into_multiple_requests_each_within_limit():
+    # Patch the module constant to a small value so the test data stays readable.
+    with patch.object(log_writer, "_MAX_CHUNK_BYTES", 10), \
+         patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome="APPENDED")
+        writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+        writer.append(b"y" * 25)
+        assert mock_post.call_count == 3  # 10 + 10 + 5 bytes
+        for call in mock_post.call_args_list:
+            body = call.kwargs["data"]
+            assert len(body) <= 10
+        assert writer._seq == 3
+        assert not writer.stopped
+
+
+def test_append_increments_seq_only_on_appended_or_duplicate():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome="APPENDED")
+        writer.append(b"hello\n")
+        assert writer._seq == 1
+        mock_post.return_value = _response(outcome="DUPLICATE")
+        writer.append(b"hello again\n")
+        assert writer._seq == 2
+
+
+def test_append_resyncs_seq_on_seq_gap():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    writer._seq = 5
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome="SEQ_GAP", expected_seq=9)
+        writer.append(b"data\n")
+    assert writer._seq == 9
+    assert not writer.stopped
+
+
+def test_append_sends_the_correct_headers():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok-abc", 3, headers={"X-Tenant": "t1"})
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome="APPENDED")
+        writer.append(b"hello\n")
+    _, kwargs = mock_post.call_args
+    assert kwargs["headers"]["X-Quapp-Log-Token"] == "tok-abc"
+    assert kwargs["headers"]["X-Log-Seq"] == "0"
+    assert kwargs["headers"]["X-Log-Dropped"] == "0"
+    assert kwargs["headers"]["Content-Type"] == "application/octet-stream"
+    assert kwargs["headers"]["X-Tenant"] == "t1"
+    assert "/writers/" in mock_post.call_args[0][0]
+    assert "attempt=3" in mock_post.call_args[0][0]
+
+
+# ── best-effort error handling ────────────────────────────────────────────────
+
+
+def test_transient_network_error_then_success_recovers_without_stopping():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.side_effect = [
+            requests.exceptions.Timeout("boom"),
+            _response(outcome="APPENDED"),
+        ]
+        writer.append(b"hello\n")
+    assert mock_post.call_count == 2
+    assert writer._seq == 1
+    assert not writer.stopped
+
+
+def test_persistent_network_error_exhausts_retries_then_stops():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.side_effect = requests.exceptions.ConnectionError("down")
+        writer.append(b"hello\n")
+    assert mock_post.call_count == log_writer._MAX_ATTEMPTS
+    assert writer.stopped
+    assert writer._seq == 0
+
+
+def test_transient_status_5xx_is_retried_then_recovers():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.side_effect = [_response(status_code=503), _response(outcome="APPENDED")]
+        writer.append(b"hello\n")
+    assert mock_post.call_count == 2
+    assert not writer.stopped
+    assert writer._seq == 1
+
+
+def test_persistent_transient_status_exhausts_retries_then_stops():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(status_code=500)
+        writer.append(b"hello\n")
+    assert mock_post.call_count == log_writer._MAX_ATTEMPTS
+    assert writer.stopped
+
+
+def test_401_stops_immediately_without_retrying():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(status_code=401)
+        writer.append(b"hello\n")
+    assert mock_post.call_count == 1
+    assert writer.stopped
+
+
+@pytest.mark.parametrize("outcome", ["SEALED", "TOO_LATE", "TRUNCATED"])
+def test_stop_outcomes_halt_further_sends(outcome):
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome=outcome)
+        writer.append(b"hello\n")
+        assert writer.stopped
+        writer.append(b"more\n")  # must not call the backend again
+        assert mock_post.call_count == 1
+
+
+def test_append_on_already_stopped_writer_only_counts_dropped_bytes():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    writer._stopped = True
+    with patch.object(log_writer.requests, "post") as mock_post:
+        writer.append(b"12345")
+    mock_post.assert_not_called()
+    assert writer._dropped_bytes == 5
+
+
+# ── open / close ──────────────────────────────────────────────────────────────
+
+
+def test_close_sends_last_seq_and_marks_stopped():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome="APPENDED")
+        writer.append(b"hello\n")  # seq -> 1
+        mock_post.return_value = _response(outcome="CLOSED")
+        writer.close(exit_code=0)
+    _, kwargs = mock_post.call_args
+    assert kwargs["json"]["lastSeq"] == 0
+    assert kwargs["json"]["exitCode"] == 0
+    assert writer.stopped
+
+
+def test_close_without_any_append_sends_last_seq_minus_one():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome="CLOSED")
+        writer.close(exit_code=1)
+    _, kwargs = mock_post.call_args
+    assert kwargs["json"]["lastSeq"] == -1
+
+
+def test_close_is_safe_to_call_on_an_already_stopped_writer():
+    writer = JobLogWriter("http://backend/jobs/j1/logs", "tok", 1)
+    writer._stopped = True
+    with patch.object(log_writer.requests, "post") as mock_post:
+        mock_post.return_value = _response(outcome="CLOSED")
+        writer.close(exit_code=0)  # must not raise
+    assert writer.stopped
+
+
+# ── from_input ────────────────────────────────────────────────────────────────
+
+
+def test_from_input_returns_none_when_log_ingest_missing():
+    assert log_writer.from_input({"x": 1}) is None
+
+
+def test_from_input_returns_none_for_non_dict_input():
+    assert log_writer.from_input("not a dict") is None
+    assert log_writer.from_input(None) is None
+
+
+def test_from_input_returns_none_when_required_fields_missing():
+    assert log_writer.from_input({"logIngest": {"baseUrl": "http://x"}}) is None
+
+
+def test_from_input_builds_writer_from_a_valid_block():
+    writer = log_writer.from_input(
+        {
+            "logIngest": {
+                "baseUrl": "http://backend/jobs/j1/logs",
+                "token": "tok",
+                "attempt": 2,
+                "headers": {"X-Tenant": "t1"},
+            }
+        }
+    )
+    assert isinstance(writer, JobLogWriter)
+    assert writer._base_url == "http://backend/jobs/j1/logs"
+    assert writer._attempt == 2
+    assert writer._extra_headers == {"X-Tenant": "t1"}

quapp_hpc-0.0.1.dev11/tests/test_stream_exec.py

@@ -0,0 +1,99 @@
+"""Tests for quapp_slurm_hpc.stream_exec — the incremental subprocess-capture loop shared by
+runner.py (tier 1) and host_wrapper.py (tier 2). No test infra existed for this before.
+"""
+import io
+import sys
+
+from quapp_slurm_hpc import stream_exec
+
+
+class _FakeWriter:
+    """Records every chunk `append()` receives, in order — stands in for JobLogWriter."""
+
+    def __init__(self):
+        self.chunks = []
+
+    def append(self, data: bytes) -> None:
+        self.chunks.append(data)
+
+
+def _always_flush(_elapsed: float) -> float:
+    # A near-zero interval makes every poll tick (every 0.2s) flush — the tests don't need to
+    # wait out the real flush_interval() schedule to see output land on the writer.
+    return 0.0
+
+
+def test_exit_code_and_merged_output():
+    exit_code, combined, stderr = stream_exec.run_streamed(
+        ["bash", "-c", "echo out1; echo err1 >&2; echo out2"],
+        merge_stderr=True,
+    )
+    assert exit_code == 0
+    assert stderr == b""
+    assert combined.count(b"\n") == 3
+    assert b"out1" in combined and b"err1" in combined and b"out2" in combined
+
+
+def test_exit_code_and_separate_stdout_stderr():
+    exit_code, stdout, stderr = stream_exec.run_streamed(
+        ["bash", "-c", "echo out1; echo err1 >&2"],
+    )
+    assert exit_code == 0
+    assert stdout == b"out1\n"
+    assert stderr == b"err1\n"
+
+
+def test_nonzero_exit_code_is_reported():
+    exit_code, _, _ = stream_exec.run_streamed(["bash", "-c", "exit 7"])
+    assert exit_code == 7
+
+
+def test_writer_receives_flushed_chunks():
+    writer = _FakeWriter()
+    stream_exec.run_streamed(
+        ["bash", "-c", "echo hello"], writer, flush_interval_fn=_always_flush,
+    )
+    assert b"".join(writer.chunks) == b"hello\n"
+
+
+def test_tee_to_writes_every_line_as_it_arrives():
+    sink = io.BytesIO()
+    exit_code, combined, _ = stream_exec.run_streamed(
+        ["bash", "-c", "echo line1; echo line2"],
+        merge_stderr=True,
+        tee_to=sink,
+    )
+    assert exit_code == 0
+    assert sink.getvalue() == b"line1\nline2\n" == combined
+
+
+def test_keep_output_false_discards_buffer_but_keeps_exit_code():
+    sink = io.BytesIO()
+    exit_code, stdout, stderr = stream_exec.run_streamed(
+        ["bash", "-c", "echo should-not-be-buffered"],
+        merge_stderr=True,
+        tee_to=sink,
+        keep_output=False,
+    )
+    assert exit_code == 0
+    assert stdout == b""
+    assert stderr == b""
+    # The content isn't lost — it went to tee_to, just not also held in memory.
+    assert sink.getvalue() == b"should-not-be-buffered\n"
+
+
+def test_tee_to_receives_output_even_when_process_is_killed_mid_run():
+    # Regression test for a wrapper that only wrote its tee target *after* the process exited:
+    # a killed process (walltime/scancel/OOM in production) must not lose what it already
+    # printed before being killed. `timeout` sends SIGTERM partway through, same as Slurm hitting
+    # a job's walltime — the run must stay well under the 5s `sleep` it never reaches the end of.
+    sink = io.BytesIO()
+    exit_code, _, _ = stream_exec.run_streamed(
+        ["timeout", "0.5", "bash", "-c", "echo before-kill; sleep 5; echo should-never-print"],
+        merge_stderr=True,
+        tee_to=sink,
+        keep_output=False,
+    )
+    assert exit_code != 0  # killed by timeout, never reached its own `exit 0`
+    assert b"before-kill" in sink.getvalue()
+    assert b"should-never-print" not in sink.getvalue()