quapp-hpc 0.0.1.dev7__tar.gz → 0.0.1.dev8__tar.gz

{quapp_hpc-0.0.1.dev7/quapp_hpc.egg-info → quapp_hpc-0.0.1.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev7
+Version: 0.0.1.dev8
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quapp-hpc"
-version = "0.0.1.dev7"
+version = "0.0.1.dev8"
 description = "Quapp HPC library — Slurm integration for Quapp Platform"
 readme = "README.md"
 authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev8/quapp_hpc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev7
+Version: 0.0.1.dev8
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev8}/quapp_hpc.egg-info/SOURCES.txt

@@ -24,4 +24,5 @@ quapp_hpc/model/device/slurm_device.py
 quapp_hpc/model/provider/__init__.py
 quapp_hpc/model/provider/slurm_provider.py
 quapp_slurm_hpc/__init__.py
+quapp_slurm_hpc/log_writer.py
 quapp_slurm_hpc/runner.py

quapp_hpc-0.0.1.dev8/quapp_slurm_hpc/log_writer.py

@@ -0,0 +1,132 @@
+"""Pushes a job's stdout/stderr to the backend's live log pipeline (PQF-23860) as it runs,
+instead of only via the final output.json snapshot uploaded at the end.
+
+No client for this protocol exists anywhere yet (checked qapp-lib / quapp-libs) — this is the
+first one, reverse-engineered directly from the backend's ingest code:
+
+    JobLogIngestControllerV1  -- POST {base_url}/writers/{writer_id}/open?attempt=N
+                              -- POST {base_url}/writers/{writer_id}/chunks?attempt=N
+                              -- POST {base_url}/writers/{writer_id}/close?attempt=N
+    JobLogWriteResult         -- the response body, wrapped by the backend's global envelope
+                                 as {"data": {"outcome": ..., "endOffset": ..., "expectedSeq": ...}}
+
+Every call carries the token the backend issued for this job attempt in the `logIngest` block
+of the invocation input (see JobLogIngestPayloadProvider on the backend). Best-effort only: a
+failure here must never raise into the handler's own execution — the job still has to run to
+completion and still produce output.json exactly as before.
+"""
+import sys
+import uuid
+from typing import Optional
+
+import requests
+
+_TOKEN_HEADER = "X-Quapp-Log-Token"
+_SEQ_HEADER = "X-Log-Seq"
+_DROPPED_HEADER = "X-Log-Dropped"
+_REQUEST_TIMEOUT_SECONDS = 5
+_STOP_OUTCOMES = {"SEALED", "TOO_LATE", "TRUNCATED"}
+
+
+class JobLogWriter:
+    """A single log writer for one job attempt. Not thread-safe — call from one thread only."""
+
+    def __init__(self, base_url: str, token: str, attempt: int, headers: Optional[dict] = None):
+        self._base_url = base_url.rstrip("/")
+        self._token = token
+        self._attempt = attempt
+        self._extra_headers = headers or {}
+        self._writer_id = str(uuid.uuid4())
+        self._seq = 0
+        self._dropped_bytes = 0
+        self._stopped = False
+
+    @property
+    def stopped(self) -> bool:
+        """True once the backend told us to stop (sealed/too-late/truncated) or a request
+        failed — callers should skip further append()/close() calls once this is set, though
+        each method is itself a safe no-op either way."""
+        return self._stopped
+
+    def open(self) -> None:
+        if self._stopped:
+            return
+        self._post("open")
+
+    def append(self, data: bytes) -> None:
+        """Sends one chunk of new output. Silently drops it (and counts it as dropped) once
+        streaming has stopped — the handler's own run must never block or fail on this."""
+        if not data:
+            return
+        if self._stopped:
+            self._dropped_bytes += len(data)
+            return
+        outcome, expected_seq = self._post(
+            "chunks",
+            data=data,
+            extra_headers={
+                _SEQ_HEADER: str(self._seq),
+                _DROPPED_HEADER: str(self._dropped_bytes),
+                # The backend's /chunks route is declared `consumes = APPLICATION_OCTET_STREAM`
+                # and rejects anything else with 415 — requests does not set this on its own
+                # for a raw bytes body.
+                "Content-Type": "application/octet-stream",
+            },
+        )
+        if outcome in ("APPENDED", "DUPLICATE"):
+            self._seq += 1
+            self._dropped_bytes = 0
+        elif outcome == "SEQ_GAP" and expected_seq is not None:
+            # The backend tells us exactly what it's missing — resync instead of retrying the
+            # same seq forever (which would never succeed and would spin indefinitely).
+            self._seq = int(expected_seq)
+            self._dropped_bytes += len(data)
+
+    def close(self, exit_code: int, abandoned: bool = False, discarded: bool = False) -> None:
+        """Closes the writer. Safe to call even if streaming already stopped earlier (e.g. the
+        backend sealed it, or a request failed), so a crash can still attempt a clean close
+        from a `finally` block without extra branching."""
+        self._post(
+            "close",
+            json={
+                "lastSeq": self._seq - 1,
+                "droppedBytes": self._dropped_bytes,
+                "abandoned": abandoned,
+                "discarded": discarded,
+                "exitCode": exit_code,
+            },
+        )
+        self._stopped = True
+
+    def _post(self, segment: str, *, json=None, data=None, extra_headers: Optional[dict] = None):
+        url = f"{self._base_url}/writers/{self._writer_id}/{segment}?attempt={self._attempt}"
+        headers = {_TOKEN_HEADER: self._token, **self._extra_headers, **(extra_headers or {})}
+        try:
+            response = requests.post(
+                url, json=json, data=data, headers=headers, timeout=_REQUEST_TIMEOUT_SECONDS
+            )
+            response.raise_for_status()
+            payload = (response.json() or {}).get("data") or {}
+            outcome = payload.get("outcome")
+            if outcome in _STOP_OUTCOMES:
+                self._stopped = True
+            return outcome, payload.get("expectedSeq")
+        except Exception as exc:  # best-effort: never let log streaming break the job itself
+            print(f"[log_writer] stopping log stream, request failed: {exc}", file=sys.stderr, flush=True)
+            self._stopped = True
+            return None, None
+
+
+def from_input(invocation_input) -> Optional[JobLogWriter]:
+    """Builds a writer from the `logIngest` block the backend injects into the job input, or
+    None when push logging is disabled or the job predates this feature. Callers must treat
+    None as "don't stream this run" — not as an error."""
+    log_ingest = invocation_input.get("logIngest") if isinstance(invocation_input, dict) else None
+    if not isinstance(log_ingest, dict):
+        return None
+    base_url = log_ingest.get("baseUrl")
+    token = log_ingest.get("token")
+    attempt = log_ingest.get("attempt")
+    if not base_url or not token or attempt is None:
+        return None
+    return JobLogWriter(base_url, token, int(attempt), log_ingest.get("headers"))

{quapp_hpc-0.0.1.dev7 → quapp_hpc-0.0.1.dev8}/quapp_slurm_hpc/runner.py

@@ -11,7 +11,8 @@ This module:
   1. Reads the invocation input from --input-file (JSON written by the sbatch script).
   2. Loads handler.py from --handler-dir.
   3. Calls handler.processing(invocation_input) → bash script string.
-  4. Executes the bash script, captures stdout / stderr / exit_code.
+  4. Executes the bash script, streaming stdout/stderr live to the backend as it runs (when the
+     input carries a `logIngest` block) while also capturing it in full for the final result.
   5. Calls handler.post_processing(raw_result) if it exists.
   6. Writes the final result dict to --output-file.
   7. Uploads output.json to S3 (key: {QUAPP_JOB_ID}/output.json) when SLURM_S3_BUCKET is set.
@@ -23,10 +24,16 @@ import os
 import subprocess
 import sys
 import tempfile
+import threading
+import time
 import traceback
 from pathlib import Path
 from types import ModuleType
 
+from quapp_slurm_hpc import log_writer
+
+_LOG_FLUSH_INTERVAL_SECONDS = 2.0
+
 
 def _load_handler(handler_dir: str) -> ModuleType:
     handler_path = Path(handler_dir) / "handler.py"
@@ -43,23 +50,74 @@ def _load_handler(handler_dir: str) -> ModuleType:
     return module
 
 
-def _run_script(script: str) -> dict:
-    """Write script to a temp file and execute it with bash, capturing output."""
+def _run_script(script: str, writer: "log_writer.JobLogWriter | None" = None) -> dict:
+    """Write script to a temp file and execute it with bash, capturing output.
+
+    Reads stdout/stderr incrementally as the script runs (instead of blocking until it exits)
+    so `writer`, when given, can push new output to the backend every
+    `_LOG_FLUSH_INTERVAL_SECONDS` seconds — the final returned dict is unchanged either way; the
+    full stdout/stderr are still captured in full for `output.json`, same as before.
+    """
     with tempfile.NamedTemporaryFile(
         mode="w", suffix=".sh", delete=False, encoding="utf-8"
     ) as f:
         f.write(script)
         tmp_path = f.name
     try:
-        result = subprocess.run(
+        process = subprocess.Popen(
             ["bash", tmp_path],
-            capture_output=True,
-            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
+
+        stdout_parts: list[bytes] = []
+        stderr_parts: list[bytes] = []
+        pending_parts: list[bytes] = []
+        lock = threading.Lock()
+
+        def _drain(stream, sink: list[bytes]) -> None:
+            for line in iter(stream.readline, b""):
+                with lock:
+                    sink.append(line)
+                    pending_parts.append(line)
+            stream.close()
+
+        readers = [
+            threading.Thread(target=_drain, args=(process.stdout, stdout_parts), daemon=True),
+            threading.Thread(target=_drain, args=(process.stderr, stderr_parts), daemon=True),
+        ]
+        for reader in readers:
+            reader.start()
+
+        def _flush() -> None:
+            if writer is None:
+                return
+            with lock:
+                if not pending_parts:
+                    return
+                chunk = b"".join(pending_parts)
+                pending_parts.clear()
+            writer.append(chunk)
+
+        last_flush = time.monotonic()
+        while True:
+            finished = process.poll() is not None and not any(r.is_alive() for r in readers)
+            now = time.monotonic()
+            if finished or now - last_flush >= _LOG_FLUSH_INTERVAL_SECONDS:
+                _flush()
+                last_flush = now
+            if finished:
+                break
+            time.sleep(0.2)
+
+        for reader in readers:
+            reader.join()
+        exit_code = process.wait()
+
         return {
-            "exit_code": result.returncode,
-            "stdout": result.stdout,
-            "stderr": result.stderr,
+            "exit_code": exit_code,
+            "stdout": b"".join(stdout_parts).decode("utf-8", errors="replace"),
+            "stderr": b"".join(stderr_parts).decode("utf-8", errors="replace"),
         }
     finally:
         Path(tmp_path).unlink(missing_ok=True)
@@ -75,10 +133,18 @@ def main() -> None:
     args = parser.parse_args()
 
     output: dict = {}
+    writer = None
+    exit_code_for_close = 1
     try:
         with open(args.input_file, encoding="utf-8") as f:
             invocation_input = json.load(f)
 
+        # Absent when log push is disabled or the job predates this feature — _run_script and
+        # writer.append/close below all treat `writer is None` as "don't stream", not an error.
+        writer = log_writer.from_input(invocation_input)
+        if writer is not None:
+            writer.open()
+
         handler = _load_handler(args.handler_dir)
 
         if not hasattr(handler, "processing"):
@@ -92,7 +158,8 @@ def main() -> None:
                 f"handler.processing() must return a str (bash script), got {type(script).__name__}"
             )
 
-        raw_result = _run_script(script)
+        raw_result = _run_script(script, writer)
+        exit_code_for_close = raw_result.get("exit_code", 1)
 
         if hasattr(handler, "post_processing"):
             output = handler.post_processing(raw_result)
@@ -105,6 +172,12 @@ def main() -> None:
             "stdout": "",
             "stderr": traceback.format_exc(),
         }
+        exit_code_for_close = output["exit_code"]
+    finally:
+        # Close even when the script never ran (e.g. handler.py failed to load) so a writer
+        # that was opened above is never left dangling on the backend.
+        if writer is not None:
+            writer.close(exit_code_for_close)
 
     with open(args.output_file, "w", encoding="utf-8") as f:
         json.dump(output, f)