quapp-hpc 0.0.1.dev6__tar.gz → 0.0.1.dev8__tar.gz

{quapp_hpc-0.0.1.dev6/quapp_hpc.egg-info → quapp_hpc-0.0.1.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev6
+Version: 0.0.1.dev8
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev6 → quapp_hpc-0.0.1.dev8}/pyproject.toml

@@ -1,33 +1,33 @@
-[build-system]
-requires = ["setuptools>=61.0.0", "wheel==0.45.1"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "quapp-hpc"
-version = "0.0.1.dev6"
-description = "Quapp HPC library — Slurm integration for Quapp Platform"
-readme = "README.md"
-authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]
-license = { file = "LICENSE" }
-classifiers = [
-    "License :: OSI Approved :: MIT License",
-    "Programming Language :: Python",
-    "Programming Language :: Python :: 3",
-]
-keywords = ["quapp", "quapp-hpc", "slurm", "hpc"]
-dependencies = [
-    "quapp-common==0.0.12.dev12",
-    "requests>=2.31.0",
-    "boto3>=1.28.0",
-]
-requires-python = ">=3.10,<3.13"
-
-[project.optional-dependencies]
-dev = ["black", "bumpver", "isort", "pip-tools", "pytest"]
-
-[tool.setuptools.packages.find]
-include = ["quapp_hpc*", "quapp_slurm_hpc*"]
-exclude = ["*.md", "*.yml", "*.yaml", "*.toml", "tests*", ".gitignore"]
-
-[project.urls]
-Homepage = "https://quapp.cloud/"
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel==0.45.1"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "quapp-hpc"
+version = "0.0.1.dev8"
+description = "Quapp HPC library — Slurm integration for Quapp Platform"
+readme = "README.md"
+authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]
+license = { file = "LICENSE" }
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+]
+keywords = ["quapp", "quapp-hpc", "slurm", "hpc"]
+dependencies = [
+    "quapp-common==0.0.12.dev12",
+    "requests>=2.31.0",
+    "boto3>=1.28.0",
+]
+requires-python = ">=3.10,<3.13"
+
+[project.optional-dependencies]
+dev = ["black", "bumpver", "isort", "pip-tools", "pytest"]
+
+[tool.setuptools.packages.find]
+include = ["quapp_hpc*", "quapp_slurm_hpc*"]
+exclude = ["*.md", "*.yml", "*.yaml", "*.toml", "tests*", ".gitignore"]
+
+[project.urls]
+Homepage = "https://quapp.cloud/"

{quapp_hpc-0.0.1.dev6 → quapp_hpc-0.0.1.dev8/quapp_hpc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev6
+Version: 0.0.1.dev8
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev6 → quapp_hpc-0.0.1.dev8}/quapp_hpc.egg-info/SOURCES.txt

@@ -24,4 +24,5 @@ quapp_hpc/model/device/slurm_device.py
 quapp_hpc/model/provider/__init__.py
 quapp_hpc/model/provider/slurm_provider.py
 quapp_slurm_hpc/__init__.py
+quapp_slurm_hpc/log_writer.py
 quapp_slurm_hpc/runner.py

quapp_hpc-0.0.1.dev8/quapp_slurm_hpc/log_writer.py

@@ -0,0 +1,132 @@
+"""Pushes a job's stdout/stderr to the backend's live log pipeline (PQF-23860) as it runs,
+instead of only via the final output.json snapshot uploaded at the end.
+
+No client for this protocol exists anywhere yet (checked qapp-lib / quapp-libs) — this is the
+first one, reverse-engineered directly from the backend's ingest code:
+
+    JobLogIngestControllerV1  -- POST {base_url}/writers/{writer_id}/open?attempt=N
+                              -- POST {base_url}/writers/{writer_id}/chunks?attempt=N
+                              -- POST {base_url}/writers/{writer_id}/close?attempt=N
+    JobLogWriteResult         -- the response body, wrapped by the backend's global envelope
+                                 as {"data": {"outcome": ..., "endOffset": ..., "expectedSeq": ...}}
+
+Every call carries the token the backend issued for this job attempt in the `logIngest` block
+of the invocation input (see JobLogIngestPayloadProvider on the backend). Best-effort only: a
+failure here must never raise into the handler's own execution — the job still has to run to
+completion and still produce output.json exactly as before.
+"""
+import sys
+import uuid
+from typing import Optional
+
+import requests
+
+_TOKEN_HEADER = "X-Quapp-Log-Token"
+_SEQ_HEADER = "X-Log-Seq"
+_DROPPED_HEADER = "X-Log-Dropped"
+_REQUEST_TIMEOUT_SECONDS = 5
+_STOP_OUTCOMES = {"SEALED", "TOO_LATE", "TRUNCATED"}
+
+
+class JobLogWriter:
+    """A single log writer for one job attempt. Not thread-safe — call from one thread only."""
+
+    def __init__(self, base_url: str, token: str, attempt: int, headers: Optional[dict] = None):
+        self._base_url = base_url.rstrip("/")
+        self._token = token
+        self._attempt = attempt
+        self._extra_headers = headers or {}
+        self._writer_id = str(uuid.uuid4())
+        self._seq = 0
+        self._dropped_bytes = 0
+        self._stopped = False
+
+    @property
+    def stopped(self) -> bool:
+        """True once the backend told us to stop (sealed/too-late/truncated) or a request
+        failed — callers should skip further append()/close() calls once this is set, though
+        each method is itself a safe no-op either way."""
+        return self._stopped
+
+    def open(self) -> None:
+        if self._stopped:
+            return
+        self._post("open")
+
+    def append(self, data: bytes) -> None:
+        """Sends one chunk of new output. Silently drops it (and counts it as dropped) once
+        streaming has stopped — the handler's own run must never block or fail on this."""
+        if not data:
+            return
+        if self._stopped:
+            self._dropped_bytes += len(data)
+            return
+        outcome, expected_seq = self._post(
+            "chunks",
+            data=data,
+            extra_headers={
+                _SEQ_HEADER: str(self._seq),
+                _DROPPED_HEADER: str(self._dropped_bytes),
+                # The backend's /chunks route is declared `consumes = APPLICATION_OCTET_STREAM`
+                # and rejects anything else with 415 — requests does not set this on its own
+                # for a raw bytes body.
+                "Content-Type": "application/octet-stream",
+            },
+        )
+        if outcome in ("APPENDED", "DUPLICATE"):
+            self._seq += 1
+            self._dropped_bytes = 0
+        elif outcome == "SEQ_GAP" and expected_seq is not None:
+            # The backend tells us exactly what it's missing — resync instead of retrying the
+            # same seq forever (which would never succeed and would spin indefinitely).
+            self._seq = int(expected_seq)
+            self._dropped_bytes += len(data)
+
+    def close(self, exit_code: int, abandoned: bool = False, discarded: bool = False) -> None:
+        """Closes the writer. Safe to call even if streaming already stopped earlier (e.g. the
+        backend sealed it, or a request failed), so a crash can still attempt a clean close
+        from a `finally` block without extra branching."""
+        self._post(
+            "close",
+            json={
+                "lastSeq": self._seq - 1,
+                "droppedBytes": self._dropped_bytes,
+                "abandoned": abandoned,
+                "discarded": discarded,
+                "exitCode": exit_code,
+            },
+        )
+        self._stopped = True
+
+    def _post(self, segment: str, *, json=None, data=None, extra_headers: Optional[dict] = None):
+        url = f"{self._base_url}/writers/{self._writer_id}/{segment}?attempt={self._attempt}"
+        headers = {_TOKEN_HEADER: self._token, **self._extra_headers, **(extra_headers or {})}
+        try:
+            response = requests.post(
+                url, json=json, data=data, headers=headers, timeout=_REQUEST_TIMEOUT_SECONDS
+            )
+            response.raise_for_status()
+            payload = (response.json() or {}).get("data") or {}
+            outcome = payload.get("outcome")
+            if outcome in _STOP_OUTCOMES:
+                self._stopped = True
+            return outcome, payload.get("expectedSeq")
+        except Exception as exc:  # best-effort: never let log streaming break the job itself
+            print(f"[log_writer] stopping log stream, request failed: {exc}", file=sys.stderr, flush=True)
+            self._stopped = True
+            return None, None
+
+
+def from_input(invocation_input) -> Optional[JobLogWriter]:
+    """Builds a writer from the `logIngest` block the backend injects into the job input, or
+    None when push logging is disabled or the job predates this feature. Callers must treat
+    None as "don't stream this run" — not as an error."""
+    log_ingest = invocation_input.get("logIngest") if isinstance(invocation_input, dict) else None
+    if not isinstance(log_ingest, dict):
+        return None
+    base_url = log_ingest.get("baseUrl")
+    token = log_ingest.get("token")
+    attempt = log_ingest.get("attempt")
+    if not base_url or not token or attempt is None:
+        return None
+    return JobLogWriter(base_url, token, int(attempt), log_ingest.get("headers"))

quapp_hpc-0.0.1.dev8/quapp_slurm_hpc/runner.py

@@ -0,0 +1,215 @@
+"""CLI runner — invoked by the sbatch script inside the Apptainer container.
+
+The Java SlurmPlatformManager generates a sbatch script that runs:
+
+    apptainer exec docker://<image> python -m quapp_slurm_hpc.runner \
+        --input-file  /data/jobs/<jobId>/input.json  \
+        --output-file /data/jobs/<jobId>/output.json \
+        --handler-dir /home/app/function
+
+This module:
+  1. Reads the invocation input from --input-file (JSON written by the sbatch script).
+  2. Loads handler.py from --handler-dir.
+  3. Calls handler.processing(invocation_input) → bash script string.
+  4. Executes the bash script, streaming stdout/stderr live to the backend as it runs (when the
+     input carries a `logIngest` block) while also capturing it in full for the final result.
+  5. Calls handler.post_processing(raw_result) if it exists.
+  6. Writes the final result dict to --output-file.
+  7. Uploads output.json to S3 (key: {QUAPP_JOB_ID}/output.json) when SLURM_S3_BUCKET is set.
+"""
+import argparse
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import traceback
+from pathlib import Path
+from types import ModuleType
+
+from quapp_slurm_hpc import log_writer
+
+_LOG_FLUSH_INTERVAL_SECONDS = 2.0
+
+
+def _load_handler(handler_dir: str) -> ModuleType:
+    handler_path = Path(handler_dir) / "handler.py"
+    if not handler_path.exists():
+        raise FileNotFoundError(
+            f"handler.py not found in {handler_dir!r}. "
+            "Ensure the function image was built with the handler in /home/app/function."
+        )
+    if str(handler_dir) not in sys.path:
+        sys.path.insert(0, str(handler_dir))
+    spec = importlib.util.spec_from_file_location("handler", handler_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _run_script(script: str, writer: "log_writer.JobLogWriter | None" = None) -> dict:
+    """Write script to a temp file and execute it with bash, capturing output.
+
+    Reads stdout/stderr incrementally as the script runs (instead of blocking until it exits)
+    so `writer`, when given, can push new output to the backend every
+    `_LOG_FLUSH_INTERVAL_SECONDS` seconds — the final returned dict is unchanged either way; the
+    full stdout/stderr are still captured in full for `output.json`, same as before.
+    """
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".sh", delete=False, encoding="utf-8"
+    ) as f:
+        f.write(script)
+        tmp_path = f.name
+    try:
+        process = subprocess.Popen(
+            ["bash", tmp_path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        stdout_parts: list[bytes] = []
+        stderr_parts: list[bytes] = []
+        pending_parts: list[bytes] = []
+        lock = threading.Lock()
+
+        def _drain(stream, sink: list[bytes]) -> None:
+            for line in iter(stream.readline, b""):
+                with lock:
+                    sink.append(line)
+                    pending_parts.append(line)
+            stream.close()
+
+        readers = [
+            threading.Thread(target=_drain, args=(process.stdout, stdout_parts), daemon=True),
+            threading.Thread(target=_drain, args=(process.stderr, stderr_parts), daemon=True),
+        ]
+        for reader in readers:
+            reader.start()
+
+        def _flush() -> None:
+            if writer is None:
+                return
+            with lock:
+                if not pending_parts:
+                    return
+                chunk = b"".join(pending_parts)
+                pending_parts.clear()
+            writer.append(chunk)
+
+        last_flush = time.monotonic()
+        while True:
+            finished = process.poll() is not None and not any(r.is_alive() for r in readers)
+            now = time.monotonic()
+            if finished or now - last_flush >= _LOG_FLUSH_INTERVAL_SECONDS:
+                _flush()
+                last_flush = now
+            if finished:
+                break
+            time.sleep(0.2)
+
+        for reader in readers:
+            reader.join()
+        exit_code = process.wait()
+
+        return {
+            "exit_code": exit_code,
+            "stdout": b"".join(stdout_parts).decode("utf-8", errors="replace"),
+            "stderr": b"".join(stderr_parts).decode("utf-8", errors="replace"),
+        }
+    finally:
+        Path(tmp_path).unlink(missing_ok=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Quapp HPC runner — executes handler.py inside an Apptainer container"
+    )
+    parser.add_argument("--input-file",  required=True, help="Path to input JSON file")
+    parser.add_argument("--output-file", required=True, help="Path to write output JSON")
+    parser.add_argument("--handler-dir", required=True, help="Directory containing handler.py")
+    args = parser.parse_args()
+
+    output: dict = {}
+    writer = None
+    exit_code_for_close = 1
+    try:
+        with open(args.input_file, encoding="utf-8") as f:
+            invocation_input = json.load(f)
+
+        # Absent when log push is disabled or the job predates this feature — _run_script and
+        # writer.append/close below all treat `writer is None` as "don't stream", not an error.
+        writer = log_writer.from_input(invocation_input)
+        if writer is not None:
+            writer.open()
+
+        handler = _load_handler(args.handler_dir)
+
+        if not hasattr(handler, "processing"):
+            raise AttributeError(
+                "handler.py must define a processing(invocation_input: dict) -> str function"
+            )
+
+        script = handler.processing(invocation_input)
+        if not isinstance(script, str):
+            raise TypeError(
+                f"handler.processing() must return a str (bash script), got {type(script).__name__}"
+            )
+
+        raw_result = _run_script(script, writer)
+        exit_code_for_close = raw_result.get("exit_code", 1)
+
+        if hasattr(handler, "post_processing"):
+            output = handler.post_processing(raw_result)
+        else:
+            output = raw_result
+
+    except Exception:
+        output = {
+            "exit_code": 1,
+            "stdout": "",
+            "stderr": traceback.format_exc(),
+        }
+        exit_code_for_close = output["exit_code"]
+    finally:
+        # Close even when the script never ran (e.g. handler.py failed to load) so a writer
+        # that was opened above is never left dangling on the backend.
+        if writer is not None:
+            writer.close(exit_code_for_close)
+
+    with open(args.output_file, "w", encoding="utf-8") as f:
+        json.dump(output, f)
+
+    _upload_to_s3(args.output_file)
+
+
+def _upload_to_s3(output_file: str) -> None:
+    """Upload output.json to S3 so the Java backend can read the result.
+
+    Requires SLURM_S3_BUCKET and QUAPP_JOB_ID env vars (injected by sbatch script).
+    Skips silently when the bucket is not configured.
+    """
+    s3_bucket = os.getenv("SLURM_S3_BUCKET", "")
+    job_uuid = os.getenv("QUAPP_JOB_ID", "")
+    if not s3_bucket or not job_uuid:
+        return
+
+    try:
+        import boto3
+        s3 = boto3.client(
+            "s3",
+            region_name=os.getenv("SLURM_S3_REGION", "ap-southeast-1"),
+            aws_access_key_id=os.getenv("SLURM_S3_ACCESS_KEY"),
+            aws_secret_access_key=os.getenv("SLURM_S3_SECRET_KEY"),
+        )
+        s3_key = f"{job_uuid}/output.json"
+        s3.upload_file(output_file, s3_bucket, s3_key)
+        print(f"[runner] Uploaded output to s3://{s3_bucket}/{s3_key}", flush=True)
+    except Exception as exc:
+        print(f"[runner] WARNING: S3 upload failed: {exc}", file=sys.stderr, flush=True)
+
+
+if __name__ == "__main__":
+    main()

quapp_hpc-0.0.1.dev6/quapp_slurm_hpc/runner.py

@@ -1,112 +0,0 @@
-"""CLI runner — invoked by the sbatch script inside the Apptainer container.
-
-The Java SlurmPlatformManager generates a sbatch script that runs:
-
-    apptainer exec docker://<image> python -m quapp_slurm_hpc.runner \
-        --input-file  /data/jobs/<jobId>/input.json  \
-        --output-file /data/jobs/<jobId>/output.json \
-        --handler-dir /home/app/function
-
-This module:
-  1. Reads the invocation input from --input-file (JSON written by the sbatch script).
-  2. Loads handler.py from --handler-dir.
-  3. Calls handler.processing(invocation_input) → bash script string.
-  4. Executes the bash script, captures stdout / stderr / exit_code.
-  5. Calls handler.post_processing(raw_result) if it exists.
-  6. Writes the final result dict to --output-file.
-"""
-import argparse
-import importlib.util
-import json
-import subprocess
-import sys
-import tempfile
-import traceback
-from pathlib import Path
-from types import ModuleType
-
-
-def _load_handler(handler_dir: str) -> ModuleType:
-    handler_path = Path(handler_dir) / "handler.py"
-    if not handler_path.exists():
-        raise FileNotFoundError(
-            f"handler.py not found in {handler_dir!r}. "
-            "Ensure the function image was built with the handler in /home/app/function."
-        )
-    if str(handler_dir) not in sys.path:
-        sys.path.insert(0, str(handler_dir))
-    spec = importlib.util.spec_from_file_location("handler", handler_path)
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-def _run_script(script: str) -> dict:
-    """Write script to a temp file and execute it with bash, capturing output."""
-    with tempfile.NamedTemporaryFile(
-        mode="w", suffix=".sh", delete=False, encoding="utf-8"
-    ) as f:
-        f.write(script)
-        tmp_path = f.name
-    try:
-        result = subprocess.run(
-            ["bash", tmp_path],
-            capture_output=True,
-            text=True,
-        )
-        return {
-            "exit_code": result.returncode,
-            "stdout": result.stdout,
-            "stderr": result.stderr,
-        }
-    finally:
-        Path(tmp_path).unlink(missing_ok=True)
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Quapp HPC runner — executes handler.py inside an Apptainer container"
-    )
-    parser.add_argument("--input-file",  required=True, help="Path to input JSON file")
-    parser.add_argument("--output-file", required=True, help="Path to write output JSON")
-    parser.add_argument("--handler-dir", required=True, help="Directory containing handler.py")
-    args = parser.parse_args()
-
-    output: dict = {}
-    try:
-        with open(args.input_file, encoding="utf-8") as f:
-            invocation_input = json.load(f)
-
-        handler = _load_handler(args.handler_dir)
-
-        if not hasattr(handler, "processing"):
-            raise AttributeError(
-                "handler.py must define a processing(invocation_input: dict) -> str function"
-            )
-
-        script = handler.processing(invocation_input)
-        if not isinstance(script, str):
-            raise TypeError(
-                f"handler.processing() must return a str (bash script), got {type(script).__name__}"
-            )
-
-        raw_result = _run_script(script)
-
-        if hasattr(handler, "post_processing"):
-            output = handler.post_processing(raw_result)
-        else:
-            output = raw_result
-
-    except Exception:
-        output = {
-            "exit_code": 1,
-            "stdout": "",
-            "stderr": traceback.format_exc(),
-        }
-
-    with open(args.output_file, "w", encoding="utf-8") as f:
-        json.dump(output, f)
-
-
-if __name__ == "__main__":
-    main()