quapp-hpc 0.0.1.dev2__tar.gz → 0.0.1.dev4__tar.gz

{quapp_hpc-0.0.1.dev2/quapp_hpc.egg-info → quapp_hpc-0.0.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev2
+Version: 0.0.1.dev4
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev2 → quapp_hpc-0.0.1.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quapp-hpc"
-version = "0.0.1.dev2"
+version = "0.0.1.dev4"
 description = "Quapp HPC library — Slurm integration for Quapp Platform"
 readme = "README.md"
 authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]

{quapp_hpc-0.0.1.dev2 → quapp_hpc-0.0.1.dev4}/quapp_hpc/component/backend/hpc_invocation.py

@@ -12,12 +12,14 @@ class HpcInvocation(Invocation):
         super().__init__(request_data)
         raw = request_data.input or {}
         job = raw.get("job", {})
+        # s3Bucket intentionally dropped — S3 bucket is system config, read by
+        # SlurmDevice from the S3_BUCKET env var (slurm-credentials secret).
+        # User input must not override system fields (HPC design philosophy).
         self._hpc_config = {
             "resources":      raw.get("resources", {}),
             "container":      raw.get("container", {}),
             "environment":    job.get("environment", {}),
             "input_s3_paths": job.get("input_s3_paths", []),
-            "s3_bucket":      raw.get("s3Bucket", ""),
         }
 
     def _export_circuit(self, circuit):

{quapp_hpc-0.0.1.dev2 → quapp_hpc-0.0.1.dev4}/quapp_hpc/component/backend/slurm_job_fetching.py

@@ -14,11 +14,7 @@ SLURM_JWT  = os.getenv("SLURM_JWT", "")
 S3_BUCKET  = os.getenv("S3_BUCKET",  "quapp-slurm-output-dev")
 AWS_REGION = os.getenv("AWS_REGION", "ap-southeast-1")
 
-_SLURM_TO_JOB_STATUS = {
-    "PENDING":     JobStatus.RUNNING.value,
-    "CONFIGURING": JobStatus.RUNNING.value,
-    "RUNNING":     JobStatus.RUNNING.value,
-    "COMPLETING":  JobStatus.RUNNING.value,
+_SLURM_TERMINAL_TO_JOB_STATUS = {
     "COMPLETED":   JobStatus.DONE.value,
     "FAILED":      JobStatus.ERROR.value,
     "CANCELLED":   JobStatus.ERROR.value,
@@ -85,7 +81,7 @@ class SlurmJobFetching(JobFetching):
 
         jobs = data.get("jobs", [data])
         if not jobs:
-            return JobStatus.RUNNING.value
+            return "UNKNOWN"
 
         raw_state = jobs[0].get("job_state", "UNKNOWN")
         if isinstance(raw_state, list):
@@ -93,7 +89,7 @@ class SlurmJobFetching(JobFetching):
 
         state = str(raw_state).strip()
         self._logger.info(f"Slurm job {slurm_job_id} state: {state}")
-        return _SLURM_TO_JOB_STATUS.get(state, JobStatus.RUNNING.value)
+        return _SLURM_TERMINAL_TO_JOB_STATUS.get(state, state)
 
     def _get_job_result(self, job: dict) -> _SlurmJobResult:
         return _SlurmJobResult(job_uuid=self.job_id, s3_bucket=S3_BUCKET, aws_region=AWS_REGION)

{quapp_hpc-0.0.1.dev2 → quapp_hpc-0.0.1.dev4}/quapp_hpc/model/device/slurm_device.py

@@ -14,20 +14,25 @@ from quapp_common.model.provider.provider import Provider
 
 from ..provider.slurm_provider import SlurmProvider, SLURM_ACCOUNT
 
+# ── System config (read from K8s secret slurm-credentials at ksvc startup) ───
 S3_BUCKET         = os.getenv("S3_BUCKET",               "quapp-slurm-output-dev")
 AWS_REGION        = os.getenv("AWS_REGION",               "ap-southeast-1")
 SLURM_POLL_SEC    = int(os.getenv("SLURM_POLL_SEC",      "30"))
 SLURM_TIMEOUT_SEC = int(os.getenv("SLURM_TIMEOUT_SEC",   "21600"))  # 6 hours
 SLURM_TIME_LIMIT  = int(os.getenv("SLURM_TIME_LIMIT_MIN", "60"))
 
+# ── Compute runtime config (injected by builder when creating ksvc) ──────────
+# Builder sets these env vars based on the function's qapp_compute_runtime row.
+# The path is resolved per-function: /data/containers/functions/<fn_hash>_<fn_tag>.sif
+# CTS knative_builder/function/handler.sh adds these via --env to `kn ksvc create`.
+COMPUTE_SIF_PATH   = os.getenv("COMPUTE_SIF_PATH")
+COMPUTE_ENTRYPOINT = os.getenv("COMPUTE_ENTRYPOINT", "python3 /tmp/quapp_job.py")
+COMPUTE_SCRIPT_EXT = os.getenv("COMPUTE_SCRIPT_EXT", "py")
+
 _TERMINAL_STATES = {"COMPLETED", "FAILED", "CANCELLED", "TIMEOUT", "NODE_FAIL", "PREEMPTED"}
 _DONE_STATE      = "COMPLETED"
 
-_SLURM_TO_JOB_STATUS = {
-    "PENDING":     JobStatus.RUNNING.value,
-    "CONFIGURING": JobStatus.RUNNING.value,
-    "RUNNING":     JobStatus.RUNNING.value,
-    "COMPLETING":  JobStatus.RUNNING.value,
+_SLURM_TERMINAL_TO_JOB_STATUS = {
     "COMPLETED":   JobStatus.DONE.value,
     "FAILED":      JobStatus.ERROR.value,
     "CANCELLED":   JobStatus.ERROR.value,
@@ -48,6 +53,53 @@ _COLLECT_PY = (
 )
 
 
+def _resolve_compute_container(container_cfg: dict) -> tuple[str, str, str]:
+    """Resolve Apptainer image path, entrypoint command, and script extension.
+
+    Resolution order (first match wins):
+      1. User override via invocation `container`:
+           container.type = 'sif'    + container.image  →  /data/containers/<image>
+           container.type = 'docker' + container.image  →  docker://<image>
+      2. Builder-injected env vars (per-function SIF from build step):
+           COMPUTE_SIF_PATH + COMPUTE_ENTRYPOINT + COMPUTE_SCRIPT_EXT
+      3. Hard error — no valid configuration. Container is mandatory; running user
+         scripts on bare metal is not supported (see HPC design philosophy).
+    """
+    container_type = (container_cfg or {}).get("type", "").lower()
+    user_image     = (container_cfg or {}).get("image", "")
+    user_entry     = (container_cfg or {}).get("entrypoint")
+    user_ext       = (container_cfg or {}).get("script_extension")
+
+    # 1. User override — explicit container choice in invocation input
+    if container_type in ("sif", "docker"):
+        if not user_image:
+            raise ValueError(
+                f"container.type={container_type!r} requires container.image"
+            )
+        if container_type == "sif":
+            sif_path = f"/data/containers/{user_image}"
+        else:
+            sif_path = f"docker://{user_image}"
+        return (
+            sif_path,
+            user_entry or COMPUTE_ENTRYPOINT,
+            user_ext   or COMPUTE_SCRIPT_EXT,
+        )
+
+    # 2. Builder-injected default — per-function SIF built at deploy time
+    if COMPUTE_SIF_PATH:
+        return (COMPUTE_SIF_PATH, COMPUTE_ENTRYPOINT, COMPUTE_SCRIPT_EXT)
+
+    # 3. No path resolved → fail explicitly. Running outside a container is not
+    # supported per the Quapp-HPC design philosophy (container-first sandboxing).
+    raise RuntimeError(
+        "No compute container configured. Either the function pod is missing the "
+        "COMPUTE_SIF_PATH env var (builder did not produce a per-function SIF — "
+        "rebuild required) or the invocation must explicitly provide "
+        "container.type and container.image."
+    )
+
+
 class SlurmDevice(Device):
 
     def __init__(
@@ -62,7 +114,9 @@ class SlurmDevice(Device):
         self.slurm:     SlurmProvider = provider
         self.logger                  = job_logger(job_uuid)
         self.hpc_config              = hpc_config or {}
-        self.s3_bucket               = self.hpc_config.get("s3_bucket") or S3_BUCKET
+        # S3 bucket is a system field — read only from env (no input override).
+        # Per HPC design philosophy: system fields are not user-controlled.
+        self.s3_bucket               = S3_BUCKET
 
     # ── Abstract method implementations ──────────────────────────────────────
 
@@ -115,7 +169,7 @@ class SlurmDevice(Device):
 
     def _get_job_status(self, job: dict) -> str:
         state = self._fetch_slurm_state(job["slurm_job_id"])
-        return _SLURM_TO_JOB_STATUS.get(state, JobStatus.RUNNING.value)
+        return _SLURM_TERMINAL_TO_JOB_STATUS.get(state, state)
 
     def _get_job_result(self, job: dict):
         """Block until Slurm job finishes, then download result from S3."""
@@ -147,12 +201,35 @@ class SlurmDevice(Device):
     # ── SBATCH script builder ─────────────────────────────────────────────────
 
     def _build_sbatch_script(self, user_script: str) -> str:
-        """Build the full SBATCH bash script that wraps the user's computation."""
+        """Build the full SBATCH bash script that wraps the user's computation.
+
+        Container resolution (see _resolve_compute_container):
+          - Default: per-function SIF built at deploy time, identified by env
+            COMPUTE_SIF_PATH + COMPUTE_ENTRYPOINT + COMPUTE_SCRIPT_EXT
+          - Override: invocation `container` block can point at a shared SIF or
+            pull a Docker image at runtime
+
+        The user script string is base64-encoded into the SBATCH script and
+        written to /tmp/quapp_job.<ext> on the compute node, then executed
+        through Apptainer with the configured entrypoint.
+        """
         resources      = self.hpc_config.get("resources", {})
         container      = self.hpc_config.get("container", {})
         environment    = self.hpc_config.get("environment", {})
         input_s3_paths = self.hpc_config.get("input_s3_paths", [])
 
+        sif_path, entrypoint, script_ext = _resolve_compute_container(container)
+        script_file = f"/tmp/quapp_job.{script_ext}"
+
+        # Entrypoint may reference the canonical filename; rewrite to actual extension
+        entrypoint_resolved = entrypoint.replace("/tmp/quapp_job.py", script_file) \
+                                       .replace("/tmp/quapp_job.sh", script_file)
+
+        self.logger.info(
+            "Compute container resolved: sif=%s entrypoint=%s script_file=%s",
+            sif_path, entrypoint_resolved, script_file,
+        )
+
         lines = ["#!/bin/bash"]
 
         # ── SBATCH directives ─────────────────────────────────────────────────
@@ -191,27 +268,16 @@ class SlurmDevice(Device):
         if environment:
             lines.append("")
 
-        # ── Container exec prefix ─────────────────────────────────────────────
-        container_type = container.get("type", "none")
-        image          = container.get("image", "")
-        if container_type == "sif" and image:
-            exec_prefix = f"apptainer exec /data/containers/{image}"
-        elif container_type == "docker" and image:
-            exec_prefix = f"apptainer exec docker://{image}"
-        else:
-            exec_prefix = ""
-
-        # ── Write and run user script ─────────────────────────────────────────
+        # ── Write and run user script inside Apptainer ────────────────────────
         b64 = base64.b64encode(user_script.encode()).decode()
+        exec_prefix = f"apptainer exec {shlex.quote(sif_path)}" \
+            if not sif_path.startswith("docker://") \
+            else f"apptainer exec {sif_path}"
         lines += [
-            f"echo {shlex.quote(b64)} | base64 -d > /tmp/quapp_job.sh",
-            "chmod +x /tmp/quapp_job.sh",
+            f"echo {shlex.quote(b64)} | base64 -d > {script_file}",
+            f"chmod +x {script_file}",
             "",
-        ]
-        run_cmd  = "bash /tmp/quapp_job.sh"
-        full_cmd = f"{exec_prefix} {run_cmd}".strip()
-        lines += [
-            f"{full_cmd} > /tmp/quapp_stdout.txt 2>/tmp/quapp_stderr.txt || true",
+            f"{exec_prefix} {entrypoint_resolved} > /tmp/quapp_stdout.txt 2>/tmp/quapp_stderr.txt || true",
             "echo $? > /tmp/quapp_exit_code.txt",
             "",
         ]

{quapp_hpc-0.0.1.dev2 → quapp_hpc-0.0.1.dev4/quapp_hpc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev2
+Version: 0.0.1.dev4
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)