quapp-hpc 0.0.1.dev3__tar.gz → 0.0.1.dev5__tar.gz

{quapp_hpc-0.0.1.dev3/quapp_hpc.egg-info → quapp_hpc-0.0.1.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev3
+Version: 0.0.1.dev5
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.13,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: quapp-common==0.0.12.dev3
+Requires-Dist: quapp-common==0.0.12.dev12
 Requires-Dist: requests>=2.31.0
 Requires-Dist: boto3>=1.28.0
 Provides-Extra: dev

{quapp_hpc-0.0.1.dev3 → quapp_hpc-0.0.1.dev5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "quapp-hpc"
-version = "0.0.1.dev3"
+version = "0.0.1.dev5"
 description = "Quapp HPC library — Slurm integration for Quapp Platform"
 readme = "README.md"
 authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]
@@ -16,7 +16,7 @@ classifiers = [
 ]
 keywords = ["quapp", "quapp-hpc", "slurm", "hpc"]
 dependencies = [
-    "quapp-common==0.0.12.dev3",
+    "quapp-common==0.0.12.dev12",
     "requests>=2.31.0",
     "boto3>=1.28.0",
 ]

{quapp_hpc-0.0.1.dev3 → quapp_hpc-0.0.1.dev5}/quapp_hpc/component/backend/hpc_invocation.py

@@ -12,12 +12,14 @@ class HpcInvocation(Invocation):
         super().__init__(request_data)
         raw = request_data.input or {}
         job = raw.get("job", {})
+        # s3Bucket intentionally dropped — S3 bucket is system config, read by
+        # SlurmDevice from the S3_BUCKET env var (slurm-credentials secret).
+        # User input must not override system fields (HPC design philosophy).
         self._hpc_config = {
             "resources":      raw.get("resources", {}),
             "container":      raw.get("container", {}),
             "environment":    job.get("environment", {}),
             "input_s3_paths": job.get("input_s3_paths", []),
-            "s3_bucket":      raw.get("s3Bucket", ""),
         }
 
     def _export_circuit(self, circuit):

{quapp_hpc-0.0.1.dev3 → quapp_hpc-0.0.1.dev5}/quapp_hpc/factory/hpc_provider_factory.py

@@ -15,7 +15,7 @@ class HpcProviderFactory:
     def create_provider(provider_type: ProviderTag, authentication: dict):
         logger.debug(f"Creating HPC provider: {provider_type}")
 
-        if provider_type == ProviderTag.SLURM_HPC:
+        if provider_type == ProviderTag.QUAPP_HPC:
             jwt = authentication.get("slurm_jwt") or SLURM_JWT
             if not jwt:
                 raise ValueError("SLURM_JWT not set — cannot authenticate with Slurm API")

{quapp_hpc-0.0.1.dev3 → quapp_hpc-0.0.1.dev5}/quapp_hpc/model/device/slurm_device.py

@@ -14,12 +14,21 @@ from quapp_common.model.provider.provider import Provider
 
 from ..provider.slurm_provider import SlurmProvider, SLURM_ACCOUNT
 
+# ── System config (read from K8s secret slurm-credentials at ksvc startup) ───
 S3_BUCKET         = os.getenv("S3_BUCKET",               "quapp-slurm-output-dev")
 AWS_REGION        = os.getenv("AWS_REGION",               "ap-southeast-1")
 SLURM_POLL_SEC    = int(os.getenv("SLURM_POLL_SEC",      "30"))
 SLURM_TIMEOUT_SEC = int(os.getenv("SLURM_TIMEOUT_SEC",   "21600"))  # 6 hours
 SLURM_TIME_LIMIT  = int(os.getenv("SLURM_TIME_LIMIT_MIN", "60"))
 
+# ── Compute runtime config (injected by builder when creating ksvc) ──────────
+# Builder sets these env vars based on the function's qapp_compute_runtime row.
+# The path is resolved per-function: /data/containers/functions/<fn_hash>_<fn_tag>.sif
+# CTS knative_builder/function/handler.sh adds these via --env to `kn ksvc create`.
+COMPUTE_SIF_PATH   = os.getenv("COMPUTE_SIF_PATH")
+COMPUTE_ENTRYPOINT = os.getenv("COMPUTE_ENTRYPOINT", "python3 /tmp/quapp_job.py")
+COMPUTE_SCRIPT_EXT = os.getenv("COMPUTE_SCRIPT_EXT", "py")
+
 _TERMINAL_STATES = {"COMPLETED", "FAILED", "CANCELLED", "TIMEOUT", "NODE_FAIL", "PREEMPTED"}
 _DONE_STATE      = "COMPLETED"
 
@@ -44,6 +53,53 @@ _COLLECT_PY = (
 )
 
 
+def _resolve_compute_container(container_cfg: dict) -> tuple[str, str, str]:
+    """Resolve Apptainer image path, entrypoint command, and script extension.
+
+    Resolution order (first match wins):
+      1. User override via invocation `container`:
+           container.type = 'sif'    + container.image  →  /data/containers/<image>
+           container.type = 'docker' + container.image  →  docker://<image>
+      2. Builder-injected env vars (per-function SIF from build step):
+           COMPUTE_SIF_PATH + COMPUTE_ENTRYPOINT + COMPUTE_SCRIPT_EXT
+      3. Hard error — no valid configuration. Container is mandatory; running user
+         scripts on bare metal is not supported (see HPC design philosophy).
+    """
+    container_type = (container_cfg or {}).get("type", "").lower()
+    user_image     = (container_cfg or {}).get("image", "")
+    user_entry     = (container_cfg or {}).get("entrypoint")
+    user_ext       = (container_cfg or {}).get("script_extension")
+
+    # 1. User override — explicit container choice in invocation input
+    if container_type in ("sif", "docker"):
+        if not user_image:
+            raise ValueError(
+                f"container.type={container_type!r} requires container.image"
+            )
+        if container_type == "sif":
+            sif_path = f"/data/containers/{user_image}"
+        else:
+            sif_path = f"docker://{user_image}"
+        return (
+            sif_path,
+            user_entry or COMPUTE_ENTRYPOINT,
+            user_ext   or COMPUTE_SCRIPT_EXT,
+        )
+
+    # 2. Builder-injected default — per-function SIF built at deploy time
+    if COMPUTE_SIF_PATH:
+        return (COMPUTE_SIF_PATH, COMPUTE_ENTRYPOINT, COMPUTE_SCRIPT_EXT)
+
+    # 3. No path resolved → fail explicitly. Running outside a container is not
+    # supported per the Quapp-HPC design philosophy (container-first sandboxing).
+    raise RuntimeError(
+        "No compute container configured. Either the function pod is missing the "
+        "COMPUTE_SIF_PATH env var (builder did not produce a per-function SIF — "
+        "rebuild required) or the invocation must explicitly provide "
+        "container.type and container.image."
+    )
+
+
 class SlurmDevice(Device):
 
     def __init__(
@@ -58,7 +114,9 @@ class SlurmDevice(Device):
         self.slurm:     SlurmProvider = provider
         self.logger                  = job_logger(job_uuid)
         self.hpc_config              = hpc_config or {}
-        self.s3_bucket               = self.hpc_config.get("s3_bucket") or S3_BUCKET
+        # S3 bucket is a system field — read only from env (no input override).
+        # Per HPC design philosophy: system fields are not user-controlled.
+        self.s3_bucket               = S3_BUCKET
 
     # ── Abstract method implementations ──────────────────────────────────────
 
@@ -143,12 +201,35 @@ class SlurmDevice(Device):
     # ── SBATCH script builder ─────────────────────────────────────────────────
 
     def _build_sbatch_script(self, user_script: str) -> str:
-        """Build the full SBATCH bash script that wraps the user's computation."""
+        """Build the full SBATCH bash script that wraps the user's computation.
+
+        Container resolution (see _resolve_compute_container):
+          - Default: per-function SIF built at deploy time, identified by env
+            COMPUTE_SIF_PATH + COMPUTE_ENTRYPOINT + COMPUTE_SCRIPT_EXT
+          - Override: invocation `container` block can point at a shared SIF or
+            pull a Docker image at runtime
+
+        The user script string is base64-encoded into the SBATCH script and
+        written to /tmp/quapp_job.<ext> on the compute node, then executed
+        through Apptainer with the configured entrypoint.
+        """
         resources      = self.hpc_config.get("resources", {})
         container      = self.hpc_config.get("container", {})
         environment    = self.hpc_config.get("environment", {})
         input_s3_paths = self.hpc_config.get("input_s3_paths", [])
 
+        sif_path, entrypoint, script_ext = _resolve_compute_container(container)
+        script_file = f"/tmp/quapp_job.{script_ext}"
+
+        # Entrypoint may reference the canonical filename; rewrite to actual extension
+        entrypoint_resolved = entrypoint.replace("/tmp/quapp_job.py", script_file) \
+                                       .replace("/tmp/quapp_job.sh", script_file)
+
+        self.logger.info(
+            "Compute container resolved: sif=%s entrypoint=%s script_file=%s",
+            sif_path, entrypoint_resolved, script_file,
+        )
+
         lines = ["#!/bin/bash"]
 
         # ── SBATCH directives ─────────────────────────────────────────────────
@@ -187,27 +268,16 @@ class SlurmDevice(Device):
         if environment:
             lines.append("")
 
-        # ── Container exec prefix ─────────────────────────────────────────────
-        container_type = container.get("type", "none")
-        image          = container.get("image", "")
-        if container_type == "sif" and image:
-            exec_prefix = f"apptainer exec /data/containers/{image}"
-        elif container_type == "docker" and image:
-            exec_prefix = f"apptainer exec docker://{image}"
-        else:
-            exec_prefix = ""
-
-        # ── Write and run user script ─────────────────────────────────────────
+        # ── Write and run user script inside Apptainer ────────────────────────
         b64 = base64.b64encode(user_script.encode()).decode()
+        exec_prefix = f"apptainer exec {shlex.quote(sif_path)}" \
+            if not sif_path.startswith("docker://") \
+            else f"apptainer exec {sif_path}"
         lines += [
-            f"echo {shlex.quote(b64)} | base64 -d > /tmp/quapp_job.sh",
-            "chmod +x /tmp/quapp_job.sh",
+            f"echo {shlex.quote(b64)} | base64 -d > {script_file}",
+            f"chmod +x {script_file}",
             "",
-        ]
-        run_cmd  = "bash /tmp/quapp_job.sh"
-        full_cmd = f"{exec_prefix} {run_cmd}".strip()
-        lines += [
-            f"{full_cmd} > /tmp/quapp_stdout.txt 2>/tmp/quapp_stderr.txt || true",
+            f"{exec_prefix} {entrypoint_resolved} > /tmp/quapp_stdout.txt 2>/tmp/quapp_stderr.txt || true",
             "echo $? > /tmp/quapp_exit_code.txt",
             "",
         ]

{quapp_hpc-0.0.1.dev3 → quapp_hpc-0.0.1.dev5}/quapp_hpc/model/provider/slurm_provider.py

@@ -15,7 +15,7 @@ SLURM_ACCOUNT  = os.getenv("SLURM_ACCOUNT",  "quapp")
 class SlurmProvider(Provider):
 
     def __init__(self, jwt_token: str):
-        super().__init__(ProviderTag.SLURM_HPC)
+        super().__init__(ProviderTag.QUAPP_HPC)
         self.jwt_token = jwt_token
         self.base_url  = f"{SLURM_API_URL}/slurm/{SLURM_API_VER}"
 

{quapp_hpc-0.0.1.dev3 → quapp_hpc-0.0.1.dev5/quapp_hpc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev3
+Version: 0.0.1.dev5
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: <3.13,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: quapp-common==0.0.12.dev3
+Requires-Dist: quapp-common==0.0.12.dev12
 Requires-Dist: requests>=2.31.0
 Requires-Dist: boto3>=1.28.0
 Provides-Extra: dev

{quapp_hpc-0.0.1.dev3 → quapp_hpc-0.0.1.dev5}/quapp_hpc.egg-info/requires.txt

@@ -1,4 +1,4 @@
-quapp-common==0.0.12.dev3
+quapp-common==0.0.12.dev12
 requests>=2.31.0
 boto3>=1.28.0