quapp-hpc 0.0.1.dev5__tar.gz → 0.0.1.dev7__tar.gz

{quapp_hpc-0.0.1.dev5/quapp_hpc.egg-info → quapp_hpc-0.0.1.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev5
+Version: 0.0.1.dev7
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev5 → quapp_hpc-0.0.1.dev7}/pyproject.toml

@@ -1,33 +1,33 @@
-[build-system]
-requires = ["setuptools>=61.0.0", "wheel==0.45.1"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "quapp-hpc"
-version = "0.0.1.dev5"
-description = "Quapp HPC library — Slurm integration for Quapp Platform"
-readme = "README.md"
-authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]
-license = { file = "LICENSE" }
-classifiers = [
-    "License :: OSI Approved :: MIT License",
-    "Programming Language :: Python",
-    "Programming Language :: Python :: 3",
-]
-keywords = ["quapp", "quapp-hpc", "slurm", "hpc"]
-dependencies = [
-    "quapp-common==0.0.12.dev12",
-    "requests>=2.31.0",
-    "boto3>=1.28.0",
-]
-requires-python = ">=3.10,<3.13"
-
-[project.optional-dependencies]
-dev = ["black", "bumpver", "isort", "pip-tools", "pytest"]
-
-[tool.setuptools.packages.find]
-include = ["quapp_hpc*"]
-exclude = ["*.md", "*.yml", "*.yaml", "*.toml", "tests*", ".gitignore"]
-
-[project.urls]
-Homepage = "https://quapp.cloud/"
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel==0.45.1"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "quapp-hpc"
+version = "0.0.1.dev7"
+description = "Quapp HPC library — Slurm integration for Quapp Platform"
+readme = "README.md"
+authors = [{ name = "CITYNOW Co. Ltd.", email = "corp@citynow.vn" }]
+license = { file = "LICENSE" }
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+]
+keywords = ["quapp", "quapp-hpc", "slurm", "hpc"]
+dependencies = [
+    "quapp-common==0.0.12.dev12",
+    "requests>=2.31.0",
+    "boto3>=1.28.0",
+]
+requires-python = ">=3.10,<3.13"
+
+[project.optional-dependencies]
+dev = ["black", "bumpver", "isort", "pip-tools", "pytest"]
+
+[tool.setuptools.packages.find]
+include = ["quapp_hpc*", "quapp_slurm_hpc*"]
+exclude = ["*.md", "*.yml", "*.yaml", "*.toml", "tests*", ".gitignore"]
+
+[project.urls]
+Homepage = "https://quapp.cloud/"

{quapp_hpc-0.0.1.dev5 → quapp_hpc-0.0.1.dev7/quapp_hpc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quapp-hpc
-Version: 0.0.1.dev5
+Version: 0.0.1.dev7
 Summary: Quapp HPC library — Slurm integration for Quapp Platform
 Author-email: "CITYNOW Co. Ltd." <corp@citynow.vn>
 License: The MIT License (MIT)

{quapp_hpc-0.0.1.dev5 → quapp_hpc-0.0.1.dev7}/quapp_hpc.egg-info/SOURCES.txt

@@ -22,4 +22,6 @@ quapp_hpc/model/__init__.py
 quapp_hpc/model/device/__init__.py
 quapp_hpc/model/device/slurm_device.py
 quapp_hpc/model/provider/__init__.py
-quapp_hpc/model/provider/slurm_provider.py
+quapp_hpc/model/provider/slurm_provider.py
+quapp_slurm_hpc/__init__.py
+quapp_slurm_hpc/runner.py

quapp_hpc-0.0.1.dev7/quapp_hpc.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+quapp_hpc
+quapp_slurm_hpc

quapp_hpc-0.0.1.dev7/quapp_slurm_hpc/runner.py

@@ -0,0 +1,142 @@
+"""CLI runner — invoked by the sbatch script inside the Apptainer container.
+
+The Java SlurmPlatformManager generates a sbatch script that runs:
+
+    apptainer exec docker://<image> python -m quapp_slurm_hpc.runner \
+        --input-file  /data/jobs/<jobId>/input.json  \
+        --output-file /data/jobs/<jobId>/output.json \
+        --handler-dir /home/app/function
+
+This module:
+  1. Reads the invocation input from --input-file (JSON written by the sbatch script).
+  2. Loads handler.py from --handler-dir.
+  3. Calls handler.processing(invocation_input) → bash script string.
+  4. Executes the bash script, captures stdout / stderr / exit_code.
+  5. Calls handler.post_processing(raw_result) if it exists.
+  6. Writes the final result dict to --output-file.
+  7. Uploads output.json to S3 (key: {QUAPP_JOB_ID}/output.json) when SLURM_S3_BUCKET is set.
+"""
+import argparse
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import traceback
+from pathlib import Path
+from types import ModuleType
+
+
+def _load_handler(handler_dir: str) -> ModuleType:
+    handler_path = Path(handler_dir) / "handler.py"
+    if not handler_path.exists():
+        raise FileNotFoundError(
+            f"handler.py not found in {handler_dir!r}. "
+            "Ensure the function image was built with the handler in /home/app/function."
+        )
+    if str(handler_dir) not in sys.path:
+        sys.path.insert(0, str(handler_dir))
+    spec = importlib.util.spec_from_file_location("handler", handler_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _run_script(script: str) -> dict:
+    """Write script to a temp file and execute it with bash, capturing output."""
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".sh", delete=False, encoding="utf-8"
+    ) as f:
+        f.write(script)
+        tmp_path = f.name
+    try:
+        result = subprocess.run(
+            ["bash", tmp_path],
+            capture_output=True,
+            text=True,
+        )
+        return {
+            "exit_code": result.returncode,
+            "stdout": result.stdout,
+            "stderr": result.stderr,
+        }
+    finally:
+        Path(tmp_path).unlink(missing_ok=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Quapp HPC runner — executes handler.py inside an Apptainer container"
+    )
+    parser.add_argument("--input-file",  required=True, help="Path to input JSON file")
+    parser.add_argument("--output-file", required=True, help="Path to write output JSON")
+    parser.add_argument("--handler-dir", required=True, help="Directory containing handler.py")
+    args = parser.parse_args()
+
+    output: dict = {}
+    try:
+        with open(args.input_file, encoding="utf-8") as f:
+            invocation_input = json.load(f)
+
+        handler = _load_handler(args.handler_dir)
+
+        if not hasattr(handler, "processing"):
+            raise AttributeError(
+                "handler.py must define a processing(invocation_input: dict) -> str function"
+            )
+
+        script = handler.processing(invocation_input)
+        if not isinstance(script, str):
+            raise TypeError(
+                f"handler.processing() must return a str (bash script), got {type(script).__name__}"
+            )
+
+        raw_result = _run_script(script)
+
+        if hasattr(handler, "post_processing"):
+            output = handler.post_processing(raw_result)
+        else:
+            output = raw_result
+
+    except Exception:
+        output = {
+            "exit_code": 1,
+            "stdout": "",
+            "stderr": traceback.format_exc(),
+        }
+
+    with open(args.output_file, "w", encoding="utf-8") as f:
+        json.dump(output, f)
+
+    _upload_to_s3(args.output_file)
+
+
+def _upload_to_s3(output_file: str) -> None:
+    """Upload output.json to S3 so the Java backend can read the result.
+
+    Requires SLURM_S3_BUCKET and QUAPP_JOB_ID env vars (injected by sbatch script).
+    Skips silently when the bucket is not configured.
+    """
+    s3_bucket = os.getenv("SLURM_S3_BUCKET", "")
+    job_uuid = os.getenv("QUAPP_JOB_ID", "")
+    if not s3_bucket or not job_uuid:
+        return
+
+    try:
+        import boto3
+        s3 = boto3.client(
+            "s3",
+            region_name=os.getenv("SLURM_S3_REGION", "ap-southeast-1"),
+            aws_access_key_id=os.getenv("SLURM_S3_ACCESS_KEY"),
+            aws_secret_access_key=os.getenv("SLURM_S3_SECRET_KEY"),
+        )
+        s3_key = f"{job_uuid}/output.json"
+        s3.upload_file(output_file, s3_bucket, s3_key)
+        print(f"[runner] Uploaded output to s3://{s3_bucket}/{s3_key}", flush=True)
+    except Exception as exc:
+        print(f"[runner] WARNING: S3 upload failed: {exc}", file=sys.stderr, flush=True)
+
+
+if __name__ == "__main__":
+    main()

quapp_hpc-0.0.1.dev5/quapp_hpc.egg-info/top_level.txt

@@ -1 +0,0 @@
-quapp_hpc