py-cluster-api 0.8.0__tar.gz → 0.8.1__tar.gz

{py_cluster_api-0.8.0 → py_cluster_api-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: py-cluster-api
-Version: 0.8.0
+Version: 0.8.1
 Summary: Generic Python library for running jobs on HPC clusters
 Project-URL: Homepage, https://github.com/JaneliaSciComp/py-cluster-api
 Project-URL: Repository, https://github.com/JaneliaSciComp/py-cluster-api

{py_cluster_api-0.8.0 → py_cluster_api-0.8.1}/cluster_api/executors/lsf.py

@@ -36,7 +36,7 @@ _LSF_STATUS_MAP: dict[str, JobStatus] = {
 }
 
 _BJOBS_FIELDS = (
-    "jobid job_name stat exit_code exec_host max_mem "
+    "jobid jobindex job_name stat exit_code exec_host max_mem "
     "submit_time start_time finish_time"
 )
 
@@ -305,6 +305,12 @@ class LSFExecutor(Executor):
             job_id = str(rec.get("JOBID", "")).strip()
             if not job_id:
                 continue
+            # bjobs reports the bare parent ID for every array element; the
+            # index lives in JOBINDEX ("0" for non-array jobs). Re-key as
+            # "parent[idx]" so poll()/reconnect() can tell elements apart.
+            job_index = str(rec.get("JOBINDEX", "")).strip()
+            if job_index and job_index != "0" and "[" not in job_id:
+                job_id = f"{job_id}[{job_index}]"
 
             stat = rec.get("STAT", "").strip()
             status = _LSF_STATUS_MAP.get(stat)

{py_cluster_api-0.8.0 → py_cluster_api-0.8.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "py-cluster-api"
-version = "0.8.0"
+version = "0.8.1"
 description = "Generic Python library for running jobs on HPC clusters"
 readme = "README.md"
 license = { file = "LICENSE" }

{py_cluster_api-0.8.0 → py_cluster_api-0.8.1}/tests/test_lsf.py

@@ -599,10 +599,10 @@ class TestParseArrayElements:
     def test_element_ids_as_keys(self, lsf_config):
         executor = LSFExecutor(lsf_config)
         output = self._make_bjobs_json([
-            {"JOBID": "12345[1]", "STAT": "DONE", "EXIT_CODE": "0",
+            {"JOBID": "12345", "JOBINDEX": "1", "STAT": "DONE", "EXIT_CODE": "0",
              "EXEC_HOST": "node01", "MAX_MEM": "128 MB",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
-            {"JOBID": "12345[2]", "STAT": "RUN", "EXIT_CODE": "-",
+            {"JOBID": "12345", "JOBINDEX": "2", "STAT": "RUN", "EXIT_CODE": "-",
              "EXEC_HOST": "node02", "MAX_MEM": "-",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
         ])
@@ -618,7 +618,7 @@ class TestParseArrayElements:
             {"JOBID": "12345", "STAT": "RUN", "EXIT_CODE": "-",
              "EXEC_HOST": "-", "MAX_MEM": "-",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
-            {"JOBID": "12345[1]", "STAT": "DONE", "EXIT_CODE": "0",
+            {"JOBID": "12345", "JOBINDEX": "1", "STAT": "DONE", "EXIT_CODE": "0",
              "EXEC_HOST": "node01", "MAX_MEM": "256 MB",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
         ])
@@ -627,6 +627,24 @@ class TestParseArrayElements:
         assert "12345[1]" in result
 
 
+    def test_real_lsf_format_uses_jobindex(self, lsf_config):
+        """Real bjobs -json reports the bare parent ID in JOBID and the
+        element index only in JOBINDEX (regression for issue #8)."""
+        executor = LSFExecutor(lsf_config)
+        output = self._make_bjobs_json([
+            {"JOBID": "154541712", "JOBINDEX": "2", "STAT": "RUN",
+             "JOB_NAME": "wells[2]", "EXIT_CODE": "-"},
+            {"JOBID": "154541712", "JOBINDEX": "1", "STAT": "DONE",
+             "JOB_NAME": "wells[1]", "EXIT_CODE": ""},
+            {"JOBID": "99", "JOBINDEX": "0", "STAT": "RUN",
+             "JOB_NAME": "single", "EXIT_CODE": "-"},
+        ])
+        result = executor._parse_job_statuses(output)
+        assert set(result) == {"154541712[1]", "154541712[2]", "99"}
+        assert result["154541712[1]"][0] == JobStatus.DONE
+        assert result["154541712[2]"][0] == JobStatus.RUNNING
+
+
 class TestArrayStatusComputation:
     def test_no_elements(self):
         record = JobRecord(
@@ -730,13 +748,13 @@ class TestArrayElementPolling:
         assert len(job.array_elements) == 0
 
         bjobs_output = json.dumps({"RECORDS": [
-            {"JOBID": "12345[1]", "STAT": "DONE", "EXIT_CODE": "0",
+            {"JOBID": "12345", "JOBINDEX": "1", "STAT": "DONE", "EXIT_CODE": "0",
              "EXEC_HOST": "node01", "MAX_MEM": "128 MB",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
-            {"JOBID": "12345[2]", "STAT": "RUN", "EXIT_CODE": "-",
+            {"JOBID": "12345", "JOBINDEX": "2", "STAT": "RUN", "EXIT_CODE": "-",
              "EXEC_HOST": "node02", "MAX_MEM": "-",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
-            {"JOBID": "12345[3]", "STAT": "PEND", "EXIT_CODE": "-",
+            {"JOBID": "12345", "JOBINDEX": "3", "STAT": "PEND", "EXIT_CODE": "-",
              "EXEC_HOST": "-", "MAX_MEM": "-",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
         ]})
@@ -752,6 +770,40 @@ class TestArrayElementPolling:
         assert job.status == JobStatus.RUNNING
         assert statuses["12345"] == JobStatus.RUNNING
 
+    async def test_poll_real_lsf_format_one_element_done(self, lsf_config, work_dir):
+        """Issue #8: bjobs lists DONE elements last and reports the bare parent
+        ID for every element; the parent must stay RUNNING until all are done."""
+        executor = LSFExecutor(lsf_config)
+        with patch.object(
+            executor, "_call",
+            new_callable=AsyncMock,
+            return_value="Job <154541712> is submitted to queue <normal>.",
+        ):
+            job = await executor.submit_array(
+                command="echo hello", name="wells", array_range=(1, 4),
+                resources=ResourceSpec(work_dir=work_dir),
+            )
+
+        def rec(idx, stat):
+            return {"JOBID": "154541712", "JOBINDEX": str(idx), "STAT": stat,
+                    "JOB_NAME": f"wells[{idx}]", "EXIT_CODE": "-",
+                    "EXEC_HOST": "-", "MAX_MEM": "-",
+                    "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"}
+
+        bjobs_output = json.dumps({"RECORDS": [
+            rec(2, "RUN"), rec(3, "RUN"), rec(4, "RUN"), rec(1, "DONE"),
+        ]})
+        with patch.object(executor, "_call", new_callable=AsyncMock, return_value=bjobs_output) as m:
+            await executor.poll()
+            args = m.call_args.args[0]
+            assert "jobindex" in args[args.index("-o") + 1]
+
+        assert job.status == JobStatus.RUNNING
+        assert not job.is_terminal
+        assert len(job.array_elements) == 4
+        assert job.array_elements[1].status == JobStatus.DONE
+        assert job.array_elements[4].status == JobStatus.RUNNING
+
     async def test_poll_all_done(self, lsf_config, work_dir):
         executor = LSFExecutor(lsf_config)
         with patch.object(
@@ -767,10 +819,10 @@ class TestArrayElementPolling:
             )
 
         bjobs_output = json.dumps({"RECORDS": [
-            {"JOBID": "12345[1]", "STAT": "DONE", "EXIT_CODE": "0",
+            {"JOBID": "12345", "JOBINDEX": "1", "STAT": "DONE", "EXIT_CODE": "0",
              "EXEC_HOST": "node01", "MAX_MEM": "128 MB",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
-            {"JOBID": "12345[2]", "STAT": "DONE", "EXIT_CODE": "0",
+            {"JOBID": "12345", "JOBINDEX": "2", "STAT": "DONE", "EXIT_CODE": "0",
              "EXEC_HOST": "node02", "MAX_MEM": "256 MB",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
         ]})
@@ -796,13 +848,13 @@ class TestArrayElementPolling:
             )
 
         bjobs_output = json.dumps({"RECORDS": [
-            {"JOBID": "12345[1]", "STAT": "DONE", "EXIT_CODE": "0",
+            {"JOBID": "12345", "JOBINDEX": "1", "STAT": "DONE", "EXIT_CODE": "0",
              "EXEC_HOST": "node01", "MAX_MEM": "128 MB",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
-            {"JOBID": "12345[2]", "STAT": "EXIT", "EXIT_CODE": "1",
+            {"JOBID": "12345", "JOBINDEX": "2", "STAT": "EXIT", "EXIT_CODE": "1",
              "EXEC_HOST": "node02", "MAX_MEM": "-",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
-            {"JOBID": "12345[3]", "STAT": "DONE", "EXIT_CODE": "0",
+            {"JOBID": "12345", "JOBINDEX": "3", "STAT": "DONE", "EXIT_CODE": "0",
              "EXEC_HOST": "node03", "MAX_MEM": "-",
              "SUBMIT_TIME": "-", "START_TIME": "-", "FINISH_TIME": "-"},
         ]})

{py_cluster_api-0.8.0 → py_cluster_api-0.8.1}/tests/test_reconnect.py

@@ -27,8 +27,15 @@ def _make_record(
     start_time="Jan  5 10:01:00 2024",
     finish_time="-",
 ):
+    # Real bjobs -json: elements carry the bare parent ID in JOBID, the
+    # index in JOBINDEX, and "name[idx]" in JOB_NAME.
+    job_index = "0"
+    if "[" in job_id:
+        job_id, job_index = job_id.rstrip("]").split("[")
+        job_name = f"{job_name}[{job_index}]"
     return {
         "JOBID": job_id,
+        "JOBINDEX": job_index,
         "JOB_NAME": job_name,
         "STAT": stat,
         "EXIT_CODE": exit_code,