pipen-cli-gbatch 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

pipen_cli_gbatch/__init__.py

@@ -79,7 +79,7 @@ from pipen.cli import CLIPlugin
 from pipen.scheduler import GbatchScheduler
 from pipen_poplog import LogsPopulator
 
-__version__ = "0.0.2"
+__version__ = "0.0.4"
 __all__ = ("CliGbatchPlugin", "CliGbatchDaemon")
 
 
@@ -117,6 +117,11 @@ class CliGbatchDaemon:
 
         self.config.prescript = self.config.get("prescript", None) or ""
         self.config.postscript = self.config.get("postscript", None) or ""
+        if "labels" in self.config and isinstance(self.config.labels, list):
+            self.config.labels = {
+                key: val
+                for key, val in (item.split("=", 1) for item in self.config.labels)
+            }
         self.command = command
 
     def _get_arg_from_command(self, arg: str) -> str | None:
@@ -183,7 +188,7 @@ class CliGbatchDaemon:
         """
         mount = self.config.get("mount", [])
         # mount the workdir
-        mount.append(f'{source}:{target}')
+        mount.append(f"{source}:{target}")
 
         self.config["mount"] = mount
 
@@ -197,7 +202,7 @@ class CliGbatchDaemon:
             SystemExit: If workdir is not a valid Google Storage bucket path.
         """
         command_workdir = self._get_arg_from_command("workdir")
-        workdir = self.config.get("workdir", command_workdir)
+        workdir = self.config.get("workdir", None) or command_workdir
 
         if not workdir or not isinstance(AnyPath(workdir), GSPath):
             print(
@@ -312,7 +317,7 @@ class CliGbatchDaemon:
 
     def _show_scheduler_opts(self):
         """Log the scheduler options for debugging purposes."""
-        logger.debug("Scheduler Options:")
+        logger.info("Scheduler Options:")
         for key, val in self.config.items():
             if key in (
                 "workdir",
@@ -332,7 +337,7 @@ class CliGbatchDaemon:
             ):
                 continue
 
-            logger.debug(f"- {key}: {val}")
+            logger.info(f"- {key}: {val}")
 
     async def _run_wait(self):  # pragma: no cover
         """Run the pipeline and wait for completion.
@@ -493,7 +498,7 @@ class CliGbatchDaemon:
                 )
                 sys.exit(1)
 
-            if 'name' not in self.config:
+            if "name" not in self.config or not self.config.name:
                 self.config["name"] = "PipenCliGbatchDaemon"
 
     async def run(self):  # pragma: no cover
@@ -553,6 +558,17 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             logger.error(f"/STDERR {self.stderr_populator.residue}")
             self.stderr_populator.residue = ""
 
+    @plugin.impl
+    async def on_job_init(self, scheduler, job):
+        """Handle job initialization event.
+
+        Args:
+            scheduler: The scheduler instance.
+            job: The job being initialized.
+        """
+        self.stdout_populator.logfile = scheduler.workdir.joinpath("0", "job.stdout")
+        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
+
     @plugin.impl
     async def on_job_started(self, scheduler, job):
         """Handle job start event by setting up log file paths.
@@ -564,8 +580,6 @@ class XquteCliGbatchPlugin:  # pragma: no cover
         if not self.log_start:
             return
 
-        self.stdout_populator.logfile = scheduler.workdir.joinpath("0", "job.stdout")
-        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
         logger.info("Job is picked up by Google Batch, pulling stdout/stderr...")
 
     @plugin.impl
@@ -693,29 +707,29 @@ class CliGbatchPlugin(CLIPlugin):  # pragma: no cover
         super().__init__(parser, subparser)
         subparser.epilog = """\033[1;4mExamples\033[0m:
 
-  \u200B
+  \u200b
   # Run a command and wait for it to complete
   > pipen gbatch --workdir gs://my-bucket/workdir -- \\
     python myscript.py --input input.txt --output output.txt
 
-  \u200B
+  \u200b
   # Use named mounts
   > pipen gbatch --workdir gs://my-bucket/workdir --mount INFILE=gs://bucket/path/to/file \\
     --mount OUTDIR=gs://bucket/path/to/outdir -- \\
     bash -c 'cat $INFILE > $OUTDIR/output.txt'
 
-  \u200B
+  \u200b
   # Run a command in a detached mode
   > pipen gbatch --nowait --project $PROJECT --location $LOCATION \\
     --workdir gs://my-bucket/workdir -- \\
     python myscript.py --input input.txt --output output.txt
 
-  \u200B
+  \u200b
   # If you have a profile defined in ~/.pipen.toml or ./.pipen.toml
   > pipen gbatch --profile myprofile -- \\
     python myscript.py --input input.txt --output output.txt
 
-  \u200B
+  \u200b
   # View the logs of a previously run command
   > pipen gbatch --view-logs all --name my-daemon-name \\
     --workdir gs://my-bucket/workdir
@@ -753,12 +767,36 @@ class CliGbatchPlugin(CLIPlugin):  # pragma: no cover
             CONFIG_FILES,
             known_parsed.profile,
         )
+
+        def is_valid(val: Any) -> bool:
+            """Check if a value is valid (not None, not empty string, not empty list).
+            """
+            if val is None:
+                return False
+            if isinstance(val, bool):
+                return True
+            return bool(val)
+
         # update parsed with the defaults
         for key, val in defaults.items():
+            if (
+                key == "mount"
+                and val
+                and getattr(known_parsed, key, None)
+            ):
+                if not isinstance(val, (tuple, list)):
+                    val = [val]
+                val = list(val)
+
+                kp_mount = getattr(known_parsed, key)
+                val.extend(kp_mount)
+                setattr(known_parsed, key, val)
+                continue
+
             if (
                 key == "command"
                 or val is None
-                or getattr(known_parsed, key, None) is not None
+                or is_valid(getattr(known_parsed, key, None))
             ):
                 continue
 

pipen_cli_gbatch/daemon_args.toml

@@ -120,7 +120,7 @@ help = "The location to run the job."
 flags = ["--mount"]
 # type = "list"
 default = []
-action = "clear_append"
+action = "append"
 help = """The list of mounts to mount to the VM, each in the format of SOURCE:TARGET, where SOURCE must be either a Google Storage Bucket path (gs://...).
 You can also use named mounts like `INDIR=gs://my-bucket/inputs` and the directory will be mounted to `/mnt/disks/INDIR` in the VM;
 then you can use environment variable `$INDIR` in the command/script to refer to the mounted path.
@@ -186,20 +186,21 @@ and positive values mean to run after the main command."""
 [[groups.arguments]]
 flags = ["--allocationPolicy"]
 type = "json"
-default = "{}"
+default = {}
 help = "The JSON string of extra settings of allocationPolicy add to the job.json. Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#AllocationPolicy for details."
 
 [[groups.arguments]]
 flags = ["--taskGroups"]
 type = "json"
-default = "[]"
+default = []
 help = "The JSON string of extra settings of taskGroups add to the job.json. Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#TaskGroup for details."
 
 [[groups.arguments]]
 flags = ["--labels"]
-type = "json"
-default = "{}"
-help = "The JSON string of labels to add to the job. Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Job.FIELDS.labels for details."
+# type = "json"
+default = []
+action = "clear_append"
+help = "The strings of labels to add to the job (key=value). Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Job.FIELDS.labels for details."
 
 [[groups.arguments]]
 flags = ["--gcloud"]

{pipen_cli_gbatch-0.0.2.dist-info → pipen_cli_gbatch-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: pipen-cli-gbatch
-Version: 0.0.2
+Version: 0.0.4
 Summary: A pipen cli plugin to run command via Google Cloud Batch
 License: MIT
 Author: pwwang
@@ -13,9 +13,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: google-cloud-storage (>=3.0.0,<4.0.0)
 Requires-Dist: pipen (>=0.17.19,<0.18.0)
 Requires-Dist: pipen-poplog (>=0.3.6,<0.4.0)
+Project-URL: Homepage, https://github.com/pwwang/pipen-cli-gbatch
+Project-URL: Repository, https://github.com/pwwang/pipen-cli-gbatch
 Description-Content-Type: text/markdown
 
 # pipen-cli-gbatch

pipen_cli_gbatch-0.0.4.dist-info/RECORD

@@ -0,0 +1,6 @@
+pipen_cli_gbatch/__init__.py,sha256=x5R1wU-TyXS_Z1EqJ1jWhFn66sKLt_fg8eyXtz9k7rg,29508
+pipen_cli_gbatch/daemon_args.toml,sha256=YzqLumFjIY8mRMzRt3M5D1fEjK93QvQJ0XynDnimPIo,6873
+pipen_cli_gbatch-0.0.4.dist-info/METADATA,sha256=5i6JjljULy5Km96zlHt0dcI97Wleq73yzYpakX5j8Zw,11301
+pipen_cli_gbatch-0.0.4.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+pipen_cli_gbatch-0.0.4.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
+pipen_cli_gbatch-0.0.4.dist-info/RECORD,,

{pipen_cli_gbatch-0.0.2.dist-info → pipen_cli_gbatch-0.0.4.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any

pipen_cli_gbatch-0.0.2.dist-info/RECORD

@@ -1,6 +0,0 @@
-pipen_cli_gbatch/__init__.py,sha256=cTX5Sdt-42el8s4mW2daAUQcCieP_2kOiJpzR0zzags,28294
-pipen_cli_gbatch/daemon_args.toml,sha256=Q_fbj1Eesj02A74RDteSCXGE8s9iaRF9pAzYZOyF8ek,6851
-pipen_cli_gbatch-0.0.2.dist-info/METADATA,sha256=y7v2ah3bPm8RzFeSqvA78jC7GUU8g4SRPPzSOgMp9vk,11116
-pipen_cli_gbatch-0.0.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-pipen_cli_gbatch-0.0.2.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
-pipen_cli_gbatch-0.0.2.dist-info/RECORD,,