pipen-cli-gbatch 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

pipen_cli_gbatch/__init__.py

@@ -79,8 +79,9 @@ from pipen.cli import CLIPlugin
 from pipen.scheduler import GbatchScheduler
 from pipen_poplog import LogsPopulator
 
-__version__ = "0.0.4"
+__version__ = "0.0.6"
 __all__ = ("CliGbatchPlugin", "CliGbatchDaemon")
+MOUNTED_CWD = "/mnt/disks/.cwd"
 
 
 class CliGbatchDaemon:
@@ -115,6 +116,17 @@ class CliGbatchDaemon:
         else:
             self.config = Diot(config)
 
+        self.mount_as_cwd = self.config.pop("mount_as_cwd", None)
+        if self.mount_as_cwd:
+            if self.config.cwd:
+                print(
+                    "\033[1;4mError\033[0m: --mount-as-cwd cannot be used with "
+                    "--cwd at the same time.\n"
+                )
+                sys.exit(1)
+            self.config.cwd = MOUNTED_CWD
+            self._add_mount(self.mount_as_cwd, MOUNTED_CWD)
+
         self.config.prescript = self.config.get("prescript", None) or ""
         self.config.postscript = self.config.get("postscript", None) or ""
         if "labels" in self.config and isinstance(self.config.labels, list):
@@ -201,6 +213,11 @@ class CliGbatchDaemon:
         Raises:
             SystemExit: If workdir is not a valid Google Storage bucket path.
         """
+        command_name = self._get_arg_from_command("name") or self.config["name"]
+        from_mount_as_cwd = self.mount_as_cwd and not self.config.workdir
+        if from_mount_as_cwd:
+            self.config.workdir = f"{self.mount_as_cwd}/.pipen/{command_name}"
+
         command_workdir = self._get_arg_from_command("workdir")
         workdir = self.config.get("workdir", None) or command_workdir
 
@@ -212,11 +229,14 @@ class CliGbatchDaemon:
             sys.exit(1)
 
         self.config["workdir"] = workdir
-        # If command workdir is different from config workdir, we need to mount it
-        self._add_mount(workdir, GbatchScheduler.MOUNTED_METADIR)
+        if from_mount_as_cwd:  # already mounted
+            self._replace_arg_in_command("workdir", f"{MOUNTED_CWD}/.pipen")
+        else:
+            # If command workdir is different from config workdir, we need to mount it
+            self._add_mount(workdir, GbatchScheduler.MOUNTED_METADIR)
 
-        # replace --workdir value with the mounted workdir in the command
-        self._replace_arg_in_command("workdir", GbatchScheduler.MOUNTED_METADIR)
+            # replace --workdir value with the mounted workdir in the command
+            self._replace_arg_in_command("workdir", GbatchScheduler.MOUNTED_METADIR)
 
     def _handle_outdir(self):
         """Handle output directory configuration and mounting.
@@ -229,6 +249,12 @@ class CliGbatchDaemon:
         if command_outdir:
             self._add_mount(command_outdir, GbatchScheduler.MOUNTED_OUTDIR)
             self._replace_arg_in_command("outdir", GbatchScheduler.MOUNTED_OUTDIR)
+        elif self.mount_as_cwd:
+            command_name = self._get_arg_from_command("name") or self.config.name
+            self._replace_arg_in_command(
+                "outdir",
+                f"{MOUNTED_CWD}/{command_name}-output",
+            )
 
     def _infer_name(self):
         """Infer the daemon name from configuration or command arguments.
@@ -303,6 +329,7 @@ class CliGbatchDaemon:
                     "version",
                     "loglevel",
                     "mounts",
+                    "mount_as_cwd",
                     "plain",
                 )
             },
@@ -333,6 +360,7 @@ class CliGbatchDaemon:
                 "version",
                 "loglevel",
                 "mounts",
+                "mount_as_cwd",
                 "plain",
             ):
                 continue
@@ -483,9 +511,9 @@ class CliGbatchDaemon:
         logger.setLevel(self.config.loglevel.upper())
 
         if not self.config.plain:
+            self._infer_name()
             self._handle_workdir()
             self._handle_outdir()
-            self._infer_name()
             self._infer_jobname_prefix()
         else:
             if not self.config.workdir or not isinstance(
@@ -551,10 +579,10 @@ class XquteCliGbatchPlugin:  # pragma: no cover
 
     def _clear_residues(self):
         """Clear any remaining log residues and display them."""
-        if self.stdout_populator.residue:
+        if self.stdout_populator and self.stdout_populator.residue:
             logger.info(f"/STDOUT {self.stdout_populator.residue}")
             self.stdout_populator.residue = ""
-        if self.stderr_populator.residue:
+        if self.stderr_populator and self.stderr_populator.residue:
             logger.error(f"/STDERR {self.stderr_populator.residue}")
             self.stderr_populator.residue = ""
 
@@ -595,15 +623,17 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             # Make it less frequent
             return
 
-        stdout_lines = self.stdout_populator.populate()
-        self.stdout_populator.increment_counter(len(stdout_lines))
-        for line in stdout_lines:
-            logger.info(f"/STDOUT {line}")
-
-        stderr_lines = self.stderr_populator.populate()
-        self.stderr_populator.increment_counter(len(stderr_lines))
-        for line in stderr_lines:
-            logger.error(f"/STDERR {line}")
+        if self.stderr_populator:
+            stdout_lines = self.stdout_populator.populate()
+            self.stdout_populator.increment_counter(len(stdout_lines))
+            for line in stdout_lines:
+                logger.info(f"/STDOUT {line}")
+
+        if self.stderr_populator:
+            stderr_lines = self.stderr_populator.populate()
+            self.stderr_populator.increment_counter(len(stderr_lines))
+            for line in stderr_lines:
+                logger.error(f"/STDERR {line}")
 
     @plugin.impl
     async def on_job_killed(self, scheduler, job):

pipen_cli_gbatch/daemon_args.toml

@@ -128,6 +128,16 @@ You can also mount a file like `INFILE=gs://my-bucket/inputs/file.txt`. The pare
 and the file will be available at `/mnt/disks/INFILE/inputs/file.txt` in the VM. `$INFILE` can also be used in the command/script to refer to the mounted path.
 """
 
+[[groups.arguments]]
+flags = ["--mount-as-cwd"]
+type = "str"
+help = """The directory to mount as the current working directory of the command.
+This is a shortcut for `--mount <cloudpath>:/mnt/disks/.cwd --cwd /mnt/disks/.cwd`.
+The <cloudpath> must be a Google Storage Bucket path (gs://...). When this option is used,
+and `--workdir` is not provided, the workdir will be set to `<cloudpath>/.pipen/<command_name>`,
+where <command_name> is the name of the command (or the value of `--name` if provided).
+"""
+
 [[groups.arguments]]
 flags = ["--service-account"]
 type = "str"

{pipen_cli_gbatch-0.0.4.dist-info → pipen_cli_gbatch-0.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipen-cli-gbatch
-Version: 0.0.4
+Version: 0.0.6
 Summary: A pipen cli plugin to run command via Google Cloud Batch
 License: MIT
 Author: pwwang

pipen_cli_gbatch-0.0.6.dist-info/RECORD

@@ -0,0 +1,6 @@
+pipen_cli_gbatch/__init__.py,sha256=5ChKkEGzq1S2irrwHpNts2oqezfB1zKs5IA2i3YJd-c,30862
+pipen_cli_gbatch/daemon_args.toml,sha256=XrCDwTaJ7xPgGLtZev4qikjrZWqixdE8tqSsFnIvmjc,7381
+pipen_cli_gbatch-0.0.6.dist-info/METADATA,sha256=ZvlRbQNOG6k2EIe2UeE4f469VP8AZM_IcLDlkuw91wg,11301
+pipen_cli_gbatch-0.0.6.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+pipen_cli_gbatch-0.0.6.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
+pipen_cli_gbatch-0.0.6.dist-info/RECORD,,

pipen_cli_gbatch-0.0.4.dist-info/RECORD

@@ -1,6 +0,0 @@
-pipen_cli_gbatch/__init__.py,sha256=x5R1wU-TyXS_Z1EqJ1jWhFn66sKLt_fg8eyXtz9k7rg,29508
-pipen_cli_gbatch/daemon_args.toml,sha256=YzqLumFjIY8mRMzRt3M5D1fEjK93QvQJ0XynDnimPIo,6873
-pipen_cli_gbatch-0.0.4.dist-info/METADATA,sha256=5i6JjljULy5Km96zlHt0dcI97Wleq73yzYpakX5j8Zw,11301
-pipen_cli_gbatch-0.0.4.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-pipen_cli_gbatch-0.0.4.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
-pipen_cli_gbatch-0.0.4.dist-info/RECORD,,