pipen-cli-gbatch 0.0.6__py3-none-any.whl → 0.1.0__py3-none-any.whl

pipen_cli_gbatch/__init__.py

@@ -79,7 +79,7 @@ from pipen.cli import CLIPlugin
 from pipen.scheduler import GbatchScheduler
 from pipen_poplog import LogsPopulator
 
-__version__ = "0.0.6"
+__version__ = "0.1.0"
 __all__ = ("CliGbatchPlugin", "CliGbatchDaemon")
 MOUNTED_CWD = "/mnt/disks/.cwd"
 
@@ -135,6 +135,7 @@ class CliGbatchDaemon:
                 for key, val in (item.split("=", 1) for item in self.config.labels)
             }
         self.command = command
+        self._command_workdir = None
 
     def _get_arg_from_command(self, arg: str) -> str | None:
         """Get the value of the given argument from the command line.
@@ -218,8 +219,10 @@ class CliGbatchDaemon:
         if from_mount_as_cwd:
             self.config.workdir = f"{self.mount_as_cwd}/.pipen/{command_name}"
 
-        command_workdir = self._get_arg_from_command("workdir")
-        workdir = self.config.get("workdir", None) or command_workdir
+        # self._command_workdir to save the original command workdir
+        self._command_workdir = workdir = (
+            self.config.get("workdir", None) or self._get_arg_from_command("workdir")
+        )
 
         if not workdir or not isinstance(AnyPath(workdir), GSPath):
             print(
@@ -247,8 +250,16 @@ class CliGbatchDaemon:
         command_outdir = self._get_arg_from_command("outdir")
 
         if command_outdir:
-            self._add_mount(command_outdir, GbatchScheduler.MOUNTED_OUTDIR)
-            self._replace_arg_in_command("outdir", GbatchScheduler.MOUNTED_OUTDIR)
+            coudir = AnyPath(command_outdir)
+            if (
+                not isinstance(coudir, GSPath)
+                and not coudir.is_absolute()
+                and self.mount_as_cwd
+            ):
+                self._replace_arg_in_command("outdir", f"{MOUNTED_CWD}/{coudir}")
+            else:
+                self._add_mount(command_outdir, GbatchScheduler.MOUNTED_OUTDIR)
+                self._replace_arg_in_command("outdir", GbatchScheduler.MOUNTED_OUTDIR)
         elif self.mount_as_cwd:
             command_name = self._get_arg_from_command("name") or self.config.name
             self._replace_arg_in_command(
@@ -304,7 +315,13 @@ class CliGbatchDaemon:
             and not self.config.view_logs
             and "logging" not in plugin.get_all_plugin_names()
         ):
-            plugins.append(XquteCliGbatchPlugin())
+            if self.config.plain:
+                # use the stdout file from daemon
+                stdout_file = None
+            else:
+                stdout_file = AnyPath(f"{self._command_workdir}/run-latest.log")
+
+            plugins.append(XquteCliGbatchPlugin(stdout_file=stdout_file))
 
         return Xqute(
             "gbatch",
@@ -516,6 +533,12 @@ class CliGbatchDaemon:
             self._handle_outdir()
             self._infer_jobname_prefix()
         else:
+            if "name" not in self.config or not self.config.name:
+                self.config["name"] = "PipenCliGbatchDaemon"
+
+            if not self.config.workdir and self.mount_as_cwd:
+                self.config.workdir = f"{self.mount_as_cwd}/.pipen"
+
             if not self.config.workdir or not isinstance(
                 AnyPath(self.config.workdir),
                 GSPath,
@@ -526,9 +549,6 @@ class CliGbatchDaemon:
                 )
                 sys.exit(1)
 
-            if "name" not in self.config or not self.config.name:
-                self.config["name"] = "PipenCliGbatchDaemon"
-
     async def run(self):  # pragma: no cover
         """Execute the daemon pipeline based on configuration.
 
@@ -560,12 +580,15 @@ class XquteCliGbatchPlugin:  # pragma: no cover
 
     Attributes:
         name (str): The plugin name.
-        log_start (bool): Whether to start logging when job starts.
         stdout_populator (LogsPopulator): Handles stdout log population.
         stderr_populator (LogsPopulator): Handles stderr log population.
     """
 
-    def __init__(self, name: str = "logging", log_start: bool = True):
+    def __init__(
+        self,
+        name: str = "logging",
+        stdout_file: str | Path | GSPath | None = None,
+    ):
         """Initialize the logging plugin.
 
         Args:
@@ -573,7 +596,7 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             log_start: Whether to start logging when job starts.
         """
         self.name = name
-        self.log_start = log_start
+        self.stdout_file = stdout_file
         self.stdout_populator = LogsPopulator()
         self.stderr_populator = LogsPopulator()
 
@@ -586,17 +609,6 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             logger.error(f"/STDERR {self.stderr_populator.residue}")
             self.stderr_populator.residue = ""
 
-    @plugin.impl
-    async def on_job_init(self, scheduler, job):
-        """Handle job initialization event.
-
-        Args:
-            scheduler: The scheduler instance.
-            job: The job being initialized.
-        """
-        self.stdout_populator.logfile = scheduler.workdir.joinpath("0", "job.stdout")
-        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
-
     @plugin.impl
     async def on_job_started(self, scheduler, job):
         """Handle job start event by setting up log file paths.
@@ -605,10 +617,31 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             scheduler: The scheduler instance.
             job: The job that started.
         """
-        if not self.log_start:
-            return
-
         logger.info("Job is picked up by Google Batch, pulling stdout/stderr...")
+        if not self.stdout_file:
+            self.stdout_populator.logfile = scheduler.workdir.joinpath(
+                "0", "job.stdout"
+            )
+        elif not self.stdout_file.exists():
+            await asyncio.sleep(3)  # wait a bit for the file to be created
+            if not self.stdout_file.exists():
+                logger.warning(f"Running logs not found: {self.stdout_file}")
+                logger.warning(
+                    "Make sure pipen-log2file plugin is enabled for your pipeline."
+                )
+                logger.warning("Falling back to pull logs from daemon...")
+                self.stdout_populator.logfile = scheduler.workdir.joinpath(
+                    "0", "job.stdout"
+                )
+            else:
+                self.stdout_populator.logfile = (
+                    self.stdout_file.resolve()
+                    if self.stdout_file.is_symlink()
+                    else self.stdout_file
+                )
+        else:
+            self.stdout_populator.logfile = self.stdout_file
+        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
 
     @plugin.impl
     async def on_job_polling(self, scheduler, job, counter):
@@ -809,11 +842,7 @@ class CliGbatchPlugin(CLIPlugin):  # pragma: no cover
 
         # update parsed with the defaults
         for key, val in defaults.items():
-            if (
-                key == "mount"
-                and val
-                and getattr(known_parsed, key, None)
-            ):
+            if key == "mount" and val and getattr(known_parsed, key, None):
                 if not isinstance(val, (tuple, list)):
                     val = [val]
                 val = list(val)

{pipen_cli_gbatch-0.0.6.dist-info → pipen_cli_gbatch-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipen-cli-gbatch
-Version: 0.0.6
+Version: 0.1.0
 Summary: A pipen cli plugin to run command via Google Cloud Batch
 License: MIT
 Author: pwwang

pipen_cli_gbatch-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+pipen_cli_gbatch/__init__.py,sha256=e2nJIiWPFzlzNAbOxzTanCtXg0oot28HrxHpH7KGyxI,32264
+pipen_cli_gbatch/daemon_args.toml,sha256=XrCDwTaJ7xPgGLtZev4qikjrZWqixdE8tqSsFnIvmjc,7381
+pipen_cli_gbatch-0.1.0.dist-info/METADATA,sha256=sNcCQTI7p7ibwuWIFT-SQo8RT-gLbjlE8TO2Bv592Q0,11301
+pipen_cli_gbatch-0.1.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+pipen_cli_gbatch-0.1.0.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
+pipen_cli_gbatch-0.1.0.dist-info/RECORD,,

pipen_cli_gbatch-0.0.6.dist-info/RECORD

@@ -1,6 +0,0 @@
-pipen_cli_gbatch/__init__.py,sha256=5ChKkEGzq1S2irrwHpNts2oqezfB1zKs5IA2i3YJd-c,30862
-pipen_cli_gbatch/daemon_args.toml,sha256=XrCDwTaJ7xPgGLtZev4qikjrZWqixdE8tqSsFnIvmjc,7381
-pipen_cli_gbatch-0.0.6.dist-info/METADATA,sha256=ZvlRbQNOG6k2EIe2UeE4f469VP8AZM_IcLDlkuw91wg,11301
-pipen_cli_gbatch-0.0.6.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-pipen_cli_gbatch-0.0.6.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
-pipen_cli_gbatch-0.0.6.dist-info/RECORD,,