pipen-cli-gbatch 0.0.7__py3-none-any.whl → 0.1.1__py3-none-any.whl

pipen_cli_gbatch/__init__.py

@@ -79,7 +79,7 @@ from pipen.cli import CLIPlugin
 from pipen.scheduler import GbatchScheduler
 from pipen_poplog import LogsPopulator
 
-__version__ = "0.0.7"
+__version__ = "0.1.1"
 __all__ = ("CliGbatchPlugin", "CliGbatchDaemon")
 MOUNTED_CWD = "/mnt/disks/.cwd"
 
@@ -135,6 +135,7 @@ class CliGbatchDaemon:
                 for key, val in (item.split("=", 1) for item in self.config.labels)
             }
         self.command = command
+        self._command_workdir = None
 
     def _get_arg_from_command(self, arg: str) -> str | None:
         """Get the value of the given argument from the command line.
@@ -218,8 +219,10 @@ class CliGbatchDaemon:
         if from_mount_as_cwd:
             self.config.workdir = f"{self.mount_as_cwd}/.pipen/{command_name}"
 
-        command_workdir = self._get_arg_from_command("workdir")
-        workdir = self.config.get("workdir", None) or command_workdir
+        # self._command_workdir to save the original command workdir
+        self._command_workdir = workdir = (
+            self.config.get("workdir", None) or self._get_arg_from_command("workdir")
+        )
 
         if not workdir or not isinstance(AnyPath(workdir), GSPath):
             print(
@@ -312,7 +315,13 @@ class CliGbatchDaemon:
             and not self.config.view_logs
             and "logging" not in plugin.get_all_plugin_names()
         ):
-            plugins.append(XquteCliGbatchPlugin())
+            if self.config.plain:
+                # use the stdout file from daemon
+                stdout_file = None
+            else:
+                stdout_file = AnyPath(f"{self._command_workdir}/run-latest.log")
+
+            plugins.append(XquteCliGbatchPlugin(stdout_file=stdout_file))
 
         return Xqute(
             "gbatch",
@@ -540,7 +549,6 @@ class CliGbatchDaemon:
                 )
                 sys.exit(1)
 
-
     async def run(self):  # pragma: no cover
         """Execute the daemon pipeline based on configuration.
 
@@ -572,12 +580,15 @@ class XquteCliGbatchPlugin:  # pragma: no cover
 
     Attributes:
         name (str): The plugin name.
-        log_start (bool): Whether to start logging when job starts.
         stdout_populator (LogsPopulator): Handles stdout log population.
         stderr_populator (LogsPopulator): Handles stderr log population.
     """
 
-    def __init__(self, name: str = "logging", log_start: bool = True):
+    def __init__(
+        self,
+        name: str = "logging",
+        stdout_file: str | Path | GSPath | None = None,
+    ):
         """Initialize the logging plugin.
 
         Args:
@@ -585,7 +596,7 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             log_start: Whether to start logging when job starts.
         """
         self.name = name
-        self.log_start = log_start
+        self.stdout_file = stdout_file
         self.stdout_populator = LogsPopulator()
         self.stderr_populator = LogsPopulator()
 
@@ -598,17 +609,6 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             logger.error(f"/STDERR {self.stderr_populator.residue}")
             self.stderr_populator.residue = ""
 
-    @plugin.impl
-    async def on_job_init(self, scheduler, job):
-        """Handle job initialization event.
-
-        Args:
-            scheduler: The scheduler instance.
-            job: The job being initialized.
-        """
-        self.stdout_populator.logfile = scheduler.workdir.joinpath("0", "job.stdout")
-        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
-
     @plugin.impl
     async def on_job_started(self, scheduler, job):
         """Handle job start event by setting up log file paths.
@@ -617,10 +617,46 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             scheduler: The scheduler instance.
             job: The job that started.
         """
-        if not self.log_start:
-            return
+        logger.info("Job is picked up by Google Batch, pulling stdout/stderr ...")
+        if not self.stdout_file:
+            self.stdout_populator.logfile = scheduler.workdir.joinpath(
+                "0", "job.stdout"
+            )
+        elif not self.stdout_file.exists():
+            logger.warning(f"Running logs file not found: {self.stdout_file}")
+            logger.warning("  Waiting for it to be created ...")
+            i = 0
+            while not self.stdout_file.exists():
+                await asyncio.sleep(3)
+                i += 1
+                if i >= 20:
+                    break
+
+            if not self.stdout_file.exists():
+                logger.warning(
+                    "  Still not found, falling back to pull logs from daemon ..."
+                )
+                logger.warning(
+                    "  Make sure pipen-log2file plugin is enabled for your pipeline."
+                )
+                self.stdout_populator.logfile = scheduler.workdir.joinpath(
+                    "0", "job.stdout"
+                )
+            else:
+                logger.info("  Found the running logs, pulling ...")
+                self.stdout_populator.logfile = (
+                    self.stdout_file.resolve()
+                    if self.stdout_file.is_symlink()
+                    else self.stdout_file
+                )
+        else:
+            self.stdout_populator.logfile = (
+                self.stdout_file.resolve()
+                if self.stdout_file.is_symlink()
+                else self.stdout_file
+            )
 
-        logger.info("Job is picked up by Google Batch, pulling stdout/stderr...")
+        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
 
     @plugin.impl
     async def on_job_polling(self, scheduler, job, counter):
@@ -821,11 +857,7 @@ class CliGbatchPlugin(CLIPlugin):  # pragma: no cover
 
         # update parsed with the defaults
         for key, val in defaults.items():
-            if (
-                key == "mount"
-                and val
-                and getattr(known_parsed, key, None)
-            ):
+            if key == "mount" and val and getattr(known_parsed, key, None):
                 if not isinstance(val, (tuple, list)):
                     val = [val]
                 val = list(val)

{pipen_cli_gbatch-0.0.7.dist-info → pipen_cli_gbatch-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipen-cli-gbatch
-Version: 0.0.7
+Version: 0.1.1
 Summary: A pipen cli plugin to run command via Google Cloud Batch
 License: MIT
 Author: pwwang

pipen_cli_gbatch-0.1.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+pipen_cli_gbatch/__init__.py,sha256=_8YcTVxCzFRfD-mf9bXP4tplcEhMkFFlOuVkla-jFas,32698
+pipen_cli_gbatch/daemon_args.toml,sha256=XrCDwTaJ7xPgGLtZev4qikjrZWqixdE8tqSsFnIvmjc,7381
+pipen_cli_gbatch-0.1.1.dist-info/METADATA,sha256=kvLiPVSXeqLJeNvFuR2TS_o-lQw9uoSjNBfYLcg_MoI,11301
+pipen_cli_gbatch-0.1.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+pipen_cli_gbatch-0.1.1.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
+pipen_cli_gbatch-0.1.1.dist-info/RECORD,,

pipen_cli_gbatch-0.0.7.dist-info/RECORD

@@ -1,6 +0,0 @@
-pipen_cli_gbatch/__init__.py,sha256=xiYBNz3z3Eu8vmFECPFpOt1EE501T_gZx_iPUDajCd4,31309
-pipen_cli_gbatch/daemon_args.toml,sha256=XrCDwTaJ7xPgGLtZev4qikjrZWqixdE8tqSsFnIvmjc,7381
-pipen_cli_gbatch-0.0.7.dist-info/METADATA,sha256=pxmFaIpwMG6GU4wZF_1EBqtWa9l44KI-m1Lhi3TI1Dw,11301
-pipen_cli_gbatch-0.0.7.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-pipen_cli_gbatch-0.0.7.dist-info/entry_points.txt,sha256=Z9NLeCpRo-rb8wss5mB5TBcG-_RbdlPA49b8Ma5pvQA,57
-pipen_cli_gbatch-0.0.7.dist-info/RECORD,,