pipen-cli-gbatch 0.0.6__tar.gz → 0.1.0__tar.gz

{pipen_cli_gbatch-0.0.6 → pipen_cli_gbatch-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipen-cli-gbatch
-Version: 0.0.6
+Version: 0.1.0
 Summary: A pipen cli plugin to run command via Google Cloud Batch
 License: MIT
 Author: pwwang

{pipen_cli_gbatch-0.0.6 → pipen_cli_gbatch-0.1.0}/pipen_cli_gbatch/__init__.py

@@ -79,7 +79,7 @@ from pipen.cli import CLIPlugin
 from pipen.scheduler import GbatchScheduler
 from pipen_poplog import LogsPopulator
 
-__version__ = "0.0.6"
+__version__ = "0.1.0"
 __all__ = ("CliGbatchPlugin", "CliGbatchDaemon")
 MOUNTED_CWD = "/mnt/disks/.cwd"
 
@@ -135,6 +135,7 @@ class CliGbatchDaemon:
                 for key, val in (item.split("=", 1) for item in self.config.labels)
             }
         self.command = command
+        self._command_workdir = None
 
     def _get_arg_from_command(self, arg: str) -> str | None:
         """Get the value of the given argument from the command line.
@@ -218,8 +219,10 @@ class CliGbatchDaemon:
         if from_mount_as_cwd:
             self.config.workdir = f"{self.mount_as_cwd}/.pipen/{command_name}"
 
-        command_workdir = self._get_arg_from_command("workdir")
-        workdir = self.config.get("workdir", None) or command_workdir
+        # self._command_workdir to save the original command workdir
+        self._command_workdir = workdir = (
+            self.config.get("workdir", None) or self._get_arg_from_command("workdir")
+        )
 
         if not workdir or not isinstance(AnyPath(workdir), GSPath):
             print(
@@ -247,8 +250,16 @@ class CliGbatchDaemon:
         command_outdir = self._get_arg_from_command("outdir")
 
         if command_outdir:
-            self._add_mount(command_outdir, GbatchScheduler.MOUNTED_OUTDIR)
-            self._replace_arg_in_command("outdir", GbatchScheduler.MOUNTED_OUTDIR)
+            coudir = AnyPath(command_outdir)
+            if (
+                not isinstance(coudir, GSPath)
+                and not coudir.is_absolute()
+                and self.mount_as_cwd
+            ):
+                self._replace_arg_in_command("outdir", f"{MOUNTED_CWD}/{coudir}")
+            else:
+                self._add_mount(command_outdir, GbatchScheduler.MOUNTED_OUTDIR)
+                self._replace_arg_in_command("outdir", GbatchScheduler.MOUNTED_OUTDIR)
         elif self.mount_as_cwd:
             command_name = self._get_arg_from_command("name") or self.config.name
             self._replace_arg_in_command(
@@ -304,7 +315,13 @@ class CliGbatchDaemon:
             and not self.config.view_logs
             and "logging" not in plugin.get_all_plugin_names()
         ):
-            plugins.append(XquteCliGbatchPlugin())
+            if self.config.plain:
+                # use the stdout file from daemon
+                stdout_file = None
+            else:
+                stdout_file = AnyPath(f"{self._command_workdir}/run-latest.log")
+
+            plugins.append(XquteCliGbatchPlugin(stdout_file=stdout_file))
 
         return Xqute(
             "gbatch",
@@ -516,6 +533,12 @@ class CliGbatchDaemon:
             self._handle_outdir()
             self._infer_jobname_prefix()
         else:
+            if "name" not in self.config or not self.config.name:
+                self.config["name"] = "PipenCliGbatchDaemon"
+
+            if not self.config.workdir and self.mount_as_cwd:
+                self.config.workdir = f"{self.mount_as_cwd}/.pipen"
+
             if not self.config.workdir or not isinstance(
                 AnyPath(self.config.workdir),
                 GSPath,
@@ -526,9 +549,6 @@ class CliGbatchDaemon:
                 )
                 sys.exit(1)
 
-            if "name" not in self.config or not self.config.name:
-                self.config["name"] = "PipenCliGbatchDaemon"
-
     async def run(self):  # pragma: no cover
         """Execute the daemon pipeline based on configuration.
 
@@ -560,12 +580,15 @@ class XquteCliGbatchPlugin:  # pragma: no cover
 
     Attributes:
         name (str): The plugin name.
-        log_start (bool): Whether to start logging when job starts.
         stdout_populator (LogsPopulator): Handles stdout log population.
         stderr_populator (LogsPopulator): Handles stderr log population.
     """
 
-    def __init__(self, name: str = "logging", log_start: bool = True):
+    def __init__(
+        self,
+        name: str = "logging",
+        stdout_file: str | Path | GSPath | None = None,
+    ):
         """Initialize the logging plugin.
 
         Args:
@@ -573,7 +596,7 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             log_start: Whether to start logging when job starts.
         """
         self.name = name
-        self.log_start = log_start
+        self.stdout_file = stdout_file
         self.stdout_populator = LogsPopulator()
         self.stderr_populator = LogsPopulator()
 
@@ -586,17 +609,6 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             logger.error(f"/STDERR {self.stderr_populator.residue}")
             self.stderr_populator.residue = ""
 
-    @plugin.impl
-    async def on_job_init(self, scheduler, job):
-        """Handle job initialization event.
-
-        Args:
-            scheduler: The scheduler instance.
-            job: The job being initialized.
-        """
-        self.stdout_populator.logfile = scheduler.workdir.joinpath("0", "job.stdout")
-        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
-
     @plugin.impl
     async def on_job_started(self, scheduler, job):
         """Handle job start event by setting up log file paths.
@@ -605,10 +617,31 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             scheduler: The scheduler instance.
             job: The job that started.
         """
-        if not self.log_start:
-            return
-
         logger.info("Job is picked up by Google Batch, pulling stdout/stderr...")
+        if not self.stdout_file:
+            self.stdout_populator.logfile = scheduler.workdir.joinpath(
+                "0", "job.stdout"
+            )
+        elif not self.stdout_file.exists():
+            await asyncio.sleep(3)  # wait a bit for the file to be created
+            if not self.stdout_file.exists():
+                logger.warning(f"Running logs not found: {self.stdout_file}")
+                logger.warning(
+                    "Make sure pipen-log2file plugin is enabled for your pipeline."
+                )
+                logger.warning("Falling back to pull logs from daemon...")
+                self.stdout_populator.logfile = scheduler.workdir.joinpath(
+                    "0", "job.stdout"
+                )
+            else:
+                self.stdout_populator.logfile = (
+                    self.stdout_file.resolve()
+                    if self.stdout_file.is_symlink()
+                    else self.stdout_file
+                )
+        else:
+            self.stdout_populator.logfile = self.stdout_file
+        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
 
     @plugin.impl
     async def on_job_polling(self, scheduler, job, counter):
@@ -809,11 +842,7 @@ class CliGbatchPlugin(CLIPlugin):  # pragma: no cover
 
         # update parsed with the defaults
         for key, val in defaults.items():
-            if (
-                key == "mount"
-                and val
-                and getattr(known_parsed, key, None)
-            ):
+            if key == "mount" and val and getattr(known_parsed, key, None):
                 if not isinstance(val, (tuple, list)):
                     val = [val]
                 val = list(val)

{pipen_cli_gbatch-0.0.6 → pipen_cli_gbatch-0.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pipen-cli-gbatch"
-version = "0.0.6"
+version = "0.1.0"
 description = "A pipen cli plugin to run command via Google Cloud Batch"
 authors = ["pwwang <pwwang@pwwang.com>"]
 license = "MIT"

{pipen_cli_gbatch-0.0.6 → pipen_cli_gbatch-0.1.0}/setup.py

@@ -17,7 +17,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'pipen-cli-gbatch',
-    'version': '0.0.6',
+    'version': '0.1.0',
     'description': 'A pipen cli plugin to run command via Google Cloud Batch',
     'long_description': '# pipen-cli-gbatch\n\nA pipen CLI plugin to run commands via Google Cloud Batch.\n\nThe idea is to submit the command using xqute and use the gbatch scheduler to run it on Google Cloud Batch.\n\n## Installation\n\n```bash\npip install pipen-cli-gbatch\n```\n\n## Usage\n\n### Basic Command Execution\n\nTo run a command like:\n\n```bash\npython myscript.py --input input.txt --output output.txt\n```\n\nYou can run it with:\n\n```bash\npipen gbatch -- python myscript.py --input input.txt --output output.txt\n```\n\n### With Configuration File\n\nIn order to provide configurations like we do for a normal pipen pipeline, you can also provide a config file (the `[pipen-cli-gbatch]` section will be used):\n\n```bash\npipen gbatch @config.toml -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\n### Detached Mode\n\nWe can also use the `--nowait` option to run the command in a detached mode:\n\n```bash\npipen gbatch --nowait -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\nOr by default, it will wait for the command to complete:\n\n```bash\npipen gbatch -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\nWhile waiting, the running logs will be pulled and shown in the terminal.\n\n### View Logs\n\nWhen running in detached mode, one can also pull the logs later by:\n\n```bash\npipen gbatch --view-logs -- \\\n    python myscript.py --input input.txt --output output.txt\n\n# or  just provide the workdir\npipen gbatch --view-logs --workdir gs://my-bucket/workdir\n```\n\n## Configuration\n\nBecause the daemon pipeline is running on Google Cloud Batch, a Google Storage Bucket path is required for the workdir. For example: `gs://my-bucket/workdir`\n\nA unique job ID will be generated per the name (`--name`) and workdir, so that if the same command is run again with the same name and workdir, it will not start a new job, but just attach to the existing job and pull the logs.\n\nIf `--name` is not provided in the command line, it will try to grab the name (`--name`) from the command line arguments after `--`, or else use "name" from the root section of the configuration file, with a "GbatchDaemon" suffix. If nothing can be found, a default name "PipenGbatchDaemon" will be used.\n\nThen a workdir `{workdir}/<daemon pipeline name>/` will be created to store the meta information.\n\nWith `--profile` provided, the scheduler options (`scheduler_opts`) defined in `~/.pipen.toml` and `./.pipen.toml` will be used as default.\n\n## All Options\n\n```bash\n> pipen gbatch --help\nUsage: pipen gbatch [-h] [--nowait | --view-logs {all,stdout,stderr}] [--workdir WORKDIR]\n                    [--error-strategy {retry,halt}] [--num-retries NUM_RETRIES] [--prescript PRESCRIPT]\n                    [--postscript POSTSCRIPT] [--jobname-prefix JOBNAME_PREFIX] [--recheck-interval RECHECK_INTERVAL]\n                    [--cwd CWD] [--project PROJECT] [--location LOCATION] [--mount MOUNT]\n                    [--service-account SERVICE_ACCOUNT] [--network NETWORK] [--subnetwork SUBNETWORK]\n                    [--no-external-ip-address] [--machine-type MACHINE_TYPE] [--provisioning-model {STANDARD,SPOT}]\n                    [--image-uri IMAGE_URI] [--entrypoint ENTRYPOINT] [--commands COMMANDS] [--runnables RUNNABLES]\n                    [--allocationPolicy ALLOCATIONPOLICY] [--taskGroups TASKGROUPS] [--labels LABELS] [--gcloud GCLOUD]\n                    [--name NAME] [--profile PROFILE] [--version]\n                    [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL,debug,info,warning,error,critical}]\n                    ...\n\nSimplify running commands via Google Cloud Batch.\n\nKey Options:\n  The key options to run the command.\n\n  --workdir WORKDIR     The workdir (a Google Storage Bucket path is required) to store the meta information of the\n                        daemon pipeline.\n                        If not provided, the one from the command will be used.\n  command               The command passed after `--` to run, with all its arguments. Note that the command should be\n                        provided after `--`.\n\nScheduler Options:\n  The options to configure the gbatch scheduler.\n\n  --error-strategy {retry,halt}\n                        The strategy when there is error happened [default: halt]\n  --num-retries NUM_RETRIES\n                        The number of retries when there is error happened. Only valid when --error-strategy is \'retry\'.\n                        [default: 0]\n  --prescript PRESCRIPT\n                        The prescript to run before the main command.\n  --postscript POSTSCRIPT\n                        The postscript to run after the main command.\n  --jobname-prefix JOBNAME_PREFIX\n                        The prefix of the name prefix of the daemon job.\n                        If not provided, try to generate one from the command to run.\n                        If the command is also not provided, use \'pipen-gbatch-daemon\' as the prefix.\n  --recheck-interval RECHECK_INTERVAL\n                        The interval to recheck the job status, each takes about 0.1 seconds. [default: 600]\n  --cwd CWD             The working directory to run the command. If not provided, the current directory is used. You\n                        can pass either a mounted path (inside the VM) or a Google Storage Bucket path (gs://...). If a\n                        Google Storage Bucket path is provided, the mounted path will be inferred from the mounted paths\n                        of the VM.\n  --project PROJECT     The Google Cloud project to run the job.\n  --location LOCATION   The location to run the job.\n  --mount MOUNT         The list of mounts to mount to the VM, each in the format of SOURCE:TARGET, where SOURCE must be\n                        either a Google Storage Bucket path (gs://...). [default: []]\n  --service-account SERVICE_ACCOUNT\n                        The service account to run the job.\n  --network NETWORK     The network to run the job.\n  --subnetwork SUBNETWORK\n                        The subnetwork to run the job.\n  --no-external-ip-address\n                        Whether to disable external IP address for the VM.\n  --machine-type MACHINE_TYPE\n                        The machine type of the VM.\n  --provisioning-model {STANDARD,SPOT}\n                        The provisioning model of the VM.\n  --image-uri IMAGE_URI\n                        The custom image URI of the VM.\n  --entrypoint ENTRYPOINT\n                        The entry point of the container to run the command.\n  --commands COMMANDS   The list of commands to run in the container, each as a separate string. [default: []]\n  --runnables RUNNABLES\n                        The JSON string of extra settings of runnables add to the job.json.\n                        Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Runnable\n                        for details.\n                        You can have an extra key \'order\' for each runnable, where negative values mean to run before\n                        the main command,\n                        and positive values mean to run after the main command.\n  --allocationPolicy ALLOCATIONPOLICY\n                        The JSON string of extra settings of allocationPolicy add to the job.json. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#AllocationPolicy\n                        for details. [default: {}]\n  --taskGroups TASKGROUPS\n                        The JSON string of extra settings of taskGroups add to the job.json. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#TaskGroup for\n                        details. [default: []]\n  --labels LABELS       The JSON string of labels to add to the job. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Job.FIELDS.labels\n                        for details. [default: {}]\n  --gcloud GCLOUD       The path to the gcloud command. [default: gcloud]\n\nOptions:\n  -h, --help            show this help message and exit\n  --nowait              Run the command in a detached mode without waiting for its completion. [default: False]\n  --view-logs {all,stdout,stderr}\n                        View the logs of a job.\n  --name NAME           The name of the daemon pipeline.\n                        If not provided, try to generate one from the command to run.\n                        If the command is also not provided, use \'PipenCliGbatchDaemon\' as the name.\n  --profile PROFILE     Use the `scheduler_opts` as the Scheduler Options of a given profile from pipen configuration\n                        files,\n                        including ~/.pipen.toml and ./pipen.toml.\n                        Note that if not provided, nothing will be loaded from the configuration files.\n  --version             Show the version of the pipen-cli-gbatch package. [default: False]\n  --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL,debug,info,warning,error,critical}\n                        Set the logging level for the daemon process. [default: INFO]\n\nExamples:\n  \u200b\n  # Run a command and wait for it to complete\n  > pipen gbatch --workdir gs://my-bucket/workdir -- \\\n    python myscript.py --input input.txt --output output.txt\n\n  # Use named mounts\n  > pipen gbatch --workdir gs://my-bucket/workdir --mount INFILE=gs://bucket/path/to/file \\\n    --mount OUTDIR=gs://bucket/path/to/outdir -- \\\n    cat $INFILE > $OUTDIR/output.txt\n  \u200b\n  # Run a command in a detached mode\n  > pipen gbatch --nowait --project $PROJECT --location $LOCATION \\\n    --workdir gs://my-bucket/workdir -- \\\n    python myscript.py --input input.txt --output output.txt\n  \u200b\n  # If you have a profile defined in ~/.pipen.toml or ./.pipen.toml\n  > pipen gbatch --profile myprofile -- \\\n    python myscript.py --input input.txt --output output.txt\n  \u200b\n  # View the logs of a previously run command\n  > pipen gbatch --view-logs all --name my-daemon-name \\\n    --workdir gs://my-bucket/workdir\n```\n\n## API\n\nThe API can also be used to run commands programmatically:\n\n```python\nimport asyncio\nfrom pipen_cli_gbatch import CliGbatchDaemon\n\npipe = CliGbatchDaemon(config_for_daemon, command)\nasyncio.run(pipe.run())\n```\n\nNote that the daemon pipeline will always be running without caching, so that the command will always be executed when the pipeline is run.\n',
     'author': 'pwwang',