pipen-cli-gbatch 0.0.4__tar.gz → 0.0.6__tar.gz

{pipen_cli_gbatch-0.0.4 → pipen_cli_gbatch-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pipen-cli-gbatch
-Version: 0.0.4
+Version: 0.0.6
 Summary: A pipen cli plugin to run command via Google Cloud Batch
 License: MIT
 Author: pwwang

{pipen_cli_gbatch-0.0.4 → pipen_cli_gbatch-0.0.6}/pipen_cli_gbatch/__init__.py

@@ -79,8 +79,9 @@ from pipen.cli import CLIPlugin
 from pipen.scheduler import GbatchScheduler
 from pipen_poplog import LogsPopulator
 
-__version__ = "0.0.4"
+__version__ = "0.0.6"
 __all__ = ("CliGbatchPlugin", "CliGbatchDaemon")
+MOUNTED_CWD = "/mnt/disks/.cwd"
 
 
 class CliGbatchDaemon:
@@ -115,6 +116,17 @@ class CliGbatchDaemon:
         else:
             self.config = Diot(config)
 
+        self.mount_as_cwd = self.config.pop("mount_as_cwd", None)
+        if self.mount_as_cwd:
+            if self.config.cwd:
+                print(
+                    "\033[1;4mError\033[0m: --mount-as-cwd cannot be used with "
+                    "--cwd at the same time.\n"
+                )
+                sys.exit(1)
+            self.config.cwd = MOUNTED_CWD
+            self._add_mount(self.mount_as_cwd, MOUNTED_CWD)
+
         self.config.prescript = self.config.get("prescript", None) or ""
         self.config.postscript = self.config.get("postscript", None) or ""
         if "labels" in self.config and isinstance(self.config.labels, list):
@@ -201,6 +213,11 @@ class CliGbatchDaemon:
         Raises:
             SystemExit: If workdir is not a valid Google Storage bucket path.
         """
+        command_name = self._get_arg_from_command("name") or self.config["name"]
+        from_mount_as_cwd = self.mount_as_cwd and not self.config.workdir
+        if from_mount_as_cwd:
+            self.config.workdir = f"{self.mount_as_cwd}/.pipen/{command_name}"
+
         command_workdir = self._get_arg_from_command("workdir")
         workdir = self.config.get("workdir", None) or command_workdir
 
@@ -212,11 +229,14 @@ class CliGbatchDaemon:
             sys.exit(1)
 
         self.config["workdir"] = workdir
-        # If command workdir is different from config workdir, we need to mount it
-        self._add_mount(workdir, GbatchScheduler.MOUNTED_METADIR)
+        if from_mount_as_cwd:  # already mounted
+            self._replace_arg_in_command("workdir", f"{MOUNTED_CWD}/.pipen")
+        else:
+            # If command workdir is different from config workdir, we need to mount it
+            self._add_mount(workdir, GbatchScheduler.MOUNTED_METADIR)
 
-        # replace --workdir value with the mounted workdir in the command
-        self._replace_arg_in_command("workdir", GbatchScheduler.MOUNTED_METADIR)
+            # replace --workdir value with the mounted workdir in the command
+            self._replace_arg_in_command("workdir", GbatchScheduler.MOUNTED_METADIR)
 
     def _handle_outdir(self):
         """Handle output directory configuration and mounting.
@@ -229,6 +249,12 @@ class CliGbatchDaemon:
         if command_outdir:
             self._add_mount(command_outdir, GbatchScheduler.MOUNTED_OUTDIR)
             self._replace_arg_in_command("outdir", GbatchScheduler.MOUNTED_OUTDIR)
+        elif self.mount_as_cwd:
+            command_name = self._get_arg_from_command("name") or self.config.name
+            self._replace_arg_in_command(
+                "outdir",
+                f"{MOUNTED_CWD}/{command_name}-output",
+            )
 
     def _infer_name(self):
         """Infer the daemon name from configuration or command arguments.
@@ -303,6 +329,7 @@ class CliGbatchDaemon:
                     "version",
                     "loglevel",
                     "mounts",
+                    "mount_as_cwd",
                     "plain",
                 )
             },
@@ -333,6 +360,7 @@ class CliGbatchDaemon:
                 "version",
                 "loglevel",
                 "mounts",
+                "mount_as_cwd",
                 "plain",
             ):
                 continue
@@ -483,9 +511,9 @@ class CliGbatchDaemon:
         logger.setLevel(self.config.loglevel.upper())
 
         if not self.config.plain:
+            self._infer_name()
             self._handle_workdir()
             self._handle_outdir()
-            self._infer_name()
             self._infer_jobname_prefix()
         else:
             if not self.config.workdir or not isinstance(
@@ -551,10 +579,10 @@ class XquteCliGbatchPlugin:  # pragma: no cover
 
     def _clear_residues(self):
         """Clear any remaining log residues and display them."""
-        if self.stdout_populator.residue:
+        if self.stdout_populator and self.stdout_populator.residue:
             logger.info(f"/STDOUT {self.stdout_populator.residue}")
             self.stdout_populator.residue = ""
-        if self.stderr_populator.residue:
+        if self.stderr_populator and self.stderr_populator.residue:
             logger.error(f"/STDERR {self.stderr_populator.residue}")
             self.stderr_populator.residue = ""
 
@@ -595,15 +623,17 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             # Make it less frequent
             return
 
-        stdout_lines = self.stdout_populator.populate()
-        self.stdout_populator.increment_counter(len(stdout_lines))
-        for line in stdout_lines:
-            logger.info(f"/STDOUT {line}")
-
-        stderr_lines = self.stderr_populator.populate()
-        self.stderr_populator.increment_counter(len(stderr_lines))
-        for line in stderr_lines:
-            logger.error(f"/STDERR {line}")
+        if self.stderr_populator:
+            stdout_lines = self.stdout_populator.populate()
+            self.stdout_populator.increment_counter(len(stdout_lines))
+            for line in stdout_lines:
+                logger.info(f"/STDOUT {line}")
+
+        if self.stderr_populator:
+            stderr_lines = self.stderr_populator.populate()
+            self.stderr_populator.increment_counter(len(stderr_lines))
+            for line in stderr_lines:
+                logger.error(f"/STDERR {line}")
 
     @plugin.impl
     async def on_job_killed(self, scheduler, job):

{pipen_cli_gbatch-0.0.4 → pipen_cli_gbatch-0.0.6}/pipen_cli_gbatch/daemon_args.toml

@@ -128,6 +128,16 @@ You can also mount a file like `INFILE=gs://my-bucket/inputs/file.txt`. The pare
 and the file will be available at `/mnt/disks/INFILE/inputs/file.txt` in the VM. `$INFILE` can also be used in the command/script to refer to the mounted path.
 """
 
+[[groups.arguments]]
+flags = ["--mount-as-cwd"]
+type = "str"
+help = """The directory to mount as the current working directory of the command.
+This is a shortcut for `--mount <cloudpath>:/mnt/disks/.cwd --cwd /mnt/disks/.cwd`.
+The <cloudpath> must be a Google Storage Bucket path (gs://...). When this option is used,
+and `--workdir` is not provided, the workdir will be set to `<cloudpath>/.pipen/<command_name>`,
+where <command_name> is the name of the command (or the value of `--name` if provided).
+"""
+
 [[groups.arguments]]
 flags = ["--service-account"]
 type = "str"

{pipen_cli_gbatch-0.0.4 → pipen_cli_gbatch-0.0.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pipen-cli-gbatch"
-version = "0.0.4"
+version = "0.0.6"
 description = "A pipen cli plugin to run command via Google Cloud Batch"
 authors = ["pwwang <pwwang@pwwang.com>"]
 license = "MIT"

{pipen_cli_gbatch-0.0.4 → pipen_cli_gbatch-0.0.6}/setup.py

@@ -17,7 +17,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'pipen-cli-gbatch',
-    'version': '0.0.4',
+    'version': '0.0.6',
     'description': 'A pipen cli plugin to run command via Google Cloud Batch',
     'long_description': '# pipen-cli-gbatch\n\nA pipen CLI plugin to run commands via Google Cloud Batch.\n\nThe idea is to submit the command using xqute and use the gbatch scheduler to run it on Google Cloud Batch.\n\n## Installation\n\n```bash\npip install pipen-cli-gbatch\n```\n\n## Usage\n\n### Basic Command Execution\n\nTo run a command like:\n\n```bash\npython myscript.py --input input.txt --output output.txt\n```\n\nYou can run it with:\n\n```bash\npipen gbatch -- python myscript.py --input input.txt --output output.txt\n```\n\n### With Configuration File\n\nIn order to provide configurations like we do for a normal pipen pipeline, you can also provide a config file (the `[pipen-cli-gbatch]` section will be used):\n\n```bash\npipen gbatch @config.toml -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\n### Detached Mode\n\nWe can also use the `--nowait` option to run the command in a detached mode:\n\n```bash\npipen gbatch --nowait -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\nOr by default, it will wait for the command to complete:\n\n```bash\npipen gbatch -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\nWhile waiting, the running logs will be pulled and shown in the terminal.\n\n### View Logs\n\nWhen running in detached mode, one can also pull the logs later by:\n\n```bash\npipen gbatch --view-logs -- \\\n    python myscript.py --input input.txt --output output.txt\n\n# or  just provide the workdir\npipen gbatch --view-logs --workdir gs://my-bucket/workdir\n```\n\n## Configuration\n\nBecause the daemon pipeline is running on Google Cloud Batch, a Google Storage Bucket path is required for the workdir. For example: `gs://my-bucket/workdir`\n\nA unique job ID will be generated per the name (`--name`) and workdir, so that if the same command is run again with the same name and workdir, it will not start a new job, but just attach to the existing job and pull the logs.\n\nIf `--name` is not provided in the command line, it will try to grab the name (`--name`) from the command line arguments after `--`, or else use "name" from the root section of the configuration file, with a "GbatchDaemon" suffix. If nothing can be found, a default name "PipenGbatchDaemon" will be used.\n\nThen a workdir `{workdir}/<daemon pipeline name>/` will be created to store the meta information.\n\nWith `--profile` provided, the scheduler options (`scheduler_opts`) defined in `~/.pipen.toml` and `./.pipen.toml` will be used as default.\n\n## All Options\n\n```bash\n> pipen gbatch --help\nUsage: pipen gbatch [-h] [--nowait | --view-logs {all,stdout,stderr}] [--workdir WORKDIR]\n                    [--error-strategy {retry,halt}] [--num-retries NUM_RETRIES] [--prescript PRESCRIPT]\n                    [--postscript POSTSCRIPT] [--jobname-prefix JOBNAME_PREFIX] [--recheck-interval RECHECK_INTERVAL]\n                    [--cwd CWD] [--project PROJECT] [--location LOCATION] [--mount MOUNT]\n                    [--service-account SERVICE_ACCOUNT] [--network NETWORK] [--subnetwork SUBNETWORK]\n                    [--no-external-ip-address] [--machine-type MACHINE_TYPE] [--provisioning-model {STANDARD,SPOT}]\n                    [--image-uri IMAGE_URI] [--entrypoint ENTRYPOINT] [--commands COMMANDS] [--runnables RUNNABLES]\n                    [--allocationPolicy ALLOCATIONPOLICY] [--taskGroups TASKGROUPS] [--labels LABELS] [--gcloud GCLOUD]\n                    [--name NAME] [--profile PROFILE] [--version]\n                    [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL,debug,info,warning,error,critical}]\n                    ...\n\nSimplify running commands via Google Cloud Batch.\n\nKey Options:\n  The key options to run the command.\n\n  --workdir WORKDIR     The workdir (a Google Storage Bucket path is required) to store the meta information of the\n                        daemon pipeline.\n                        If not provided, the one from the command will be used.\n  command               The command passed after `--` to run, with all its arguments. Note that the command should be\n                        provided after `--`.\n\nScheduler Options:\n  The options to configure the gbatch scheduler.\n\n  --error-strategy {retry,halt}\n                        The strategy when there is error happened [default: halt]\n  --num-retries NUM_RETRIES\n                        The number of retries when there is error happened. Only valid when --error-strategy is \'retry\'.\n                        [default: 0]\n  --prescript PRESCRIPT\n                        The prescript to run before the main command.\n  --postscript POSTSCRIPT\n                        The postscript to run after the main command.\n  --jobname-prefix JOBNAME_PREFIX\n                        The prefix of the name prefix of the daemon job.\n                        If not provided, try to generate one from the command to run.\n                        If the command is also not provided, use \'pipen-gbatch-daemon\' as the prefix.\n  --recheck-interval RECHECK_INTERVAL\n                        The interval to recheck the job status, each takes about 0.1 seconds. [default: 600]\n  --cwd CWD             The working directory to run the command. If not provided, the current directory is used. You\n                        can pass either a mounted path (inside the VM) or a Google Storage Bucket path (gs://...). If a\n                        Google Storage Bucket path is provided, the mounted path will be inferred from the mounted paths\n                        of the VM.\n  --project PROJECT     The Google Cloud project to run the job.\n  --location LOCATION   The location to run the job.\n  --mount MOUNT         The list of mounts to mount to the VM, each in the format of SOURCE:TARGET, where SOURCE must be\n                        either a Google Storage Bucket path (gs://...). [default: []]\n  --service-account SERVICE_ACCOUNT\n                        The service account to run the job.\n  --network NETWORK     The network to run the job.\n  --subnetwork SUBNETWORK\n                        The subnetwork to run the job.\n  --no-external-ip-address\n                        Whether to disable external IP address for the VM.\n  --machine-type MACHINE_TYPE\n                        The machine type of the VM.\n  --provisioning-model {STANDARD,SPOT}\n                        The provisioning model of the VM.\n  --image-uri IMAGE_URI\n                        The custom image URI of the VM.\n  --entrypoint ENTRYPOINT\n                        The entry point of the container to run the command.\n  --commands COMMANDS   The list of commands to run in the container, each as a separate string. [default: []]\n  --runnables RUNNABLES\n                        The JSON string of extra settings of runnables add to the job.json.\n                        Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Runnable\n                        for details.\n                        You can have an extra key \'order\' for each runnable, where negative values mean to run before\n                        the main command,\n                        and positive values mean to run after the main command.\n  --allocationPolicy ALLOCATIONPOLICY\n                        The JSON string of extra settings of allocationPolicy add to the job.json. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#AllocationPolicy\n                        for details. [default: {}]\n  --taskGroups TASKGROUPS\n                        The JSON string of extra settings of taskGroups add to the job.json. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#TaskGroup for\n                        details. [default: []]\n  --labels LABELS       The JSON string of labels to add to the job. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Job.FIELDS.labels\n                        for details. [default: {}]\n  --gcloud GCLOUD       The path to the gcloud command. [default: gcloud]\n\nOptions:\n  -h, --help            show this help message and exit\n  --nowait              Run the command in a detached mode without waiting for its completion. [default: False]\n  --view-logs {all,stdout,stderr}\n                        View the logs of a job.\n  --name NAME           The name of the daemon pipeline.\n                        If not provided, try to generate one from the command to run.\n                        If the command is also not provided, use \'PipenCliGbatchDaemon\' as the name.\n  --profile PROFILE     Use the `scheduler_opts` as the Scheduler Options of a given profile from pipen configuration\n                        files,\n                        including ~/.pipen.toml and ./pipen.toml.\n                        Note that if not provided, nothing will be loaded from the configuration files.\n  --version             Show the version of the pipen-cli-gbatch package. [default: False]\n  --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL,debug,info,warning,error,critical}\n                        Set the logging level for the daemon process. [default: INFO]\n\nExamples:\n  \u200b\n  # Run a command and wait for it to complete\n  > pipen gbatch --workdir gs://my-bucket/workdir -- \\\n    python myscript.py --input input.txt --output output.txt\n\n  # Use named mounts\n  > pipen gbatch --workdir gs://my-bucket/workdir --mount INFILE=gs://bucket/path/to/file \\\n    --mount OUTDIR=gs://bucket/path/to/outdir -- \\\n    cat $INFILE > $OUTDIR/output.txt\n  \u200b\n  # Run a command in a detached mode\n  > pipen gbatch --nowait --project $PROJECT --location $LOCATION \\\n    --workdir gs://my-bucket/workdir -- \\\n    python myscript.py --input input.txt --output output.txt\n  \u200b\n  # If you have a profile defined in ~/.pipen.toml or ./.pipen.toml\n  > pipen gbatch --profile myprofile -- \\\n    python myscript.py --input input.txt --output output.txt\n  \u200b\n  # View the logs of a previously run command\n  > pipen gbatch --view-logs all --name my-daemon-name \\\n    --workdir gs://my-bucket/workdir\n```\n\n## API\n\nThe API can also be used to run commands programmatically:\n\n```python\nimport asyncio\nfrom pipen_cli_gbatch import CliGbatchDaemon\n\npipe = CliGbatchDaemon(config_for_daemon, command)\nasyncio.run(pipe.run())\n```\n\nNote that the daemon pipeline will always be running without caching, so that the command will always be executed when the pipeline is run.\n',
     'author': 'pwwang',