pipen-cli-gbatch 0.0.2__tar.gz → 0.0.4__tar.gz

{pipen_cli_gbatch-0.0.2 → pipen_cli_gbatch-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: pipen-cli-gbatch
-Version: 0.0.2
+Version: 0.0.4
 Summary: A pipen cli plugin to run command via Google Cloud Batch
 License: MIT
 Author: pwwang
@@ -13,9 +13,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: google-cloud-storage (>=3.0.0,<4.0.0)
 Requires-Dist: pipen (>=0.17.19,<0.18.0)
 Requires-Dist: pipen-poplog (>=0.3.6,<0.4.0)
+Project-URL: Homepage, https://github.com/pwwang/pipen-cli-gbatch
+Project-URL: Repository, https://github.com/pwwang/pipen-cli-gbatch
 Description-Content-Type: text/markdown
 
 # pipen-cli-gbatch

{pipen_cli_gbatch-0.0.2 → pipen_cli_gbatch-0.0.4}/pipen_cli_gbatch/__init__.py

@@ -79,7 +79,7 @@ from pipen.cli import CLIPlugin
 from pipen.scheduler import GbatchScheduler
 from pipen_poplog import LogsPopulator
 
-__version__ = "0.0.2"
+__version__ = "0.0.4"
 __all__ = ("CliGbatchPlugin", "CliGbatchDaemon")
 
 
@@ -117,6 +117,11 @@ class CliGbatchDaemon:
 
         self.config.prescript = self.config.get("prescript", None) or ""
         self.config.postscript = self.config.get("postscript", None) or ""
+        if "labels" in self.config and isinstance(self.config.labels, list):
+            self.config.labels = {
+                key: val
+                for key, val in (item.split("=", 1) for item in self.config.labels)
+            }
         self.command = command
 
     def _get_arg_from_command(self, arg: str) -> str | None:
@@ -183,7 +188,7 @@ class CliGbatchDaemon:
         """
         mount = self.config.get("mount", [])
         # mount the workdir
-        mount.append(f'{source}:{target}')
+        mount.append(f"{source}:{target}")
 
         self.config["mount"] = mount
 
@@ -197,7 +202,7 @@ class CliGbatchDaemon:
             SystemExit: If workdir is not a valid Google Storage bucket path.
         """
         command_workdir = self._get_arg_from_command("workdir")
-        workdir = self.config.get("workdir", command_workdir)
+        workdir = self.config.get("workdir", None) or command_workdir
 
         if not workdir or not isinstance(AnyPath(workdir), GSPath):
             print(
@@ -312,7 +317,7 @@ class CliGbatchDaemon:
 
     def _show_scheduler_opts(self):
         """Log the scheduler options for debugging purposes."""
-        logger.debug("Scheduler Options:")
+        logger.info("Scheduler Options:")
         for key, val in self.config.items():
             if key in (
                 "workdir",
@@ -332,7 +337,7 @@ class CliGbatchDaemon:
             ):
                 continue
 
-            logger.debug(f"- {key}: {val}")
+            logger.info(f"- {key}: {val}")
 
     async def _run_wait(self):  # pragma: no cover
         """Run the pipeline and wait for completion.
@@ -493,7 +498,7 @@ class CliGbatchDaemon:
                 )
                 sys.exit(1)
 
-            if 'name' not in self.config:
+            if "name" not in self.config or not self.config.name:
                 self.config["name"] = "PipenCliGbatchDaemon"
 
     async def run(self):  # pragma: no cover
@@ -553,6 +558,17 @@ class XquteCliGbatchPlugin:  # pragma: no cover
             logger.error(f"/STDERR {self.stderr_populator.residue}")
             self.stderr_populator.residue = ""
 
+    @plugin.impl
+    async def on_job_init(self, scheduler, job):
+        """Handle job initialization event.
+
+        Args:
+            scheduler: The scheduler instance.
+            job: The job being initialized.
+        """
+        self.stdout_populator.logfile = scheduler.workdir.joinpath("0", "job.stdout")
+        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
+
     @plugin.impl
     async def on_job_started(self, scheduler, job):
         """Handle job start event by setting up log file paths.
@@ -564,8 +580,6 @@ class XquteCliGbatchPlugin:  # pragma: no cover
         if not self.log_start:
             return
 
-        self.stdout_populator.logfile = scheduler.workdir.joinpath("0", "job.stdout")
-        self.stderr_populator.logfile = scheduler.workdir.joinpath("0", "job.stderr")
         logger.info("Job is picked up by Google Batch, pulling stdout/stderr...")
 
     @plugin.impl
@@ -693,29 +707,29 @@ class CliGbatchPlugin(CLIPlugin):  # pragma: no cover
         super().__init__(parser, subparser)
         subparser.epilog = """\033[1;4mExamples\033[0m:
 
-  \u200B
+  \u200b
   # Run a command and wait for it to complete
   > pipen gbatch --workdir gs://my-bucket/workdir -- \\
     python myscript.py --input input.txt --output output.txt
 
-  \u200B
+  \u200b
   # Use named mounts
   > pipen gbatch --workdir gs://my-bucket/workdir --mount INFILE=gs://bucket/path/to/file \\
     --mount OUTDIR=gs://bucket/path/to/outdir -- \\
     bash -c 'cat $INFILE > $OUTDIR/output.txt'
 
-  \u200B
+  \u200b
   # Run a command in a detached mode
   > pipen gbatch --nowait --project $PROJECT --location $LOCATION \\
     --workdir gs://my-bucket/workdir -- \\
     python myscript.py --input input.txt --output output.txt
 
-  \u200B
+  \u200b
   # If you have a profile defined in ~/.pipen.toml or ./.pipen.toml
   > pipen gbatch --profile myprofile -- \\
     python myscript.py --input input.txt --output output.txt
 
-  \u200B
+  \u200b
   # View the logs of a previously run command
   > pipen gbatch --view-logs all --name my-daemon-name \\
     --workdir gs://my-bucket/workdir
@@ -753,12 +767,36 @@ class CliGbatchPlugin(CLIPlugin):  # pragma: no cover
             CONFIG_FILES,
             known_parsed.profile,
         )
+
+        def is_valid(val: Any) -> bool:
+            """Check if a value is valid (not None, not empty string, not empty list).
+            """
+            if val is None:
+                return False
+            if isinstance(val, bool):
+                return True
+            return bool(val)
+
         # update parsed with the defaults
         for key, val in defaults.items():
+            if (
+                key == "mount"
+                and val
+                and getattr(known_parsed, key, None)
+            ):
+                if not isinstance(val, (tuple, list)):
+                    val = [val]
+                val = list(val)
+
+                kp_mount = getattr(known_parsed, key)
+                val.extend(kp_mount)
+                setattr(known_parsed, key, val)
+                continue
+
             if (
                 key == "command"
                 or val is None
-                or getattr(known_parsed, key, None) is not None
+                or is_valid(getattr(known_parsed, key, None))
             ):
                 continue
 

{pipen_cli_gbatch-0.0.2 → pipen_cli_gbatch-0.0.4}/pipen_cli_gbatch/daemon_args.toml

@@ -120,7 +120,7 @@ help = "The location to run the job."
 flags = ["--mount"]
 # type = "list"
 default = []
-action = "clear_append"
+action = "append"
 help = """The list of mounts to mount to the VM, each in the format of SOURCE:TARGET, where SOURCE must be either a Google Storage Bucket path (gs://...).
 You can also use named mounts like `INDIR=gs://my-bucket/inputs` and the directory will be mounted to `/mnt/disks/INDIR` in the VM;
 then you can use environment variable `$INDIR` in the command/script to refer to the mounted path.
@@ -186,20 +186,21 @@ and positive values mean to run after the main command."""
 [[groups.arguments]]
 flags = ["--allocationPolicy"]
 type = "json"
-default = "{}"
+default = {}
 help = "The JSON string of extra settings of allocationPolicy add to the job.json. Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#AllocationPolicy for details."
 
 [[groups.arguments]]
 flags = ["--taskGroups"]
 type = "json"
-default = "[]"
+default = []
 help = "The JSON string of extra settings of taskGroups add to the job.json. Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#TaskGroup for details."
 
 [[groups.arguments]]
 flags = ["--labels"]
-type = "json"
-default = "{}"
-help = "The JSON string of labels to add to the job. Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Job.FIELDS.labels for details."
+# type = "json"
+default = []
+action = "clear_append"
+help = "The strings of labels to add to the job (key=value). Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Job.FIELDS.labels for details."
 
 [[groups.arguments]]
 flags = ["--gcloud"]

{pipen_cli_gbatch-0.0.2 → pipen_cli_gbatch-0.0.4}/pyproject.toml

@@ -1,18 +1,21 @@
-[project]
+[tool.poetry]
 name = "pipen-cli-gbatch"
-version = "0.0.2"
+version = "0.0.4"
 description = "A pipen cli plugin to run command via Google Cloud Batch"
-authors = [
-    {name = "pwwang",email = "pwwang@pwwang.com"}
-]
-license = {text = "MIT"}
+authors = ["pwwang <pwwang@pwwang.com>"]
+license = "MIT"
 readme = "README.md"
-requires-python = ">=3.9,<4.0"
-dependencies = [
-    "pipen (>=0.17.19,<0.18.0)",
-    "pipen-poplog (>=0.3.6,<0.4.0)",
-    "google-cloud-storage (>=3.0.0,<4.0.0)"
-]
+homepage = "https://github.com/pwwang/pipen-cli-gbatch"
+repository = "https://github.com/pwwang/pipen-cli-gbatch"
+
+[tool.poetry.build]
+generate-setup-file = true
+
+[tool.poetry.dependencies]
+python = "^3.9"
+pipen = "^0.17.19"
+pipen-poplog = "^0.3.6"
+google-cloud-storage = "^3.0.0"
 
 [tool.pytest.ini_options]
 addopts = "-vv -n auto --dist loadgroup -p no:benchmark -W error::UserWarning --cov-config=.coveragerc --cov=pipen_cli_gbatch --cov-report xml:.coverage.xml --cov-report term-missing"

pipen_cli_gbatch-0.0.4/setup.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+packages = \
+['pipen_cli_gbatch']
+
+package_data = \
+{'': ['*']}
+
+install_requires = \
+['google-cloud-storage>=3.0.0,<4.0.0',
+ 'pipen-poplog>=0.3.6,<0.4.0',
+ 'pipen>=0.17.19,<0.18.0']
+
+entry_points = \
+{'pipen_cli': ['cli-gbatch = pipen_cli_gbatch:CliGbatchPlugin']}
+
+setup_kwargs = {
+    'name': 'pipen-cli-gbatch',
+    'version': '0.0.4',
+    'description': 'A pipen cli plugin to run command via Google Cloud Batch',
+    'long_description': '# pipen-cli-gbatch\n\nA pipen CLI plugin to run commands via Google Cloud Batch.\n\nThe idea is to submit the command using xqute and use the gbatch scheduler to run it on Google Cloud Batch.\n\n## Installation\n\n```bash\npip install pipen-cli-gbatch\n```\n\n## Usage\n\n### Basic Command Execution\n\nTo run a command like:\n\n```bash\npython myscript.py --input input.txt --output output.txt\n```\n\nYou can run it with:\n\n```bash\npipen gbatch -- python myscript.py --input input.txt --output output.txt\n```\n\n### With Configuration File\n\nIn order to provide configurations like we do for a normal pipen pipeline, you can also provide a config file (the `[pipen-cli-gbatch]` section will be used):\n\n```bash\npipen gbatch @config.toml -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\n### Detached Mode\n\nWe can also use the `--nowait` option to run the command in a detached mode:\n\n```bash\npipen gbatch --nowait -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\nOr by default, it will wait for the command to complete:\n\n```bash\npipen gbatch -- \\\n    python myscript.py --input input.txt --output output.txt\n```\n\nWhile waiting, the running logs will be pulled and shown in the terminal.\n\n### View Logs\n\nWhen running in detached mode, one can also pull the logs later by:\n\n```bash\npipen gbatch --view-logs -- \\\n    python myscript.py --input input.txt --output output.txt\n\n# or  just provide the workdir\npipen gbatch --view-logs --workdir gs://my-bucket/workdir\n```\n\n## Configuration\n\nBecause the daemon pipeline is running on Google Cloud Batch, a Google Storage Bucket path is required for the workdir. For example: `gs://my-bucket/workdir`\n\nA unique job ID will be generated per the name (`--name`) and workdir, so that if the same command is run again with the same name and workdir, it will not start a new job, but just attach to the existing job and pull the logs.\n\nIf `--name` is not provided in the command line, it will try to grab the name (`--name`) from the command line arguments after `--`, or else use "name" from the root section of the configuration file, with a "GbatchDaemon" suffix. If nothing can be found, a default name "PipenGbatchDaemon" will be used.\n\nThen a workdir `{workdir}/<daemon pipeline name>/` will be created to store the meta information.\n\nWith `--profile` provided, the scheduler options (`scheduler_opts`) defined in `~/.pipen.toml` and `./.pipen.toml` will be used as default.\n\n## All Options\n\n```bash\n> pipen gbatch --help\nUsage: pipen gbatch [-h] [--nowait | --view-logs {all,stdout,stderr}] [--workdir WORKDIR]\n                    [--error-strategy {retry,halt}] [--num-retries NUM_RETRIES] [--prescript PRESCRIPT]\n                    [--postscript POSTSCRIPT] [--jobname-prefix JOBNAME_PREFIX] [--recheck-interval RECHECK_INTERVAL]\n                    [--cwd CWD] [--project PROJECT] [--location LOCATION] [--mount MOUNT]\n                    [--service-account SERVICE_ACCOUNT] [--network NETWORK] [--subnetwork SUBNETWORK]\n                    [--no-external-ip-address] [--machine-type MACHINE_TYPE] [--provisioning-model {STANDARD,SPOT}]\n                    [--image-uri IMAGE_URI] [--entrypoint ENTRYPOINT] [--commands COMMANDS] [--runnables RUNNABLES]\n                    [--allocationPolicy ALLOCATIONPOLICY] [--taskGroups TASKGROUPS] [--labels LABELS] [--gcloud GCLOUD]\n                    [--name NAME] [--profile PROFILE] [--version]\n                    [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL,debug,info,warning,error,critical}]\n                    ...\n\nSimplify running commands via Google Cloud Batch.\n\nKey Options:\n  The key options to run the command.\n\n  --workdir WORKDIR     The workdir (a Google Storage Bucket path is required) to store the meta information of the\n                        daemon pipeline.\n                        If not provided, the one from the command will be used.\n  command               The command passed after `--` to run, with all its arguments. Note that the command should be\n                        provided after `--`.\n\nScheduler Options:\n  The options to configure the gbatch scheduler.\n\n  --error-strategy {retry,halt}\n                        The strategy when there is error happened [default: halt]\n  --num-retries NUM_RETRIES\n                        The number of retries when there is error happened. Only valid when --error-strategy is \'retry\'.\n                        [default: 0]\n  --prescript PRESCRIPT\n                        The prescript to run before the main command.\n  --postscript POSTSCRIPT\n                        The postscript to run after the main command.\n  --jobname-prefix JOBNAME_PREFIX\n                        The prefix of the name prefix of the daemon job.\n                        If not provided, try to generate one from the command to run.\n                        If the command is also not provided, use \'pipen-gbatch-daemon\' as the prefix.\n  --recheck-interval RECHECK_INTERVAL\n                        The interval to recheck the job status, each takes about 0.1 seconds. [default: 600]\n  --cwd CWD             The working directory to run the command. If not provided, the current directory is used. You\n                        can pass either a mounted path (inside the VM) or a Google Storage Bucket path (gs://...). If a\n                        Google Storage Bucket path is provided, the mounted path will be inferred from the mounted paths\n                        of the VM.\n  --project PROJECT     The Google Cloud project to run the job.\n  --location LOCATION   The location to run the job.\n  --mount MOUNT         The list of mounts to mount to the VM, each in the format of SOURCE:TARGET, where SOURCE must be\n                        either a Google Storage Bucket path (gs://...). [default: []]\n  --service-account SERVICE_ACCOUNT\n                        The service account to run the job.\n  --network NETWORK     The network to run the job.\n  --subnetwork SUBNETWORK\n                        The subnetwork to run the job.\n  --no-external-ip-address\n                        Whether to disable external IP address for the VM.\n  --machine-type MACHINE_TYPE\n                        The machine type of the VM.\n  --provisioning-model {STANDARD,SPOT}\n                        The provisioning model of the VM.\n  --image-uri IMAGE_URI\n                        The custom image URI of the VM.\n  --entrypoint ENTRYPOINT\n                        The entry point of the container to run the command.\n  --commands COMMANDS   The list of commands to run in the container, each as a separate string. [default: []]\n  --runnables RUNNABLES\n                        The JSON string of extra settings of runnables add to the job.json.\n                        Refer to https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Runnable\n                        for details.\n                        You can have an extra key \'order\' for each runnable, where negative values mean to run before\n                        the main command,\n                        and positive values mean to run after the main command.\n  --allocationPolicy ALLOCATIONPOLICY\n                        The JSON string of extra settings of allocationPolicy add to the job.json. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#AllocationPolicy\n                        for details. [default: {}]\n  --taskGroups TASKGROUPS\n                        The JSON string of extra settings of taskGroups add to the job.json. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#TaskGroup for\n                        details. [default: []]\n  --labels LABELS       The JSON string of labels to add to the job. Refer to\n                        https://cloud.google.com/batch/docs/reference/rest/v1/projects.locations.jobs#Job.FIELDS.labels\n                        for details. [default: {}]\n  --gcloud GCLOUD       The path to the gcloud command. [default: gcloud]\n\nOptions:\n  -h, --help            show this help message and exit\n  --nowait              Run the command in a detached mode without waiting for its completion. [default: False]\n  --view-logs {all,stdout,stderr}\n                        View the logs of a job.\n  --name NAME           The name of the daemon pipeline.\n                        If not provided, try to generate one from the command to run.\n                        If the command is also not provided, use \'PipenCliGbatchDaemon\' as the name.\n  --profile PROFILE     Use the `scheduler_opts` as the Scheduler Options of a given profile from pipen configuration\n                        files,\n                        including ~/.pipen.toml and ./pipen.toml.\n                        Note that if not provided, nothing will be loaded from the configuration files.\n  --version             Show the version of the pipen-cli-gbatch package. [default: False]\n  --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL,debug,info,warning,error,critical}\n                        Set the logging level for the daemon process. [default: INFO]\n\nExamples:\n  \u200b\n  # Run a command and wait for it to complete\n  > pipen gbatch --workdir gs://my-bucket/workdir -- \\\n    python myscript.py --input input.txt --output output.txt\n\n  # Use named mounts\n  > pipen gbatch --workdir gs://my-bucket/workdir --mount INFILE=gs://bucket/path/to/file \\\n    --mount OUTDIR=gs://bucket/path/to/outdir -- \\\n    cat $INFILE > $OUTDIR/output.txt\n  \u200b\n  # Run a command in a detached mode\n  > pipen gbatch --nowait --project $PROJECT --location $LOCATION \\\n    --workdir gs://my-bucket/workdir -- \\\n    python myscript.py --input input.txt --output output.txt\n  \u200b\n  # If you have a profile defined in ~/.pipen.toml or ./.pipen.toml\n  > pipen gbatch --profile myprofile -- \\\n    python myscript.py --input input.txt --output output.txt\n  \u200b\n  # View the logs of a previously run command\n  > pipen gbatch --view-logs all --name my-daemon-name \\\n    --workdir gs://my-bucket/workdir\n```\n\n## API\n\nThe API can also be used to run commands programmatically:\n\n```python\nimport asyncio\nfrom pipen_cli_gbatch import CliGbatchDaemon\n\npipe = CliGbatchDaemon(config_for_daemon, command)\nasyncio.run(pipe.run())\n```\n\nNote that the daemon pipeline will always be running without caching, so that the command will always be executed when the pipeline is run.\n',
+    'author': 'pwwang',
+    'author_email': 'pwwang@pwwang.com',
+    'maintainer': 'None',
+    'maintainer_email': 'None',
+    'url': 'https://github.com/pwwang/pipen-cli-gbatch',
+    'packages': packages,
+    'package_data': package_data,
+    'install_requires': install_requires,
+    'entry_points': entry_points,
+    'python_requires': '>=3.9,<4.0',
+}
+
+
+setup(**setup_kwargs)