gc-batch 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gc_batch-0.2.0/.gitignore +53 -0
- gc_batch-0.2.0/CHANGELOG.md +100 -0
- gc_batch-0.2.0/CONTRIBUTING.md +144 -0
- gc_batch-0.2.0/LICENSE +21 -0
- gc_batch-0.2.0/PKG-INFO +182 -0
- gc_batch-0.2.0/README.md +149 -0
- gc_batch-0.2.0/pyproject.toml +127 -0
- gc_batch-0.2.0/src/gc_batch/__init__.py +68 -0
- gc_batch-0.2.0/src/gc_batch/_version.py +1 -0
- gc_batch-0.2.0/src/gc_batch/batch_logging.py +470 -0
- gc_batch-0.2.0/src/gc_batch/client.py +953 -0
- gc_batch-0.2.0/src/gc_batch/constants.py +190 -0
- gc_batch-0.2.0/src/gc_batch/entrypoint.py +718 -0
- gc_batch-0.2.0/src/gc_batch/gcs_logging.py +550 -0
- gc_batch-0.2.0/src/gc_batch/google_utils.py +66 -0
- gc_batch-0.2.0/src/gc_batch/logger.py +143 -0
- gc_batch-0.2.0/src/gc_batch/models/__init__.py +28 -0
- gc_batch-0.2.0/src/gc_batch/models/batch_config.py +38 -0
- gc_batch-0.2.0/src/gc_batch/models/job_request.py +422 -0
- gc_batch-0.2.0/src/gc_batch/models/job_result.py +17 -0
- gc_batch-0.2.0/src/gc_batch/py.typed +0 -0
- gc_batch-0.2.0/src/gc_batch/settings.py +229 -0
- gc_batch-0.2.0/src/gc_batch/utils.py +205 -0
- gc_batch-0.2.0/tests/test_gc_batch/__init__.py +0 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_batch_logging.py +812 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_client.py +305 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_entrypoint_create_warnings.py +113 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_entrypoint_logs.py +187 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_gcs_logging.py +480 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_job_request.py +107 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_regressions.py +29 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_settings.py +210 -0
- gc_batch-0.2.0/tests/test_gc_batch/test_utils.py +298 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
|
|
4
|
+
# Environments
|
|
5
|
+
.env
|
|
6
|
+
.venv
|
|
7
|
+
.venv-licenses
|
|
8
|
+
env/
|
|
9
|
+
venv/
|
|
10
|
+
ENV/
|
|
11
|
+
env.bak/
|
|
12
|
+
venv.bak/
|
|
13
|
+
|
|
14
|
+
# mypy
|
|
15
|
+
.mypy_cache/
|
|
16
|
+
.dmypy.json
|
|
17
|
+
dmypy.json
|
|
18
|
+
|
|
19
|
+
#python
|
|
20
|
+
*.egg-info/
|
|
21
|
+
*.egg-info/*
|
|
22
|
+
|
|
23
|
+
# uv specific
|
|
24
|
+
.uv/
|
|
25
|
+
|
|
26
|
+
# macOS
|
|
27
|
+
.DS_Store
|
|
28
|
+
|
|
29
|
+
# Temporary folders
|
|
30
|
+
tmp/
|
|
31
|
+
temp/
|
|
32
|
+
|
|
33
|
+
# Editor directories and files
|
|
34
|
+
.vscode/
|
|
35
|
+
!.vscode/extensions.json
|
|
36
|
+
.idea
|
|
37
|
+
|
|
38
|
+
# Secrets
|
|
39
|
+
*.secret.env
|
|
40
|
+
|
|
41
|
+
# Version files
|
|
42
|
+
*_version.py
|
|
43
|
+
|
|
44
|
+
dist/
|
|
45
|
+
|
|
46
|
+
# Local git worktrees
|
|
47
|
+
.worktrees/
|
|
48
|
+
|
|
49
|
+
# mkdocs build output
|
|
50
|
+
site/
|
|
51
|
+
|
|
52
|
+
# Claude Code local settings
|
|
53
|
+
.claude/
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.2.0]
|
|
11
|
+
|
|
12
|
+
First public release.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- CLI for managing Google Cloud Batch jobs: `create`, `list-jobs`, `list-my-jobs`,
|
|
17
|
+
`status`, `logs`, `cancel`, and `show-config`.
|
|
18
|
+
- `gc-batch --version`, which the contributing and security guides ask bug
|
|
19
|
+
reporters to include.
|
|
20
|
+
- `GCBatchClient`, a Python client for the same operations, and
|
|
21
|
+
`BatchJobConfig.apply_profile`, which applies a `JobProfile`'s networking and VM
|
|
22
|
+
settings to a job config — what `create --job-profile` does under the hood, also
|
|
23
|
+
reachable directly from library code.
|
|
24
|
+
- GCS bucket mounts for job input, output, and logs, and local SSD storage.
|
|
25
|
+
- Automatic `job-name`, `created-using`, `created-at`, and `created-by` labels, plus
|
|
26
|
+
filtering by label, name, status, and time window. The `created-by` label is set
|
|
27
|
+
by `GCBatchClient.create_job`, so jobs submitted through the Python client carry
|
|
28
|
+
it too and are found by `list-my-jobs`; a `created-by` passed in
|
|
29
|
+
`JobRequest.labels` takes precedence over the environment-derived value.
|
|
30
|
+
`owner_email_env_vars` defaults to
|
|
31
|
+
`["OWNER_EMAIL", "WORKBENCH_USER_EMAIL", "TERRA_USER_EMAIL", "USER"]`, so
|
|
32
|
+
`created-by` (and therefore `list-my-jobs`) works inside the All of Us Researcher
|
|
33
|
+
Workbench, which sets the first three but not `$USER`; `$USER` remains the final
|
|
34
|
+
fallback elsewhere.
|
|
35
|
+
- `GCBatchSettings` (`gc_batch.settings`), resolving configuration from constructor
|
|
36
|
+
arguments, `GC_BATCH_*` environment variables, a TOML config file, and neutral
|
|
37
|
+
defaults, in that precedence order.
|
|
38
|
+
- Named job profiles via `--job-profile`, including a built-in `all-of-us` profile
|
|
39
|
+
for the restricted-VPC All of Us Researcher Workbench environment. `create` warns
|
|
40
|
+
when a job profile declares that Cloud Logging is unreadable and no
|
|
41
|
+
`--logs-bucket` was given — the case where a job's logs are written somewhere the
|
|
42
|
+
user cannot read them. The log destination is fixed when the job is created, so
|
|
43
|
+
the warning names the only remedy: recreate with `--logs-bucket`. The `all-of-us`
|
|
44
|
+
profile sets this `cloud_logging_unreadable` flag; it defaults to false, so
|
|
45
|
+
nothing changes for other profiles. The warning goes to stderr and does not block
|
|
46
|
+
job creation, and the `Cloud Logging URL` line that `create` prints on success is
|
|
47
|
+
annotated in the same case, since that URL will show nothing.
|
|
48
|
+
- Logs written to a GCS bucket are readable. `gc-batch logs print` and
|
|
49
|
+
`logs download` detect that a job was created with `--logs-bucket` and read the
|
|
50
|
+
log objects out of the bucket instead of querying Cloud Logging. That is the only
|
|
51
|
+
working log route where the caller has no Cloud Logging read access at all — for
|
|
52
|
+
example the All of Us Researcher Workbench, where the workspace pet service
|
|
53
|
+
account gets `403 Permission denied for all log views`. The bucket and per-job
|
|
54
|
+
prefix are recovered from the job resource, so nothing extra has to be passed.
|
|
55
|
+
Logs written with `--logs-bucket` land in a per-job subfolder beneath the given
|
|
56
|
+
path, `gs://<logs-bucket-path>/<generated-job-name>/`; the path printed by
|
|
57
|
+
`create` is the parent folder, not the one holding that job's files.
|
|
58
|
+
- `GCSLogReader` (`gc_batch.gcs_logging`) for reading those logs from Python. It
|
|
59
|
+
returns entries in the same shape as the Cloud Logging reader, preserves the
|
|
60
|
+
order Batch wrote them in, reads whole objects (no result cap, so long logs are
|
|
61
|
+
not silently truncated), decodes lossily so a job printing binary output cannot
|
|
62
|
+
break log retrieval, and honours a requester-pays billing project recorded on the
|
|
63
|
+
job. Logs read from a `--logs-bucket` job preserve output order: the interleaved
|
|
64
|
+
`output-` object is read instead of separate per-stream objects, so stderr lines
|
|
65
|
+
emitted mid-run keep their original position, and the Batch agent's own lines are
|
|
66
|
+
dropped by stream tag.
|
|
67
|
+
- `gc-batch logs print` and `logs download` exit 1 when log retrieval fails, so a
|
|
68
|
+
wrapper script can tell a 403 from a job that simply produced no output; a
|
|
69
|
+
successful read of zero entries exits 0 and reports `No log entries matched the
|
|
70
|
+
query`. For the same reason `BatchLogging._query_cloud_logs`, `print_logs_for_job`,
|
|
71
|
+
and `download_logs_for_job` raise on API errors rather than swallowing them, and
|
|
72
|
+
the two public methods return the number of entries handled;
|
|
73
|
+
`get_instance_id_from_logs` returns `None` when its query fails. An unrecognised
|
|
74
|
+
`--severity` value is rejected with a clear error when reading logs from a
|
|
75
|
+
bucket, instead of silently matching nothing.
|
|
76
|
+
- `gc-batch status` on a job that logs to a bucket points at the bucket instead of
|
|
77
|
+
offering a Cloud Logging console URL with nothing behind it, and `logs filter` /
|
|
78
|
+
`logs url` say plainly that their Cloud Logging query will match nothing for such
|
|
79
|
+
a job. `gc-batch status` on a failed job that produced no logs at all adds a
|
|
80
|
+
clearly labelled inference that the container never started and the image
|
|
81
|
+
probably could not be pulled, naming the image URI. The Batch API never states
|
|
82
|
+
this reason — the only failure text is "… with exit code 1" — so it is presented
|
|
83
|
+
as an inference, not as something Batch reported.
|
|
84
|
+
- `LICENSE` (MIT), `CONTRIBUTING.md`, `SECURITY.md`, and this changelog.
|
|
85
|
+
- CI running `ruff`, `mypy` (blocking), `pytest`, and a strict docs build across
|
|
86
|
+
Python 3.10-3.13.
|
|
87
|
+
|
|
88
|
+
### Documentation
|
|
89
|
+
|
|
90
|
+
- Documentation for using gc-batch as a Python library: a [Python API
|
|
91
|
+
guide](docs/guides/python-api.md) covering submission, polling, filtering, logs,
|
|
92
|
+
mounts, settings, job profiles, fan-out, and error handling, plus library
|
|
93
|
+
examples in the README and alongside the CLI example workflows.
|
|
94
|
+
- Troubleshooting covers the two ways log retrieval comes up empty: a job that
|
|
95
|
+
failed before its container started (no task logs and no GCS objects are written,
|
|
96
|
+
and the Batch API never states the reason), and `403 Permission denied for all log
|
|
97
|
+
views`, which is the normal state in restricted environments and is solved by
|
|
98
|
+
creating jobs with `--logs-bucket`. Includes how to tell an unreachable registry
|
|
99
|
+
from a bad image reference, the `/bin/bash` entrypoint requirement, and the
|
|
100
|
+
per-task log object naming.
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# Contributing to gc-batch
|
|
2
|
+
|
|
3
|
+
Thanks for your interest in contributing. This document covers how to get set up,
|
|
4
|
+
what the review expectations are, and how to propose changes. Participation in
|
|
5
|
+
this project is governed by our [Code of Conduct](CODE_OF_CONDUCT.md).
|
|
6
|
+
|
|
7
|
+
## Getting set up
|
|
8
|
+
|
|
9
|
+
`gc-batch` uses [uv](https://docs.astral.sh/uv/) for dependency and environment
|
|
10
|
+
management, and [just](https://github.com/casey/just) as a task runner.
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
git clone https://github.com/empirico-oss/gc-batch.git
|
|
14
|
+
cd gc-batch
|
|
15
|
+
uv sync
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Supported Python versions are 3.10 through 3.13.
|
|
19
|
+
|
|
20
|
+
## Development workflow
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
just test # run the test suite
|
|
24
|
+
just lint # ruff check, ruff format --check, mypy
|
|
25
|
+
just preview-docs # serve the documentation locally
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Before opening a pull request, make sure all of the following pass, since CI runs
|
|
29
|
+
the same checks:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
uv run pytest -vv tests
|
|
33
|
+
uv run ruff check .
|
|
34
|
+
uv run ruff format --check .
|
|
35
|
+
uv run mypy src
|
|
36
|
+
uv run --group docs mkdocs build --strict
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
These map to the jobs in `.github/workflows/ci.yaml`, which runs on every pull
|
|
40
|
+
request and on pushes to `main`: `lint` (ruff), `types` (mypy, blocking), `test`
|
|
41
|
+
(pytest on Python 3.10-3.13), `licenses` (runtime dependency licenses stay
|
|
42
|
+
MIT-compatible -- `just licenses`), and `docs` (`mkdocs build --strict`, so a
|
|
43
|
+
broken link or reference fails the build).
|
|
44
|
+
|
|
45
|
+
## Building and installing
|
|
46
|
+
|
|
47
|
+
Released versions are published to [PyPI](https://pypi.org/project/gc-batch/)
|
|
48
|
+
and install with `pip install gc-batch`. To test unreleased changes -- or before
|
|
49
|
+
the first release lands -- build a wheel locally:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
uv build # writes dist/gc_batch-<version>-{py3-none-any.whl,.tar.gz}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Then install it one of three ways:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install dist/gc_batch-<version>-py3-none-any.whl # built wheel
|
|
59
|
+
uv tool install --force --no-cache ./ # local checkout
|
|
60
|
+
pip install "gc-batch @ git+https://github.com/empirico-oss/gc-batch.git@main"
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Re-running the `uv tool install` line with `--force` is also how you update an
|
|
64
|
+
existing global install. Verify with `gc-batch --version`.
|
|
65
|
+
|
|
66
|
+
Documentation is published by `.github/workflows/mkdocs.yaml`, which tracks
|
|
67
|
+
releases rather than pushes to `main` -- see
|
|
68
|
+
[Deploying Documentation](docs/development/docs.md#deploying-documentation). The
|
|
69
|
+
`docs` CI job builds the site with `--strict` on every pull request.
|
|
70
|
+
|
|
71
|
+
## Releasing
|
|
72
|
+
|
|
73
|
+
`.github/workflows/release.yaml` publishes to PyPI when a GitHub Release is
|
|
74
|
+
published. It builds the sdist and wheel, verifies them, and uploads via PyPI
|
|
75
|
+
Trusted Publishing, so no API token is stored in the repository.
|
|
76
|
+
|
|
77
|
+
The release tag is the source of truth for the version: `uv-dynamic-versioning`
|
|
78
|
+
derives the version from git, and the workflow refuses to publish if the built
|
|
79
|
+
version does not match the tag. To cut a release:
|
|
80
|
+
|
|
81
|
+
1. Move the pending `CHANGELOG.md` entries under a new version heading.
|
|
82
|
+
2. Publish a GitHub Release tagged `v<version>` (for example `v0.1.0`).
|
|
83
|
+
3. Approve the `pypi` environment if it is configured to require review.
|
|
84
|
+
|
|
85
|
+
Dispatch the same workflow manually to publish to TestPyPI first. Run it from a
|
|
86
|
+
tag ref, not a branch: off-tag builds carry PEP 440 local version metadata
|
|
87
|
+
(`+branch.sha`), which no package index accepts.
|
|
88
|
+
|
|
89
|
+
Releasing also deploys documentation for that version. The two workflows trigger
|
|
90
|
+
deliberately differently -- the release workflow runs for any published release
|
|
91
|
+
*including* pre-releases, while the docs workflow runs only for full releases --
|
|
92
|
+
so a release candidate reaches PyPI without moving the `stable` docs alias.
|
|
93
|
+
|
|
94
|
+
## Code standards
|
|
95
|
+
|
|
96
|
+
- **Type hints are required** on public functions and methods. `mypy` runs over
|
|
97
|
+
`src/` in CI.
|
|
98
|
+
- **Formatting and linting** are handled by `ruff` (line length 100). Run
|
|
99
|
+
`uv run ruff format .` to apply formatting.
|
|
100
|
+
- **Docstrings** follow the Google style, since the API documentation is generated
|
|
101
|
+
from them by `mkdocstrings`.
|
|
102
|
+
- **No deployment-specific values in source.** Project IDs and network
|
|
103
|
+
configuration belong in settings (see below), never hardcoded. This is
|
|
104
|
+
enforced by review.
|
|
105
|
+
|
|
106
|
+
## Configuration, not hardcoding
|
|
107
|
+
|
|
108
|
+
Anything that differs between deployments is resolved through `GCBatchSettings`
|
|
109
|
+
in `src/gc_batch/settings.py`, with precedence:
|
|
110
|
+
|
|
111
|
+
1. explicit constructor argument
|
|
112
|
+
2. `GC_BATCH_*` environment variables
|
|
113
|
+
3. a TOML config file (`--config-file`, `$GC_BATCH_CONFIG_FILE`, `./gc-batch.toml`,
|
|
114
|
+
or `$XDG_CONFIG_HOME/gc-batch/config.toml`)
|
|
115
|
+
4. neutral defaults in code
|
|
116
|
+
|
|
117
|
+
If you are adding a value that varies by environment, add it as a settings field
|
|
118
|
+
with a sensible neutral default rather than a literal in the code path. See
|
|
119
|
+
[Configuration](docs/guides/configuration.md).
|
|
120
|
+
|
|
121
|
+
## Tests
|
|
122
|
+
|
|
123
|
+
- New logic needs tests. Bug fixes should include a test that fails before the fix.
|
|
124
|
+
- Tests live in `tests/` and mirror the `src/gc_batch/` layout.
|
|
125
|
+
- Avoid tests that require live Google Cloud credentials; mock the client
|
|
126
|
+
boundaries as the existing tests do.
|
|
127
|
+
|
|
128
|
+
## Pull requests
|
|
129
|
+
|
|
130
|
+
- Keep pull requests focused on a single concern.
|
|
131
|
+
- Describe the motivation, not just the change.
|
|
132
|
+
- Call out any breaking change to the CLI surface or the public Python API
|
|
133
|
+
explicitly in the description, and update `CHANGELOG.md`.
|
|
134
|
+
- Update the relevant documentation under `docs/` in the same pull request.
|
|
135
|
+
|
|
136
|
+
## Reporting bugs and requesting features
|
|
137
|
+
|
|
138
|
+
Open an issue at https://github.com/empirico-oss/gc-batch/issues. For bugs, please include
|
|
139
|
+
the `gc-batch` version, your Python version, the command you ran, and the full
|
|
140
|
+
error output. Do not include credentials, access tokens, internal project
|
|
141
|
+
identifiers, or participant data in issue reports.
|
|
142
|
+
|
|
143
|
+
For security issues, follow [SECURITY.md](SECURITY.md) instead of opening a
|
|
144
|
+
public issue.
|
gc_batch-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Empirico Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
gc_batch-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: gc-batch
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Command-line interface and Python client for managing Google Cloud Batch jobs
|
|
5
|
+
Project-URL: Homepage, https://github.com/empirico-oss/gc-batch
|
|
6
|
+
Project-URL: Documentation, https://empirico-oss.github.io/gc-batch/stable/
|
|
7
|
+
Project-URL: Repository, https://github.com/empirico-oss/gc-batch
|
|
8
|
+
Project-URL: Issues, https://github.com/empirico-oss/gc-batch/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/empirico-oss/gc-batch/blob/main/CHANGELOG.md
|
|
10
|
+
Author: Empirico Inc.
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: batch,bioinformatics,cli,google-cloud,google-cloud-batch
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Environment :: Console
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: <3.14.0,>=3.10.0
|
|
26
|
+
Requires-Dist: click>=8.2.1
|
|
27
|
+
Requires-Dist: google-cloud-batch>=0.17.36
|
|
28
|
+
Requires-Dist: google-cloud-logging>=3.12.1
|
|
29
|
+
Requires-Dist: google-cloud-storage>=3.2.0
|
|
30
|
+
Requires-Dist: pydantic-settings>=2.7.0
|
|
31
|
+
Requires-Dist: pydantic>=2.12.0
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# gc-batch
|
|
35
|
+
|
|
36
|
+
A command-line interface and Python client for managing Google Cloud Batch jobs,
|
|
37
|
+
with support for job creation, monitoring, logging, GCS bucket mounts, local SSD
|
|
38
|
+
storage, and filtering by labels.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install gc-batch
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Or, for development, using [uv](https://docs.astral.sh/uv/):
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
git clone https://github.com/empirico-oss/gc-batch.git
|
|
50
|
+
cd gc-batch
|
|
51
|
+
uv sync
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Requires Python 3.10 or newer, and credentials for a Google Cloud project with the
|
|
55
|
+
Batch API enabled (`gcloud auth application-default login`).
|
|
56
|
+
|
|
57
|
+
## Quick start
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# Create a job
|
|
61
|
+
gc-batch --project-id my-project create \
|
|
62
|
+
--job-name my-first-job \
|
|
63
|
+
--docker-image gcr.io/my-project/my-image:latest \
|
|
64
|
+
--command "python /app/main.py"
|
|
65
|
+
|
|
66
|
+
# List jobs, or just your own
|
|
67
|
+
gc-batch --project-id my-project list-jobs
|
|
68
|
+
gc-batch --project-id my-project list-my-jobs
|
|
69
|
+
|
|
70
|
+
# Check status and read logs
|
|
71
|
+
gc-batch --project-id my-project status --job-name my-first-job-1234567890
|
|
72
|
+
gc-batch --project-id my-project logs --job-name my-first-job-1234567890
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The project is resolved from `--project-id`, then `$GOOGLE_PROJECT`, then the
|
|
76
|
+
`default_project_id` setting. If none is set, the command exits with an error
|
|
77
|
+
rather than guessing.
|
|
78
|
+
|
|
79
|
+
## Use as a Python library
|
|
80
|
+
|
|
81
|
+
The same functionality is importable, so scripts, notebooks, and orchestration
|
|
82
|
+
code can submit and monitor jobs without shelling out to the CLI:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import time
|
|
86
|
+
|
|
87
|
+
from gc_batch import BatchClientConfig, BatchJobConfig, GCBatchClient, JobRequest
|
|
88
|
+
from gc_batch.utils import is_job_finished
|
|
89
|
+
|
|
90
|
+
client = GCBatchClient(BatchClientConfig(project_id="my-project", location="us-central1"))
|
|
91
|
+
|
|
92
|
+
job = client.create_job(
|
|
93
|
+
JobRequest(
|
|
94
|
+
job_name="my-first-job",
|
|
95
|
+
docker_image="gcr.io/my-project/my-image:latest",
|
|
96
|
+
command="python /app/main.py",
|
|
97
|
+
config=BatchJobConfig(
|
|
98
|
+
machine_type="n2-standard-4",
|
|
99
|
+
boot_disk_type="pd-balanced",
|
|
100
|
+
input_bucket="my-bucket/datasets",
|
|
101
|
+
output_bucket="my-bucket/results",
|
|
102
|
+
),
|
|
103
|
+
labels={"team": "data-science"},
|
|
104
|
+
)
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# create_job timestamps the name, so read the submitted one off the job
|
|
108
|
+
short_name = job.name.split("/")[-1]
|
|
109
|
+
|
|
110
|
+
while not is_job_finished(job := client.get_job(short_name)):
|
|
111
|
+
time.sleep(30)
|
|
112
|
+
|
|
113
|
+
print(job.status.state.name)
|
|
114
|
+
if job.status.state.name == "FAILED":
|
|
115
|
+
print(client.get_failure_message(job))
|
|
116
|
+
print(client.get_cloud_logging_url(job, severity="ERROR"))
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Listing, cancelling, and log retrieval are all available too:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
jobs = client.list_jobs(labels={"team": "data-science"}, status="RUNNING")
|
|
123
|
+
client.cancel_job(job.name) # full resource path, not the short name
|
|
124
|
+
|
|
125
|
+
# Logs, from Cloud Logging or from the bucket when the job used logs_bucket
|
|
126
|
+
client.batch_logging.print_logs_for_job(job)
|
|
127
|
+
client.gcs_logging.print_logs_for_job(job)
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
See the [Python API guide](docs/guides/python-api.md) for job profiles, mounts,
|
|
131
|
+
local SSD, settings, fan-out, and error handling.
|
|
132
|
+
|
|
133
|
+
## Configuration
|
|
134
|
+
|
|
135
|
+
Values that differ between deployments — the job-name prefix, the
|
|
136
|
+
`created-using` label, and named job profiles — are resolved in this order,
|
|
137
|
+
highest precedence first:
|
|
138
|
+
|
|
139
|
+
1. explicit arguments
|
|
140
|
+
2. `GC_BATCH_*` environment variables
|
|
141
|
+
3. a TOML config file (`--config-file`, `$GC_BATCH_CONFIG_FILE`, `./gc-batch.toml`,
|
|
142
|
+
or `$XDG_CONFIG_HOME/gc-batch/config.toml`)
|
|
143
|
+
4. neutral defaults
|
|
144
|
+
|
|
145
|
+
### Restricted VPC environments
|
|
146
|
+
|
|
147
|
+
`gc-batch` ships a built-in `all-of-us` job profile for the
|
|
148
|
+
[All of Us](https://allofus.nih.gov/) Researcher Workbench, which runs jobs inside
|
|
149
|
+
a restricted VPC:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
gc-batch --project-id my-project create \
|
|
153
|
+
--job-profile all-of-us \
|
|
154
|
+
--job-name my-job \
|
|
155
|
+
--docker-image gcr.io/my-project/my-image:latest \
|
|
156
|
+
--command "python /app/main.py"
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
You can define additional profiles in the config file. See the
|
|
160
|
+
[Configuration guide](docs/guides/configuration.md).
|
|
161
|
+
|
|
162
|
+
## Documentation
|
|
163
|
+
|
|
164
|
+
Full documentation: https://empirico-oss.github.io/gc-batch/stable/
|
|
165
|
+
|
|
166
|
+
- [Quick Start](docs/guides/quick-start.md)
|
|
167
|
+
- [Commands Reference](docs/guides/commands.md)
|
|
168
|
+
- [Python API](docs/guides/python-api.md)
|
|
169
|
+
- [Input/Output Mounts](docs/guides/mounts.md)
|
|
170
|
+
- [Local SSD Storage](docs/guides/local-ssd.md)
|
|
171
|
+
- [Label Management](docs/guides/labels.md)
|
|
172
|
+
- [Configuration](docs/guides/configuration.md)
|
|
173
|
+
- [Troubleshooting](docs/guides/troubleshooting.md)
|
|
174
|
+
|
|
175
|
+
## Contributing
|
|
176
|
+
|
|
177
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Security issues should be reported per
|
|
178
|
+
[SECURITY.md](SECURITY.md).
|
|
179
|
+
|
|
180
|
+
## License
|
|
181
|
+
|
|
182
|
+
MIT — see [LICENSE](LICENSE).
|
gc_batch-0.2.0/README.md
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# gc-batch
|
|
2
|
+
|
|
3
|
+
A command-line interface and Python client for managing Google Cloud Batch jobs,
|
|
4
|
+
with support for job creation, monitoring, logging, GCS bucket mounts, local SSD
|
|
5
|
+
storage, and filtering by labels.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install gc-batch
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Or, for development, using [uv](https://docs.astral.sh/uv/):
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
git clone https://github.com/empirico-oss/gc-batch.git
|
|
17
|
+
cd gc-batch
|
|
18
|
+
uv sync
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Requires Python 3.10 or newer, and credentials for a Google Cloud project with the
|
|
22
|
+
Batch API enabled (`gcloud auth application-default login`).
|
|
23
|
+
|
|
24
|
+
## Quick start
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# Create a job
|
|
28
|
+
gc-batch --project-id my-project create \
|
|
29
|
+
--job-name my-first-job \
|
|
30
|
+
--docker-image gcr.io/my-project/my-image:latest \
|
|
31
|
+
--command "python /app/main.py"
|
|
32
|
+
|
|
33
|
+
# List jobs, or just your own
|
|
34
|
+
gc-batch --project-id my-project list-jobs
|
|
35
|
+
gc-batch --project-id my-project list-my-jobs
|
|
36
|
+
|
|
37
|
+
# Check status and read logs
|
|
38
|
+
gc-batch --project-id my-project status --job-name my-first-job-1234567890
|
|
39
|
+
gc-batch --project-id my-project logs --job-name my-first-job-1234567890
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The project is resolved from `--project-id`, then `$GOOGLE_PROJECT`, then the
|
|
43
|
+
`default_project_id` setting. If none is set, the command exits with an error
|
|
44
|
+
rather than guessing.
|
|
45
|
+
|
|
46
|
+
## Use as a Python library
|
|
47
|
+
|
|
48
|
+
The same functionality is importable, so scripts, notebooks, and orchestration
|
|
49
|
+
code can submit and monitor jobs without shelling out to the CLI:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import time
|
|
53
|
+
|
|
54
|
+
from gc_batch import BatchClientConfig, BatchJobConfig, GCBatchClient, JobRequest
|
|
55
|
+
from gc_batch.utils import is_job_finished
|
|
56
|
+
|
|
57
|
+
client = GCBatchClient(BatchClientConfig(project_id="my-project", location="us-central1"))
|
|
58
|
+
|
|
59
|
+
job = client.create_job(
|
|
60
|
+
JobRequest(
|
|
61
|
+
job_name="my-first-job",
|
|
62
|
+
docker_image="gcr.io/my-project/my-image:latest",
|
|
63
|
+
command="python /app/main.py",
|
|
64
|
+
config=BatchJobConfig(
|
|
65
|
+
machine_type="n2-standard-4",
|
|
66
|
+
boot_disk_type="pd-balanced",
|
|
67
|
+
input_bucket="my-bucket/datasets",
|
|
68
|
+
output_bucket="my-bucket/results",
|
|
69
|
+
),
|
|
70
|
+
labels={"team": "data-science"},
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
# create_job timestamps the name, so read the submitted one off the job
|
|
75
|
+
short_name = job.name.split("/")[-1]
|
|
76
|
+
|
|
77
|
+
while not is_job_finished(job := client.get_job(short_name)):
|
|
78
|
+
time.sleep(30)
|
|
79
|
+
|
|
80
|
+
print(job.status.state.name)
|
|
81
|
+
if job.status.state.name == "FAILED":
|
|
82
|
+
print(client.get_failure_message(job))
|
|
83
|
+
print(client.get_cloud_logging_url(job, severity="ERROR"))
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Listing, cancelling, and log retrieval are all available too:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
jobs = client.list_jobs(labels={"team": "data-science"}, status="RUNNING")
|
|
90
|
+
client.cancel_job(job.name) # full resource path, not the short name
|
|
91
|
+
|
|
92
|
+
# Logs, from Cloud Logging or from the bucket when the job used logs_bucket
|
|
93
|
+
client.batch_logging.print_logs_for_job(job)
|
|
94
|
+
client.gcs_logging.print_logs_for_job(job)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
See the [Python API guide](docs/guides/python-api.md) for job profiles, mounts,
|
|
98
|
+
local SSD, settings, fan-out, and error handling.
|
|
99
|
+
|
|
100
|
+
## Configuration
|
|
101
|
+
|
|
102
|
+
Values that differ between deployments — the job-name prefix, the
|
|
103
|
+
`created-using` label, and named job profiles — are resolved in this order,
|
|
104
|
+
highest precedence first:
|
|
105
|
+
|
|
106
|
+
1. explicit arguments
|
|
107
|
+
2. `GC_BATCH_*` environment variables
|
|
108
|
+
3. a TOML config file (`--config-file`, `$GC_BATCH_CONFIG_FILE`, `./gc-batch.toml`,
|
|
109
|
+
or `$XDG_CONFIG_HOME/gc-batch/config.toml`)
|
|
110
|
+
4. neutral defaults
|
|
111
|
+
|
|
112
|
+
### Restricted VPC environments
|
|
113
|
+
|
|
114
|
+
`gc-batch` ships a built-in `all-of-us` job profile for the
|
|
115
|
+
[All of Us](https://allofus.nih.gov/) Researcher Workbench, which runs jobs inside
|
|
116
|
+
a restricted VPC:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
gc-batch --project-id my-project create \
|
|
120
|
+
--job-profile all-of-us \
|
|
121
|
+
--job-name my-job \
|
|
122
|
+
--docker-image gcr.io/my-project/my-image:latest \
|
|
123
|
+
--command "python /app/main.py"
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
You can define additional profiles in the config file. See the
|
|
127
|
+
[Configuration guide](docs/guides/configuration.md).
|
|
128
|
+
|
|
129
|
+
## Documentation
|
|
130
|
+
|
|
131
|
+
Full documentation: https://empirico-oss.github.io/gc-batch/stable/
|
|
132
|
+
|
|
133
|
+
- [Quick Start](docs/guides/quick-start.md)
|
|
134
|
+
- [Commands Reference](docs/guides/commands.md)
|
|
135
|
+
- [Python API](docs/guides/python-api.md)
|
|
136
|
+
- [Input/Output Mounts](docs/guides/mounts.md)
|
|
137
|
+
- [Local SSD Storage](docs/guides/local-ssd.md)
|
|
138
|
+
- [Label Management](docs/guides/labels.md)
|
|
139
|
+
- [Configuration](docs/guides/configuration.md)
|
|
140
|
+
- [Troubleshooting](docs/guides/troubleshooting.md)
|
|
141
|
+
|
|
142
|
+
## Contributing
|
|
143
|
+
|
|
144
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md). Security issues should be reported per
|
|
145
|
+
[SECURITY.md](SECURITY.md).
|
|
146
|
+
|
|
147
|
+
## License
|
|
148
|
+
|
|
149
|
+
MIT — see [LICENSE](LICENSE).
|