prefect-deployments-toolkit 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect_deployments_toolkit-0.0.1/.github/workflows/cd.yml +68 -0
- prefect_deployments_toolkit-0.0.1/.github/workflows/detect-and-tag-new-version.yml +56 -0
- prefect_deployments_toolkit-0.0.1/CHANGELOG.md +23 -0
- prefect_deployments_toolkit-0.0.1/LICENSE +132 -0
- prefect_deployments_toolkit-0.0.1/PKG-INFO +114 -0
- prefect_deployments_toolkit-0.0.1/README.md +90 -0
- prefect_deployments_toolkit-0.0.1/pyproject.toml +40 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/__main__.py +145 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/deployment.py +250 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/deployment_loader.py +139 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/get_modified_deployments.py +110 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/log_buffer.py +112 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/prefect_api.py +291 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/prefect_cli.py +56 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/prefect_rest.py +286 -0
- prefect_deployments_toolkit-0.0.1/src/prefect_deployments_toolkit/yaml_utils.py +156 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
inputs:
|
|
6
|
+
version:
|
|
7
|
+
description: "The release version."
|
|
8
|
+
required: true
|
|
9
|
+
default: "v0.x.x"
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build-distribution:
|
|
13
|
+
name: Build distribution 📦
|
|
14
|
+
timeout-minutes: 5
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v6
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v6
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.13"
|
|
22
|
+
- name: Install pypa/build
|
|
23
|
+
run: python3 -m pip install build --user
|
|
24
|
+
- name: Build a binary wheel and a source tarball
|
|
25
|
+
run: python3 -m build
|
|
26
|
+
- name: Store the distribution packages
|
|
27
|
+
uses: actions/upload-artifact@v7
|
|
28
|
+
with:
|
|
29
|
+
name: python-package-distributions
|
|
30
|
+
path: dist/
|
|
31
|
+
|
|
32
|
+
publish-to-pypi:
|
|
33
|
+
name: Publish to PyPI 🚀
|
|
34
|
+
timeout-minutes: 5
|
|
35
|
+
needs:
|
|
36
|
+
- build-distribution
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
environment:
|
|
39
|
+
name: release
|
|
40
|
+
url: https://pypi.org/p/prefect-deployments-toolkit
|
|
41
|
+
permissions:
|
|
42
|
+
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
|
|
43
|
+
steps:
|
|
44
|
+
- name: Download all the dists
|
|
45
|
+
uses: actions/download-artifact@v8
|
|
46
|
+
with:
|
|
47
|
+
name: python-package-distributions
|
|
48
|
+
path: dist/
|
|
49
|
+
- name: Publish package distributions to PyPI
|
|
50
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
51
|
+
with:
|
|
52
|
+
verbose: true
|
|
53
|
+
|
|
54
|
+
create_github_release:
|
|
55
|
+
name: Create a GitHub release 🔖
|
|
56
|
+
timeout-minutes: 5
|
|
57
|
+
needs:
|
|
58
|
+
- publish-to-pypi
|
|
59
|
+
runs-on: ubuntu-latest
|
|
60
|
+
permissions:
|
|
61
|
+
contents: write
|
|
62
|
+
steps:
|
|
63
|
+
- uses: actions/checkout@v6
|
|
64
|
+
- name: Create a release
|
|
65
|
+
uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd #v1.21.0
|
|
66
|
+
with:
|
|
67
|
+
generateReleaseNotes: true
|
|
68
|
+
tag: ${{ github.event.inputs.version }}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Detect if package version has been bumped & if so, trigger a release.
|
|
2
|
+
|
|
3
|
+
# Take the version from pyproject.toml and check if a git tag for this version exists.
|
|
4
|
+
# If not, trigger the release workflow (cd.yml) by pushing a new tag to the repository.
|
|
5
|
+
|
|
6
|
+
name: Detect and tag new version
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
branches:
|
|
11
|
+
- "main"
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
check-version-and-push-tag:
|
|
16
|
+
name: Detect and create a tag for new version
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v6
|
|
21
|
+
with:
|
|
22
|
+
fetch-depth: 0
|
|
23
|
+
|
|
24
|
+
- name: Install uv
|
|
25
|
+
uses: astral-sh/setup-uv@v7
|
|
26
|
+
with:
|
|
27
|
+
enable-cache: true
|
|
28
|
+
|
|
29
|
+
- name: Get package version from pyproject.toml
|
|
30
|
+
id: get-version
|
|
31
|
+
run: |
|
|
32
|
+
VERSION=$(uv version | grep -oP '\d+\.\d+\.\d+.*')
|
|
33
|
+
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
|
34
|
+
|
|
35
|
+
- name: Check if a tag for this version already exists in the repo
|
|
36
|
+
uses: mukunku/tag-exists-action@5c39604fe8aef7e65acb6fbcf96ec580f7680313 #v1.7.0
|
|
37
|
+
id: check-tag
|
|
38
|
+
with:
|
|
39
|
+
tag: v${{ steps.get-version.outputs.version }}
|
|
40
|
+
|
|
41
|
+
- uses: fregante/setup-git-user@024bc0b8e177d7e77203b48dab6fb45666854b35 # v2.0.2
|
|
42
|
+
if: steps.check-tag.outputs.exists != 'true'
|
|
43
|
+
|
|
44
|
+
- name: Publish the new tag
|
|
45
|
+
# Trigger the cd.yml workflow if a new tag is detected.
|
|
46
|
+
id: publish-tag
|
|
47
|
+
if: steps.check-tag.outputs.exists != 'true'
|
|
48
|
+
run: |
|
|
49
|
+
git tag -a v${{ steps.get-version.outputs.version }} -m "Release v${{ steps.get-version.outputs.version }}"
|
|
50
|
+
git push origin v${{ steps.get-version.outputs.version }}
|
|
51
|
+
|
|
52
|
+
- name: Run the release workflow
|
|
53
|
+
if: steps.check-tag.outputs.exists != 'true'
|
|
54
|
+
env:
|
|
55
|
+
GH_TOKEN: ${{ github.token }}
|
|
56
|
+
run: gh workflow run cd.yml -f version=v${{ steps.get-version.outputs.version }} --ref main
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [0.0.1] - 2026-07-03
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Initial release of the toolkit.
|
|
13
|
+
- `get_modified_deployments` module to detect added, modified, and removed Prefect deployments between two git references.
|
|
14
|
+
- Unified handling of `push`, `pull_request`, and `pull_request_target` CI event types for change detection.
|
|
15
|
+
- `apply_deployments` module (via `__main__`) to create, update, or delete Prefect deployments concurrently.
|
|
16
|
+
- Dual backend support: `cli` (wraps `prefect deploy`) and `rest` (direct Prefect Cloud REST API calls).
|
|
17
|
+
- Dev environment overrides: deployment name prefixing, work pool substitution, and schedule pausing.
|
|
18
|
+
- Duplicate deployment name detection with optional automatic cleanup (`--enforce-unique-deployment-names`).
|
|
19
|
+
- Thread-safe, non-interleaved log buffering for concurrent deployment runs.
|
|
20
|
+
- YAML schedule validation (rejects singular `schedule` key, empty `schedules`, and entries missing `cron`).
|
|
21
|
+
|
|
22
|
+
[0.0.1]: https://github.com/yourusername/prefect-deployments-toolkit/releases/tag/v0.0.1
|
|
23
|
+
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
PolyForm Noncommercial License 1.0.0
|
|
2
|
+
|
|
3
|
+
<https://polyformproject.org/licenses/noncommercial/1.0.0>
|
|
4
|
+
|
|
5
|
+
Copyright (c) 2026 [Your Name or Organization]
|
|
6
|
+
|
|
7
|
+
## Acceptance
|
|
8
|
+
|
|
9
|
+
In order to get any license under these terms, you must agree
|
|
10
|
+
to them as both strict obligations and conditions to all
|
|
11
|
+
your licenses.
|
|
12
|
+
|
|
13
|
+
## Copyright License
|
|
14
|
+
|
|
15
|
+
The licensor grants you a copyright license for the software
|
|
16
|
+
to do everything you might do with the software that would
|
|
17
|
+
otherwise infringe the licensor's copyright in it for any
|
|
18
|
+
permitted purpose. However, you may only distribute the
|
|
19
|
+
software according to Distribution License and make changes
|
|
20
|
+
or new works based on the software according to Changes
|
|
21
|
+
and New Works License.
|
|
22
|
+
|
|
23
|
+
## Distribution License
|
|
24
|
+
|
|
25
|
+
The licensor grants you an additional copyright license to
|
|
26
|
+
distribute copies of the software. Your license to distribute
|
|
27
|
+
covers distributing the software with changes and new works
|
|
28
|
+
permitted by Changes and New Works License.
|
|
29
|
+
|
|
30
|
+
## Notices
|
|
31
|
+
|
|
32
|
+
You must ensure that anyone who gets a copy of any part of
|
|
33
|
+
the software from you also gets a copy of these terms or the
|
|
34
|
+
URL for them above, as well as copies of any plain-text lines
|
|
35
|
+
beginning with `Required Notice` that the licensor provided
|
|
36
|
+
with the software. For example:
|
|
37
|
+
|
|
38
|
+
> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
|
|
39
|
+
|
|
40
|
+
## Changes and New Works License
|
|
41
|
+
|
|
42
|
+
The licensor grants you an additional copyright license to
|
|
43
|
+
make changes and new works based on the software for any
|
|
44
|
+
permitted purpose.
|
|
45
|
+
|
|
46
|
+
## Patent License
|
|
47
|
+
|
|
48
|
+
The licensor grants you a patent license for the software that
|
|
49
|
+
covers patent claims the licensor can license, or becomes able
|
|
50
|
+
to license, that you would infringe by using the software.
|
|
51
|
+
|
|
52
|
+
## Noncommercial Purposes
|
|
53
|
+
|
|
54
|
+
Any noncommercial purpose is a permitted purpose.
|
|
55
|
+
|
|
56
|
+
## Personal Uses
|
|
57
|
+
|
|
58
|
+
Personal use for research, experiment, and testing for the
|
|
59
|
+
benefit of public knowledge, personal study, private
|
|
60
|
+
entertainment, hobby projects, amateur pursuits, or religious
|
|
61
|
+
observance, without any anticipated commercial application,
|
|
62
|
+
is use for a permitted purpose.
|
|
63
|
+
|
|
64
|
+
## Noncommercial Organizations
|
|
65
|
+
|
|
66
|
+
Use by any charitable organization, educational institution,
|
|
67
|
+
public research organization, public safety or health
|
|
68
|
+
organization, environmental protection organization, or
|
|
69
|
+
government institution is use for a permitted purpose
|
|
70
|
+
regardless of the source of funding or obligations resulting
|
|
71
|
+
from the funding.
|
|
72
|
+
|
|
73
|
+
## Fair Use
|
|
74
|
+
|
|
75
|
+
You may have "fair use" rights for the software under the
|
|
76
|
+
law. These terms do not limit them.
|
|
77
|
+
|
|
78
|
+
## No Other Rights
|
|
79
|
+
|
|
80
|
+
These terms do not allow you to sublicense or transfer any of
|
|
81
|
+
your licenses to anyone else, or prevent the licensor from
|
|
82
|
+
granting licenses to anyone else. These terms do not imply
|
|
83
|
+
any other licenses.
|
|
84
|
+
|
|
85
|
+
## Patent Defense
|
|
86
|
+
|
|
87
|
+
If you make any written claim that the software infringes or
|
|
88
|
+
contributes to infringement of any patent, your patent license
|
|
89
|
+
for the software granted under these terms ends immediately.
|
|
90
|
+
If your company makes such a claim, your patent license ends
|
|
91
|
+
immediately for work on behalf of your company.
|
|
92
|
+
|
|
93
|
+
## Violations
|
|
94
|
+
|
|
95
|
+
The first time you are notified in writing that you have
|
|
96
|
+
violated any of these terms, or done anything with the
|
|
97
|
+
software not covered by your licenses, your licenses can
|
|
98
|
+
nonetheless continue if you come into full compliance with
|
|
99
|
+
these terms, and take practical steps to correct past
|
|
100
|
+
violations, within 32 days of receiving notice. Otherwise, all
|
|
101
|
+
your licenses end immediately.
|
|
102
|
+
|
|
103
|
+
## No Liability
|
|
104
|
+
|
|
105
|
+
As far as the law allows, the software comes as is, without
|
|
106
|
+
any warranty or condition, and the licensor will not be liable
|
|
107
|
+
to you for any damages arising out of these terms or the use
|
|
108
|
+
or nature of the software, under any kind of legal claim.
|
|
109
|
+
|
|
110
|
+
## Definitions
|
|
111
|
+
|
|
112
|
+
The **licensor** is the individual or entity offering these
|
|
113
|
+
terms, and the **software** is the software the licensor makes
|
|
114
|
+
available under these terms.
|
|
115
|
+
|
|
116
|
+
**You** refers to the individual or entity agreeing to these
|
|
117
|
+
terms.
|
|
118
|
+
|
|
119
|
+
**Your company** is any legal entity, sole proprietorship, or
|
|
120
|
+
other kind of organization that you work for, plus all
|
|
121
|
+
organizations that have control over, are under the control of,
|
|
122
|
+
or are under common control with that organization. **Control**
|
|
123
|
+
means ownership of substantially all the assets of an entity,
|
|
124
|
+
or the power to direct its management and policies by vote,
|
|
125
|
+
contract, or otherwise. Control can be direct or indirect.
|
|
126
|
+
|
|
127
|
+
**Your licenses** are all the licenses granted to you for the
|
|
128
|
+
software under these terms.
|
|
129
|
+
|
|
130
|
+
**Use** means anything you do with the software requiring one
|
|
131
|
+
of your licenses.
|
|
132
|
+
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: prefect-deployments-toolkit
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: CLI toolkit for detecting and applying Prefect deployment changes in CI/CD pipelines
|
|
5
|
+
Project-URL: Repository, https://github.com/dyvenia/prefect-deployments-toolkit
|
|
6
|
+
Project-URL: Issues, https://github.com/dyvenia/prefect-deployments-toolkit/issues
|
|
7
|
+
Author-email: k-wolski <kwolski@dyvenia.com>
|
|
8
|
+
License: PolyForm-Noncommercial-1.0.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: ci-cd,deployment,gitlab-ci,prefect,workflow-orchestration
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Software Development :: Build Tools
|
|
18
|
+
Classifier: Topic :: System :: Systems Administration
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: httpx>=0.27
|
|
21
|
+
Requires-Dist: prefect<4,>=3.0
|
|
22
|
+
Requires-Dist: pyyaml>=6.0
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# Prefect Deployments Toolkit
|
|
26
|
+
|
|
27
|
+
A lightweight CLI toolkit for managing Prefect deployment lifecycles in CI/CD pipelines. It detects deployments that were added, modified, or removed between two git references, and applies (creates, updates, or deletes) those deployments against Prefect Cloud or a self-hosted Prefect server — via either the Prefect CLI or direct REST API calls.
|
|
28
|
+
|
|
29
|
+
Built for teams running many Prefect flows out of a monorepo, where each deployment is defined in its own YAML file alongside a shared `prefect_base.yaml`, and CI needs to figure out *which* deployments actually changed before re-registering them.
|
|
30
|
+
|
|
31
|
+
## Features
|
|
32
|
+
|
|
33
|
+
- **Change detection** — diff deployments between two git refs (a merge request source vs. target branch, or `HEAD~1` vs. `HEAD` after a merge) and get back exactly which deployments were added, modified, or removed.
|
|
34
|
+
- **Unified push / pull-request logic** — one code path handles both `pull_request` (compare local files against a fetched base branch) and `push`/`pull_request_target` (compare `HEAD~1` against `HEAD`) event styles.
|
|
35
|
+
- **Dual deploy backends** — apply deployments using the `prefect` CLI as a subprocess, or bypass it entirely with direct Prefect Cloud REST API calls for faster, more controllable deploys.
|
|
36
|
+
- **Dev/prod environment overrides** — automatic name prefixing, work pool substitution, and schedule pausing for dev environments.
|
|
37
|
+
- **Concurrent deployment application** — apply multiple deployments in parallel with clean, non-interleaved log output per deployment.
|
|
38
|
+
- **Duplicate deployment name detection** — warns (or optionally cleans up) when the same deployment name resolves to more than one flow.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install prefect-deployments-toolkit
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Requires Python >= 3.10.
|
|
47
|
+
|
|
48
|
+
## Usage
|
|
49
|
+
|
|
50
|
+
### 1. Detect modified deployments
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
python -m prefect_deployments_toolkit.get_modified_deployments \
|
|
54
|
+
--modified-by push \
|
|
55
|
+
--base-ref main \
|
|
56
|
+
--deployments-dir deployments
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Prints a comma-separated list of changed deployment names to stdout, and (if running inside GitHub Actions) writes `DEPLOYMENT_NAMES`, `NEW_OR_MODIFIED_DEPLOYMENT_NAMES`, and `REMOVED_DEPLOYMENT_NAMES` to `GITHUB_ENV`.
|
|
60
|
+
|
|
61
|
+
`--modified-by` accepts:
|
|
62
|
+
- `pull_request` — compares your local working tree against the fetched base branch (use during an open MR/PR).
|
|
63
|
+
- `push` or `pull_request_target` — compares `HEAD~1` against `HEAD` (use after a merge to a protected branch).
|
|
64
|
+
|
|
65
|
+
### 2. Apply deployments
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
python -m prefect_deployments_toolkit \
|
|
69
|
+
--deployment-names "flow_a,flow_b" \
|
|
70
|
+
--enable-schedule false \
|
|
71
|
+
--tag dev \
|
|
72
|
+
--reference feature-branch-1 \
|
|
73
|
+
--repo-name my-flows-repo \
|
|
74
|
+
--custom-image "" \
|
|
75
|
+
--deployments-dir deployments \
|
|
76
|
+
--dev-work-pool my-dev-work-pool \
|
|
77
|
+
--backend rest \
|
|
78
|
+
--enforce-unique-deployment-names false
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Expected Repository Layout
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
deployments/
|
|
85
|
+
├── prefect_base.yaml # shared config merged into every deployment
|
|
86
|
+
├── flow_a.yaml
|
|
87
|
+
├── flow_b.yaml
|
|
88
|
+
└── ...
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
Each deployment YAML file must declare a `deployments:` list with at least a `name` and `entrypoint` key, following the standard Prefect deployment YAML schema.
|
|
93
|
+
|
|
94
|
+
## Backends
|
|
95
|
+
|
|
96
|
+
| Backend | How it works | When to use |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| `cli` | Shells out to `prefect deploy` | Simplest, matches local `prefect deploy` behavior exactly |
|
|
99
|
+
| `rest` | Talks directly to the Prefect Cloud REST API | Faster for many concurrent deployments, avoids CLI subprocess overhead |
|
|
100
|
+
|
|
101
|
+
## Environment Variables
|
|
102
|
+
|
|
103
|
+
| Variable | Required | Purpose |
|
|
104
|
+
|---|---|---|
|
|
105
|
+
| `PREFECT_API_URL` | Yes | Full Prefect Cloud/server API base URL |
|
|
106
|
+
| `PREFECT_API_KEY` | Yes | API key for authentication |
|
|
107
|
+
|
|
108
|
+
## License
|
|
109
|
+
|
|
110
|
+
Licensed under [PolyForm Noncommercial 1.0.0](https://polyformproject.org/licenses/noncommercial/1.0.0). You may view, use, and modify this software for noncommercial purposes. Commercial use — including building products, services, or automation on top of it — requires a separate license. Contact [your contact info] for commercial licensing inquiries.
|
|
111
|
+
|
|
112
|
+
## Contributing
|
|
113
|
+
|
|
114
|
+
Issues and pull requests are welcome for noncommercial use cases. Please open an issue before submitting a large PR to discuss scope.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Prefect Deployments Toolkit
|
|
2
|
+
|
|
3
|
+
A lightweight CLI toolkit for managing Prefect deployment lifecycles in CI/CD pipelines. It detects deployments that were added, modified, or removed between two git references, and applies (creates, updates, or deletes) those deployments against Prefect Cloud or a self-hosted Prefect server — via either the Prefect CLI or direct REST API calls.
|
|
4
|
+
|
|
5
|
+
Built for teams running many Prefect flows out of a monorepo, where each deployment is defined in its own YAML file alongside a shared `prefect_base.yaml`, and CI needs to figure out *which* deployments actually changed before re-registering them.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- **Change detection** — diff deployments between two git refs (a merge request source vs. target branch, or `HEAD~1` vs. `HEAD` after a merge) and get back exactly which deployments were added, modified, or removed.
|
|
10
|
+
- **Unified push / pull-request logic** — one code path handles both `pull_request` (compare local files against a fetched base branch) and `push`/`pull_request_target` (compare `HEAD~1` against `HEAD`) event styles.
|
|
11
|
+
- **Dual deploy backends** — apply deployments using the `prefect` CLI as a subprocess, or bypass it entirely with direct Prefect Cloud REST API calls for faster, more controllable deploys.
|
|
12
|
+
- **Dev/prod environment overrides** — automatic name prefixing, work pool substitution, and schedule pausing for dev environments.
|
|
13
|
+
- **Concurrent deployment application** — apply multiple deployments in parallel with clean, non-interleaved log output per deployment.
|
|
14
|
+
- **Duplicate deployment name detection** — warns (or optionally cleans up) when the same deployment name resolves to more than one flow.
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install prefect-deployments-toolkit
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Requires Python >= 3.10.
|
|
23
|
+
|
|
24
|
+
## Usage
|
|
25
|
+
|
|
26
|
+
### 1. Detect modified deployments
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
python -m prefect_deployments_toolkit.get_modified_deployments \
|
|
30
|
+
--modified-by push \
|
|
31
|
+
--base-ref main \
|
|
32
|
+
--deployments-dir deployments
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Prints a comma-separated list of changed deployment names to stdout, and (if running inside GitHub Actions) writes `DEPLOYMENT_NAMES`, `NEW_OR_MODIFIED_DEPLOYMENT_NAMES`, and `REMOVED_DEPLOYMENT_NAMES` to `GITHUB_ENV`.
|
|
36
|
+
|
|
37
|
+
`--modified-by` accepts:
|
|
38
|
+
- `pull_request` — compares your local working tree against the fetched base branch (use during an open MR/PR).
|
|
39
|
+
- `push` or `pull_request_target` — compares `HEAD~1` against `HEAD` (use after a merge to a protected branch).
|
|
40
|
+
|
|
41
|
+
### 2. Apply deployments
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python -m prefect_deployments_toolkit \
|
|
45
|
+
--deployment-names "flow_a,flow_b" \
|
|
46
|
+
--enable-schedule false \
|
|
47
|
+
--tag dev \
|
|
48
|
+
--reference feature-branch-1 \
|
|
49
|
+
--repo-name my-flows-repo \
|
|
50
|
+
--custom-image "" \
|
|
51
|
+
--deployments-dir deployments \
|
|
52
|
+
--dev-work-pool my-dev-work-pool \
|
|
53
|
+
--backend rest \
|
|
54
|
+
--enforce-unique-deployment-names false
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Expected Repository Layout
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
deployments/
|
|
61
|
+
├── prefect_base.yaml # shared config merged into every deployment
|
|
62
|
+
├── flow_a.yaml
|
|
63
|
+
├── flow_b.yaml
|
|
64
|
+
└── ...
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
Each deployment YAML file must declare a `deployments:` list with at least a `name` and `entrypoint` key, following the standard Prefect deployment YAML schema.
|
|
69
|
+
|
|
70
|
+
## Backends
|
|
71
|
+
|
|
72
|
+
| Backend | How it works | When to use |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| `cli` | Shells out to `prefect deploy` | Simplest, matches local `prefect deploy` behavior exactly |
|
|
75
|
+
| `rest` | Talks directly to the Prefect Cloud REST API | Faster for many concurrent deployments, avoids CLI subprocess overhead |
|
|
76
|
+
|
|
77
|
+
## Environment Variables
|
|
78
|
+
|
|
79
|
+
| Variable | Required | Purpose |
|
|
80
|
+
|---|---|---|
|
|
81
|
+
| `PREFECT_API_URL` | Yes | Full Prefect Cloud/server API base URL |
|
|
82
|
+
| `PREFECT_API_KEY` | Yes | API key for authentication |
|
|
83
|
+
|
|
84
|
+
## License
|
|
85
|
+
|
|
86
|
+
Licensed under [PolyForm Noncommercial 1.0.0](https://polyformproject.org/licenses/noncommercial/1.0.0). You may view, use, and modify this software for noncommercial purposes. Commercial use — including building products, services, or automation on top of it — requires a separate license. Contact [your contact info] for commercial licensing inquiries.
|
|
87
|
+
|
|
88
|
+
## Contributing
|
|
89
|
+
|
|
90
|
+
Issues and pull requests are welcome for noncommercial use cases. Please open an issue before submitting a large PR to discuss scope.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "prefect-deployments-toolkit"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "CLI toolkit for detecting and applying Prefect deployment changes in CI/CD pipelines"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = { text = "PolyForm-Noncommercial-1.0.0" }
|
|
7
|
+
requires-python = ">=3.10"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "k-wolski", email = "kwolski@dyvenia.com" }
|
|
10
|
+
]
|
|
11
|
+
keywords = ["prefect", "deployment", "ci-cd", "gitlab-ci", "workflow-orchestration"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.10",
|
|
15
|
+
"Programming Language :: Python :: 3.11",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Programming Language :: Python :: 3.13",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Topic :: Software Development :: Build Tools",
|
|
20
|
+
"Topic :: System :: Systems Administration",
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"httpx>=0.27",
|
|
24
|
+
"pyyaml>=6.0",
|
|
25
|
+
"prefect>=3.0,<4",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Repository = "https://github.com/dyvenia/prefect-deployments-toolkit"
|
|
30
|
+
Issues = "https://github.com/dyvenia/prefect-deployments-toolkit/issues"
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
prefect-deployments-toolkit = "prefect_deployments_toolkit.__main__:main"
|
|
34
|
+
|
|
35
|
+
[build-system]
|
|
36
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
37
|
+
build-backend = "hatchling.build"
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.wheel]
|
|
40
|
+
packages = ["src/prefect_deployments_toolkit"]
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Entry point: parse args and apply all deployments concurrently.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python -m apply_deployments \
|
|
5
|
+
--deployment-names "hello_world3,hello_world_4" \
|
|
6
|
+
--enable-schedule false \
|
|
7
|
+
--tag dev \
|
|
8
|
+
--reference feature_branch1 \
|
|
9
|
+
--repo-name edp-flows \
|
|
10
|
+
--custom-image "" \
|
|
11
|
+
--deployments-dir prefect/deployments \
|
|
12
|
+
--dev-work-pool lapp-dev-work-pool-prefect3 \
|
|
13
|
+
--backend cli \
|
|
14
|
+
--enforce-unique-deployment-names false
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import logging
|
|
19
|
+
import sys
|
|
20
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from .deployment import DeploymentContext, apply_single_deployment
|
|
24
|
+
from .log_buffer import buffered_deployment_log
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
MAX_WORKERS = 8
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _parse_args() -> argparse.Namespace:
|
|
32
|
+
parser = argparse.ArgumentParser(description="Apply Prefect deployments.")
|
|
33
|
+
parser.add_argument("--deployment-names", required=True,
|
|
34
|
+
help="Comma-separated list of deployment names.")
|
|
35
|
+
parser.add_argument("--enable-schedule", required=True,
|
|
36
|
+
choices=["true", "false"],
|
|
37
|
+
help="Whether to enable schedules after deploy.")
|
|
38
|
+
parser.add_argument("--tag", required=True,
|
|
39
|
+
help="Image/environment tag (e.g. 'dev', 'v1.2.3').")
|
|
40
|
+
parser.add_argument("--reference", required=True,
|
|
41
|
+
help="Git branch or tag reference (e.g. 'main', 'feature_branch1').")
|
|
42
|
+
parser.add_argument("--repo-name", required=True,
|
|
43
|
+
help="Repository name used in job variable paths.")
|
|
44
|
+
parser.add_argument("--custom-image", default="",
|
|
45
|
+
help="Full custom image reference, or empty string to use pool default.")
|
|
46
|
+
parser.add_argument("--deployments-dir", default="deployments",
|
|
47
|
+
help="Path to the deployments directory.")
|
|
48
|
+
parser.add_argument("--dev-work-pool", default="",
|
|
49
|
+
help="Dev work pool name. When set with --tag=dev, activates dev overrides.")
|
|
50
|
+
parser.add_argument("--max-workers", type=int, default=MAX_WORKERS,
|
|
51
|
+
help=f"Max concurrent deployments (default: {MAX_WORKERS}).")
|
|
52
|
+
parser.add_argument("--backend", default="cli", choices=["cli", "rest"],
|
|
53
|
+
help="Deployment backend: 'cli' (prefect CLI, default) or "
|
|
54
|
+
"'rest' (direct Prefect Cloud REST API calls).")
|
|
55
|
+
parser.add_argument("--enforce-unique-deployment-names", default="false",
|
|
56
|
+
choices=["true", "false"],
|
|
57
|
+
help=(
|
|
58
|
+
"If false (default), only WARN in logs when a deployment name is used by"
|
|
59
|
+
"more than one flow — nothing is deleted. If true, delete every duplicate"
|
|
60
|
+
"deployment record except the one matching the flow currently resolved from"
|
|
61
|
+
"its entrypoint, enforcing globally unique deployment names."
|
|
62
|
+
))
|
|
63
|
+
return parser.parse_args()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _run_deployment(
|
|
67
|
+
deployment_name: str,
|
|
68
|
+
index: int,
|
|
69
|
+
total: int,
|
|
70
|
+
ctx: DeploymentContext,
|
|
71
|
+
) -> str | None:
|
|
72
|
+
"""Run a single deployment inside a buffered log context.
|
|
73
|
+
|
|
74
|
+
Returns the deployment_name if it failed, None on success.
|
|
75
|
+
"""
|
|
76
|
+
separator = "#" * 60
|
|
77
|
+
with buffered_deployment_log(deployment_name):
|
|
78
|
+
logger.info("%s [%d/%d] %s %s", separator, index, total, deployment_name, separator)
|
|
79
|
+
try:
|
|
80
|
+
apply_single_deployment(deployment_name, ctx)
|
|
81
|
+
logger.info(">>> DONE [%d/%d] %s", index, total, deployment_name)
|
|
82
|
+
return None
|
|
83
|
+
except Exception as exc: # noqa: BLE001
|
|
84
|
+
logger.error(">>> FAILED [%d/%d] %s: %s", index, total, deployment_name, exc)
|
|
85
|
+
return deployment_name
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def main() -> None:
|
|
89
|
+
logging.basicConfig(
|
|
90
|
+
level=logging.INFO,
|
|
91
|
+
format="%(asctime)s [%(levelname)s] %(message)s",
|
|
92
|
+
)
|
|
93
|
+
args = _parse_args()
|
|
94
|
+
names = [n.strip() for n in args.deployment_names.split(",") if n.strip()]
|
|
95
|
+
|
|
96
|
+
if not names:
|
|
97
|
+
logger.error("No deployment names provided.")
|
|
98
|
+
sys.exit(1)
|
|
99
|
+
|
|
100
|
+
ctx = DeploymentContext(
|
|
101
|
+
deployment_names=names,
|
|
102
|
+
enable_schedule=args.enable_schedule == "true",
|
|
103
|
+
tag=args.tag,
|
|
104
|
+
reference=args.reference,
|
|
105
|
+
repo_name=args.repo_name,
|
|
106
|
+
custom_image=args.custom_image,
|
|
107
|
+
deployments_dir=Path(args.deployments_dir),
|
|
108
|
+
dev_prefect_work_pool_name=args.dev_work_pool,
|
|
109
|
+
backend=args.backend,
|
|
110
|
+
enforce_unique_deployment_names=args.enforce_unique_deployment_names == "true",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
total = len(names)
|
|
114
|
+
workers = min(args.max_workers, total)
|
|
115
|
+
logger.info(
|
|
116
|
+
"Applying %d deployment(s) with up to %d concurrent workers (backend=%s).",
|
|
117
|
+
total, workers, ctx.backend,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
failed: list[str] = []
|
|
121
|
+
|
|
122
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
123
|
+
futures = {
|
|
124
|
+
executor.submit(_run_deployment, name, i, total, ctx): name
|
|
125
|
+
for i, name in enumerate(names, start=1)
|
|
126
|
+
}
|
|
127
|
+
for future in as_completed(futures):
|
|
128
|
+
result = future.result() # re-raises only unexpected executor errors
|
|
129
|
+
if result is not None:
|
|
130
|
+
failed.append(result)
|
|
131
|
+
|
|
132
|
+
if failed:
|
|
133
|
+
logger.error(
|
|
134
|
+
"%d deployment(s) failed: %s",
|
|
135
|
+
len(failed),
|
|
136
|
+
", ".join(failed),
|
|
137
|
+
)
|
|
138
|
+
sys.exit(1)
|
|
139
|
+
|
|
140
|
+
logger.info("All %d deployment(s) applied successfully.", total)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
if __name__ == "__main__":
|
|
144
|
+
main()
|
|
145
|
+
|