generic-gitlab-cicd 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- generic_gitlab_cicd-0.3.2/PKG-INFO +170 -0
- generic_gitlab_cicd-0.3.2/README.md +159 -0
- generic_gitlab_cicd-0.3.2/generic_ci/__init__.py +3 -0
- generic_gitlab_cicd-0.3.2/generic_ci/__main__.py +3 -0
- generic_gitlab_cicd-0.3.2/generic_ci/cli.py +110 -0
- generic_gitlab_cicd-0.3.2/generic_ci/compiler.py +242 -0
- generic_gitlab_cicd-0.3.2/generic_ci/config.py +148 -0
- generic_gitlab_cicd-0.3.2/generic_ci/dependencies.py +186 -0
- generic_gitlab_cicd-0.3.2/generic_ci/models.py +189 -0
- generic_gitlab_cicd-0.3.2/generic_ci/runtime.py +482 -0
- generic_gitlab_cicd-0.3.2/generic_ci/sources.py +314 -0
- generic_gitlab_cicd-0.3.2/generic_ci/workflows/__init__.py +1 -0
- generic_gitlab_cicd-0.3.2/generic_ci/workflows/compiler.py +316 -0
- generic_gitlab_cicd-0.3.2/generic_ci/workflows/ecosystems.py +185 -0
- generic_gitlab_cicd-0.3.2/generic_ci/workflows/helm.py +168 -0
- generic_gitlab_cicd-0.3.2/generic_ci/workflows/models.py +222 -0
- generic_gitlab_cicd-0.3.2/generic_ci/workflows/publication.py +82 -0
- generic_gitlab_cicd-0.3.2/generic_ci/workflows/runtime.py +421 -0
- generic_gitlab_cicd-0.3.2/generic_gitlab_cicd.egg-info/PKG-INFO +170 -0
- generic_gitlab_cicd-0.3.2/generic_gitlab_cicd.egg-info/SOURCES.txt +31 -0
- generic_gitlab_cicd-0.3.2/generic_gitlab_cicd.egg-info/dependency_links.txt +1 -0
- generic_gitlab_cicd-0.3.2/generic_gitlab_cicd.egg-info/entry_points.txt +2 -0
- generic_gitlab_cicd-0.3.2/generic_gitlab_cicd.egg-info/requires.txt +4 -0
- generic_gitlab_cicd-0.3.2/generic_gitlab_cicd.egg-info/top_level.txt +1 -0
- generic_gitlab_cicd-0.3.2/pyproject.toml +17 -0
- generic_gitlab_cicd-0.3.2/setup.cfg +4 -0
- generic_gitlab_cicd-0.3.2/tests/test_chart.py +120 -0
- generic_gitlab_cicd-0.3.2/tests/test_framework.py +146 -0
- generic_gitlab_cicd-0.3.2/tests/test_product.py +172 -0
- generic_gitlab_cicd-0.3.2/tests/test_publication.py +176 -0
- generic_gitlab_cicd-0.3.2/tests/test_review.py +145 -0
- generic_gitlab_cicd-0.3.2/tests/test_sources.py +102 -0
- generic_gitlab_cicd-0.3.2/tests/test_workflows.py +170 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: generic-gitlab-cicd
|
|
3
|
+
Version: 0.3.2
|
|
4
|
+
Summary: Validated project configuration and reproducible GitLab delivery pipelines
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: pydantic<3,>=2.10
|
|
8
|
+
Requires-Dist: PyYAML<7,>=6
|
|
9
|
+
Requires-Dist: tomlkit<1,>=0.13
|
|
10
|
+
Requires-Dist: packaging<27,>=25
|
|
11
|
+
|
|
12
|
+
Testing commands and the local/E2E split: [testing — revision one](docs/testing-revision-one.md).
|
|
13
|
+
|
|
14
|
+
Start with the [documentation map](docs/README.md), [AI authoring guide](docs/ai-authoring-revision-three.md), or portable [generic-ci-authoring skill](skills/generic-ci-authoring/SKILL.md).
|
|
15
|
+
|
|
16
|
+
Organization defaults and Git-backed starter templates are now available: see [configuration sources — revision two](docs/configuration-sources-revision-two.md).
|
|
17
|
+
|
|
18
|
+
> Feature-branch workflow interface (0.3.1): see [implemented revision-one guide](docs/workflows-revision-one.md), [configuration examples](examples/workflows), and [JSON Schema](schemas/workflows.schema.json). This is a review build with documented integration gates. Older prototype CLI commands now require `--format legacy`; existing GitLab components remain available.
|
|
19
|
+
|
|
20
|
+
# Generic GitLab CI components — revision one
|
|
21
|
+
|
|
22
|
+
A self-contained component repository for GitLab, including an offline-capable runtime-image factory and a generic Kubernetes Helm chart. The recommended Python entry point is `examples/uv-airgap.yml`.
|
|
23
|
+
|
|
24
|
+
This is an implementation to configure and validate on your installation. It has not been run against your GitLab, Artifactory or Kubernetes cluster. Infrastructure addresses, credentials, approved base images and application deployment settings must be supplied by your organization.
|
|
25
|
+
|
|
26
|
+
## Components
|
|
27
|
+
|
|
28
|
+
| Component | Purpose |
|
|
29
|
+
| --- | --- |
|
|
30
|
+
| `workflow` | Stages, cancellation of superseded work, suppression of duplicate push/MR pipelines |
|
|
31
|
+
| `task` | Language-neutral lint, test, build or verification commands; reports, caches and artifacts |
|
|
32
|
+
| `uv-test` | Locked uv validation, optional Git dependency replacement and commit provenance |
|
|
33
|
+
| `version-check` | PEP 440 version increase against the current target branch; tag/version matching |
|
|
34
|
+
| `python-package` | Build wheel/sdist once, check metadata, retain distribution artifacts |
|
|
35
|
+
| `python-publish` | Publish those artifacts to PyPI, TestPyPI, GitLab or Artifactory |
|
|
36
|
+
| `container-preview-build` | Same-project MR builds restricted to a dedicated preview repository |
|
|
37
|
+
| `container-build` | Rootless BuildKit build, digest metadata, optional protected-ref registry push |
|
|
38
|
+
| `deploy` / `preview` | Provider-neutral command adapters with deployment locking and preview cleanup |
|
|
39
|
+
| `helm-deploy` / `helm-preview` | Helm rollout with failure rollback and isolated review environments |
|
|
40
|
+
| `release` | Create a GitLab release for an existing protected tag |
|
|
41
|
+
|
|
42
|
+
Each YAML file documents its own typed inputs. Include components multiple times with unique job names. Secrets remain GitLab CI variables; they are never component inputs.
|
|
43
|
+
|
|
44
|
+
## Bootstrap
|
|
45
|
+
|
|
46
|
+
1. Import this directory into a GitLab project such as `platform/ci-components` on your own GitLab instance. Component includes cannot directly cross to an unrelated GitLab instance; mirror this repository internally.
|
|
47
|
+
2. Configure the variables in `docs/airgap.md`, and supply approved internal Python, BuildKit and Helm runtime images. The repository's own pipeline uses those images. Do not switch all consumers until the runtime images are available.
|
|
48
|
+
3. Adapt non-secret files under `images/python/config/`, add approved CA certificates, and review `images/python/requirements.lock`. Run `images.gitlab-ci.yml` to build the Python runtime from your internal package index and push it to Artifactory. Initial BuildKit and Python base images must already be mirrored and trusted.
|
|
49
|
+
4. Run this repository's validation pipeline. Publish/tag an immutable component version, e.g. `1.0.0`. Optionally mark the project as a CI/CD Catalog resource and create a GitLab release. Consumers may pin a full commit SHA before the first release.
|
|
50
|
+
5. Copy the relevant example as the consuming project's `.gitlab-ci.yml`. Replace `platform/ci-components`, the release ref and deployment placeholders.
|
|
51
|
+
6. Validate the expanded pipeline with your GitLab CI Lint, then run MR, default-branch and protected-tag scenarios before enabling production publishing.
|
|
52
|
+
|
|
53
|
+
Use a supported GitLab with CI/CD components and array inputs (GitLab 17+ syntax baseline). The optional native release job also requires a GitLab-compatible glab runtime; verify server/CLI compatibility. Runtime jobs target Linux container runners, not Windows shell runners.
|
|
54
|
+
|
|
55
|
+
## Minimal uv project
|
|
56
|
+
|
|
57
|
+
```yaml
|
|
58
|
+
variables:
|
|
59
|
+
CI_DEPENDENCY_REPO: ""
|
|
60
|
+
CI_DEPENDENCY_REF: ""
|
|
61
|
+
CI_DEPENDENCY_PACKAGE: ""
|
|
62
|
+
CI_DEPENDENCY_OVERRIDES: '[]'
|
|
63
|
+
|
|
64
|
+
include:
|
|
65
|
+
- component: $CI_SERVER_FQDN/platform/ci-components/workflow@1.0.0
|
|
66
|
+
- component: $CI_SERVER_FQDN/platform/ci-components/uv-test@1.0.0
|
|
67
|
+
inputs:
|
|
68
|
+
name: api-test
|
|
69
|
+
repo: $CI_DEPENDENCY_REPO
|
|
70
|
+
ref: $CI_DEPENDENCY_REF
|
|
71
|
+
package: $CI_DEPENDENCY_PACKAGE
|
|
72
|
+
overrides-json: $CI_DEPENDENCY_OVERRIDES
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Set `PYTHON_CI_IMAGE` to your internal runtime image. Commit `uv.lock`. Put pytest and application test dependencies in the project's dependency groups. `uv-test` syncs all groups; define mutually compatible groups. Tests run with `uv run --no-sync` so uv does not undo a candidate installation.
|
|
76
|
+
|
|
77
|
+
## Cross-repository dependencies
|
|
78
|
+
|
|
79
|
+
For an unreleased dependency, start a pipeline with:
|
|
80
|
+
|
|
81
|
+
```text
|
|
82
|
+
CI_DEPENDENCY_REPO=https://gitlab.internal/team/shared-sdk.git
|
|
83
|
+
CI_DEPENDENCY_REF=feature/new-api
|
|
84
|
+
CI_DEPENDENCY_PACKAGE=shared-sdk
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
For several dependencies or a package in a repository subdirectory:
|
|
88
|
+
|
|
89
|
+
```json
|
|
90
|
+
[
|
|
91
|
+
{"repo":"https://gitlab.internal/team/sdk.git", "ref":"feature/new-api", "package":"shared-sdk"},
|
|
92
|
+
{"repo":"https://gitlab.internal/team/core.git", "ref":"0123456789012345678901234567890123456789", "package":"core-utils", "subdirectory":"packages/utils"}
|
|
93
|
+
]
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Set that array as `CI_DEPENDENCY_OVERRIDES`. Both mechanisms can be combined if package names do not overlap.
|
|
97
|
+
|
|
98
|
+
The helper resolves each branch/tag to a commit, applies temporary `tool.uv.override-dependencies`, removes conflicting uv source mappings for the named distribution, re-resolves, and verifies installed `direct_url.json` commit metadata. Original pyproject and lock bytes are restored even if sync fails. A successful override run saves requested refs and actual commits as a provenance artifact. A branch can move between pipelines; use the recorded SHA for reproducible reruns.
|
|
99
|
+
|
|
100
|
+
The normal path uses `uv sync --locked --all-groups`. The override path deliberately re-resolves and can change transitive dependencies; it tests candidate compatibility rather than certifying the production lock. Overrides must name an installed distribution; a typo or irrelevant package fails verification.
|
|
101
|
+
|
|
102
|
+
Configure read-only Git authentication in the job's `setup` or runner credential helper. Use credential-free repository URLs. For CI_JOB_TOKEN access, allowlist the consumer project in each dependency project. Restrict the credential helper to the intended host. Do not put credentials in URLs, artifacts, images, or tracked config.
|
|
103
|
+
|
|
104
|
+
Run a uv workspace from its root; its virtual environment must be `.venv` there. For independent monorepo projects, include one `uv-test` per project and use unique report/provenance paths. Custom UV_PROJECT_ENVIRONMENT layouts require adapting the helper's virtual-environment path.
|
|
105
|
+
|
|
106
|
+
## Coordinated releases
|
|
107
|
+
|
|
108
|
+
Candidate testing solves the validation deadlock. It does not make multiple registry publications transactional.
|
|
109
|
+
|
|
110
|
+
1. Validate the application and dependency candidates together using immutable overrides.
|
|
111
|
+
2. Publish the dependency's real version to the internal package repository.
|
|
112
|
+
3. Update/validate the application's production lock against that registry, with overrides cleared.
|
|
113
|
+
4. Build once, publish/deploy the application, then create its GitLab release.
|
|
114
|
+
|
|
115
|
+
A dependency package can usually be published before the application is deployed; publishing the wheel does not deploy a running service. For stronger coordination, use an internal candidate repository and promote approved packages before deployment. Two independent GitLab projects still need an orchestration policy; this framework does not promise atomic cross-project rollback. Multi-project trigger/approval coordination is an extension point, not implemented here.
|
|
116
|
+
|
|
117
|
+
The standard publish and production deploy jobs reject nonempty `CI_DEPENDENCY_*` override variables. Do not hard-code candidate inputs in a production pipeline. Preview artifacts may intentionally contain candidate dependencies; never promote those to production without a clean release validation.
|
|
118
|
+
|
|
119
|
+
## Versioning, tags and releases
|
|
120
|
+
|
|
121
|
+
`version-check` reads static `[project].version`. In branch/MR pipelines it fetches the current target branch and requires a strict PEP 440 increase. On tags it requires the tag suffix to match the package version. The default prefix is `v`; use `sdk-v` for independent package tags. Dynamic SCM versions require a custom command via `task`.
|
|
122
|
+
|
|
123
|
+
Use change rules to restrict bump checks to meaningful package changes if documentation-only MRs should not bump a version. The default checks every selected non-default branch pipeline. `allow-new-package` explicitly permits a manifest absent from the target branch.
|
|
124
|
+
|
|
125
|
+
Tags are an input to the release pipeline. Create and protect them through your established process; CI does not silently create or push tags. Package publishing and production deployment are blocking manual jobs on protected tags by default. The GitLab release runs last, using `CHANGELOG.md` or a configured notes file. Existing GitLab release retries require deliberate handling; no overwrite or package `--skip-existing` behavior hides duplicate releases.
|
|
126
|
+
|
|
127
|
+
A tag check validates manifest metadata, not the version assigned by every possible custom build backend. The build defaults use static PEP 621 metadata. Custom build commands should add wheel metadata/version assertions if they derive or rewrite versions.
|
|
128
|
+
|
|
129
|
+
## Monorepos and cost
|
|
130
|
+
|
|
131
|
+
`examples/monorepo.yml` demonstrates independent service rules, shared dependency paths and service-specific tag names. Include every shared source/config/lock path that can affect a service; transitive dependency graph discovery is not automatic. `CI_FULL_PIPELINE=true` and schedules run full verification. Tags rebuild all package units in that example, so a release never consumes artifacts from a different pipeline.
|
|
132
|
+
|
|
133
|
+
The defaults preserve stage barriers: lint → test → build → verify → publish → deploy → release. Jobs use `dependencies` only to select artifacts. No default `needs: []` bypasses validation. All enabled tests must pass before any build proceeds. Stage barriers are conservative across unrelated services; customize explicit DAG dependencies only after keeping release gates intact.
|
|
134
|
+
|
|
135
|
+
Tests fail normally; only runner/system failures retry once. Deploy/publish operations never retry automatically. Superseded interruptible work can be cancelled. Use `task` cache controls for package download caches, with lockfile-derived keys; keep protected/unprotected cache separation enabled. uv's download cache is safe to rebuild and is not proof of dependency correctness.
|
|
136
|
+
|
|
137
|
+
## Helm deployment
|
|
138
|
+
|
|
139
|
+
Use `charts/generic-app`, `examples/helm-values.yaml`, and `docs/kubernetes.md`. Helm manages the application release; Terraform is appropriate for cluster, DNS, network and registry infrastructure outside this repository's scope.
|
|
140
|
+
|
|
141
|
+
Deploy digest-pinned images from the exact build artifacts. `helm-deploy.image-map-json` maps application names to image repositories and BuildKit metadata files; the helper extracts digests into an overlay passed to Helm. One Helm release may deploy several services, or use separate releases for independently deployable services.
|
|
142
|
+
|
|
143
|
+
## Validation
|
|
144
|
+
|
|
145
|
+
```sh
|
|
146
|
+
python -m pip install -r requirements-dev.txt
|
|
147
|
+
python scripts/sync_embedded.py --check
|
|
148
|
+
python -m unittest discover -s tests -v
|
|
149
|
+
python tests/integration_uv.py
|
|
150
|
+
helm lint charts/generic-app -f examples/helm-values.yaml --strict
|
|
151
|
+
helm template smoke charts/generic-app -f examples/helm-values.yaml
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
The integration test uses local Git repositories and URL rewriting; it does not contact a real dependency host or publish anything. The lightweight component expander tests inputs, names and artifact references. It is not GitLab's server-side compiler. Do not interpret local tests as proof that a specific runner, RBAC policy, Artifactory endpoint or cluster deployment works.
|
|
155
|
+
|
|
156
|
+
## Official references
|
|
157
|
+
|
|
158
|
+
- GitLab components: https://docs.gitlab.com/ci/components/
|
|
159
|
+
- GitLab input types: https://docs.gitlab.com/ci/inputs/
|
|
160
|
+
- GitLab environment teardown: https://docs.gitlab.com/ci/environments/
|
|
161
|
+
- Rootless BuildKit: https://docs.gitlab.com/ci/docker/using_buildkit/
|
|
162
|
+
- GitLab releases: https://docs.gitlab.com/user/project/releases/release_cicd_examples/
|
|
163
|
+
- uv overrides: https://docs.astral.sh/uv/concepts/resolution/
|
|
164
|
+
- uv configuration: https://docs.astral.sh/uv/reference/settings/
|
|
165
|
+
- PyPI trusted publishing: https://docs.pypi.org/trusted-publishers/using-a-publisher/
|
|
166
|
+
- Helm upgrade: https://helm.sh/docs/helm/helm_upgrade/
|
|
167
|
+
|
|
168
|
+
## Publishing this toolkit to PyPI
|
|
169
|
+
|
|
170
|
+
The PyPI distribution is `generic-gitlab-cicd`; its CLI remains `generic-ci`. The [Publish to PyPI workflow](.github/workflows/publish.yml) uses Trusted Publishing on a published GitHub release or a manual run. See [publisher configuration and release steps](docs/pypi-release.md).
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
Testing commands and the local/E2E split: [testing — revision one](docs/testing-revision-one.md).
|
|
2
|
+
|
|
3
|
+
Start with the [documentation map](docs/README.md), [AI authoring guide](docs/ai-authoring-revision-three.md), or portable [generic-ci-authoring skill](skills/generic-ci-authoring/SKILL.md).
|
|
4
|
+
|
|
5
|
+
Organization defaults and Git-backed starter templates are now available: see [configuration sources — revision two](docs/configuration-sources-revision-two.md).
|
|
6
|
+
|
|
7
|
+
> Feature-branch workflow interface (0.3.1): see [implemented revision-one guide](docs/workflows-revision-one.md), [configuration examples](examples/workflows), and [JSON Schema](schemas/workflows.schema.json). This is a review build with documented integration gates. Older prototype CLI commands now require `--format legacy`; existing GitLab components remain available.
|
|
8
|
+
|
|
9
|
+
# Generic GitLab CI components — revision one
|
|
10
|
+
|
|
11
|
+
A self-contained component repository for GitLab, including an offline-capable runtime-image factory and a generic Kubernetes Helm chart. The recommended Python entry point is `examples/uv-airgap.yml`.
|
|
12
|
+
|
|
13
|
+
This is an implementation to configure and validate on your installation. It has not been run against your GitLab, Artifactory or Kubernetes cluster. Infrastructure addresses, credentials, approved base images and application deployment settings must be supplied by your organization.
|
|
14
|
+
|
|
15
|
+
## Components
|
|
16
|
+
|
|
17
|
+
| Component | Purpose |
|
|
18
|
+
| --- | --- |
|
|
19
|
+
| `workflow` | Stages, cancellation of superseded work, suppression of duplicate push/MR pipelines |
|
|
20
|
+
| `task` | Language-neutral lint, test, build or verification commands; reports, caches and artifacts |
|
|
21
|
+
| `uv-test` | Locked uv validation, optional Git dependency replacement and commit provenance |
|
|
22
|
+
| `version-check` | PEP 440 version increase against the current target branch; tag/version matching |
|
|
23
|
+
| `python-package` | Build wheel/sdist once, check metadata, retain distribution artifacts |
|
|
24
|
+
| `python-publish` | Publish those artifacts to PyPI, TestPyPI, GitLab or Artifactory |
|
|
25
|
+
| `container-preview-build` | Same-project MR builds restricted to a dedicated preview repository |
|
|
26
|
+
| `container-build` | Rootless BuildKit build, digest metadata, optional protected-ref registry push |
|
|
27
|
+
| `deploy` / `preview` | Provider-neutral command adapters with deployment locking and preview cleanup |
|
|
28
|
+
| `helm-deploy` / `helm-preview` | Helm rollout with failure rollback and isolated review environments |
|
|
29
|
+
| `release` | Create a GitLab release for an existing protected tag |
|
|
30
|
+
|
|
31
|
+
Each YAML file documents its own typed inputs. Include components multiple times with unique job names. Secrets remain GitLab CI variables; they are never component inputs.
|
|
32
|
+
|
|
33
|
+
## Bootstrap
|
|
34
|
+
|
|
35
|
+
1. Import this directory into a GitLab project such as `platform/ci-components` on your own GitLab instance. Component includes cannot directly cross to an unrelated GitLab instance; mirror this repository internally.
|
|
36
|
+
2. Configure the variables in `docs/airgap.md`, and supply approved internal Python, BuildKit and Helm runtime images. The repository's own pipeline uses those images. Do not switch all consumers until the runtime images are available.
|
|
37
|
+
3. Adapt non-secret files under `images/python/config/`, add approved CA certificates, and review `images/python/requirements.lock`. Run `images.gitlab-ci.yml` to build the Python runtime from your internal package index and push it to Artifactory. Initial BuildKit and Python base images must already be mirrored and trusted.
|
|
38
|
+
4. Run this repository's validation pipeline. Publish/tag an immutable component version, e.g. `1.0.0`. Optionally mark the project as a CI/CD Catalog resource and create a GitLab release. Consumers may pin a full commit SHA before the first release.
|
|
39
|
+
5. Copy the relevant example as the consuming project's `.gitlab-ci.yml`. Replace `platform/ci-components`, the release ref and deployment placeholders.
|
|
40
|
+
6. Validate the expanded pipeline with your GitLab CI Lint, then run MR, default-branch and protected-tag scenarios before enabling production publishing.
|
|
41
|
+
|
|
42
|
+
Use a supported GitLab with CI/CD components and array inputs (GitLab 17+ syntax baseline). The optional native release job also requires a GitLab-compatible glab runtime; verify server/CLI compatibility. Runtime jobs target Linux container runners, not Windows shell runners.
|
|
43
|
+
|
|
44
|
+
## Minimal uv project
|
|
45
|
+
|
|
46
|
+
```yaml
|
|
47
|
+
variables:
|
|
48
|
+
CI_DEPENDENCY_REPO: ""
|
|
49
|
+
CI_DEPENDENCY_REF: ""
|
|
50
|
+
CI_DEPENDENCY_PACKAGE: ""
|
|
51
|
+
CI_DEPENDENCY_OVERRIDES: '[]'
|
|
52
|
+
|
|
53
|
+
include:
|
|
54
|
+
- component: $CI_SERVER_FQDN/platform/ci-components/workflow@1.0.0
|
|
55
|
+
- component: $CI_SERVER_FQDN/platform/ci-components/uv-test@1.0.0
|
|
56
|
+
inputs:
|
|
57
|
+
name: api-test
|
|
58
|
+
repo: $CI_DEPENDENCY_REPO
|
|
59
|
+
ref: $CI_DEPENDENCY_REF
|
|
60
|
+
package: $CI_DEPENDENCY_PACKAGE
|
|
61
|
+
overrides-json: $CI_DEPENDENCY_OVERRIDES
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Set `PYTHON_CI_IMAGE` to your internal runtime image. Commit `uv.lock`. Put pytest and application test dependencies in the project's dependency groups. `uv-test` syncs all groups; define mutually compatible groups. Tests run with `uv run --no-sync` so uv does not undo a candidate installation.
|
|
65
|
+
|
|
66
|
+
## Cross-repository dependencies
|
|
67
|
+
|
|
68
|
+
For an unreleased dependency, start a pipeline with:
|
|
69
|
+
|
|
70
|
+
```text
|
|
71
|
+
CI_DEPENDENCY_REPO=https://gitlab.internal/team/shared-sdk.git
|
|
72
|
+
CI_DEPENDENCY_REF=feature/new-api
|
|
73
|
+
CI_DEPENDENCY_PACKAGE=shared-sdk
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
For several dependencies or a package in a repository subdirectory:
|
|
77
|
+
|
|
78
|
+
```json
|
|
79
|
+
[
|
|
80
|
+
{"repo":"https://gitlab.internal/team/sdk.git", "ref":"feature/new-api", "package":"shared-sdk"},
|
|
81
|
+
{"repo":"https://gitlab.internal/team/core.git", "ref":"0123456789012345678901234567890123456789", "package":"core-utils", "subdirectory":"packages/utils"}
|
|
82
|
+
]
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Set that array as `CI_DEPENDENCY_OVERRIDES`. Both mechanisms can be combined if package names do not overlap.
|
|
86
|
+
|
|
87
|
+
The helper resolves each branch/tag to a commit, applies temporary `tool.uv.override-dependencies`, removes conflicting uv source mappings for the named distribution, re-resolves, and verifies installed `direct_url.json` commit metadata. Original pyproject and lock bytes are restored even if sync fails. A successful override run saves requested refs and actual commits as a provenance artifact. A branch can move between pipelines; use the recorded SHA for reproducible reruns.
|
|
88
|
+
|
|
89
|
+
The normal path uses `uv sync --locked --all-groups`. The override path deliberately re-resolves and can change transitive dependencies; it tests candidate compatibility rather than certifying the production lock. Overrides must name an installed distribution; a typo or irrelevant package fails verification.
|
|
90
|
+
|
|
91
|
+
Configure read-only Git authentication in the job's `setup` or runner credential helper. Use credential-free repository URLs. For CI_JOB_TOKEN access, allowlist the consumer project in each dependency project. Restrict the credential helper to the intended host. Do not put credentials in URLs, artifacts, images, or tracked config.
|
|
92
|
+
|
|
93
|
+
Run a uv workspace from its root; its virtual environment must be `.venv` there. For independent monorepo projects, include one `uv-test` per project and use unique report/provenance paths. Custom UV_PROJECT_ENVIRONMENT layouts require adapting the helper's virtual-environment path.
|
|
94
|
+
|
|
95
|
+
## Coordinated releases
|
|
96
|
+
|
|
97
|
+
Candidate testing solves the validation deadlock. It does not make multiple registry publications transactional.
|
|
98
|
+
|
|
99
|
+
1. Validate the application and dependency candidates together using immutable overrides.
|
|
100
|
+
2. Publish the dependency's real version to the internal package repository.
|
|
101
|
+
3. Update/validate the application's production lock against that registry, with overrides cleared.
|
|
102
|
+
4. Build once, publish/deploy the application, then create its GitLab release.
|
|
103
|
+
|
|
104
|
+
A dependency package can usually be published before the application is deployed; publishing the wheel does not deploy a running service. For stronger coordination, use an internal candidate repository and promote approved packages before deployment. Two independent GitLab projects still need an orchestration policy; this framework does not promise atomic cross-project rollback. Multi-project trigger/approval coordination is an extension point, not implemented here.
|
|
105
|
+
|
|
106
|
+
The standard publish and production deploy jobs reject nonempty `CI_DEPENDENCY_*` override variables. Do not hard-code candidate inputs in a production pipeline. Preview artifacts may intentionally contain candidate dependencies; never promote those to production without a clean release validation.
|
|
107
|
+
|
|
108
|
+
## Versioning, tags and releases
|
|
109
|
+
|
|
110
|
+
`version-check` reads static `[project].version`. In branch/MR pipelines it fetches the current target branch and requires a strict PEP 440 increase. On tags it requires the tag suffix to match the package version. The default prefix is `v`; use `sdk-v` for independent package tags. Dynamic SCM versions require a custom command via `task`.
|
|
111
|
+
|
|
112
|
+
Use change rules to restrict bump checks to meaningful package changes if documentation-only MRs should not bump a version. The default checks every selected non-default branch pipeline. `allow-new-package` explicitly permits a manifest absent from the target branch.
|
|
113
|
+
|
|
114
|
+
Tags are an input to the release pipeline. Create and protect them through your established process; CI does not silently create or push tags. Package publishing and production deployment are blocking manual jobs on protected tags by default. The GitLab release runs last, using `CHANGELOG.md` or a configured notes file. Existing GitLab release retries require deliberate handling; no overwrite or package `--skip-existing` behavior hides duplicate releases.
|
|
115
|
+
|
|
116
|
+
A tag check validates manifest metadata, not the version assigned by every possible custom build backend. The build defaults use static PEP 621 metadata. Custom build commands should add wheel metadata/version assertions if they derive or rewrite versions.
|
|
117
|
+
|
|
118
|
+
## Monorepos and cost
|
|
119
|
+
|
|
120
|
+
`examples/monorepo.yml` demonstrates independent service rules, shared dependency paths and service-specific tag names. Include every shared source/config/lock path that can affect a service; transitive dependency graph discovery is not automatic. `CI_FULL_PIPELINE=true` and schedules run full verification. Tags rebuild all package units in that example, so a release never consumes artifacts from a different pipeline.
|
|
121
|
+
|
|
122
|
+
The defaults preserve stage barriers: lint → test → build → verify → publish → deploy → release. Jobs use `dependencies` only to select artifacts. No default `needs: []` bypasses validation. All enabled tests must pass before any build proceeds. Stage barriers are conservative across unrelated services; customize explicit DAG dependencies only after keeping release gates intact.
|
|
123
|
+
|
|
124
|
+
Tests fail normally; only runner/system failures retry once. Deploy/publish operations never retry automatically. Superseded interruptible work can be cancelled. Use `task` cache controls for package download caches, with lockfile-derived keys; keep protected/unprotected cache separation enabled. uv's download cache is safe to rebuild and is not proof of dependency correctness.
|
|
125
|
+
|
|
126
|
+
## Helm deployment
|
|
127
|
+
|
|
128
|
+
Use `charts/generic-app`, `examples/helm-values.yaml`, and `docs/kubernetes.md`. Helm manages the application release; Terraform is appropriate for cluster, DNS, network and registry infrastructure outside this repository's scope.
|
|
129
|
+
|
|
130
|
+
Deploy digest-pinned images from the exact build artifacts. `helm-deploy.image-map-json` maps application names to image repositories and BuildKit metadata files; the helper extracts digests into an overlay passed to Helm. One Helm release may deploy several services, or use separate releases for independently deployable services.
|
|
131
|
+
|
|
132
|
+
## Validation
|
|
133
|
+
|
|
134
|
+
```sh
|
|
135
|
+
python -m pip install -r requirements-dev.txt
|
|
136
|
+
python scripts/sync_embedded.py --check
|
|
137
|
+
python -m unittest discover -s tests -v
|
|
138
|
+
python tests/integration_uv.py
|
|
139
|
+
helm lint charts/generic-app -f examples/helm-values.yaml --strict
|
|
140
|
+
helm template smoke charts/generic-app -f examples/helm-values.yaml
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The integration test uses local Git repositories and URL rewriting; it does not contact a real dependency host or publish anything. The lightweight component expander tests inputs, names and artifact references. It is not GitLab's server-side compiler. Do not interpret local tests as proof that a specific runner, RBAC policy, Artifactory endpoint or cluster deployment works.
|
|
144
|
+
|
|
145
|
+
## Official references
|
|
146
|
+
|
|
147
|
+
- GitLab components: https://docs.gitlab.com/ci/components/
|
|
148
|
+
- GitLab input types: https://docs.gitlab.com/ci/inputs/
|
|
149
|
+
- GitLab environment teardown: https://docs.gitlab.com/ci/environments/
|
|
150
|
+
- Rootless BuildKit: https://docs.gitlab.com/ci/docker/using_buildkit/
|
|
151
|
+
- GitLab releases: https://docs.gitlab.com/user/project/releases/release_cicd_examples/
|
|
152
|
+
- uv overrides: https://docs.astral.sh/uv/concepts/resolution/
|
|
153
|
+
- uv configuration: https://docs.astral.sh/uv/reference/settings/
|
|
154
|
+
- PyPI trusted publishing: https://docs.pypi.org/trusted-publishers/using-a-publisher/
|
|
155
|
+
- Helm upgrade: https://helm.sh/docs/helm/helm_upgrade/
|
|
156
|
+
|
|
157
|
+
## Publishing this toolkit to PyPI
|
|
158
|
+
|
|
159
|
+
The PyPI distribution is `generic-gitlab-cicd`; its CLI remains `generic-ci`. The [Publish to PyPI workflow](.github/workflows/publish.yml) uses Trusted Publishing on a published GitHub release or a manual run. See [publisher configuration and release steps](docs/pypi-release.md).
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Offline authoring CLI: schema, validate, explain, render and drift checking."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
|
|
9
|
+
from .compiler import compile_pipeline, render, source_hashes
|
|
10
|
+
from .config import load
|
|
11
|
+
from .models import Pipeline, Platform
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main(argv=None):
|
|
15
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
16
|
+
if argv and argv[0] == 'source':
|
|
17
|
+
from .sources import source_main
|
|
18
|
+
try:
|
|
19
|
+
return source_main(argv[1:])
|
|
20
|
+
except (ValueError, OSError, ValidationError) as error:
|
|
21
|
+
print(f'generic-ci: {error}', file=sys.stderr)
|
|
22
|
+
return 1
|
|
23
|
+
parser = argparse.ArgumentParser(prog="generic-ci")
|
|
24
|
+
parser.add_argument("command", choices=["schema", "validate", "explain", "render", "init"])
|
|
25
|
+
parser.add_argument("--config")
|
|
26
|
+
parser.add_argument("--format", choices=["workflows", "legacy"], default="workflows")
|
|
27
|
+
parser.add_argument("--ecosystem", choices=["python", "npm", "pnpm", "bun"], default="python")
|
|
28
|
+
parser.add_argument("--platform")
|
|
29
|
+
parser.add_argument("--template")
|
|
30
|
+
parser.add_argument("--source")
|
|
31
|
+
parser.add_argument("--repo")
|
|
32
|
+
parser.add_argument("--ref")
|
|
33
|
+
parser.add_argument("--offline", action="store_true")
|
|
34
|
+
parser.add_argument("--root", default=".")
|
|
35
|
+
parser.add_argument("--output", "-o")
|
|
36
|
+
parser.add_argument("--check", action="store_true", help="Fail if output differs; never modify output")
|
|
37
|
+
parser.add_argument("--platform-schema", action="store_true")
|
|
38
|
+
options = parser.parse_args(argv)
|
|
39
|
+
try:
|
|
40
|
+
root = Path(options.root).resolve()
|
|
41
|
+
use_source = options.format == 'workflows' and (root / 'generic-ci.yml').exists()
|
|
42
|
+
from .sources import home
|
|
43
|
+
default_source = (home() / 'sources.json').exists()
|
|
44
|
+
if options.command == 'init' and (options.template or options.source or options.repo or default_source):
|
|
45
|
+
from .sources import initialize
|
|
46
|
+
initialize(root, options.template, options.source, options.repo, options.ref, options.offline, options.config)
|
|
47
|
+
return 0
|
|
48
|
+
config_path = root / (options.config or 'delivery.yml')
|
|
49
|
+
platform_path = root / (options.platform or 'ci-platform.yml')
|
|
50
|
+
if options.format == "workflows":
|
|
51
|
+
from .workflows import compiler as workflow_compiler
|
|
52
|
+
from .workflows.models import Pipeline as WorkflowPipeline, Platform as WorkflowPlatform
|
|
53
|
+
if options.command == "init":
|
|
54
|
+
import yaml
|
|
55
|
+
if config_path.exists():
|
|
56
|
+
raise ValueError("configuration exists; init will not overwrite it")
|
|
57
|
+
ecosystem = {"python": {}} if options.ecosystem == "python" else {"node": {"package-manager": options.ecosystem}}
|
|
58
|
+
command = "uv run --no-sync pytest" if options.ecosystem == "python" else options.ecosystem + " run test"
|
|
59
|
+
example = {"version": 1, "projects": {"app": {"path": ".", **ecosystem,
|
|
60
|
+
"checks": {"unit": {"script": [command]}},
|
|
61
|
+
"workflows": {"push": {"checks": ["unit"]}, "merge-request": {"checks": ["unit"]}}}}}
|
|
62
|
+
config_path.write_text(yaml.safe_dump(example, sort_keys=False))
|
|
63
|
+
print(f"Created {config_path}; provide an internal platform configuration with --platform")
|
|
64
|
+
return 0
|
|
65
|
+
if options.command == "schema":
|
|
66
|
+
model = (WorkflowPlatform if options.platform_schema else WorkflowPipeline) if options.format == "workflows" else (Platform if options.platform_schema else Pipeline)
|
|
67
|
+
result = json.dumps(model.model_json_schema(by_alias=True), indent=2) + "\n"
|
|
68
|
+
else:
|
|
69
|
+
if options.format == "workflows":
|
|
70
|
+
if use_source:
|
|
71
|
+
from .sources import load_project
|
|
72
|
+
pipeline, platform, origins, inputs = load_project(root, options.config, options.platform, options.offline)
|
|
73
|
+
else:
|
|
74
|
+
pipeline, platform = workflow_compiler.load(config_path, platform_path)
|
|
75
|
+
origins = {"checks": "developer-defined; no hidden suites"}
|
|
76
|
+
inputs = [config_path, platform_path]
|
|
77
|
+
else:
|
|
78
|
+
pipeline, platform, origins = load(config_path, platform_path)
|
|
79
|
+
inputs = [config_path, platform_path]
|
|
80
|
+
sources = source_hashes(root, inputs)
|
|
81
|
+
jobs, payload = (workflow_compiler.compile_pipeline(pipeline, platform, sources=sources) if options.format == "workflows" else compile_pipeline(pipeline, platform, sources=sources))
|
|
82
|
+
if options.command == "validate":
|
|
83
|
+
result = f"Valid: {len(pipeline.projects)} projects; {len(payload['nodes']) + 1} jobs. Target GitLab CI Lint is still required.\n"
|
|
84
|
+
elif options.command == "explain":
|
|
85
|
+
result = json.dumps({"generation": "committed top-level GitLab CI", "origins": origins,
|
|
86
|
+
"projects": payload["pipeline"]["projects"], "platform": payload["platform"], "jobs": payload["nodes"]}, indent=2) + "\n"
|
|
87
|
+
else:
|
|
88
|
+
if options.format == "workflows":
|
|
89
|
+
import yaml
|
|
90
|
+
result = "# Generated by generic-ci; edit delivery configuration and render again.\n" + yaml.safe_dump(jobs, sort_keys=False)
|
|
91
|
+
else:
|
|
92
|
+
result = render(pipeline, platform, sources=sources)
|
|
93
|
+
if options.check:
|
|
94
|
+
if not options.output or not Path(options.output).is_file() or Path(options.output).read_text() != result:
|
|
95
|
+
raise ValueError("generated output differs; render again and commit it")
|
|
96
|
+
return 0
|
|
97
|
+
if options.output:
|
|
98
|
+
target = Path(options.output)
|
|
99
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
100
|
+
target.write_text(result)
|
|
101
|
+
else:
|
|
102
|
+
print(result, end="")
|
|
103
|
+
return 0
|
|
104
|
+
except (ValueError, OSError, ValidationError) as error:
|
|
105
|
+
print(f"generic-ci: {error}", file=sys.stderr)
|
|
106
|
+
return 1
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
if __name__ == "__main__":
|
|
110
|
+
raise SystemExit(main())
|