c2cciutils 1.7.0.dev174__tar.gz → 1.8.0.dev45__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of c2cciutils might be problematic. Click here for more details.
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/LICENSE +1 -1
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/PKG-INFO +29 -58
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/README.md +16 -44
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/__init__.py +15 -230
- c2cciutils-1.8.0.dev45/c2cciutils/applications-versions.yaml +4 -0
- c2cciutils-1.8.0.dev45/c2cciutils/applications_definition.py +51 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/configuration.py +83 -554
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/env.py +8 -31
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/lib/docker.py +2 -8
- c2cciutils-1.8.0.dev45/c2cciutils/lib/oidc.py +188 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/package-lock.json +115 -127
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/package.json +1 -1
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/publish.py +26 -44
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/schema.json +3 -230
- c2cciutils-1.8.0.dev45/c2cciutils/scripts/__init__.py +1 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/clean.py +4 -11
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/docker_logs.py +4 -4
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/docker_versions_gen.py +0 -1
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/download_applications.py +0 -2
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/env.py +2 -6
- c2cciutils-1.8.0.dev45/c2cciutils/scripts/k8s/__init__.py +1 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/k8s/wait.py +2 -2
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/main.py +4 -16
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/publish.py +45 -31
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/trigger_image_update.py +3 -8
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/version.py +5 -4
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/pyproject.toml +22 -32
- c2cciutils-1.7.0.dev174/c2cciutils/applications-versions.yaml +0 -4
- c2cciutils-1.7.0.dev174/c2cciutils/applications_definition.py +0 -53
- c2cciutils-1.7.0.dev174/c2cciutils/audit.py +0 -229
- c2cciutils-1.7.0.dev174/c2cciutils/pr_checks.py +0 -286
- c2cciutils-1.7.0.dev174/c2cciutils/scripts/__init__.py +0 -3
- c2cciutils-1.7.0.dev174/c2cciutils/scripts/audit.py +0 -41
- c2cciutils-1.7.0.dev174/c2cciutils/scripts/docker_versions_update.py +0 -85
- c2cciutils-1.7.0.dev174/c2cciutils/scripts/k8s/__init__.py +0 -3
- c2cciutils-1.7.0.dev174/c2cciutils/scripts/pr_checks.py +0 -78
- c2cciutils-1.7.0.dev174/c2cciutils/security.py +0 -59
- c2cciutils-1.7.0.dev174/setup.py +0 -79
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/applications.yaml +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/branches.graphql +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/commits.graphql +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/default_branch.graphql +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/schema-applications.json +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/k8s/db.py +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/k8s/install.py +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/k8s/logs.py +0 -0
- {c2cciutils-1.7.0.dev174 → c2cciutils-1.8.0.dev45}/c2cciutils/scripts/pin_pipenv.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: c2cciutils
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.8.0.dev45
|
|
4
4
|
Summary: Common utilities for Camptocamp CI
|
|
5
5
|
Home-page: https://github.com/camptocamp/c2cciutils
|
|
6
6
|
License: FreeBSD
|
|
@@ -18,29 +18,28 @@ Classifier: Programming Language :: Python
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.9
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
24
|
Classifier: Typing :: Typed
|
|
22
25
|
Provides-Extra: audit
|
|
23
26
|
Provides-Extra: checks
|
|
24
|
-
Provides-Extra:
|
|
27
|
+
Provides-Extra: pr-checks
|
|
25
28
|
Provides-Extra: publish
|
|
26
|
-
Provides-Extra:
|
|
29
|
+
Provides-Extra: publish-plugins
|
|
27
30
|
Provides-Extra: version
|
|
28
|
-
Requires-Dist: Markdown (>=3.0.0,<4.0.0)
|
|
29
31
|
Requires-Dist: PyYAML (>=6.0.0,<7.0.0)
|
|
30
|
-
Requires-Dist: codespell (==2.2.6); extra == "pr_checks"
|
|
31
32
|
Requires-Dist: debian-inspector (>=31.0.0,<32.0.0)
|
|
32
33
|
Requires-Dist: defusedxml (>=0.0.0,<1.0.0)
|
|
33
|
-
Requires-Dist: google-api-python-client (>=2.0.0,<3.0.0); extra == "publish"
|
|
34
|
-
Requires-Dist: google-auth-httplib2 (>=0.0.0,<1.0.0); extra == "publish"
|
|
35
|
-
Requires-Dist: google-auth-oauthlib (>=1.0.0,<2.0.0); extra == "publish"
|
|
36
|
-
Requires-Dist:
|
|
37
|
-
Requires-Dist: multi-repo-automation (>=1.0.0,<2.0.0); extra == "version"
|
|
38
|
-
Requires-Dist: python-magic (>=0.0.0,<1.0.0)
|
|
34
|
+
Requires-Dist: google-api-python-client (>=2.0.0,<3.0.0) ; extra == "publish"
|
|
35
|
+
Requires-Dist: google-auth-httplib2 (>=0.0.0,<1.0.0) ; extra == "publish"
|
|
36
|
+
Requires-Dist: google-auth-oauthlib (>=1.0.0,<2.0.0) ; extra == "publish"
|
|
37
|
+
Requires-Dist: id (>=1.0.0,<2.0.0) ; extra == "publish"
|
|
38
|
+
Requires-Dist: multi-repo-automation (>=1.0.0,<2.0.0) ; extra == "version"
|
|
39
39
|
Requires-Dist: requests (>=2.0.0,<3.0.0)
|
|
40
40
|
Requires-Dist: ruamel.yaml (>=0.0.0,<1.0.0)
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist: twine (>=5.0.0,<6.0.0); extra == "publish"
|
|
41
|
+
Requires-Dist: security-md (>=1.0.0,<2.0.0)
|
|
42
|
+
Requires-Dist: twine (>=6.0.0,<7.0.0) ; extra == "publish"
|
|
44
43
|
Project-URL: Repository, https://github.com/camptocamp/c2cciutils
|
|
45
44
|
Description-Content-Type: text/markdown
|
|
46
45
|
|
|
@@ -48,47 +47,14 @@ Description-Content-Type: text/markdown
|
|
|
48
47
|
|
|
49
48
|
## Publishing
|
|
50
49
|
|
|
51
|
-
The main goals of C2C CI utils is to offer the commands
|
|
52
|
-
|
|
53
|
-
Have stabilization branches named by default `<major>.<minor>`.
|
|
54
|
-
Have the release named by default `<major>.<minor>.<patch>`.
|
|
55
|
-
|
|
56
|
-
With C2C CI utils you can publish a python package and a Docker image from the same repository.
|
|
57
|
-
|
|
58
|
-
The default publishing are:
|
|
59
|
-
|
|
60
|
-
- Push on the `<major>.<minor>` branch will publish Docker images.
|
|
61
|
-
- Create the tag `<major>.<minor>.<patch>` will publish the Docker images, and the Python package.
|
|
62
|
-
- Push on a feature branch (whatever other name) will publish the Docker images.
|
|
63
|
-
- Delete a feature branch will delete the Docker images.
|
|
64
|
-
- Push on the `master` branch will publish the Docker images with the master tag (Publishing a python package is also possible).
|
|
65
|
-
- The version at the last line of the `SECURITY.md` of the `master` branch will be also published using the `latest` tag,
|
|
66
|
-
this will respect the `tags` present in the configuration
|
|
67
|
-
- In the `SECURITY.md` file of the `master` branch we can also add a column `Alternate Tag` to publish the Docker images with another tag,
|
|
68
|
-
this will respect the `tags` present in the configuration (only for Docker).
|
|
69
|
-
|
|
70
|
-
The Docker images are published on Docker Hub and GitHub Container Registry.
|
|
71
|
-
|
|
72
|
-
You can run the publishing locally in dry-run mode:
|
|
73
|
-
|
|
74
|
-
```bash
|
|
75
|
-
GITHUB_REF=... c2cciutils-publish --dry-run ...
|
|
76
|
-
```
|
|
50
|
+
The main goals of C2C CI utils is to offer the commands to publish the project,
|
|
51
|
+
see the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing).
|
|
77
52
|
|
|
78
53
|
## Changelog
|
|
79
54
|
|
|
80
55
|
When we create a tag by default with the `changelog` workflow a release is created on GitHub, a changelog is
|
|
81
56
|
generated and added to the release.
|
|
82
57
|
|
|
83
|
-
## Security
|
|
84
|
-
|
|
85
|
-
The security is managed by the `c2cciutils-audit` command with Snyk, it will audit the dependencies of the project on every
|
|
86
|
-
stabilization branches, if possible a pull request is created automatically to update the dependencies.
|
|
87
|
-
|
|
88
|
-
When we publish a Docker image the generated image is monitored by Snyk, this means that Snyk will search
|
|
89
|
-
for all the dependencies and send the list to the Snyk web site to be monitored.
|
|
90
|
-
We also do a test of the image and log the result (This will never cause the build to fail).
|
|
91
|
-
|
|
92
58
|
## Checks
|
|
93
59
|
|
|
94
60
|
C2C CI utils will no more provide a tool to do a check of the project, this is replaced by `pre-commit`,
|
|
@@ -123,21 +89,16 @@ workflow will delete the workflows older than 500 days.
|
|
|
123
89
|
|
|
124
90
|
C2cciutils make easier to have those workflows in a project:
|
|
125
91
|
|
|
126
|
-
- `audit.yaml`: Audit the stabilization branches of the application against vulnerabilities in the python and node dependency
|
|
127
92
|
- `auto-review.yaml`: Auto review the Renovate pull requests
|
|
128
93
|
- `backport.yaml`: Trigger the backports (work with labels)
|
|
129
94
|
- `clean.yaml`: Clean the Docker images related on a deleted feature branch
|
|
130
95
|
- `main.yaml`: Main workflow especially with the c2cciutils-checks command
|
|
131
|
-
- `changelog.yaml`: Generate the changelog and create the release on GitHub
|
|
132
|
-
- `delete-old-workflows-run.yaml`: Delete the old workflows
|
|
133
|
-
- `pr-checks.yaml`: Run the checks on the pull requests
|
|
134
96
|
|
|
135
97
|
All the provided commands used in the workflow:
|
|
136
98
|
|
|
137
99
|
- `c2cciutils`: some generic tools.
|
|
138
100
|
- `c2cciutils-version`: Create a new version of the project.
|
|
139
|
-
- `c2cciutils-
|
|
140
|
-
- `c2cciutils-audit`: Do the audit, the main difference with checks is that it can change between runs on the same code.
|
|
101
|
+
- `c2cciutils-env`: Print some environment information.
|
|
141
102
|
- `c2cciutils-publish`: Publish the project.
|
|
142
103
|
- `c2cciutils-clean`: Delete Docker images on Docker Hub after corresponding branch have been deleted.
|
|
143
104
|
|
|
@@ -198,7 +159,6 @@ You can override the configuration with the file `ci/config.yaml`.
|
|
|
198
159
|
At the base of the configuration you have:
|
|
199
160
|
|
|
200
161
|
- `version`: Contains some regular expressions to find the versions branches and tags, and to convert them into application versions.
|
|
201
|
-
- `audit`: The audit configuration, see `c2cciutils/audit.py` for more information.
|
|
202
162
|
- `publish`: The publishing configuration, see `c2cciutils/publish.py` for more information.
|
|
203
163
|
|
|
204
164
|
Many actions can be disabled by setting the corresponding configuration part to `False`.
|
|
@@ -294,11 +254,22 @@ Then by default:
|
|
|
294
254
|
- Commit on `master` branch after the tag 1.3.0 => release `1.4.0.dev1`
|
|
295
255
|
- Commit on `1.3` branch after the tag 1.3.0 => release `1.3.1.dev1`
|
|
296
256
|
|
|
257
|
+
#### Authentication
|
|
258
|
+
|
|
259
|
+
If the file `~/.pypirc` exists we consider that we ar already logged in also
|
|
260
|
+
we will do the login with the `pypi` server with OpenID Connect (OIDC).
|
|
261
|
+
|
|
262
|
+
The OIDC login is recommended because it didn't needs any additional secrets,
|
|
263
|
+
but it need some configuration on pypi in the package,
|
|
264
|
+
see the [GitHub Documentation](https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-pypi#adding-the-identity-provider-to-pypi).
|
|
265
|
+
|
|
266
|
+
#### Integration if the package directly in a Docker image
|
|
267
|
+
|
|
297
268
|
To make it working in the `Dockerfile` you should have in the `poetry` stage:
|
|
298
269
|
|
|
299
270
|
```Dockerfile
|
|
300
271
|
ENV POETRY_DYNAMIC_VERSIONING_BYPASS=dev
|
|
301
|
-
RUN poetry export --extras=checks --extras=publish --
|
|
272
|
+
RUN poetry export --extras=checks --extras=publish --output=requirements.txt \
|
|
302
273
|
&& poetry export --with=dev --output=requirements-dev.txt
|
|
303
274
|
```
|
|
304
275
|
|
|
@@ -336,7 +307,7 @@ repository:
|
|
|
336
307
|
# List of kinds of versions you want to publish, that can be: rebuild (specified using --type),
|
|
337
308
|
# version_tag, version_branch, feature_branch, feature_tag (for pull request)
|
|
338
309
|
version:
|
|
339
|
-
# List of tags we want to publish interpreted with `
|
|
310
|
+
# List of tags we want to publish interpreted with `format(version=version)`
|
|
340
311
|
# e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag
|
|
341
312
|
# (that should be built by the application build) is `latest-lite`, and it will be published
|
|
342
313
|
# with the tag `1.2.3-lite`.
|
|
@@ -2,47 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
## Publishing
|
|
4
4
|
|
|
5
|
-
The main goals of C2C CI utils is to offer the commands
|
|
6
|
-
|
|
7
|
-
Have stabilization branches named by default `<major>.<minor>`.
|
|
8
|
-
Have the release named by default `<major>.<minor>.<patch>`.
|
|
9
|
-
|
|
10
|
-
With C2C CI utils you can publish a python package and a Docker image from the same repository.
|
|
11
|
-
|
|
12
|
-
The default publishing are:
|
|
13
|
-
|
|
14
|
-
- Push on the `<major>.<minor>` branch will publish Docker images.
|
|
15
|
-
- Create the tag `<major>.<minor>.<patch>` will publish the Docker images, and the Python package.
|
|
16
|
-
- Push on a feature branch (whatever other name) will publish the Docker images.
|
|
17
|
-
- Delete a feature branch will delete the Docker images.
|
|
18
|
-
- Push on the `master` branch will publish the Docker images with the master tag (Publishing a python package is also possible).
|
|
19
|
-
- The version at the last line of the `SECURITY.md` of the `master` branch will be also published using the `latest` tag,
|
|
20
|
-
this will respect the `tags` present in the configuration
|
|
21
|
-
- In the `SECURITY.md` file of the `master` branch we can also add a column `Alternate Tag` to publish the Docker images with another tag,
|
|
22
|
-
this will respect the `tags` present in the configuration (only for Docker).
|
|
23
|
-
|
|
24
|
-
The Docker images are published on Docker Hub and GitHub Container Registry.
|
|
25
|
-
|
|
26
|
-
You can run the publishing locally in dry-run mode:
|
|
27
|
-
|
|
28
|
-
```bash
|
|
29
|
-
GITHUB_REF=... c2cciutils-publish --dry-run ...
|
|
30
|
-
```
|
|
5
|
+
The main goals of C2C CI utils is to offer the commands to publish the project,
|
|
6
|
+
see the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing).
|
|
31
7
|
|
|
32
8
|
## Changelog
|
|
33
9
|
|
|
34
10
|
When we create a tag by default with the `changelog` workflow a release is created on GitHub, a changelog is
|
|
35
11
|
generated and added to the release.
|
|
36
12
|
|
|
37
|
-
## Security
|
|
38
|
-
|
|
39
|
-
The security is managed by the `c2cciutils-audit` command with Snyk, it will audit the dependencies of the project on every
|
|
40
|
-
stabilization branches, if possible a pull request is created automatically to update the dependencies.
|
|
41
|
-
|
|
42
|
-
When we publish a Docker image the generated image is monitored by Snyk, this means that Snyk will search
|
|
43
|
-
for all the dependencies and send the list to the Snyk web site to be monitored.
|
|
44
|
-
We also do a test of the image and log the result (This will never cause the build to fail).
|
|
45
|
-
|
|
46
13
|
## Checks
|
|
47
14
|
|
|
48
15
|
C2C CI utils will no more provide a tool to do a check of the project, this is replaced by `pre-commit`,
|
|
@@ -77,21 +44,16 @@ workflow will delete the workflows older than 500 days.
|
|
|
77
44
|
|
|
78
45
|
C2cciutils make easier to have those workflows in a project:
|
|
79
46
|
|
|
80
|
-
- `audit.yaml`: Audit the stabilization branches of the application against vulnerabilities in the python and node dependency
|
|
81
47
|
- `auto-review.yaml`: Auto review the Renovate pull requests
|
|
82
48
|
- `backport.yaml`: Trigger the backports (work with labels)
|
|
83
49
|
- `clean.yaml`: Clean the Docker images related on a deleted feature branch
|
|
84
50
|
- `main.yaml`: Main workflow especially with the c2cciutils-checks command
|
|
85
|
-
- `changelog.yaml`: Generate the changelog and create the release on GitHub
|
|
86
|
-
- `delete-old-workflows-run.yaml`: Delete the old workflows
|
|
87
|
-
- `pr-checks.yaml`: Run the checks on the pull requests
|
|
88
51
|
|
|
89
52
|
All the provided commands used in the workflow:
|
|
90
53
|
|
|
91
54
|
- `c2cciutils`: some generic tools.
|
|
92
55
|
- `c2cciutils-version`: Create a new version of the project.
|
|
93
|
-
- `c2cciutils-
|
|
94
|
-
- `c2cciutils-audit`: Do the audit, the main difference with checks is that it can change between runs on the same code.
|
|
56
|
+
- `c2cciutils-env`: Print some environment information.
|
|
95
57
|
- `c2cciutils-publish`: Publish the project.
|
|
96
58
|
- `c2cciutils-clean`: Delete Docker images on Docker Hub after corresponding branch have been deleted.
|
|
97
59
|
|
|
@@ -152,7 +114,6 @@ You can override the configuration with the file `ci/config.yaml`.
|
|
|
152
114
|
At the base of the configuration you have:
|
|
153
115
|
|
|
154
116
|
- `version`: Contains some regular expressions to find the versions branches and tags, and to convert them into application versions.
|
|
155
|
-
- `audit`: The audit configuration, see `c2cciutils/audit.py` for more information.
|
|
156
117
|
- `publish`: The publishing configuration, see `c2cciutils/publish.py` for more information.
|
|
157
118
|
|
|
158
119
|
Many actions can be disabled by setting the corresponding configuration part to `False`.
|
|
@@ -248,11 +209,22 @@ Then by default:
|
|
|
248
209
|
- Commit on `master` branch after the tag 1.3.0 => release `1.4.0.dev1`
|
|
249
210
|
- Commit on `1.3` branch after the tag 1.3.0 => release `1.3.1.dev1`
|
|
250
211
|
|
|
212
|
+
#### Authentication
|
|
213
|
+
|
|
214
|
+
If the file `~/.pypirc` exists we consider that we ar already logged in also
|
|
215
|
+
we will do the login with the `pypi` server with OpenID Connect (OIDC).
|
|
216
|
+
|
|
217
|
+
The OIDC login is recommended because it didn't needs any additional secrets,
|
|
218
|
+
but it need some configuration on pypi in the package,
|
|
219
|
+
see the [GitHub Documentation](https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-pypi#adding-the-identity-provider-to-pypi).
|
|
220
|
+
|
|
221
|
+
#### Integration if the package directly in a Docker image
|
|
222
|
+
|
|
251
223
|
To make it working in the `Dockerfile` you should have in the `poetry` stage:
|
|
252
224
|
|
|
253
225
|
```Dockerfile
|
|
254
226
|
ENV POETRY_DYNAMIC_VERSIONING_BYPASS=dev
|
|
255
|
-
RUN poetry export --extras=checks --extras=publish --
|
|
227
|
+
RUN poetry export --extras=checks --extras=publish --output=requirements.txt \
|
|
256
228
|
&& poetry export --with=dev --output=requirements-dev.txt
|
|
257
229
|
```
|
|
258
230
|
|
|
@@ -290,7 +262,7 @@ repository:
|
|
|
290
262
|
# List of kinds of versions you want to publish, that can be: rebuild (specified using --type),
|
|
291
263
|
# version_tag, version_branch, feature_branch, feature_tag (for pull request)
|
|
292
264
|
version:
|
|
293
|
-
# List of tags we want to publish interpreted with `
|
|
265
|
+
# List of tags we want to publish interpreted with `format(version=version)`
|
|
294
266
|
# e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag
|
|
295
267
|
# (that should be built by the application build) is `latest-lite`, and it will be published
|
|
296
268
|
# with the tag `1.2.3-lite`.
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
c2cciutils shared utils function.
|
|
3
|
-
"""
|
|
1
|
+
"""c2cciutils shared utils function."""
|
|
4
2
|
|
|
5
3
|
import glob
|
|
6
4
|
import json
|
|
@@ -11,7 +9,6 @@ import sys
|
|
|
11
9
|
from re import Match, Pattern
|
|
12
10
|
from typing import Any, Optional, TypedDict, cast
|
|
13
11
|
|
|
14
|
-
import magic
|
|
15
12
|
import requests
|
|
16
13
|
import ruamel.yaml
|
|
17
14
|
|
|
@@ -19,10 +16,7 @@ import c2cciutils.configuration
|
|
|
19
16
|
|
|
20
17
|
|
|
21
18
|
def get_repository() -> str:
|
|
22
|
-
"""
|
|
23
|
-
Get the current GitHub repository like `organization/project`.
|
|
24
|
-
"""
|
|
25
|
-
|
|
19
|
+
"""Get the current GitHub repository like `organization/project`."""
|
|
26
20
|
if "GITHUB_REPOSITORY" in os.environ:
|
|
27
21
|
return os.environ["GITHUB_REPOSITORY"]
|
|
28
22
|
|
|
@@ -46,12 +40,12 @@ def merge(default_config: Any, config: Any) -> Any:
|
|
|
46
40
|
Arguments:
|
|
47
41
|
default_config: The default config that will be applied
|
|
48
42
|
config: The base config, will be modified
|
|
49
|
-
"""
|
|
50
43
|
|
|
44
|
+
"""
|
|
51
45
|
if not isinstance(default_config, dict) or not isinstance(config, dict):
|
|
52
46
|
return config
|
|
53
47
|
|
|
54
|
-
for key in default_config
|
|
48
|
+
for key in default_config:
|
|
55
49
|
if key not in config:
|
|
56
50
|
config[key] = default_config[key]
|
|
57
51
|
else:
|
|
@@ -76,10 +70,7 @@ def get_master_branch(repo: list[str]) -> tuple[str, bool]:
|
|
|
76
70
|
|
|
77
71
|
|
|
78
72
|
def get_config() -> c2cciutils.configuration.Configuration:
|
|
79
|
-
"""
|
|
80
|
-
Get the configuration, with project and auto detections.
|
|
81
|
-
"""
|
|
82
|
-
|
|
73
|
+
"""Get the configuration, with project and auto detections."""
|
|
83
74
|
config: c2cciutils.configuration.Configuration = {}
|
|
84
75
|
if os.path.exists("ci/config.yaml"):
|
|
85
76
|
with open("ci/config.yaml", encoding="utf-8") as open_file:
|
|
@@ -129,8 +120,6 @@ def get_config() -> c2cciutils.configuration.Configuration:
|
|
|
129
120
|
|
|
130
121
|
default_config = {
|
|
131
122
|
"publish": publish_config,
|
|
132
|
-
"pr-checks": c2cciutils.configuration.PULL_REQUEST_CHECKS_DEFAULT,
|
|
133
|
-
"audit": c2cciutils.configuration.AUDIT_DEFAULT,
|
|
134
123
|
}
|
|
135
124
|
merge(default_config, config)
|
|
136
125
|
|
|
@@ -161,6 +150,7 @@ def error(
|
|
|
161
150
|
line: The line number of the error
|
|
162
151
|
col: The column number of the error
|
|
163
152
|
error_type: The kind of error (error or warning)
|
|
153
|
+
|
|
164
154
|
"""
|
|
165
155
|
result = ""
|
|
166
156
|
on_ci = os.environ.get("CI", "false").lower() == "true"
|
|
@@ -203,6 +193,7 @@ def compile_re(config: c2cciutils.configuration.VersionTransform, prefix: str =
|
|
|
203
193
|
prefix: The version prefix
|
|
204
194
|
|
|
205
195
|
Return the compiled transform config.
|
|
196
|
+
|
|
206
197
|
"""
|
|
207
198
|
result = []
|
|
208
199
|
for conf in config:
|
|
@@ -232,6 +223,7 @@ def match(
|
|
|
232
223
|
|
|
233
224
|
Returns the re match object, the matched config and the value as a tuple
|
|
234
225
|
On no match it returns None, value
|
|
226
|
+
|
|
235
227
|
"""
|
|
236
228
|
for conf in config:
|
|
237
229
|
matched = conf["from"].match(value)
|
|
@@ -249,6 +241,7 @@ def does_match(value: str, config: list[VersionTransform]) -> bool:
|
|
|
249
241
|
config: The result of `compile`
|
|
250
242
|
|
|
251
243
|
Returns True it it does match else False
|
|
244
|
+
|
|
252
245
|
"""
|
|
253
246
|
matched, _, _ = match(value, config)
|
|
254
247
|
return matched is not None
|
|
@@ -268,6 +261,7 @@ def get_value(matched: Optional[Match[str]], config: Optional[VersionTransform],
|
|
|
268
261
|
value: The default value on returned no match
|
|
269
262
|
|
|
270
263
|
Return the value
|
|
264
|
+
|
|
271
265
|
"""
|
|
272
266
|
return matched.expand(config.get("to", r"\1")) if matched is not None and config is not None else value
|
|
273
267
|
|
|
@@ -278,8 +272,8 @@ def print_versions(config: c2cciutils.configuration.PrintVersions) -> bool:
|
|
|
278
272
|
|
|
279
273
|
Arguments:
|
|
280
274
|
config: The print configuration
|
|
281
|
-
"""
|
|
282
275
|
|
|
276
|
+
"""
|
|
283
277
|
for version in config.get("versions", c2cciutils.configuration.PRINT_VERSIONS_VERSIONS_DEFAULT):
|
|
284
278
|
try:
|
|
285
279
|
sys.stdout.flush()
|
|
@@ -317,6 +311,7 @@ def gopass(key: str, default: Optional[str] = None) -> Optional[str]:
|
|
|
317
311
|
default: the value to return if gopass is not found
|
|
318
312
|
|
|
319
313
|
Return the value
|
|
314
|
+
|
|
320
315
|
"""
|
|
321
316
|
try:
|
|
322
317
|
return subprocess.check_output(["gopass", "show", key]).strip().decode()
|
|
@@ -333,6 +328,7 @@ def gopass_put(secret: str, key: str) -> None:
|
|
|
333
328
|
Arguments:
|
|
334
329
|
secret: The secret value
|
|
335
330
|
key: The key
|
|
331
|
+
|
|
336
332
|
"""
|
|
337
333
|
subprocess.check_output(["gopass", "insert", "--force", key], input=secret.encode())
|
|
338
334
|
|
|
@@ -345,6 +341,7 @@ def add_authorization_header(headers: dict[str, str]) -> dict[str, str]:
|
|
|
345
341
|
headers: The headers
|
|
346
342
|
|
|
347
343
|
Return the headers (to be chained)
|
|
344
|
+
|
|
348
345
|
"""
|
|
349
346
|
try:
|
|
350
347
|
token = (
|
|
@@ -382,8 +379,8 @@ def graphql(query_file: str, variables: dict[str, Any], default: Any = None) ->
|
|
|
382
379
|
|
|
383
380
|
Return the data result
|
|
384
381
|
In case of error it throw an exception
|
|
385
|
-
"""
|
|
386
382
|
|
|
383
|
+
"""
|
|
387
384
|
with open(os.path.join(os.path.dirname(__file__), query_file), encoding="utf-8") as query_open:
|
|
388
385
|
query = query_open.read()
|
|
389
386
|
|
|
@@ -416,164 +413,8 @@ def graphql(query_file: str, variables: dict[str, Any], default: Any = None) ->
|
|
|
416
413
|
return cast(dict[str, Any], json_response["data"])
|
|
417
414
|
|
|
418
415
|
|
|
419
|
-
def get_git_files_mime(
|
|
420
|
-
mime_type: Optional[list[str]] = None,
|
|
421
|
-
extensions: Optional[list[str]] = None,
|
|
422
|
-
ignore_patterns_re: Optional[list[str]] = None,
|
|
423
|
-
) -> list[str]:
|
|
424
|
-
"""
|
|
425
|
-
Get list of paths from git with all the files that have the specified mime type.
|
|
426
|
-
|
|
427
|
-
Arguments:
|
|
428
|
-
mime_type: The considered MIME type
|
|
429
|
-
extensions: The considered extensions
|
|
430
|
-
ignore_patterns_re: A list of regular expressions of files that we should ignore
|
|
431
|
-
"""
|
|
432
|
-
if mime_type is None:
|
|
433
|
-
mime_type = ["text/x-python", "text/x-script.python"]
|
|
434
|
-
if extensions is None:
|
|
435
|
-
extensions = [".py"]
|
|
436
|
-
ignore_patterns_compiled = [re.compile(p) for p in ignore_patterns_re or []]
|
|
437
|
-
result = []
|
|
438
|
-
|
|
439
|
-
for filename in subprocess.check_output(["git", "ls-files"]).decode().strip().split("\n"):
|
|
440
|
-
if os.path.isfile(filename) and (
|
|
441
|
-
os.path.splitext(filename)[1] in extensions or magic.from_file(filename, mime=True) in mime_type
|
|
442
|
-
):
|
|
443
|
-
accept = True
|
|
444
|
-
for pattern in ignore_patterns_compiled:
|
|
445
|
-
if pattern.search(filename):
|
|
446
|
-
accept = False
|
|
447
|
-
break
|
|
448
|
-
if accept:
|
|
449
|
-
result.append(filename)
|
|
450
|
-
return result
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
def get_branch(branch: Optional[str], master_branch: str = "master") -> str:
|
|
454
|
-
"""
|
|
455
|
-
Get the branch name.
|
|
456
|
-
|
|
457
|
-
Arguments:
|
|
458
|
-
branch: The forced to use branch name
|
|
459
|
-
master_branch: The master branch name, can be used as default value
|
|
460
|
-
|
|
461
|
-
Return the branch name
|
|
462
|
-
"""
|
|
463
|
-
|
|
464
|
-
if branch is not None:
|
|
465
|
-
return branch
|
|
466
|
-
try:
|
|
467
|
-
branch = (
|
|
468
|
-
subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], check=True, stdout=subprocess.PIPE)
|
|
469
|
-
.stdout.decode()
|
|
470
|
-
.strip()
|
|
471
|
-
)
|
|
472
|
-
except subprocess.CalledProcessError as exception:
|
|
473
|
-
print(f"Error getting branch: {exception}")
|
|
474
|
-
branch = "HEAD"
|
|
475
|
-
|
|
476
|
-
if branch == "HEAD":
|
|
477
|
-
branch = os.environ.get("GITHUB_HEAD_REF", master_branch)
|
|
478
|
-
assert branch is not None
|
|
479
|
-
return branch
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
def get_based_on_master(
|
|
483
|
-
repo: list[str],
|
|
484
|
-
override_current_branch: Optional[str],
|
|
485
|
-
master_branch: str,
|
|
486
|
-
config: c2cciutils.configuration.Configuration,
|
|
487
|
-
) -> bool:
|
|
488
|
-
"""
|
|
489
|
-
Check that we are not on a release branch (to avoid errors in versions check).
|
|
490
|
-
|
|
491
|
-
This function will check the last 20 commits in current branch,
|
|
492
|
-
and for each other branch (max 50) check if any commit in last 10 commits is the current one.
|
|
493
|
-
|
|
494
|
-
Arguments:
|
|
495
|
-
repo: The repository [<organization>, <name>]
|
|
496
|
-
override_current_branch: The branch to use instead of the current one
|
|
497
|
-
master_branch: The master branch name
|
|
498
|
-
config: The full configuration
|
|
499
|
-
"""
|
|
500
|
-
if os.environ.get("GITHUB_REF", "").startswith("refs/tags/"):
|
|
501
|
-
# The tags are never consider as based on master
|
|
502
|
-
return False
|
|
503
|
-
current_branch = get_branch(override_current_branch, master_branch)
|
|
504
|
-
if current_branch == master_branch:
|
|
505
|
-
return True
|
|
506
|
-
branches_re = compile_re(config["version"].get("branch_to_version_re", []))
|
|
507
|
-
if does_match(current_branch, branches_re):
|
|
508
|
-
return False
|
|
509
|
-
if os.environ.get("GITHUB_BASE_REF"):
|
|
510
|
-
return os.environ.get("GITHUB_BASE_REF") == master_branch
|
|
511
|
-
commits_repository_json = graphql(
|
|
512
|
-
"commits.graphql", {"name": repo[1], "owner": repo[0], "branch": current_branch}
|
|
513
|
-
).get("repository", {})
|
|
514
|
-
commits_json = (
|
|
515
|
-
commits_repository_json.get("ref", {}).get("target", {}).get("history", {}).get("nodes", [])
|
|
516
|
-
if commits_repository_json.get("ref")
|
|
517
|
-
else []
|
|
518
|
-
)
|
|
519
|
-
branches_json = [
|
|
520
|
-
branch
|
|
521
|
-
for branch in (
|
|
522
|
-
graphql("branches.graphql", {"name": repo[1], "owner": repo[0]})["repository"]["refs"]["nodes"]
|
|
523
|
-
)
|
|
524
|
-
if branch["name"] != current_branch and does_match(branch["name"], branches_re)
|
|
525
|
-
]
|
|
526
|
-
based_branch = master_branch
|
|
527
|
-
found = False
|
|
528
|
-
for commit in commits_json:
|
|
529
|
-
for branch in branches_json:
|
|
530
|
-
commits = [
|
|
531
|
-
branch_commit
|
|
532
|
-
for branch_commit in branch["target"]["history"]["nodes"]
|
|
533
|
-
if commit["oid"] == branch_commit["oid"]
|
|
534
|
-
]
|
|
535
|
-
if commits:
|
|
536
|
-
based_branch = branch["name"]
|
|
537
|
-
found = True
|
|
538
|
-
break
|
|
539
|
-
if found:
|
|
540
|
-
break
|
|
541
|
-
return based_branch == master_branch
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
def get_codespell_command(config: c2cciutils.configuration.Configuration, fix: bool = False) -> list[str]:
|
|
545
|
-
"""
|
|
546
|
-
Get the codespell command.
|
|
547
|
-
|
|
548
|
-
Arguments:
|
|
549
|
-
config: The full configuration
|
|
550
|
-
fix: If we should fix the errors
|
|
551
|
-
"""
|
|
552
|
-
codespell_config = config.get("codespell", {})
|
|
553
|
-
codespell_config = codespell_config if isinstance(codespell_config, dict) else {}
|
|
554
|
-
command = ["codespell"]
|
|
555
|
-
if fix:
|
|
556
|
-
command.append("--write-changes")
|
|
557
|
-
for spell_ignore_file in (
|
|
558
|
-
".github/spell-ignore-words.txt",
|
|
559
|
-
"spell-ignore-words.txt",
|
|
560
|
-
".spell-ignore-words.txt",
|
|
561
|
-
):
|
|
562
|
-
if os.path.exists(spell_ignore_file):
|
|
563
|
-
command.append(f"--ignore-words={spell_ignore_file}")
|
|
564
|
-
break
|
|
565
|
-
dictionaries = codespell_config.get(
|
|
566
|
-
"internal_dictionaries", c2cciutils.configuration.CODESPELL_DICTIONARIES_DEFAULT
|
|
567
|
-
)
|
|
568
|
-
if dictionaries:
|
|
569
|
-
command.append("--builtin=" + ",".join(dictionaries))
|
|
570
|
-
command += codespell_config.get("arguments", c2cciutils.configuration.CODESPELL_ARGUMENTS_DEFAULT)
|
|
571
|
-
return command
|
|
572
|
-
|
|
573
|
-
|
|
574
416
|
def snyk_exec() -> tuple[str, dict[str, str]]:
|
|
575
417
|
"""Get the Snyk cli executable path."""
|
|
576
|
-
|
|
577
418
|
if not os.path.exists(os.path.join(os.path.dirname(__file__), "node_modules")):
|
|
578
419
|
subprocess.run(["npm", "install"], cwd=os.path.dirname(__file__), check=True) # nosec
|
|
579
420
|
|
|
@@ -587,59 +428,3 @@ def snyk_exec() -> tuple[str, dict[str, str]]:
|
|
|
587
428
|
subprocess.run(["snyk", "config", "set", f"org={env['SNYK_ORG']}"], check=True, env=env)
|
|
588
429
|
|
|
589
430
|
return os.path.join(os.path.dirname(os.path.abspath(__file__)), "node_modules/snyk/bin/snyk"), env
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
def create_pull_request_if_needed(
|
|
593
|
-
current_branch: str,
|
|
594
|
-
new_branch: str,
|
|
595
|
-
commit_message: str,
|
|
596
|
-
pull_request_extra_arguments: Optional[list[str]] = None,
|
|
597
|
-
) -> bool:
|
|
598
|
-
"""
|
|
599
|
-
Create a pull request if there are changes.
|
|
600
|
-
"""
|
|
601
|
-
|
|
602
|
-
if pull_request_extra_arguments is None:
|
|
603
|
-
pull_request_extra_arguments = ["--fill"]
|
|
604
|
-
|
|
605
|
-
diff_proc = subprocess.run(["git", "diff", "--quiet"]) # pylint: disable=subprocess-run-check
|
|
606
|
-
if diff_proc.returncode != 0:
|
|
607
|
-
print("::group::Diff")
|
|
608
|
-
sys.stdout.flush()
|
|
609
|
-
sys.stderr.flush()
|
|
610
|
-
subprocess.run(["git", "diff"], check=True)
|
|
611
|
-
print("::endgroup::")
|
|
612
|
-
|
|
613
|
-
git_hash = subprocess.run(
|
|
614
|
-
["git", "rev-parse", "HEAD"], check=True, stdout=subprocess.PIPE, encoding="utf-8"
|
|
615
|
-
).stdout.strip()
|
|
616
|
-
subprocess.run(["git", "checkout", "-b", new_branch], check=True)
|
|
617
|
-
subprocess.run(["git", "add", "--all"], check=True)
|
|
618
|
-
subprocess.run(["git", "commit", f"--message={commit_message}"], check=True)
|
|
619
|
-
if os.environ.get("TEST") != "TRUE":
|
|
620
|
-
subprocess.run(
|
|
621
|
-
["git", "push", "--force", "origin", new_branch],
|
|
622
|
-
check=True,
|
|
623
|
-
)
|
|
624
|
-
env = os.environ.copy()
|
|
625
|
-
if "GH_TOKEN" not in env:
|
|
626
|
-
if "GITHUB_TOKEN" in env:
|
|
627
|
-
env["GH_TOKEN"] = env["GITHUB_TOKEN"]
|
|
628
|
-
else:
|
|
629
|
-
env["GH_TOKEN"] = str(c2cciutils.gopass("gs/ci/github/token/gopass"))
|
|
630
|
-
subprocess.run(
|
|
631
|
-
[
|
|
632
|
-
"gh",
|
|
633
|
-
"pr",
|
|
634
|
-
"create",
|
|
635
|
-
f"--base={current_branch}",
|
|
636
|
-
*pull_request_extra_arguments,
|
|
637
|
-
],
|
|
638
|
-
check=True,
|
|
639
|
-
env=env,
|
|
640
|
-
)
|
|
641
|
-
else:
|
|
642
|
-
subprocess.run(["git", "reset", "--hard"], check=True)
|
|
643
|
-
subprocess.run(["git", "checkout", git_hash], check=True)
|
|
644
|
-
|
|
645
|
-
return diff_proc.returncode != 0
|