c2cciutils 1.6.0.dev1__tar.gz → 1.6.0.dev5__tar.gz

{c2cciutils-1.6.0.dev1 → c2cciutils-1.6.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: c2cciutils
-Version: 1.6.0.dev1
+Version: 1.6.0.dev5
 Summary: Common utilities for Camptocamp CI
 Home-page: https://github.com/camptocamp/c2cciutils
 License: FreeBSD

{c2cciutils-1.6.0.dev1 → c2cciutils-1.6.0.dev5}/c2cciutils/configuration.py

@@ -813,6 +813,14 @@ PUBLISH_DOCKER_REPOSITORY_VERSIONS_DEFAULT = ["version_tag", "version_branch", "
 """Default value of the field path 'Publish Docker repository versions'"""
 
 
+PUBLISH_DOCKER_SNYK_MONITOR_ARGS_DEFAULT = ["--app-vulns"]
+"""Default value of the field path 'Publish Docker config snyk monitor_args'"""
+
+
+PUBLISH_DOCKER_SNYK_TEST_ARGS_DEFAULT = ["--app-vulns", "--severity-threshold=critical"]
+"""Default value of the field path 'Publish Docker config snyk test_args'"""
+
+
 PUBLISH_GOOGLE_CALENDAR_DEFAULT: Dict[str, Any] = {}
 """Default value of the field path 'Publish google_calendar'"""
 
@@ -984,6 +992,12 @@ class PublishDockerConfig(TypedDict, total=False):
     oneOf
     """
 
+    snyk: "_PublishDockerConfigSnyk"
+    """
+    WARNING: The required are not correctly taken in account,
+    See: https://github.com/camptocamp/jsonschema-gentypes/issues/6
+    """
+
 
 class PublishDockerImage(TypedDict, total=False):
     """Publish Docker image."""
@@ -1338,6 +1352,35 @@ class _PrintVersionsVersionsItem(TypedDict, total=False):
     """Prefix added when we print the version"""
 
 
+class _PublishDockerConfigSnyk(TypedDict, total=False):
+    """Checks the published images with Snyk"""
+
+    monitor_args: Union[List[str], Literal[False]]
+    """
+    Publish docker snyk monitor args.
+
+    The arguments to pass to the Snyk container monitor command
+
+    default:
+      - --app-vulns
+
+    oneOf
+    """
+
+    test_args: Union[List[str], Literal[False]]
+    """
+    Publish docker snyk test args.
+
+    The arguments to pass to the Snyk container test command
+
+    default:
+      - --app-vulns
+      - --severity-threshold=critical
+
+    oneOf
+    """
+
+
 _VersionTransformItem = TypedDict(
     "_VersionTransformItem",
     {

{c2cciutils-1.6.0.dev1 → c2cciutils-1.6.0.dev5}/c2cciutils/schema.json

@@ -469,6 +469,40 @@
                 },
                 { "const": false }
               ]
+            },
+            "snyk": {
+              "description": "Checks the published images with Snyk",
+              "type": "object",
+              "properties": {
+                "monitor_args": {
+                  "description": "The arguments to pass to the Snyk container monitor command",
+                  "title": "Publish docker snyk monitor args",
+                  "default": ["--app-vulns"],
+                  "oneOf": [
+                    {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    { "const": false }
+                  ]
+                },
+                "test_args": {
+                  "description": "The arguments to pass to the Snyk container test command",
+                  "title": "Publish docker snyk test args",
+                  "default": ["--app-vulns", "--severity-threshold=critical"],
+                  "oneOf": [
+                    {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    { "const": false }
+                  ]
+                }
+              }
             }
           }
         },

{c2cciutils-1.6.0.dev1 → c2cciutils-1.6.0.dev5}/c2cciutils/scripts/publish.py

@@ -305,28 +305,43 @@ def main() -> None:
 
         snyk_exec, env = c2cciutils.snyk_exec()
         for image in images_snyk:
-            if version_type in ("version_branch", "version_tag"):
-                subprocess.run(  # pylint: disable=subprocess-run-check
-                    [
-                        snyk_exec,
-                        "container",
-                        "monitor",
-                        "--app-vulns",
-                        # Available only on the business plan
-                        # f"--project-tags=tag={image.split(':')[-1]}",
-                        image,
-                    ],
-                    env=env,
+            print(f"::group::Snyk check {image}")
+            sys.stdout.flush()
+            sys.stderr.flush()
+            try:
+                if version_type in ("version_branch", "version_tag"):
+                    monitor_args = docker_config.get("snyk", {}).get(
+                        "monitor_args",
+                        c2cciutils.configuration.PUBLISH_DOCKER_SNYK_MONITOR_ARGS_DEFAULT,
+                    )
+                    if monitor_args is not False:
+                        subprocess.run(  # pylint: disable=subprocess-run-check
+                            [
+                                snyk_exec,
+                                "container",
+                                "monitor",
+                                *monitor_args,
+                                # Available only on the business plan
+                                # f"--project-tags=tag={image.split(':')[-1]}",
+                                image,
+                            ],
+                            env=env,
+                        )
+                test_args = docker_config.get("snyk", {}).get(
+                    "test_args", c2cciutils.configuration.PUBLISH_DOCKER_SNYK_TEST_ARGS_DEFAULT
                 )
-            # Currently just for information
-            subprocess.run(  # pylint: disable=subprocess-run-check
-                [snyk_exec, "container", "test", "--app-vulns", "--severity-threshold=high", image], env=env
-            )
-            subprocess.run(
-                [snyk_exec, "container", "test", "--app-vulns", "--severity-threshold=critical", image],
-                check=not based_on_master and version_type == "version_branch",
-                env=env,
-            )
+                if test_args is not False:
+                    subprocess.run(
+                        [snyk_exec, "container", "test", *test_args, image],
+                        check=not based_on_master and version_type == "version_branch",
+                        env=env,
+                    )
+
+                print("::endgroup::")
+            except subprocess.CalledProcessError as exception:
+                print(f"Error: {exception}")
+                print("::endgroup::")
+                print("::error::With error")
 
         versions_config = c2cciutils.lib.docker.get_versions_config()
         dpkg_success = True

{c2cciutils-1.6.0.dev1 → c2cciutils-1.6.0.dev5}/pyproject.toml

@@ -19,7 +19,7 @@ strict = true
 
 [tool.poetry]
 name = "c2cciutils"
-version = "1.6.0.dev1"
+version = "1.6.0.dev5"
 description = "Common utilities for Camptocamp CI"
 readme = "README.md"
 authors = ["Camptocamp <info@camptocamp.com>"]

{c2cciutils-1.6.0.dev1 → c2cciutils-1.6.0.dev5}/setup.py

@@ -994,7 +994,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'c2cciutils',
-    'version': '1.6.0.dev1',
+    'version': '1.6.0.dev5',
     'description': 'Common utilities for Camptocamp CI',
     'long_description': '# C2C CI utils\n\nThe goals of C2C CI utils are:\n\n- Have some global checks that\'s didn\'t request any dependency related to the application:\n  this commands return 3 types of results:\n  - Print some useful information:\n    - The version of some packages\n    - The used configuration (with default and autodetect)\n    - The environment variables\n    - The GitHub event file\n  - Check that some configuration where correct:\n    - Git attributes\n    - That the timeout is present in the GitHub workflow files\n    - That the stabilization version (get from the Security.md) are used everywhere it\'s needed\n  - Check the code style:\n    - End of files\n    - Black\n    - Isort\n    - Code spell\n    - Prettier\n  - Snyk tests\n    - Test (never failed)\n    - Code test (never failed, disabled by default)\n    - Iac Test (never failed, disabled by default)\n    - Fix (For information only, disabled by default)\n\nEvery check can be disabled with the following config (the configuration is `ci/config.yaml`):\n\n```yaml\nchecks:\n  <check name>: false\n```\n\nIt make easier to place the following workflows:\n\n- `audit.yaml`: Audit the stabilization branches of the application against vulnerabilities in the python and node dependency\n- `auto-review.yaml`: Auto review the Renovate pull requests\n- `backport.yaml`: Trigger the backports (work with labels)\n- `clean.yaml`: Clean the Docker images related on a deleted feature branch\n- `codeql.yaml`: Run a GitHub CodeQL check\n- `main.yaml`: Main workflow especially with the c2cciutils-checks command\n- `rebuild.yaml`: Daily rebuild of the Docker images on the stabilization branches.\n\nAll the provided commands:\n\n- `c2cciutils`: some generic tools.\n- `c2cciutils-checks`: Run the checks on the code (those checks don\'t need any project dependencies).\n- `c2cciutils-audit`: Do the audit, the main difference with checks is that it can change between runs on the same code.\n- `c2cciutils-publish`: Publish the project.\n- `c2cciutils-clean`: Delete Docker images on Docker Hub after corresponding branch have been deleted.\n- `c2cciutils-google-calendar`: Tool to test the Google credentials for calendar API and refresh them if needed. See `c2cciutils-google-calendar -h` for more information.\n- `c2cciutils-k8s-install`: Install a k3d / k3s cluster, see below.\n- `c2cciutils-k8s-db`: Create a database in the k8s cluster, see below.\n- `c2cciutils-k8s-wait`: Wait that the application started correctly in the cluster, see below.\n- `c2cciutils-k8s-logs`: Display the logs of the application in the k8s cluster, see below.\n- `c2cciutils-pin-pipenv`: Display all the dependencies that\'s in the `Pipenv.lock` but not in the `Pipenv` to be able to pin them.\n- `c2cciutils-docker-logs`: Display the logs of the application in Docker (compose).\n- `c2cciutils-trigger-image-update`: Trigger the ArgoCD repository about image update on the CI (automatically done in the publishing).\n- `c2cciutils-download-applications`: Download the applications with version managed by Renovate, see below.\n- `c2cciutils-docker-versions-gen`: Generate the Docker package versions file (`ci/dpkg-versions.yaml`), see below.\n\n## New project\n\nThe content of `example-project` can be a good base for a new project.\n\n## Secrets\n\nIn the CI we need to have the following secrets::\n\n- `HAS_SECRETS` to be set to \'HAS_SECRETS\', to avoid error errors from external\n  pull requests, already set globally on Camptocamp organization.\n- `GOPASS_CI_GITHUB_TOKEN` and `CI_GPG_PRIVATE_KEY` required to initialize the gopass password store,\n  the secrets exists in the Camptocamp organization but not shared on all project, then you should add\n  your project to the shared list.\n\n## Use locally, in the projects that use c2cciutils\n\nInstall it: `python3 -m pip install --user --requirement ci/requirements.txt`\nRun the checkers: `c2cciutils-checks [--fix] [--stop] [--check CHECK]`\nDry run publish: `GITHUB_REF=... c2cciutils-publish --dry-run ...`\n\n## Configuration\n\nYou can get the current configuration with `c2cciutils --get-config`, the default configuration depends on your project.\nNote that it didn\'t contain the default defined the schema and visible in the [generated documentation](./config.md).\n\nYou can override the configuration with the file `ci/config.yaml`.\n\nAt the base of the configuration you have:\n\n- `version`: Contains some regular expressions to find the versions branches and tags, and to convert them into application versions.\n- `checks`: The checker\'s configuration, see `c2cciutils/checks.py` for more information.\n- `audit`: The audit configuration, see `c2cciutils/audit.py` for more information.\n- `publish`: The publishing configuration, see `c2cciutils/publish.py` for more information.\n\nMany actions can be disabled by setting the corresponding configuration part to `False`.\n\n## SECURITY.md\n\nThe `SECURITY.md` file should contain the security policy of the repository, especially the end of\nsupport dates.\n\nFor compatibility with `c2cciutils` it should contain an array with at least the columns\n`Version` and `Supported Until`. The `Version` column will contain the concerned version.\nThe `Supported Until` will contain the date of end of support `dd/mm/yyyy`.\nIt can also contain the following sentences:\n\n- `Unsupported`: no longer supported => no audit, no rebuild.\n- `Best effort`: the support is ended, it is still rebuilt and audited, but this can be stopped without any notice.\n- `To be defined`: not yet released or the date will be set related of another project release date (like for GeoMapFish).\n\nSee also [GitHub Documentation](https://docs.github.com/en/github/managing-security-vulnerabilities/adding-a-security-policy-to-your-repository)\n\n## IDE\n\nThe IDE should be configured as:\n\n- using `black` and `isort` without any arguments,\n- using the `editorconfig` configuration.\n\n### VScode\n\n- Recommend extensions to work well with c2cciutils:\n  - [Prettier](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) And use EditorConfig\n  - [shell-format](https://marketplace.visualstudio.com/items?itemName=foxundermoon.shell-format) With the configuration\n    `"shellformat.flag": "-bn"`.\n  - [Better TOML](https://marketplace.visualstudio.com/items?itemName=bodil.prettier-toml)\n- Other recommend extensions:\n  - [hadolint](https://marketplace.visualstudio.com/items?itemName=exiasr.hadolint)\n  - [Code Spell Checker](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker)\n\nSelect a formatter:\n\n- `CTRL+MAJ+P`\n- Format document With...\n- Configure Default Formatter...\n- Select the formatter\n\n## Publishing\n\n### To pypi\n\nThe config is like this:\n\n```yaml\nversions:\n  # List of kinds of versions you want to publish, that can be:\n  # rebuild (specified with --type),\n  # version_tag, version_branch, feature_branch, feature_tag (for pull request)\n```\n\nIt we have a `setup.py` file, we will be in legacy mode:\nWhen publishing, the version computed from arguments or `GITHUB_REF` is put in environment variable `VERSION`, thus you should use it in `setup.py`, example:\n\n```python\nVERSION = os.environ.get("VERSION", "1.0.0")\n```\n\nAlso we consider that we use `poetry` with [poetry-dynamic-versioning](https://pypi.org/project/poetry-dynamic-versioning/) to manage the version, and [poetry-plugin-tweak-dependencies-version](https://pypi.org/project/poetry-plugin-tweak-dependencies-version/) to manage the dependencies versions.\n\nExample of configuration:\n\n```toml\n[tool.poetry-dynamic-versioning]\nenable = true\nvcs = "git"\npattern = "^(?P<base>\\\\d+(\\\\.\\\\d+)*)"\nformat-jinja = """\n{%- if env.get("VERSION_TYPE") == "version_branch" -%}\n{{serialize_pep440(bump_version(base, 1 if env.get("IS_MASTER") == "TRUE" else 2), dev=distance)}}\n{%- elif distance == 0 -%}\n{{serialize_pep440(base)}}\n{%- else -%}\n{{serialize_pep440(bump_version(base), dev=distance)}}\n{%- endif -%}\n"""\n\n```\n\nNote that we can access to the environment variables `VERSION`,`VERSION_TYPE` and `IS_MASTER`.\n\nThen by default:\n\n- Tag with `1.2.3` => release `1.2.3`\n- Commit on feature branch just do a validation\n- Commit on `master` branch after the tag 1.3.0 => release `1.4.0.dev1`\n- Commit on `1.3` branch after the tag 1.3.0 => release `1.3.1.dev1`\n\nTo make it working in the `Dockerfile` you should have in the `poetry` stage:\n\n```Dockerfile\nENV POETRY_DYNAMIC_VERSIONING_BYPASS=dev\nRUN poetry export --extras=checks --extras=publish --extras=audit --output=requirements.txt \\\n    && poetry export --with=dev --output=requirements-dev.txt\n```\n\nAnd in the `run` stage\n\n```Dockerfile\nARG VERSION=dev\nRUN --mount=type=cache,target=/root/.cache \\\n    POETRY_DYNAMIC_VERSIONING_BYPASS=${VERSION} python3 -m pip install --disable-pip-version-check --no-deps --editable=.\n```\n\nAnd in the `Makefile`:\n\n```Makefile\nVERSION = $(strip $(shell poetry version --short))\n\n.PHONY: build\nbuild: ## Build the Docker images\n    docker build --build-arg=VERSION=$(VERSION) --tag=$(GITHUB_REPOSITORY) .\n```\n\n### To Docker registry\n\nThe config is like this:\n\n```yaml\nlatest: True\nimages:\n  - name: # The base name of the image we want to publish\nrepository:\n  <internal_name>:\n    \'server\': # The fqdn name of the server if not Docker hub\n    \'version\':# List of kinds of versions you want to publish, that can be: rebuild (specified using --type),\n      # version_tag, version_branch, feature_branch, feature_tag (for pull request)\n    \'tags\':# List of tags we want to publish interpreted with `template(version=version)`\n      # e.g. if you use `{version}-lite` when you publish the version `1.2.3` the source tag\n      # (that should be built by the application build) is `latest-lite`, and it will be published\n      # with the tag `1.2.3-lite`.\n    \'group\':# If your images are published by different jobs you can separate them in different groups\n      # and publish them with `c2cciutils-publish --group=<group>`\n```\n\nBy default, the last line of the `SECURITY.md` file will be published (`docker`) with the tag\n`latest`. Set `latest` to `False` to disable it.\n\nWith the `c2cciutils-clean` the images on Docker hub for `feature_branch` will be removed on branch removing.\n\n## Download applications\n\nIn case some executables or applications from GitHub releases or any other URLs are required on the CI host\nand are not handled by any dependency manager, we provide a set of tools to install them and manage upgrades\nthrough Renovate.\n\nCreate an application file (e.-g. `applications.yaml`) with:\n\n```yaml\n# yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/c2cciutils/master/c2cciutils/schema-applications.json\n\n# Application from GitHub release\n<organization>/<project>:\n  get-file-name: <file name present in the release>\n  to-file-name: <The file name you want to create in ~/.local/bin>\n  finish-command: # The command you want to run after the file is downloaded\n    - - chmod # To be executable (usually required)\n      - +x\n      - <to-file-name>\n    - - <to-file-name> # Print the version of the application\n      - --version\n# Application from GitHub release in a tar file (or tar.gz)\n<organization>/<project>:\n  get-file-name: <file name present in the release>\n  type: tar\n  tar-file-name: <The file name available in the tar file>\n  to-file-name: <The file name you want to create in ~/.local/bin>\n  finish-command: [...] # The command you want to run after the file is downloaded\n# Application from an URL\n<application reference name>:\n  url-pattern: <The URL used to download the application>\n  to-file-name: <The file name you want to create in ~/.local/bin>\n  finish-command: [...] # The command you want to run after the file is downloaded\n```\n\nIn the attributes `url-pattern`, `get-file-name` you can use the following variables:\n\n- `{version}`: The version of the application present in the version file.\n- `{version_quote}`: The URL encoded version.\n- `{short_version}`: The version without the `v` prefix.\n\nThe `applications-versions.yaml` file is a map of applications and their versions.\n\nAdd in your Renovate configuration:\n\n```json5\n\n  regexManagers: [\n    {\n      fileMatch: [\'^applications-versions.yaml$\'],\n      matchStrings: [\n        \'(?<depName>[^\\\\s]+): (?<currentValue>[^\\\\s]+) # (?<datasource>[^\\\\s]+)\',\n      ],\n    },\n  ],\n```\n\nNow you need to call `c2cciutils-download-applications --applications-file=applications.yaml --versions-file=applications-version.yaml`\nto install required applications on CI host before using them (an already installed application is installed only if needed).\n\n## Use Renovate to trigger a new build instead of the legacy rebuild\n\nRun the command `c2cciutils-docker-versions-gen camptocamp/image[:tag]` to generate a file that is a kind of package lock of the Debian packages in the file `ci/dpkg-versions.yaml`.\n\nAdd in your renovate configuration:\n\n```javascript\n  regexManagers: [\n    {\n      fileMatch: [\'^ci/dpkg-versions.yaml$\'],\n      matchStrings: [" *(?<depName>[^\'\\\\s]+): \'?(?<currentValue>[^\'\\\\s/]*[0-9][^\'\\\\s/]*)\'?"],\n      datasourceTemplate: \'repology\',\n      versioningTemplate: \'loose\',\n    },\n  ],\n```\n\nWhen a new version of a Debian package will be available:\n\n- Renovate will automatically open a pull request to update the file `ci/dpkg-versions.yaml`.\n- And the continuous integration will build a new fresh Docker image with latest versions of all Debian packages.\n\n## Kubernetes\n\nC2cciutils provide some commands for Kubernetes.\n\nYou can define a workflow like that:\n\n```yaml\n- name: Install k3s/k3d (Kubernetes cluster)\n  run: c2cciutils-k8s-install\n\n- name: Create a database to do the tests\n  run: c2cciutils-k8s-db --script=<my_script>.sql\n\n- name: Install the application in the Kubernetes cluster\n  run: kubectl apply -f <my_application>.yaml\n\n- name: Wait that the application is ready\n  run: c2cciutils-k8s-wait\n- name: Print the application status and logs\n  run: c2cciutils-k8s-logs\n  if: always()\n\n- name: Uninstall the application\n  run: kubectl delete -f <my_application>.yaml || true\n\n- name: Cleanup the database\n  run: c2cciutils-k8s-db --cleanup\n```\n\n`c2cciutils-k8s-install` can be configured in the `ci/config.yaml` file, in section `k8s/k3d/install-commands`, default is:\n\n```yaml\n- - k3d\n    cluster\n    create\n    test-cluster\n    --no-lb\n    --no-rollback\n```\n\nSee also: [K3d cluster create documentation](https://k3d.io/v4.4.8/usage/commands/k3d_cluster_create/).\n\n`c2cciutils-k8s-db` can be configured in the `ci/config.yaml` file, in section `k8s/db/chart-options`, default is:\n\n```yaml\npersistence.enabled: \'false\'\ntls.enabled: \'true\'\ntls.autoGenerated: \'true\'\npostgresqlPassword: mySuperTestingPassword\nvolumePermissions.enabled: \'true\'\n```\n\nSee also: [Parameters documentations](https://github.com/bitnami/charts/tree/master/bitnami/postgresql#parameters).\n',
     'author': 'Camptocamp',