cici-tools 0.15.0__tar.gz → 0.16.0__tar.gz

cici_tools-0.16.0/.cici/README.md.j2

@@ -0,0 +1,97 @@
+# cici pipeline
+
+Build and publish CI/CD pipeline components with cici.
+
+cici, short for Continuous Integration Catalog Interface, is a framework and
+toolkit for managing the integration and lifecycle of packaged CI/CD
+components in a software delivery pipeline.
+
+cici enables the efficient sharing of CI/CD code in an organization, and
+eliminates a major source of friction that otherwise leads to poor adoption of
+automation and DevOps practices.
+
+cici is a foundational component of [saferatday0](https://saferatday0.dev/)
+and powers the [saferatday0 library](https://gitlab.com/saferatday0/library).
+
+## Targets
+
+| Name  | [GitLab include](https://docs.gitlab.com/ee/ci/yaml/includes.html) | [pre-commit hook](https://pre-commit.com/) | Description |
+| ----- | ------- | ---- | ----------- |
+| [cici-bundle](#cici-bundle) | ✓ | ✓ | Bundle GitLab CI/CD includes into single files. |
+| [cici-readme](#cici-readme) | ✓ | ✓ | Generate READMEs for CI pipelines. |
+| [cici-update](#cici-update) | ✓ | ✓ | Update GitLab CI/CD includes to latest released versions. |
+
+### `cici-bundle`
+
+Bundle GitLab CI/CD includes into single files.
+
+As a GitLab include:
+
+```yaml
+# .gitlab-ci.yml
+include:
+  - project: saferatday0/cici
+    file:
+      - cici-bundle.yml
+```
+
+As a pre-commit hook:
+
+```yaml
+# .pre-commit-config.yaml
+repos:
+  - repo: https://gitlab.com/saferatday0/cici
+    rev: ""
+    hooks:
+      - id: cici-bundle
+```
+
+### `cici-readme`
+
+Generate READMEs for CI pipelines.
+
+As a GitLab include:
+
+```yaml
+# .gitlab-ci.yml
+include:
+  - project: saferatday0/cici
+    file:
+      - cici-readme.yml
+```
+
+As a pre-commit hook:
+
+```yaml
+# .pre-commit-config.yaml
+repos:
+  - repo: https://gitlab.com/saferatday0/cici
+    rev: ""
+    hooks:
+      - id: cici-readme
+```
+
+### `cici-update`
+
+Update GitLab CI/CD includes to latest released versions.
+
+As a GitLab include:
+
+```yaml
+# .gitlab-ci.yml
+include:
+  - project: saferatday0/cici
+    file:
+      - cici-update.yml
+```
+
+As a pre-commit hook:
+
+```yaml
+# .pre-commit-config.yaml
+repos:
+  - repo: https://gitlab.com/saferatday0/cici
+    rev: ""
+    hooks:
+      - id: cici-update
+```

{cici_tools-0.15.0 → cici_tools-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cici-tools
-Version: 0.15.0
+Version: 0.16.0
 Summary: Continuous Integration Catalog Interface
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{cici_tools-0.15.0 → cici_tools-0.16.0}/cici/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.15.0'
-__version_tuple__ = version_tuple = (0, 15, 0)
+__version__ = version = '0.16.0'
+__version_tuple__ = version_tuple = (0, 16, 0)
 
-__commit_id__ = commit_id = 'g858d2662b'
+__commit_id__ = commit_id = 'gc5321d184'

{cici_tools-0.15.0 → cici_tools-0.16.0}/cici/cli/bundle.py

@@ -1,18 +1,22 @@
 # SPDX-FileCopyrightText: UL Research Institutes
 # SPDX-License-Identifier: Apache-2.0
 
+
 import logging
 import re
 from importlib import import_module
 from io import StringIO
 from pathlib import Path
+from typing import Optional
 
 from msgspec.structs import replace
 from termcolor import colored
 
-from ..config.project.serializers import load as load_cici_config
+from cici.config.project import models as cici_config
+from cici.config.project import serializers as cici_config_serializers
+
 from ..constants import DEFAULT_PROVIDER
-from ..paths import get_cici_config_file_path, get_cici_config_path
+from ..paths import get_cici_config_path
 
 
 def get_bundle_name(text):
@@ -34,16 +38,40 @@ def get_bundle_content(provider, bundle, cici_config_file=None):
     return content.getvalue()
 
 
+def load_cici_or_nothing(base_path: Path) -> Optional[cici_config.File]:
+    # try to load and resolve .cici/config.yaml
+    # - if config.yaml is there, load it, if not load nothing
+    # - if config.yaml does not exist return None
+
+    base_path = Path(base_path).resolve()
+
+    if base_path.name == ".cici":
+        base_path = base_path.parent
+
+    config_path_root = base_path / ".cici" / "config.yaml"
+
+    # load config.yaml if it exists
+    config_path = None
+    if config_path_root.exists():
+        config_path = config_path_root
+
+    if config_path:
+        config_file = cici_config_serializers.load(config_path)
+        # if load() succeeds then resove targets
+        return cici_config_serializers.resolve_targets(config_file, config_path)
+
+    return None
+
+
 def bundle_command(parser, args):
     logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
 
     provider = import_module(f".{DEFAULT_PROVIDER}", "cici.providers")
 
-    # preserve backwards compatibility (i.e. no .cici/config.yaml)
-    try:
-        cici_config_file = load_cici_config(get_cici_config_file_path())
-    except FileNotFoundError:
-        cici_config_file = None
+    cici_config_file = load_cici_or_nothing(args.config_path)
+
+    if not cici_config_file:
+        logging.warning(f"No config.yaml found in .cici")
 
     ci_file_path = args.config_path / provider.CI_FILE
 

cici_tools-0.16.0/cici/config/project/serializers.py

@@ -0,0 +1,192 @@
+# SPDX-FileCopyrightText: UL Research Institutes
+# SPDX-License-Identifier: Apache-2.0
+
+# Loads .cici/config.yaml
+# Defines targes, variables and metadata about the project.
+
+
+import logging
+import re
+from pathlib import Path
+from typing import Any, Optional, Union
+
+import msgspec
+import ruamel.yaml
+from msgspec.structs import replace
+
+from . import models as cici_config
+
+decoder = msgspec.json.Decoder(type=cici_config.File)
+target_decoder = msgspec.json.Decoder(type=cici_config.Target)
+image_fqdn_regex = re.compile(r"^[\w\.-]+/")
+
+
+# make sure each variable has its 'name' field set from its key
+def inject_variable_names(variables: dict[str, Union[str, dict]]) -> dict[str, dict]:
+    patched = {}
+    for key, value in variables.items():
+        if isinstance(value, str):
+            value = {"value": value}
+        # if value is None or not a dict, treat it as empty dict
+        if not isinstance(value, dict):
+            raise TypeError(f"Expected dict for variable {key}, got {type(value)}")
+        # only add name if missing
+        value.setdefault("name", key)
+        patched[key] = value
+    return patched
+
+
+def load_targets_from_dir(target_dir: Path) -> list[cici_config.Target]:
+    if not target_dir.exists() or not target_dir.is_dir():
+        return []
+
+    yaml = ruamel.yaml.YAML(typ="safe")
+    targets: list[cici_config.Target] = []
+
+    # checking for .yaml
+    for file in sorted(target_dir.glob("*.yaml")):
+        try:
+            with open(file, "r", encoding="utf-8") as f:
+                data = yaml.load(f) or {}
+
+            # make sure name exists before decoding
+            data.setdefault("name", file.stem)
+
+            # decode using msgspec for validation and type safety
+            target_obj = target_decoder.decode(msgspec.json.encode(data))
+            targets.append(target_obj)
+
+        except msgspec.ValidationError as e:
+            logging.warning(f"Validation failed for {file.name}: {e}")
+        except Exception as e:
+            logging.warning(f"Failed to load target {file.name}: {e}")
+
+    logging.info(f"Loaded {len(targets)} valid targets from {target_dir}")
+    return targets
+
+
+def resolve_targets(
+    cici_config_file: Optional[cici_config.File], config_path: Path
+) -> Optional[cici_config.File]:
+    # Resolve and merge targets from .cici/config.yaml and from .cici/targets/*.yaml
+
+    # - Directory targets override config.yaml targets with the same name
+    # - If no targets dir exists, use the targets defined in the config.yaml
+    # - returns updated 'cici_config.File' with merged targets
+
+    if not cici_config_file:
+        return None
+
+    # get directory that could exist targets
+    targets_dir = config_path.parent / "targets"
+
+    # get targets from config.yaml
+    config_targets: list[cici_config.Target] = list(cici_config_file.targets)
+
+    # load targets from '.cici/targets/' if there are any
+    dir_targets: list[cici_config.Target] = []
+    if targets_dir.exists() and any(targets_dir.glob("*.yaml")):
+        logging.info(f"loading targets from directory: {targets_dir}")
+        dir_targets = load_targets_from_dir(targets_dir)
+
+    # combine all targets
+    all_targets = list(config_targets) + list(dir_targets)
+
+    # get target names
+    target_names = [
+        target.name for target in all_targets if getattr(target, "name", None)
+    ]
+
+    # check duplicates
+    if len(target_names) != len(set(target_names)):
+        raise ValueError(
+            f"Duplicate target names found: {', '.join(sorted(target_names))}"
+        )
+
+    # create new file with merged target list
+    resolved_file = replace(cici_config_file, targets=all_targets)
+    return resolved_file
+
+
+def patch_image(image: str, container_proxy: str = "${CONTAINER_PROXY}") -> str:
+    """Patch in $CONTAINER_PROXY to image unless the following are true:
+
+    A: Does the image URL contain ${CONTAINER_PROXY} (the literal string)
+    B: Is the image URL a fully-qualified container URL?
+    C: Does the image already start with a variable?
+    """
+
+    if not image:
+        return image
+    if container_proxy in image:
+        return image
+
+    if image_fqdn_regex.match(image):
+        return image
+
+    if image.startswith("$"):
+        return image
+
+    return f"{container_proxy}{image}"
+
+
+def loads(
+    text: str,
+    gitlab_ci_jobs: Optional[dict[str, Any]] = None,
+    precommit_hooks: Optional[dict[str, Any]] = None,
+) -> cici_config.File:
+    # parse YAML into fully-typed File object
+    if gitlab_ci_jobs is None:
+        gitlab_ci_jobs = {}
+    if precommit_hooks is None:
+        precommit_hooks = {}
+
+    yaml = ruamel.yaml.YAML(typ="safe")
+    data = yaml.load(text)
+
+    # verify targets exists even if empty
+    data.setdefault("targets", [])
+
+    # Inject precommit/gitlab includes into each target
+
+    # Debug to test injection
+    # print("Before injection:", data["targets"])
+
+    for target in data["targets"]:
+        target["precommit_hook"] = {"name": target["name"]}
+        target["gitlab_include"] = {"name": target["name"]}
+
+    # Debug to test injection
+    # print("After injection:", data["targets"])
+
+    if "variables" in data:
+        data["variables"] = inject_variable_names(data["variables"])
+
+    # decode into file_struct
+    file_struct = decoder.decode(msgspec.json.encode(data))
+
+    # post process to patch CONTAINER_PROXY to container images
+    patched_targets = []
+    for target in file_struct.targets:
+        if target.container is not None:
+            patched_container = replace(
+                target.container,
+                image=patch_image(target.container.image),
+            )
+            patched_targets.append(replace(target, container=patched_container))
+        else:
+            patched_targets.append(target)
+
+    return replace(file_struct, targets=patched_targets)
+
+
+def load(
+    file: Union[str, Path],
+    gitlab_ci_jobs: Optional[dict[str, Any]] = None,
+    precommit_hooks: Optional[dict[str, Any]] = None,
+) -> cici_config.File:
+    return loads(
+        open(file).read(),
+        gitlab_ci_jobs=gitlab_ci_jobs,
+        precommit_hooks=precommit_hooks,
+    )

{cici_tools-0.15.0 → cici_tools-0.16.0}/cici/providers/gitlab/serializers.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: UL Research Institutes
 # SPDX-License-Identifier: Apache-2.0
 
+# Loads and serializes .gitlab-ci.yml and CI job definitions
+
 import io
 import typing
 from pathlib import Path
@@ -249,41 +251,33 @@ def loads(
     yaml = ruamel.yaml.YAML()
     data = yaml.load(text)
 
-    # 1. Normalize scalars
+    # Normalize scalars
     data = normalize_scalars(data)
 
-    # 2. merge top-level config variables
+    # merge top-level config variables
     if cici_config_file:
         data = add_config_variables(data, cici_config_file=cici_config_file)
 
-    # DEBUG
-    # print("CONFIG FILE TYPE:", type(cici_config_file))
-    # print("CONFIG FILE CONTENT:", getattr(cici_config_file, "variables", None))
-
-    # 4. normalize jobs
+    # normalize jobs
     data = normalize_jobs_in_data(data)
 
-    # 5. normalize variables format
+    # normalize variables format
     data = normalize_variables(data)
 
-    # 6. inject variable names
+    # inject variable names
     if "variables" in data:
         data["variables"] = inject_variable_names(data["variables"])
 
-    # 7. pack the jobs
+    # pack the jobs
     data = pack_jobs(data)
 
-    # DEBUG
-    # print("Keys at top level before decode:", data.keys())
-    # print("Variables at top level:", data.get("variables"))
-
-    # 8. decode into msgspec struct
+    # decode into msgspec struct
     file_struct = decode_file(data)
 
-    # 9. expand extends and anchors
+    # expand extends and anchors
     file_struct = expand_jobs(file_struct)
 
-    # 10. inject container defaults
+    # inject container defaults
     return inject_container_into_job(file_struct, cici_config_file)
 
 

{cici_tools-0.15.0 → cici_tools-0.16.0}/cici_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cici-tools
-Version: 0.15.0
+Version: 0.16.0
 Summary: Continuous Integration Catalog Interface
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{cici_tools-0.15.0 → cici_tools-0.16.0}/cici_tools.egg-info/SOURCES.txt

@@ -20,6 +20,7 @@ mkdocs.yml
 pyproject.toml
 requirements.txt
 .cici/.gitlab-ci.yml
+.cici/README.md.j2
 .cici/config.yaml
 cici/__init__.py
 cici/__main__.py
@@ -81,6 +82,7 @@ scripts/gen-api-doc.py
 tests/test_build.py
 tests/test_cli.py
 tests/test_import.py
+tests/test_resolve_targets.py
 tests/fixtures/gitlab/extends/simple-job.yml
 tests/fixtures/gitlab/extends/.cici/.gitlab-ci.yml
 tests/fixtures/gitlab/helm/helm-cm-push.yml
@@ -140,6 +142,54 @@ tests/fixtures/gitlab/simple/simple-job.yml
 tests/fixtures/gitlab/simple/.cici/.gitlab-ci.yml
 tests/fixtures/gitlab/stages/simple-job.yml
 tests/fixtures/gitlab/stages/.cici/.gitlab-ci.yml
+tests/fixtures/gitlab/targets-dir/opentofu-development-apply.yml
+tests/fixtures/gitlab/targets-dir/opentofu-development-auto-apply.yml
+tests/fixtures/gitlab/targets-dir/opentofu-development-destroy.yml
+tests/fixtures/gitlab/targets-dir/opentofu-development-plan.yml
+tests/fixtures/gitlab/targets-dir/opentofu-development-trivy.yml
+tests/fixtures/gitlab/targets-dir/opentofu-development-validate.yml
+tests/fixtures/gitlab/targets-dir/opentofu-fmt.yml
+tests/fixtures/gitlab/targets-dir/opentofu-module-build.yml
+tests/fixtures/gitlab/targets-dir/opentofu-module-release.yml
+tests/fixtures/gitlab/targets-dir/opentofu-production-apply.yml
+tests/fixtures/gitlab/targets-dir/opentofu-production-auto-apply.yml
+tests/fixtures/gitlab/targets-dir/opentofu-production-destroy.yml
+tests/fixtures/gitlab/targets-dir/opentofu-production-plan.yml
+tests/fixtures/gitlab/targets-dir/opentofu-production-trivy.yml
+tests/fixtures/gitlab/targets-dir/opentofu-production-validate.yml
+tests/fixtures/gitlab/targets-dir/opentofu-staging-apply.yml
+tests/fixtures/gitlab/targets-dir/opentofu-staging-auto-apply.yml
+tests/fixtures/gitlab/targets-dir/opentofu-staging-destroy.yml
+tests/fixtures/gitlab/targets-dir/opentofu-staging-plan.yml
+tests/fixtures/gitlab/targets-dir/opentofu-staging-trivy.yml
+tests/fixtures/gitlab/targets-dir/opentofu-staging-validate.yml
+tests/fixtures/gitlab/targets-dir/opentofu-trivy.yml
+tests/fixtures/gitlab/targets-dir/opentofu-validate.yml
+tests/fixtures/gitlab/targets-dir/.cici/.gitlab-ci.yml
+tests/fixtures/gitlab/targets-dir/.cici/config.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-development-apply.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-development-auto-apply.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-development-destroy.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-development-plan.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-development-validate.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-fmt.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-module-build.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-module-release.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-production-apply.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-production-auto-apply.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-production-destroy.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-production-plan.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-production-trivy.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-production-validate.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-staging-apply.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-staging-auto-apply.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-staging-destroy.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-staging-plan.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-staging-trivy.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-staging-validate.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-trivy.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/opentofu-validate.yaml
+tests/fixtures/gitlab/targets-dir/.cici/targets/tofu-development-trivy.yaml
 tests/fixtures/gitlab/terraform/terraform-fmt.yml
 tests/fixtures/gitlab/terraform/terraform-production-apply.yml
 tests/fixtures/gitlab/terraform/terraform-production-plan.yml

{cici_tools-0.15.0 → cici_tools-0.16.0}/pyproject.toml

@@ -64,6 +64,9 @@ cici = "cici.__main__:main"
 [tool.isort]
 profile = "black"
 
+[tool.mypy]
+python_version = "3.10"
+
 [tool.setuptools.package-data]
 "cici" = ["py.typed"]
 

{cici_tools-0.15.0 → cici_tools-0.16.0}/requirements.txt

@@ -2,7 +2,7 @@
 #    uv pip compile --python-version 3.11 -o requirements.txt pyproject.toml
 appdirs==1.4.4
     # via cici-tools (pyproject.toml)
-attrs==25.3.0
+attrs==25.4.0
     # via
     #   jsonschema
     #   referencing
@@ -20,19 +20,19 @@ msgspec==0.19.0
     # via cici-tools (pyproject.toml)
 python-decouple==3.8
     # via cici-tools (pyproject.toml)
-referencing==0.36.2
+referencing==0.37.0
     # via
     #   jsonschema
     #   jsonschema-specifications
-rpds-py==0.27.1
+rpds-py==0.28.0
     # via
     #   jsonschema
     #   referencing
-ruamel-yaml==0.18.15
+ruamel-yaml==0.18.16
     # via cici-tools (pyproject.toml)
 ruamel-yaml-clib==0.2.14
     # via ruamel-yaml
-termcolor==3.1.0
+termcolor==3.2.0
     # via cici-tools (pyproject.toml)
 typing-extensions==4.15.0
     # via referencing