cici-tools 0.15.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

cici/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.15.0'
-__version_tuple__ = version_tuple = (0, 15, 0)
+__version__ = version = '0.16.0'
+__version_tuple__ = version_tuple = (0, 16, 0)
 
-__commit_id__ = commit_id = 'g858d2662b'
+__commit_id__ = commit_id = 'gc5321d184'

cici/cli/bundle.py

@@ -1,18 +1,22 @@
 # SPDX-FileCopyrightText: UL Research Institutes
 # SPDX-License-Identifier: Apache-2.0
 
+
 import logging
 import re
 from importlib import import_module
 from io import StringIO
 from pathlib import Path
+from typing import Optional
 
 from msgspec.structs import replace
 from termcolor import colored
 
-from ..config.project.serializers import load as load_cici_config
+from cici.config.project import models as cici_config
+from cici.config.project import serializers as cici_config_serializers
+
 from ..constants import DEFAULT_PROVIDER
-from ..paths import get_cici_config_file_path, get_cici_config_path
+from ..paths import get_cici_config_path
 
 
 def get_bundle_name(text):
@@ -34,16 +38,40 @@ def get_bundle_content(provider, bundle, cici_config_file=None):
     return content.getvalue()
 
 
+def load_cici_or_nothing(base_path: Path) -> Optional[cici_config.File]:
+    # try to load and resolve .cici/config.yaml
+    # - if config.yaml is there, load it, if not load nothing
+    # - if config.yaml does not exist return None
+
+    base_path = Path(base_path).resolve()
+
+    if base_path.name == ".cici":
+        base_path = base_path.parent
+
+    config_path_root = base_path / ".cici" / "config.yaml"
+
+    # load config.yaml if it exists
+    config_path = None
+    if config_path_root.exists():
+        config_path = config_path_root
+
+    if config_path:
+        config_file = cici_config_serializers.load(config_path)
+        # if load() succeeds then resove targets
+        return cici_config_serializers.resolve_targets(config_file, config_path)
+
+    return None
+
+
 def bundle_command(parser, args):
     logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
 
     provider = import_module(f".{DEFAULT_PROVIDER}", "cici.providers")
 
-    # preserve backwards compatibility (i.e. no .cici/config.yaml)
-    try:
-        cici_config_file = load_cici_config(get_cici_config_file_path())
-    except FileNotFoundError:
-        cici_config_file = None
+    cici_config_file = load_cici_or_nothing(args.config_path)
+
+    if not cici_config_file:
+        logging.warning(f"No config.yaml found in .cici")
 
     ci_file_path = args.config_path / provider.CI_FILE
 

cici/config/project/serializers.py

@@ -1,7 +1,11 @@
 # SPDX-FileCopyrightText: UL Research Institutes
 # SPDX-License-Identifier: Apache-2.0
 
+# Loads .cici/config.yaml
+# Defines targes, variables and metadata about the project.
 
+
+import logging
 import re
 from pathlib import Path
 from typing import Any, Optional, Union
@@ -13,22 +17,97 @@ from msgspec.structs import replace
 from . import models as cici_config
 
 decoder = msgspec.json.Decoder(type=cici_config.File)
+target_decoder = msgspec.json.Decoder(type=cici_config.Target)
 image_fqdn_regex = re.compile(r"^[\w\.-]+/")
 
 
-def inject_variable_names(variables: dict[str, dict]) -> dict[str, dict]:
-    # make sure each variable has its 'name' field set from its key
+# make sure each variable has its 'name' field set from its key
+def inject_variable_names(variables: dict[str, Union[str, dict]]) -> dict[str, dict]:
     patched = {}
     for key, value in variables.items():
+        if isinstance(value, str):
+            value = {"value": value}
         # if value is None or not a dict, treat it as empty dict
         if not isinstance(value, dict):
-            value = {}
+            raise TypeError(f"Expected dict for variable {key}, got {type(value)}")
         # only add name if missing
         value.setdefault("name", key)
         patched[key] = value
     return patched
 
 
+def load_targets_from_dir(target_dir: Path) -> list[cici_config.Target]:
+    if not target_dir.exists() or not target_dir.is_dir():
+        return []
+
+    yaml = ruamel.yaml.YAML(typ="safe")
+    targets: list[cici_config.Target] = []
+
+    # checking for .yaml
+    for file in sorted(target_dir.glob("*.yaml")):
+        try:
+            with open(file, "r", encoding="utf-8") as f:
+                data = yaml.load(f) or {}
+
+            # make sure name exists before decoding
+            data.setdefault("name", file.stem)
+
+            # decode using msgspec for validation and type safety
+            target_obj = target_decoder.decode(msgspec.json.encode(data))
+            targets.append(target_obj)
+
+        except msgspec.ValidationError as e:
+            logging.warning(f"Validation failed for {file.name}: {e}")
+        except Exception as e:
+            logging.warning(f"Failed to load target {file.name}: {e}")
+
+    logging.info(f"Loaded {len(targets)} valid targets from {target_dir}")
+    return targets
+
+
+def resolve_targets(
+    cici_config_file: Optional[cici_config.File], config_path: Path
+) -> Optional[cici_config.File]:
+    # Resolve and merge targets from .cici/config.yaml and from .cici/targets/*.yaml
+
+    # - Directory targets override config.yaml targets with the same name
+    # - If no targets dir exists, use the targets defined in the config.yaml
+    # - returns updated 'cici_config.File' with merged targets
+
+    if not cici_config_file:
+        return None
+
+    # get directory that could exist targets
+    targets_dir = config_path.parent / "targets"
+
+    # get targets from config.yaml
+    config_targets: list[cici_config.Target] = list(cici_config_file.targets)
+
+    # load targets from '.cici/targets/' if there are any
+    dir_targets: list[cici_config.Target] = []
+    if targets_dir.exists() and any(targets_dir.glob("*.yaml")):
+        logging.info(f"loading targets from directory: {targets_dir}")
+        dir_targets = load_targets_from_dir(targets_dir)
+
+    # combine all targets
+    all_targets = list(config_targets) + list(dir_targets)
+
+    # get target names
+    target_names = [
+        target.name for target in all_targets if getattr(target, "name", None)
+    ]
+
+    # check duplicates
+    if len(target_names) != len(set(target_names)):
+        raise ValueError(
+            f"Duplicate target names found: {', '.join(sorted(target_names))}"
+        )
+
+    # create new file with merged target list
+    resolved_file = replace(cici_config_file, targets=all_targets)
+    return resolved_file
+
+
 def patch_image(image: str, container_proxy: str = "${CONTAINER_PROXY}") -> str:
     """Patch in $CONTAINER_PROXY to image unless the following are true:
 
@@ -64,15 +143,21 @@ def loads(
 
     yaml = ruamel.yaml.YAML(typ="safe")
     data = yaml.load(text)
+
     # verify targets exists even if empty
     data.setdefault("targets", [])
 
     # Inject precommit/gitlab includes into each target
+
+    # Debug to test injection
+    # print("Before injection:", data["targets"])
+
     for target in data["targets"]:
-        if target["name"] in precommit_hooks:
-            target["precommit_hook"] = {"name": target["name"]}
-        if target["name"] in gitlab_ci_jobs:
-            target["gitlab_include"] = {"name": target["name"]}
+        target["precommit_hook"] = {"name": target["name"]}
+        target["gitlab_include"] = {"name": target["name"]}
+
+    # Debug to test injection
+    # print("After injection:", data["targets"])
 
     if "variables" in data:
         data["variables"] = inject_variable_names(data["variables"])

cici/providers/gitlab/serializers.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: UL Research Institutes
 # SPDX-License-Identifier: Apache-2.0
 
+# Loads and serializes .gitlab-ci.yml and CI job definitions
+
 import io
 import typing
 from pathlib import Path
@@ -249,41 +251,33 @@ def loads(
     yaml = ruamel.yaml.YAML()
     data = yaml.load(text)
 
-    # 1. Normalize scalars
+    # Normalize scalars
     data = normalize_scalars(data)
 
-    # 2. merge top-level config variables
+    # merge top-level config variables
     if cici_config_file:
         data = add_config_variables(data, cici_config_file=cici_config_file)
 
-    # DEBUG
-    # print("CONFIG FILE TYPE:", type(cici_config_file))
-    # print("CONFIG FILE CONTENT:", getattr(cici_config_file, "variables", None))
-
-    # 4. normalize jobs
+    # normalize jobs
     data = normalize_jobs_in_data(data)
 
-    # 5. normalize variables format
+    # normalize variables format
     data = normalize_variables(data)
 
-    # 6. inject variable names
+    # inject variable names
     if "variables" in data:
         data["variables"] = inject_variable_names(data["variables"])
 
-    # 7. pack the jobs
+    # pack the jobs
     data = pack_jobs(data)
 
-    # DEBUG
-    # print("Keys at top level before decode:", data.keys())
-    # print("Variables at top level:", data.get("variables"))
-
-    # 8. decode into msgspec struct
+    # decode into msgspec struct
     file_struct = decode_file(data)
 
-    # 9. expand extends and anchors
+    # expand extends and anchors
     file_struct = expand_jobs(file_struct)
 
-    # 10. inject container defaults
+    # inject container defaults
     return inject_container_into_job(file_struct, cici_config_file)
 
 

{cici_tools-0.15.0.dist-info → cici_tools-0.16.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cici-tools
-Version: 0.15.0
+Version: 0.16.0
 Summary: Continuous Integration Catalog Interface
 Author-email: Digital Safety Research Institute <contact@dsri.org>
 License: Apache-2.0

{cici_tools-0.15.0.dist-info → cici_tools-0.16.0.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cici/__init__.py,sha256=wdYYpZKxK2omc_mRJgPj86Ec-QwoB4noZzlk71sBjnk,87
 cici/__main__.py,sha256=TWDBrcS5vXSclLWLm0Iu3uUXOkDw0PzcJ8a0zd7_ClU,150
-cici/_version.py,sha256=yVM3H6SCPkszqQN_Lm-MropXHFmwVrg3EdCRYTwK0gw,714
+cici/_version.py,sha256=nFweIwOtVfmijqw6TBaMFicDb1QpkX-juuqHfIWXZJw,714
 cici/constants.py,sha256=ydShHTAbuanWWXeg9tJZ5GLm65M41AKmIibUheKV5nw,913
 cici/exceptions.py,sha256=JMpcD5YjYwKdn65D0iOEchXiAgIVsKApAyngCTwVgI4,139
 cici/main.py,sha256=ceH9s7i2XdB-mpUtDWuzLVn9TZHiwmsq-daZOgB9oRI,748
@@ -8,7 +8,7 @@ cici/paths.py,sha256=R-zBlX70uUJKR-ttGmZ7gI8iIKdxz9hLN1bP1pSXZxI,1103
 cici/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cici/utils.py,sha256=E6GrD6qCvsFqaR-fhsuBf8qotxyUO2ZdOoKj2vRUUHs,1917
 cici/cli/__init__.py,sha256=wdYYpZKxK2omc_mRJgPj86Ec-QwoB4noZzlk71sBjnk,87
-cici/cli/bundle.py,sha256=Xv0mlUidcrwx3cf4k2awRZcvQYDgz4d9EjDFq2BYsPw,3152
+cici/cli/bundle.py,sha256=XTsyXdlJHTK4Ca0i0S637pOIendaAI6hra5GdRgrYsw,3928
 cici/cli/readme.py,sha256=th7PjmsckF3d0roVmV6Y4-1Azz9TGbaDXcyvPX3tjZQ,2496
 cici/cli/schema.py,sha256=tQNirJrMyUf96iaA8XBmALeo4EvdRGqWLkTKCtvWR5M,1623
 cici/cli/update.py,sha256=8PZpd6HqJucoZDhLLX8eHAMSFh-Ngox6UYiVqsy6ZCs,4825
@@ -16,13 +16,13 @@ cici/config/__init__.py,sha256=TIewA-VCl2VYl_d0EviE3EEQSmBtDwjU26R68pDT38E,141
 cici/config/user.py,sha256=UBz21P-Vj41rKy2DAZwdAp5vcliAEEQlEQeEfHk-EZ4,841
 cici/config/project/__init__.py,sha256=wdYYpZKxK2omc_mRJgPj86Ec-QwoB4noZzlk71sBjnk,87
 cici/config/project/models.py,sha256=Ao8MGliXihoYRgqplWQAIyC3r4Jb0aj53F7EhgZGcu4,5873
-cici/config/project/serializers.py,sha256=sQxRzMlXTWfUbkJBiu5huvkUc8z5xYVfsoYs8W55Aqw,3202
+cici/config/project/serializers.py,sha256=Mlqodt4j2Q4KXK-LjxAA8jdopCenbBMokQ-LfITtJRc,6119
 cici/providers/__init__.py,sha256=wdYYpZKxK2omc_mRJgPj86Ec-QwoB4noZzlk71sBjnk,87
 cici/providers/gitlab/__init__.py,sha256=8ej6k3hd25UFOGFhHymLa2mvz9hEwMjdxyu66hnJwOQ,259
 cici/providers/gitlab/constants.py,sha256=nA36aqGvwBBood8lTFJueEY8lKEsOVjkp6ciMjAeKJA,1786
 cici/providers/gitlab/models.py,sha256=zPOLSfb811yVYgkKNirAEhcAPkoBwE-r_fmxMfxn8GU,7084
 cici/providers/gitlab/normalizers.py,sha256=EFQiJzRDkoahes0zwIMahuVC3DOy9A97eZDhMXdamno,4762
-cici/providers/gitlab/serializers.py,sha256=RHqy5ZI061U41PzMxBeOITOGIErxRIS6GW1iWOzTKFw,10835
+cici/providers/gitlab/serializers.py,sha256=WmIFhztf5-oLhitJwzezKcKp4hMdctZgm9BaJWbELtw,10581
 cici/providers/gitlab/utils.py,sha256=NUZ25rAosSWQyWh7m3xhiHRotqVl1krckZL58yR0LKQ,651
 cici/providers/gitlab/yaml_style.py,sha256=Obqm6JxmGgksCG_KYOFL9JH3S-bwzH9qFRvtD68mNjs,13409
 cici/schema/LICENSE.gitlab,sha256=0N7FHuinuoGBqZE6aUlD84yw3CGylx8lyfzertgmtY0,2001
@@ -39,10 +39,10 @@ cici/templates/target-table.md.j2,sha256=NYspLbP3ap1706WnehZRtu6a2uxL_mKf7zfm3y0
 cici/templates/targets.md.j2,sha256=k-lkfReFJMCu9t2SjuD5YJ-Q_KECJxdY7v-0TWDTaCM,116
 cici/templates/variable-list.md.j2,sha256=M9r3j7d8H171okQNj8j4pc92kMdeWSRoeZQ5QB3NUTQ,613
 cici/templates/variables.md.j2,sha256=2DquXagNupqBkD95_3ZI4guJOK-eG_igaaMdRIUDrcE,84
-cici_tools-0.15.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-cici_tools-0.15.0.dist-info/licenses/NOTICE,sha256=dIBr-7sfvnoYXYvgHIozlRJTLD4EHORRtRJtIe4znW8,55
-cici_tools-0.15.0.dist-info/METADATA,sha256=sRoea8HJWGpWRw_55L3T4-uAHgOxWWuDPR-rSf-bdKk,4178
-cici_tools-0.15.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cici_tools-0.15.0.dist-info/entry_points.txt,sha256=CuBAwYG3z7qOKm4IYARjZzdX5fytMLKyr59HLLJx_Is,44
-cici_tools-0.15.0.dist-info/top_level.txt,sha256=sv8xIjFuuqtyBMoyzueczNvZo_--q12r64Zc0lGKKF8,5
-cici_tools-0.15.0.dist-info/RECORD,,
+cici_tools-0.16.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+cici_tools-0.16.0.dist-info/licenses/NOTICE,sha256=dIBr-7sfvnoYXYvgHIozlRJTLD4EHORRtRJtIe4znW8,55
+cici_tools-0.16.0.dist-info/METADATA,sha256=-rzT_wwEirgiEcT2jelxH3ikrkXUuqhO91-K9b97CtM,4178
+cici_tools-0.16.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cici_tools-0.16.0.dist-info/entry_points.txt,sha256=CuBAwYG3z7qOKm4IYARjZzdX5fytMLKyr59HLLJx_Is,44
+cici_tools-0.16.0.dist-info/top_level.txt,sha256=sv8xIjFuuqtyBMoyzueczNvZo_--q12r64Zc0lGKKF8,5
+cici_tools-0.16.0.dist-info/RECORD,,