cici-tools 0.2.0__py3-none-any.whl

cici/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

cici/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

cici/cli/build.py

@@ -0,0 +1,55 @@
+import logging
+import sys
+from importlib import import_module
+from pathlib import Path
+
+from ..constants import DEFAULT_PROVIDER, PROVIDERS
+
+
+def build_command(parser, args):
+    logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
+
+    input_provider = import_module(f".{args.input_format}", "cici.providers")
+    output_provider = import_module(f".{args.output_format}", "cici.providers")
+
+    if not args.filename:
+        args.filename = input_provider.CI_FILE
+
+    if not Path(args.filename).exists():
+        parser.error(f"file not found: {args.filename}")
+
+    file = input_provider.load(args.filename)
+    output_provider.dump(file, sys.stdout)
+
+
+def build_parser(subparsers):
+    parser = subparsers.add_parser(
+        "build", help="build BrettOps CI file into target format"
+    )
+    parser.add_argument("filename", nargs="?")
+    parser.add_argument(
+        "-f",
+        "--from",
+        dest="input_format",
+        choices=PROVIDERS,
+        default=DEFAULT_PROVIDER,
+        help=f"input format [{DEFAULT_PROVIDER}]",
+    )
+    parser.add_argument(
+        "-t",
+        "--to",
+        dest="output_format",
+        choices=PROVIDERS,
+        default=DEFAULT_PROVIDER,
+        help=f"output format [{DEFAULT_PROVIDER}]",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="DIR",
+        dest="output_path",
+        type=Path,
+        default=Path.cwd().absolute(),
+    )
+    parser.set_defaults(func=build_command)
+    return parser

cici/cli/bundle.py

@@ -0,0 +1,104 @@
+import copy
+import logging
+from importlib import import_module
+from pathlib import Path
+from typing import Optional
+
+from termcolor import colored
+
+from ..constants import CONFIG_DIR_NAME, DEFAULT_PROVIDER
+
+
+def get_job_prefix(pipeline_name: str, job_name: str) -> Optional[str]:
+    if job_name.startswith("."):
+        return None
+    return f"{pipeline_name}-"
+
+
+def parse_groups(name, file, groups):
+    if groups:
+        groups = [group.strip() for group in groups.split(",")]
+
+    if not groups:
+        group_names = set()
+        for job_name in file.jobs.keys():
+            if job_name.startswith("."):
+                continue
+            group_name = job_name[len(f"{name}-") :]
+            group_name = group_name.split("-")[0]
+            group_names.add(group_name)
+        groups = sorted(list(group_names))
+    return groups
+
+
+def bundle_command(parser, args):
+    logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
+    print(colored("pipeline name:", color="yellow"), args.pipeline_name)
+
+    provider = import_module(f".{DEFAULT_PROVIDER}", "cici.providers")
+
+    ci_file_path = args.config_path / provider.CI_FILE
+
+    if not Path(ci_file_path).exists():
+        parser.error(f"file not found: {ci_file_path}")
+
+    file = provider.load(ci_file_path)
+
+    args.output_path = Path(args.output_path)
+    args.groups = parse_groups(args.pipeline_name, file, args.groups)
+    print(colored("bundle names:", color="yellow"), args.groups)
+
+    for bundle_name in args.groups:
+        pattern = f"{args.pipeline_name}-{bundle_name}"
+
+        bundle = copy.deepcopy(file)
+        bundle.jobs = {}
+
+        for job_name, job in file.jobs.items():
+            job_prefix = get_job_prefix(
+                pipeline_name=args.pipeline_name,
+                job_name=job_name,
+            )
+            if not job_prefix:
+                continue
+            assert job_name.startswith(job_prefix)
+            if job_name.startswith(pattern):
+                bundle.jobs[job_name] = job
+
+        bundle_filename = args.output_path / f"{bundle_name}.yml"
+
+        with open(bundle_filename, "w") as stream:
+            provider.dump(bundle, stream)
+        print(colored("created", "magenta"), bundle_filename.name)
+
+
+def bundle_parser(subparsers):
+    parser = subparsers.add_parser(
+        "bundle", help="distill a CI file to a job and its dependencies"
+    )
+    parser.add_argument(
+        "config_path",
+        metavar="DIR",
+        nargs="?",
+        type=Path,
+        default=(Path.cwd() / CONFIG_DIR_NAME).absolute(),
+    )
+    parser.add_argument("-g", "--groups", help="job group patterns")
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="DIR",
+        dest="output_path",
+        type=Path,
+        default=Path.cwd().absolute(),
+    )
+    parser.add_argument(
+        "-n",
+        "--name",
+        metavar="DIR",
+        dest="pipeline_name",
+        default=str(Path.cwd().name),
+        help="the name of the pipeline",
+    )
+    parser.set_defaults(func=bundle_command)
+    return parser

cici/cli/fmt.py

@@ -0,0 +1,30 @@
+from importlib import import_module
+from pathlib import Path
+
+from termcolor import colored
+
+from ..constants import DEFAULT_PROVIDER
+
+
+def fmt_command(parser, args):
+    provider = import_module(f".{DEFAULT_PROVIDER}", "cici.providers")
+
+    if not args.filenames:
+        args.filenames = [provider.CI_FILE]
+
+    for filename in args.filenames:
+        if not Path(filename).exists():
+            parser.error(f"file not found: {filename}")
+
+    for filename in args.filenames:
+        file = provider.load(filename)
+        with open(filename, "w") as stream:
+            provider.dump(file, stream)
+        print(colored(filename, "magenta"), "formatted")
+
+
+def fmt_parser(subparsers):
+    parser = subparsers.add_parser("fmt", help="Reformat CI file")
+    parser.add_argument("filenames", metavar="FILE", nargs="*")
+    parser.set_defaults(func=fmt_command)
+    return parser

cici/cli/update.py

@@ -0,0 +1,112 @@
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import ruamel.yaml
+from termcolor import colored
+
+GITLAB_URL = "https://gitlab.com"
+API_V4_URL = f"{GITLAB_URL}/api/v4"
+
+HEADERS: dict[str, str] = {}
+private_token = os.environ.get("GITLAB_PRIVATE_TOKEN")
+if private_token:
+    HEADERS.update({"PRIVATE-TOKEN": private_token})
+
+
+def to_fragment(text):
+    if not isinstance(text, str):
+        return text
+    return urllib.parse.quote(text, safe="")
+
+
+def get_apiv4_url(url):
+    try:
+        url = f"{API_V4_URL}{url}"
+        request = urllib.request.Request(url=url, headers=HEADERS)
+        response = urllib.request.urlopen(request)
+        content = response.read()
+        return json.loads(content)
+    except urllib.error.HTTPError:
+        return None
+
+
+def get_project(project_id):
+    return get_apiv4_url(f"/projects/{to_fragment(project_id)}")
+
+
+def get_latest_release(project_id):
+    return get_apiv4_url(
+        f"/projects/{to_fragment(project_id)}/releases/permalink/latest"
+    )
+
+
+def cproject(project_name):
+    return colored(project_name, "magenta")
+
+
+def ctag(tag_name):
+    return colored(tag_name, "yellow", attrs=["bold"])
+
+
+def update_include(include):
+    if any(key not in include for key in ("project", "file")):
+        return include
+
+    project = get_project(include["project"])
+    project_name = project["path_with_namespace"]
+
+    latest_release = get_latest_release(project["id"])
+    if latest_release:
+        latest_tag = latest_release["tag_name"]
+        current_tag = include.get("ref", None)
+        if current_tag:
+            if current_tag != latest_tag:
+                print(
+                    colored(project_name, "red", attrs=["bold"]),
+                    "updated from {current} to {latest}".format(
+                        current=ctag(current_tag),
+                        latest=ctag(latest_tag),
+                    ),
+                )
+            else:
+                print(cproject(project_name), "is the latest at", ctag(current_tag))
+        elif not current_tag:
+            print(cproject(project_name), "pinned to", ctag(latest_tag))
+        include["ref"] = latest_tag
+    else:
+        print(cproject(project_name), "has no releases")
+    newinclude = {}
+    newinclude["project"] = include["project"]
+    if "ref" in include:
+        newinclude["ref"] = include["ref"]
+    newinclude["file"] = include["file"]
+    return newinclude
+
+
+def update_includes(includes):
+    if not isinstance(includes, list):
+        return includes
+
+    return [update_include(include) for include in includes]
+
+
+def update_command(parser, args):
+    yaml = ruamel.yaml.YAML()
+    data = yaml.load(open(args.filename))
+
+    if "include" in data:
+        data["include"] = update_includes(data["include"])
+
+    with open(args.filename, "w") as handle:
+        yaml.indent(mapping=2, sequence=4, offset=2)
+        yaml.dump(data, handle)
+
+
+def update_parser(subparsers):
+    parser = subparsers.add_parser("update")
+    parser.add_argument("filename", nargs="?", default=".gitlab-ci.yml")
+    parser.set_defaults(func=update_command)
+    return parser

cici/constants.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+BASE_DIR = Path(__file__).parent.absolute()
+
+SCHEMA_DIR = BASE_DIR / "schema"
+
+COMMAND_DIR = BASE_DIR / "cli"
+
+COMMANDS = sorted(
+    [path.stem for path in COMMAND_DIR.glob("*.py") if not path.stem.startswith("_")]
+)
+
+PROVIDER_DIR = BASE_DIR / "providers"
+
+PROVIDERS = sorted(
+    [path.stem for path in PROVIDER_DIR.glob("*.py") if not path.stem.startswith("_")]
+    + [
+        path.stem
+        for path in PROVIDER_DIR.glob("*/")
+        if path.is_dir() and not path.stem.startswith("_")
+    ]
+)
+
+DEFAULT_PROVIDER = PROVIDERS[0]
+
+CONFIG_DIR_NAME = ".cici"

cici/main.py

@@ -0,0 +1,17 @@
+import argparse
+from importlib import import_module
+
+from .constants import COMMANDS
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers(required=True)
+
+    for command_name in COMMANDS:
+        command = import_module(f".{command_name}", "cici.cli")
+        getattr(command, f"{command_name}_parser")(subparsers=subparsers)
+
+    args = parser.parse_args()
+    args.func(parser=parser, args=args)

cici/providers/gitlab/__init__.py

@@ -0,0 +1,3 @@
+from .constants import CI_FILE
+from .models import File
+from .serializers import dump, dumps, load, loads

cici/providers/gitlab/constants.py

@@ -0,0 +1,77 @@
+ACCESS = "access"
+ALWAYS = "always"
+DEVELOPMENT = "development"
+IF_NOT_PRESENT = "if-not-present"
+NEVER = "never"
+ON_SUCCESS = "on_success"
+ON_FAILURE = "on_failure"
+OTHER = "other"
+PREPARE = "prepare"
+PRODUCTION = "production"
+PULL = "pull"
+PULL_PUSH = "pull-push"
+PUSH = "push"
+STAGING = "staging"
+START = "start"
+STOP = "stop"
+TESTING = "testing"
+VERIFY = "verify"
+
+CACHE_POLICIES = (PULL, PUSH, PULL_PUSH)
+
+DEPLOYMENT_TIERS = (
+    PRODUCTION,
+    STAGING,
+    TESTING,
+    DEVELOPMENT,
+    OTHER,
+)
+
+ENVIRONMENT_ACTIONS = (
+    START,
+    PREPARE,
+    STOP,
+    VERIFY,
+    ACCESS,
+)
+
+PULL_POLICIES = (
+    ALWAYS,
+    IF_NOT_PRESENT,
+    NEVER,
+)
+
+WHEN_CHOICES = (ON_SUCCESS, ON_FAILURE, ALWAYS)
+
+RETRY_MAX_CHOICES = (0, 1, 2)
+
+UNKNOWN_FAILURE = "unknown_failure"
+SCRIPT_FAILURE = "script_failure"
+API_FAILURE = "api_failure"
+STUCK_OR_TIMEOUT_FAILURE = "stuck_or_timeout_failure"
+RUNNER_SYSTEM_FAILURE = "runner_system_failure"
+RUNNER_UNSUPPORTED = "runner_unsupported"
+STALE_SCHEDULE = "stale_schedule"
+JOB_EXECUTION_TIMEOUT = "job_execution_timeout"
+ARCHIVED_FAILURE = "archived_failure"
+UNMET_PREREQUISITES = "unmet_prerequisites"
+SCHEDULER_FAILURE = "scheduler_failure"
+DATA_INTEGRITY_FAILURE = "data_integrity_failure"
+
+RETRY_WHEN_CHOICES = (
+    ALWAYS,
+    UNKNOWN_FAILURE,
+    SCRIPT_FAILURE,
+    API_FAILURE,
+    STUCK_OR_TIMEOUT_FAILURE,
+    RUNNER_SYSTEM_FAILURE,
+    RUNNER_UNSUPPORTED,
+    STALE_SCHEDULE,
+    JOB_EXECUTION_TIMEOUT,
+    ARCHIVED_FAILURE,
+    UNMET_PREREQUISITES,
+    SCHEDULER_FAILURE,
+    DATA_INTEGRITY_FAILURE,
+)
+
+CI_FILE = ".gitlab-ci.yml"

cici/providers/gitlab/converter.py

@@ -0,0 +1,203 @@
+import functools
+from typing import Union
+
+import cattrs
+from cattrs.gen import make_dict_structure_fn, make_dict_unstructure_fn, override
+
+from ...utils import (
+    make_quoted_dict,
+    make_quoted_list_or_string,
+    make_quoted_string,
+    make_scalar_list,
+)
+from . import models
+
+CONVERTER = cattrs.Converter(omit_if_default=True, forbid_extra_keys=True)
+
+
+make_dict_struct = functools.partial(
+    make_dict_structure_fn,
+    _cattrs_forbid_extra_keys=True,
+    _cattrs_omit_if_default=True,
+)
+
+make_dict_unstruct = functools.partial(
+    make_dict_unstructure_fn,
+    _cattrs_forbid_extra_keys=True,
+    _cattrs_omit_if_default=True,
+)
+
+
+def quoted_string_struct_hook(object, __):
+    return make_quoted_string(object)
+
+
+def quoted_list_or_string_struct_hook(object, __):
+    return make_quoted_list_or_string(object)
+
+
+def quoted_dict_struct_hook(object, __):
+    return make_quoted_dict(object)
+
+
+def structure_image(object, __):
+    if object is None:
+        return None
+    elif isinstance(object, str):
+        return make_quoted_string(CONVERTER.structure(object, str))
+    return CONVERTER.structure(object, models.Image)
+
+
+def structure_include(object, __):
+    if isinstance(object, str):
+        return CONVERTER.structure(object, str)
+    items = []
+    for item in object:
+        if "local" in item:
+            items.append(CONVERTER.structure(item, models.IncludeLocal))
+        elif "project" in item:
+            items.append(CONVERTER.structure(item, models.IncludeProject))
+        elif "remote" in item:
+            items.append(CONVERTER.structure(item, models.IncludeRemote))
+        elif "template" in item:
+            items.append(CONVERTER.structure(item, models.IncludeTemplate))
+    return items
+
+
+def structure_script(object, __):
+    if isinstance(object, str):
+        return CONVERTER.structure(object, str)
+    items = []
+    for item in object:
+        if isinstance(item, str):
+            items.append(CONVERTER.structure(item, str))
+        else:
+            for entry in item:
+                items.append(CONVERTER.structure(entry, str))
+    return make_scalar_list(items)
+
+
+def structure_service(object, __):
+    if isinstance(object, str):
+        return make_quoted_string(CONVERTER.structure(object, str))
+    return CONVERTER.structure(object, models.Service)
+
+
+def structure_variables(object, __):
+    return make_quoted_dict(object)
+
+
+CONVERTER.register_structure_hook(
+    Union[
+        str,
+        list[
+            Union[
+                models.IncludeLocal,
+                models.IncludeProject,
+                models.IncludeRemote,
+                models.IncludeTemplate,
+            ]
+        ],
+    ],
+    structure_include,
+)
+
+
+CONVERTER.register_structure_hook(
+    Union[str, list[str]], quoted_list_or_string_struct_hook
+)
+
+CONVERTER.register_structure_hook(
+    Union[str, models.Service],
+    lambda object, _: CONVERTER.structure(
+        object, str if isinstance(object, str) else models.Service
+    ),
+)
+
+CONVERTER.register_structure_hook(
+    Union[int, models.Retry],
+    lambda object, _: CONVERTER.structure(
+        object, int if isinstance(object, int) else models.Retry
+    ),
+)
+
+
+CONVERTER.register_structure_hook(
+    Union[str, list[Union[str, list[str]]]],
+    structure_script,
+)
+
+CONVERTER.register_structure_hook(
+    models.Image,
+    make_dict_struct(
+        models.Image,
+        CONVERTER,
+        name=override(struct_hook=quoted_string_struct_hook),
+        entrypoint=override(struct_hook=quoted_list_or_string_struct_hook),
+    ),
+)
+
+CONVERTER.register_structure_hook(
+    models.Service,
+    make_dict_struct(
+        models.Service,
+        CONVERTER,
+        name=override(struct_hook=quoted_string_struct_hook),
+        command=override(struct_hook=quoted_list_or_string_struct_hook),
+        entrypoint=override(struct_hook=quoted_list_or_string_struct_hook),
+    ),
+)
+
+
+def unstructure_script(cls):
+    object = CONVERTER.unstructure(cls, str if isinstance(cls, str) else list[str])
+    if isinstance(object, str):
+        return object  # make_quoted_scalar_string(object)
+    return make_scalar_list(object)
+
+
+CONVERTER.register_unstructure_hook(
+    Union[str, list[str]],
+    unstructure_script,
+)
+
+CONVERTER.register_structure_hook(
+    Union[str, models.Image, None],
+    structure_image,
+)
+
+CONVERTER.register_structure_hook(
+    models.Rule,
+    make_dict_struct(
+        models.Rule,
+        CONVERTER,
+        if_=override(rename="if"),
+    ),
+)
+CONVERTER.register_unstructure_hook(
+    models.Rule,
+    make_dict_unstruct(
+        models.Rule,
+        CONVERTER,
+        if_=override(rename="if"),
+    ),
+)
+
+
+CONVERTER.register_structure_hook(
+    models.Job,
+    make_dict_struct(
+        models.Job,
+        CONVERTER,
+        variables=override(struct_hook=quoted_dict_struct_hook),
+    ),
+)
+
+CONVERTER.register_structure_hook(
+    models.File,
+    make_dict_struct(
+        models.File,
+        CONVERTER,
+        variables=override(struct_hook=quoted_dict_struct_hook),
+    ),
+)