cici-tools 0.2.0__tar.gz

cici-tools-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 BrettOps Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cici-tools-0.2.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include cici/schema/*.json
+include cici/schema/LICENSE.*

cici-tools-0.2.0/PKG-INFO

@@ -0,0 +1,63 @@
+Metadata-Version: 2.1
+Name: cici-tools
+Version: 0.2.0
+Summary: Power tools for CI/CD.
+Home-page: https://gitlab.com/brettops/tools/cici
+Author: Brett Weir
+Author-email: brett@brettops.io
+License: MIT
+Keywords: ci pipeline python
+Platform: UNKNOWN
+Classifier: Development Status :: 1 - Planning
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# cici-tools
+
+<!-- BADGIE TIME -->
+
+[![brettops tool](https://img.shields.io/badge/brettops-tool-209cdf?labelColor=162d50)](https://brettops.io)
+[![pipeline status](https://img.shields.io/gitlab/pipeline-status/brettops/tools/cici?branch=main)](https://gitlab.com/brettops/tools/cici/-/commits/main)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![code style: black](https://img.shields.io/badge/code_style-black-000000.svg)](https://github.com/psf/black)
+[![imports: isort](https://img.shields.io/badge/imports-isort-1674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
+
+<!-- END BADGIE TIME -->
+
+> **WARNING:** `cici` is experimental and I can't even decide on a name for it.
+> Stay away!
+
+## Usage
+
+### `bundle`
+
+Flatten `extends` keywords to make zero-dependency GitLab CI/CD files.
+
+### `fmt`
+
+Normalize the style of your GitLab CI/CD files:
+
+```bash
+cici fmt
+```
+
+### `update`
+
+Update to the latest GitLab CI/CD `include` versions available.
+
+```
+cici update
+```
+
+

cici-tools-0.2.0/README.md

@@ -0,0 +1,37 @@
+# cici-tools
+
+<!-- BADGIE TIME -->
+
+[![brettops tool](https://img.shields.io/badge/brettops-tool-209cdf?labelColor=162d50)](https://brettops.io)
+[![pipeline status](https://img.shields.io/gitlab/pipeline-status/brettops/tools/cici?branch=main)](https://gitlab.com/brettops/tools/cici/-/commits/main)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![code style: black](https://img.shields.io/badge/code_style-black-000000.svg)](https://github.com/psf/black)
+[![imports: isort](https://img.shields.io/badge/imports-isort-1674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
+
+<!-- END BADGIE TIME -->
+
+> **WARNING:** `cici` is experimental and I can't even decide on a name for it.
+> Stay away!
+
+## Usage
+
+### `bundle`
+
+Flatten `extends` keywords to make zero-dependency GitLab CI/CD files.
+
+### `fmt`
+
+Normalize the style of your GitLab CI/CD files:
+
+```bash
+cici fmt
+```
+
+### `update`
+
+Update to the latest GitLab CI/CD `include` versions available.
+
+```
+cici update
+```

cici-tools-0.2.0/cici/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

cici-tools-0.2.0/cici/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

cici-tools-0.2.0/cici/cli/build.py

@@ -0,0 +1,55 @@
+import logging
+import sys
+from importlib import import_module
+from pathlib import Path
+
+from ..constants import DEFAULT_PROVIDER, PROVIDERS
+
+
+def build_command(parser, args):
+    logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
+
+    input_provider = import_module(f".{args.input_format}", "cici.providers")
+    output_provider = import_module(f".{args.output_format}", "cici.providers")
+
+    if not args.filename:
+        args.filename = input_provider.CI_FILE
+
+    if not Path(args.filename).exists():
+        parser.error(f"file not found: {args.filename}")
+
+    file = input_provider.load(args.filename)
+    output_provider.dump(file, sys.stdout)
+
+
+def build_parser(subparsers):
+    parser = subparsers.add_parser(
+        "build", help="build BrettOps CI file into target format"
+    )
+    parser.add_argument("filename", nargs="?")
+    parser.add_argument(
+        "-f",
+        "--from",
+        dest="input_format",
+        choices=PROVIDERS,
+        default=DEFAULT_PROVIDER,
+        help=f"input format [{DEFAULT_PROVIDER}]",
+    )
+    parser.add_argument(
+        "-t",
+        "--to",
+        dest="output_format",
+        choices=PROVIDERS,
+        default=DEFAULT_PROVIDER,
+        help=f"output format [{DEFAULT_PROVIDER}]",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="DIR",
+        dest="output_path",
+        type=Path,
+        default=Path.cwd().absolute(),
+    )
+    parser.set_defaults(func=build_command)
+    return parser

cici-tools-0.2.0/cici/cli/bundle.py

@@ -0,0 +1,104 @@
+import copy
+import logging
+from importlib import import_module
+from pathlib import Path
+from typing import Optional
+
+from termcolor import colored
+
+from ..constants import CONFIG_DIR_NAME, DEFAULT_PROVIDER
+
+
+def get_job_prefix(pipeline_name: str, job_name: str) -> Optional[str]:
+    if job_name.startswith("."):
+        return None
+    return f"{pipeline_name}-"
+
+
+def parse_groups(name, file, groups):
+    if groups:
+        groups = [group.strip() for group in groups.split(",")]
+
+    if not groups:
+        group_names = set()
+        for job_name in file.jobs.keys():
+            if job_name.startswith("."):
+                continue
+            group_name = job_name[len(f"{name}-") :]
+            group_name = group_name.split("-")[0]
+            group_names.add(group_name)
+        groups = sorted(list(group_names))
+    return groups
+
+
+def bundle_command(parser, args):
+    logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
+    print(colored("pipeline name:", color="yellow"), args.pipeline_name)
+
+    provider = import_module(f".{DEFAULT_PROVIDER}", "cici.providers")
+
+    ci_file_path = args.config_path / provider.CI_FILE
+
+    if not Path(ci_file_path).exists():
+        parser.error(f"file not found: {ci_file_path}")
+
+    file = provider.load(ci_file_path)
+
+    args.output_path = Path(args.output_path)
+    args.groups = parse_groups(args.pipeline_name, file, args.groups)
+    print(colored("bundle names:", color="yellow"), args.groups)
+
+    for bundle_name in args.groups:
+        pattern = f"{args.pipeline_name}-{bundle_name}"
+
+        bundle = copy.deepcopy(file)
+        bundle.jobs = {}
+
+        for job_name, job in file.jobs.items():
+            job_prefix = get_job_prefix(
+                pipeline_name=args.pipeline_name,
+                job_name=job_name,
+            )
+            if not job_prefix:
+                continue
+            assert job_name.startswith(job_prefix)
+            if job_name.startswith(pattern):
+                bundle.jobs[job_name] = job
+
+        bundle_filename = args.output_path / f"{bundle_name}.yml"
+
+        with open(bundle_filename, "w") as stream:
+            provider.dump(bundle, stream)
+        print(colored("created", "magenta"), bundle_filename.name)
+
+
+def bundle_parser(subparsers):
+    parser = subparsers.add_parser(
+        "bundle", help="distill a CI file to a job and its dependencies"
+    )
+    parser.add_argument(
+        "config_path",
+        metavar="DIR",
+        nargs="?",
+        type=Path,
+        default=(Path.cwd() / CONFIG_DIR_NAME).absolute(),
+    )
+    parser.add_argument("-g", "--groups", help="job group patterns")
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="DIR",
+        dest="output_path",
+        type=Path,
+        default=Path.cwd().absolute(),
+    )
+    parser.add_argument(
+        "-n",
+        "--name",
+        metavar="DIR",
+        dest="pipeline_name",
+        default=str(Path.cwd().name),
+        help="the name of the pipeline",
+    )
+    parser.set_defaults(func=bundle_command)
+    return parser

cici-tools-0.2.0/cici/cli/fmt.py

@@ -0,0 +1,30 @@
+from importlib import import_module
+from pathlib import Path
+
+from termcolor import colored
+
+from ..constants import DEFAULT_PROVIDER
+
+
+def fmt_command(parser, args):
+    provider = import_module(f".{DEFAULT_PROVIDER}", "cici.providers")
+
+    if not args.filenames:
+        args.filenames = [provider.CI_FILE]
+
+    for filename in args.filenames:
+        if not Path(filename).exists():
+            parser.error(f"file not found: {filename}")
+
+    for filename in args.filenames:
+        file = provider.load(filename)
+        with open(filename, "w") as stream:
+            provider.dump(file, stream)
+        print(colored(filename, "magenta"), "formatted")
+
+
+def fmt_parser(subparsers):
+    parser = subparsers.add_parser("fmt", help="Reformat CI file")
+    parser.add_argument("filenames", metavar="FILE", nargs="*")
+    parser.set_defaults(func=fmt_command)
+    return parser

cici-tools-0.2.0/cici/cli/update.py

@@ -0,0 +1,112 @@
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import ruamel.yaml
+from termcolor import colored
+
+GITLAB_URL = "https://gitlab.com"
+API_V4_URL = f"{GITLAB_URL}/api/v4"
+
+HEADERS: dict[str, str] = {}
+private_token = os.environ.get("GITLAB_PRIVATE_TOKEN")
+if private_token:
+    HEADERS.update({"PRIVATE-TOKEN": private_token})
+
+
+def to_fragment(text):
+    if not isinstance(text, str):
+        return text
+    return urllib.parse.quote(text, safe="")
+
+
+def get_apiv4_url(url):
+    try:
+        url = f"{API_V4_URL}{url}"
+        request = urllib.request.Request(url=url, headers=HEADERS)
+        response = urllib.request.urlopen(request)
+        content = response.read()
+        return json.loads(content)
+    except urllib.error.HTTPError:
+        return None
+
+
+def get_project(project_id):
+    return get_apiv4_url(f"/projects/{to_fragment(project_id)}")
+
+
+def get_latest_release(project_id):
+    return get_apiv4_url(
+        f"/projects/{to_fragment(project_id)}/releases/permalink/latest"
+    )
+
+
+def cproject(project_name):
+    return colored(project_name, "magenta")
+
+
+def ctag(tag_name):
+    return colored(tag_name, "yellow", attrs=["bold"])
+
+
+def update_include(include):
+    if any(key not in include for key in ("project", "file")):
+        return include
+
+    project = get_project(include["project"])
+    project_name = project["path_with_namespace"]
+
+    latest_release = get_latest_release(project["id"])
+    if latest_release:
+        latest_tag = latest_release["tag_name"]
+        current_tag = include.get("ref", None)
+        if current_tag:
+            if current_tag != latest_tag:
+                print(
+                    colored(project_name, "red", attrs=["bold"]),
+                    "updated from {current} to {latest}".format(
+                        current=ctag(current_tag),
+                        latest=ctag(latest_tag),
+                    ),
+                )
+            else:
+                print(cproject(project_name), "is the latest at", ctag(current_tag))
+        elif not current_tag:
+            print(cproject(project_name), "pinned to", ctag(latest_tag))
+        include["ref"] = latest_tag
+    else:
+        print(cproject(project_name), "has no releases")
+    newinclude = {}
+    newinclude["project"] = include["project"]
+    if "ref" in include:
+        newinclude["ref"] = include["ref"]
+    newinclude["file"] = include["file"]
+    return newinclude
+
+
+def update_includes(includes):
+    if not isinstance(includes, list):
+        return includes
+
+    return [update_include(include) for include in includes]
+
+
+def update_command(parser, args):
+    yaml = ruamel.yaml.YAML()
+    data = yaml.load(open(args.filename))
+
+    if "include" in data:
+        data["include"] = update_includes(data["include"])
+
+    with open(args.filename, "w") as handle:
+        yaml.indent(mapping=2, sequence=4, offset=2)
+        yaml.dump(data, handle)
+
+
+def update_parser(subparsers):
+    parser = subparsers.add_parser("update")
+    parser.add_argument("filename", nargs="?", default=".gitlab-ci.yml")
+    parser.set_defaults(func=update_command)
+    return parser

cici-tools-0.2.0/cici/constants.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+BASE_DIR = Path(__file__).parent.absolute()
+
+SCHEMA_DIR = BASE_DIR / "schema"
+
+COMMAND_DIR = BASE_DIR / "cli"
+
+COMMANDS = sorted(
+    [path.stem for path in COMMAND_DIR.glob("*.py") if not path.stem.startswith("_")]
+)
+
+PROVIDER_DIR = BASE_DIR / "providers"
+
+PROVIDERS = sorted(
+    [path.stem for path in PROVIDER_DIR.glob("*.py") if not path.stem.startswith("_")]
+    + [
+        path.stem
+        for path in PROVIDER_DIR.glob("*/")
+        if path.is_dir() and not path.stem.startswith("_")
+    ]
+)
+
+DEFAULT_PROVIDER = PROVIDERS[0]
+
+CONFIG_DIR_NAME = ".cici"

cici-tools-0.2.0/cici/main.py

@@ -0,0 +1,17 @@
+import argparse
+from importlib import import_module
+
+from .constants import COMMANDS
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers(required=True)
+
+    for command_name in COMMANDS:
+        command = import_module(f".{command_name}", "cici.cli")
+        getattr(command, f"{command_name}_parser")(subparsers=subparsers)
+
+    args = parser.parse_args()
+    args.func(parser=parser, args=args)

cici-tools-0.2.0/cici/providers/gitlab/__init__.py

@@ -0,0 +1,3 @@
+from .constants import CI_FILE
+from .models import File
+from .serializers import dump, dumps, load, loads

cici-tools-0.2.0/cici/providers/gitlab/constants.py

@@ -0,0 +1,77 @@
+ACCESS = "access"
+ALWAYS = "always"
+DEVELOPMENT = "development"
+IF_NOT_PRESENT = "if-not-present"
+NEVER = "never"
+ON_SUCCESS = "on_success"
+ON_FAILURE = "on_failure"
+OTHER = "other"
+PREPARE = "prepare"
+PRODUCTION = "production"
+PULL = "pull"
+PULL_PUSH = "pull-push"
+PUSH = "push"
+STAGING = "staging"
+START = "start"
+STOP = "stop"
+TESTING = "testing"
+VERIFY = "verify"
+
+CACHE_POLICIES = (PULL, PUSH, PULL_PUSH)
+
+DEPLOYMENT_TIERS = (
+    PRODUCTION,
+    STAGING,
+    TESTING,
+    DEVELOPMENT,
+    OTHER,
+)
+
+ENVIRONMENT_ACTIONS = (
+    START,
+    PREPARE,
+    STOP,
+    VERIFY,
+    ACCESS,
+)
+
+PULL_POLICIES = (
+    ALWAYS,
+    IF_NOT_PRESENT,
+    NEVER,
+)
+
+WHEN_CHOICES = (ON_SUCCESS, ON_FAILURE, ALWAYS)
+
+RETRY_MAX_CHOICES = (0, 1, 2)
+
+UNKNOWN_FAILURE = "unknown_failure"
+SCRIPT_FAILURE = "script_failure"
+API_FAILURE = "api_failure"
+STUCK_OR_TIMEOUT_FAILURE = "stuck_or_timeout_failure"
+RUNNER_SYSTEM_FAILURE = "runner_system_failure"
+RUNNER_UNSUPPORTED = "runner_unsupported"
+STALE_SCHEDULE = "stale_schedule"
+JOB_EXECUTION_TIMEOUT = "job_execution_timeout"
+ARCHIVED_FAILURE = "archived_failure"
+UNMET_PREREQUISITES = "unmet_prerequisites"
+SCHEDULER_FAILURE = "scheduler_failure"
+DATA_INTEGRITY_FAILURE = "data_integrity_failure"
+
+RETRY_WHEN_CHOICES = (
+    ALWAYS,
+    UNKNOWN_FAILURE,
+    SCRIPT_FAILURE,
+    API_FAILURE,
+    STUCK_OR_TIMEOUT_FAILURE,
+    RUNNER_SYSTEM_FAILURE,
+    RUNNER_UNSUPPORTED,
+    STALE_SCHEDULE,
+    JOB_EXECUTION_TIMEOUT,
+    ARCHIVED_FAILURE,
+    UNMET_PREREQUISITES,
+    SCHEDULER_FAILURE,
+    DATA_INTEGRITY_FAILURE,
+)
+
+CI_FILE = ".gitlab-ci.yml"