ichec-cicd 0.1.12__tar.gz

ichec_cicd-0.1.12/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Irish Centre for High End Computing
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ichec_cicd-0.1.12/PKG-INFO

@@ -0,0 +1,117 @@
+Metadata-Version: 2.4
+Name: ichec_cicd
+Version: 0.1.12
+Summary: A collection of utilities to manage ICHEC project CI/CD pipelines.
+Author: Irish Centre for High End Computing
+License: MIT
+Project-URL: Repository, https://git.ichec.ie/platform-engineering/modules/cicd/ichec-cicd
+Project-URL: Homepage, https://git.ichec.ie/platform-engineering/modules/cicd/ichec-cicd
+Keywords: CICD,HPC,Gitlab
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: build
+Requires-Dist: twine
+Requires-Dist: ichec_platform_core>=0.0.4
+Requires-Dist: tomlkit
+Requires-Dist: Jinja2
+Dynamic: license-file
+
+# ichec_cicd
+
+This project is a collection of utilities for managing CI/CD pipelines at ICHEC.
+
+It provides opinionated interfaces to encourage standarization of our project structures and workflows.
+
+# Install
+
+The package is available from PyPI:
+
+```sh
+pip install ichec_cicd
+```
+
+# Features #
+
+## Deploy a Package to a Repository
+
+From the package's top-level directory:
+
+```sh
+ichec_cicd deploy --token $REPO_TOKEN
+```
+
+As an example, for a Python project this might be the PyPI repository's token.
+
+## Set a Package's Version Number
+
+From the package's top-level directory:
+
+```sh
+ichec_cicd set_version $VERSION
+```
+
+## Increment a Repository's Tag ##
+
+From the repository's top-level directory, and on the branch the tag will be dervied from:
+
+``` sh
+ichec_cicd increment_tag --field patch
+```
+
+Here `semver` tag versioning is assumed with a `major.minor.patch` scheme. Note: in a CI/CD pipeline some more input options are needed to initialize the git repo for pushing the tag to. You can use the `--help` flag for more details.
+
+## Sync Content With an External Upstream ##
+
+Here we pull content from an external upstream repository and use it to update the content in a local repo. This can be useful for public mirrors that include a subset of the content in an internal repo.
+
+``` sh
+ichec_cicd sync_external_archive \
+    --source_token $SOURCE_TOKEN \ 
+    --project_id $SOURCE_PROJ_ID \
+    --sync_script $MY_SYNC_SCRIPT \
+    --asset_name $ASSET_NAME \
+    --archive_name $ARCHIVE_NAME \
+    --target_token $TARGET_TOKEN
+```
+
+The inputs are:
+
+* `SOURCE_TOKEN`: Gitlab access token for the source (private) repo - must have asset download rights
+* `SOURCE_PROJ_ID`: The numeric project ID for the source repo
+* `MY_SYNC_SCRIPT`: Path to a script that will use the extracted contents of the source archive to update the target repo
+* `ASSET_NAME`: Name of the asset corresponding to the archive in the source repo's latest release collection
+* `ARCHIVE_NAME`: Name of the downloaded file corresponding to the archive - this can differ from the asset name.
+* `TARGET_TOKEN`: OATH token for the target repo - so that it can be pushed to from a CI runner
+
+## Generate a containerfile ##
+
+Here we generate a containerfile from a template given a release manifest in a repo:
+
+``` sh
+ichec_cicd containerfile generate --manifest <path_to_manifest> --template_dir <container_templates> 
+```
+
+where `<path_to_manifest>` is the path to an ICHEC release manifest and `<container_template>` is a directory with container templates.
+
+
+## Containerize a Web App ##
+
+Here we set up a standardized container build context for web applications, i.e. Django or FastAPI apps.
+
+``` sh
+ichec_cicd wrap --project <project_to_wrap> --template <wrapping_template> 
+```
+
+where `<project_to_wrap>` is a url or filesystem path to a repository with the project to be wrapped and `<wrapping_template>` is a url or path to a repository with a template to do the wrapping. The project to be wrapped needs to have an `ichec_manifest.yml` file at its top level with package metadata and the `<wrapping_template>` should match the framework being used, e.g. (Django/FastAPI).
+
+# Licensing #
+
+This project is licensed under the MIT License. See the accompanying `LICENSE` file for details.
+

ichec_cicd-0.1.12/README.md

@@ -0,0 +1,92 @@
+# ichec_cicd
+
+This project is a collection of utilities for managing CI/CD pipelines at ICHEC.
+
+It provides opinionated interfaces to encourage standarization of our project structures and workflows.
+
+# Install
+
+The package is available from PyPI:
+
+```sh
+pip install ichec_cicd
+```
+
+# Features #
+
+## Deploy a Package to a Repository
+
+From the package's top-level directory:
+
+```sh
+ichec_cicd deploy --token $REPO_TOKEN
+```
+
+As an example, for a Python project this might be the PyPI repository's token.
+
+## Set a Package's Version Number
+
+From the package's top-level directory:
+
+```sh
+ichec_cicd set_version $VERSION
+```
+
+## Increment a Repository's Tag ##
+
+From the repository's top-level directory, and on the branch the tag will be dervied from:
+
+``` sh
+ichec_cicd increment_tag --field patch
+```
+
+Here `semver` tag versioning is assumed with a `major.minor.patch` scheme. Note: in a CI/CD pipeline some more input options are needed to initialize the git repo for pushing the tag to. You can use the `--help` flag for more details.
+
+## Sync Content With an External Upstream ##
+
+Here we pull content from an external upstream repository and use it to update the content in a local repo. This can be useful for public mirrors that include a subset of the content in an internal repo.
+
+``` sh
+ichec_cicd sync_external_archive \
+    --source_token $SOURCE_TOKEN \ 
+    --project_id $SOURCE_PROJ_ID \
+    --sync_script $MY_SYNC_SCRIPT \
+    --asset_name $ASSET_NAME \
+    --archive_name $ARCHIVE_NAME \
+    --target_token $TARGET_TOKEN
+```
+
+The inputs are:
+
+* `SOURCE_TOKEN`: Gitlab access token for the source (private) repo - must have asset download rights
+* `SOURCE_PROJ_ID`: The numeric project ID for the source repo
+* `MY_SYNC_SCRIPT`: Path to a script that will use the extracted contents of the source archive to update the target repo
+* `ASSET_NAME`: Name of the asset corresponding to the archive in the source repo's latest release collection
+* `ARCHIVE_NAME`: Name of the downloaded file corresponding to the archive - this can differ from the asset name.
+* `TARGET_TOKEN`: OATH token for the target repo - so that it can be pushed to from a CI runner
+
+## Generate a containerfile ##
+
+Here we generate a containerfile from a template given a release manifest in a repo:
+
+``` sh
+ichec_cicd containerfile generate --manifest <path_to_manifest> --template_dir <container_templates> 
+```
+
+where `<path_to_manifest>` is the path to an ICHEC release manifest and `<container_template>` is a directory with container templates.
+
+
+## Containerize a Web App ##
+
+Here we set up a standardized container build context for web applications, i.e. Django or FastAPI apps.
+
+``` sh
+ichec_cicd wrap --project <project_to_wrap> --template <wrapping_template> 
+```
+
+where `<project_to_wrap>` is a url or filesystem path to a repository with the project to be wrapped and `<wrapping_template>` is a url or path to a repository with a template to do the wrapping. The project to be wrapped needs to have an `ichec_manifest.yml` file at its top level with package metadata and the `<wrapping_template>` should match the framework being used, e.g. (Django/FastAPI).
+
+# Licensing #
+
+This project is licensed under the MIT License. See the accompanying `LICENSE` file for details.
+

ichec_cicd-0.1.12/pyproject.toml

@@ -0,0 +1,84 @@
+[project]
+name = "ichec_cicd"
+dynamic = ["version"]
+license = { text = "MIT" }
+authors = [
+  { name="Irish Centre for High End Computing"},
+]
+description = "A collection of utilities to manage ICHEC project CI/CD pipelines."
+readme = "README.md"
+requires-python = ">=3.14"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.14",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Topic :: System :: Distributed Computing"
+]
+keywords = ["CICD", "HPC", "Gitlab"]
+dependencies = [
+    "build",
+    "twine",
+    "ichec_platform_core>=0.0.4",
+    "tomlkit",
+    "Jinja2",
+]
+
+[project.urls]
+Repository = "https://git.ichec.ie/platform-engineering/modules/cicd/ichec-cicd"
+Homepage = "https://git.ichec.ie/platform-engineering/modules/cicd/ichec-cicd"
+
+[project.scripts]
+ichec_cicd = "ichec_cicd.main_cli:main_cli"
+
+[build-system]
+requires = ["setuptools>=64", "setuptools-scm>=8"]
+build-backend = "setuptools.build_meta"
+
+# Version derived from the git tag (setuptools-scm). fallback_version keeps
+# non-release jobs building on the shallow CI checkout, which has no tags.
+[tool.setuptools_scm]
+fallback_version = "0.0.0"
+
+# Development toolchain (uv-native, PEP 735). CI installs it with
+# `uv sync --group dev`. Not shipped to consumers — runtime deps live in
+# [project.dependencies].
+[dependency-groups]
+dev = [
+    "ruff",
+    "mypy",
+    "pytest",
+    "pytest-cov",
+    "pytest-sugar",
+    "pytest-mock",
+]
+
+[tool.setuptools.package-data]
+"ichec_cicd" = ["py.typed"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+# --- ruff: format + lint (supersedes black, isort, flake8, pylint) ---
+[tool.ruff]
+line-length = 88
+target-version = "py314"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+# E/W pycodestyle, F pyflakes, I import-sort (was isort), N pep8-naming,
+# UP pyupgrade, B bugbear. Extend deliberately, not by pulling in "ALL".
+select = ["E", "W", "F", "I", "N", "UP", "B"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["ichec_cicd"]
+
+[tool.mypy]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests",]
+log_cli = 1
+log_cli_level = "debug"
+addopts = "--cov=ichec_cicd --cov-report term --cov-fail-under=18 --cov-report xml:coverage.xml --cov-report html"

ichec_cicd-0.1.12/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ichec_cicd-0.1.12/src/ichec_cicd/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.0"

ichec_cicd-0.1.12/src/ichec_cicd/ci/__init__.py

@@ -0,0 +1 @@
+from .ci import *  # NOQA

ichec_cicd-0.1.12/src/ichec_cicd/ci/ci.py

@@ -0,0 +1,91 @@
+import logging
+import os
+import shutil
+from pathlib import Path
+
+from ichec_platform_core import filesystem as fs
+from ichec_platform_core.system import process
+from ichec_platform_core.version_control import (
+    GitlabClient,
+    GitlabInstance,
+    GitlabToken,
+    GitRepo,
+    GitUser,
+)
+from ichec_platform_core.version_control.gitlab_client import get_latest_release
+
+logger = logging.getLogger(__name__)
+
+
+def gitlab_ci_push(
+    user_name: str,
+    user_email: str,
+    repo_url: str,
+    token: str,
+    message: str,
+    repo_dir: Path | None = None,
+):
+    if repo_dir is None:
+        repo_dir = Path.cwd()
+
+    user = GitUser(name=user_name, email=user_email)
+    repo = GitRepo(user=user, path=repo_dir)
+
+    gitlab = GitlabClient(
+        instance=GitlabInstance(url=repo_url),
+        token=GitlabToken(value=token),
+        user=user,
+        local_repo=repo,
+    )
+
+    remote_name = "origin"
+    if token:
+        remote_name = "oath_origin"
+    gitlab.push_change(message, remote_name=remote_name)
+
+
+def sync_external_archive(
+    repo_dir: Path,
+    project_id: int,
+    asset_name: str,
+    archive_name: str,
+    sync_script: Path,
+    source_token: str = "",
+    user_name: str = "",
+    user_email: str = "",
+    url: str = "",
+    target_token: str = "",
+    target_repo_url: str = "",
+):
+
+    # Download package
+    logger.info("Fetching remote asset from %s for project %s", url, project_id)
+    work_dir = repo_dir / "_temp"
+    os.makedirs(work_dir)
+    get_latest_release(
+        project_id, url, source_token, work_dir, asset_name, "PRIVATE-TOKEN"
+    )
+
+    logger.info("Extracting archive from %s", work_dir / asset_name)
+    fs.unpack_archive(work_dir / asset_name, work_dir)
+    (work_dir / asset_name).unlink()
+
+    # Run sync script
+    logger.info("Running sync with %s", repo_dir / sync_script)
+    process.run(f"{repo_dir / sync_script} {work_dir}")
+
+    # Clean work working directory
+    shutil.rmtree(work_dir)
+
+    # Commit and push change
+    logger.info("Pushing change")
+    gitlab_ci_push(
+        user_name,
+        user_email,
+        target_repo_url,
+        target_token,
+        "External_repo_sync",
+        repo_dir,
+    )
+
+    logger.info("Finished external package sync")

ichec_cicd-0.1.12/src/ichec_cicd/ci/cli.py

@@ -0,0 +1,110 @@
+import logging
+import os
+from pathlib import Path
+
+from .ci import gitlab_ci_push, sync_external_archive
+
+logger = logging.getLogger(__name__)
+
+
+def gitlab_ci_push_cli(args):
+
+    logger.info("CI pushing state of current checkout")
+
+    gitlab_ci_push(
+        args.user_name, args.user_email, args.repo_url, args.token, args.message
+    )
+
+    logger.info("CI finished pushing state of current checkout")
+
+
+def register_ci_push(subparsers):
+
+    parser = subparsers.add_parser("ci_push")
+    parser.add_argument("--user_name", type=str, help="Name of the CI user")
+    parser.add_argument("--user_email", type=str, help="Email of the CI user")
+    parser.add_argument(
+        "--instance_url", type=str, help="Url for the target ci instance"
+    )
+    parser.add_argument(
+        "--url", type=str, help="Url for the repo relative to the ci instance"
+    )
+    parser.add_argument("--token", type=str, help="Oath access token for the repo")
+    parser.add_argument("--message", type=str, help="Commit message")
+    parser.set_defaults(func=gitlab_ci_push_cli)
+
+
+def sync_external_archive_cli(args):
+
+    logger.info("Starting external package sync")
+
+    sync_external_archive(
+        repo_dir=args.repo_dir,
+        project_id=args.project_id,
+        sync_script=args.sync_script,
+        asset_name=args.asset_name,
+        archive_name=args.archive_name,
+        source_token=args.source_token,
+        user_name=args.user_name,
+        user_email=args.user_email,
+        url=args.url,
+        target_token=args.target_token,
+        target_repo_url=args.target_repo_url,
+    )
+
+    logger.info("Finished external package sync")
+
+
+def register_sync_exterinal(subparsers):
+
+    parser = subparsers.add_parser("sync_external_archive")
+    parser.add_argument(
+        "--repo_dir",
+        type=Path,
+        default=Path(os.getcwd()),
+        help="Path to the repo to increment the tag",
+    )
+
+    parser.add_argument(
+        "--project_id",
+        type=int,
+        help="The id of the project to sync against.",
+    )
+    parser.add_argument(
+        "--source_token", type=str, default="", help="Access token for the source repo"
+    )
+
+    parser.add_argument("--user_name", type=str, default="", help="Name of the CI user")
+    parser.add_argument(
+        "--user_email", type=str, default="", help="Email of the CI user"
+    )
+
+    parser.add_argument(
+        "--url",
+        type=str,
+        default="https://git.ichec.ie",
+        help="Url for the Gitlab instance",
+    )
+    parser.add_argument(
+        "--target_token", type=str, default="", help="Oath access token for the repo"
+    )
+
+    parser.add_argument(
+        "--asset_name", type=str, help="Name of the release asset for the archive"
+    )
+    parser.add_argument(
+        "--archive_name", type=str, help="Name of the downloaded archive"
+    )
+    parser.add_argument(
+        "--sync_script", type=Path, help="Path to script to do the sync"
+    )
+    parser.add_argument(
+        "--target_repo_url", type=str, default="", help="Url for the target repo"
+    )
+    parser.set_defaults(func=sync_external_archive_cli)
+
+
+def register(subparser):
+
+    register_ci_push(subparser)
+    register_sync_exterinal(subparser)

ichec_cicd-0.1.12/src/ichec_cicd/container/cli.py

@@ -0,0 +1,61 @@
+"""
+CLI handlers for the `container` module
+"""
+
+import logging
+import os
+from pathlib import Path
+
+from ichec_platform_core.serialization import read_yaml
+
+from ..manifest import ReleaseManifest
+from .container import generate_containerfile
+
+logger = logging.getLogger(__name__)
+
+
+def containerfile_generate_cli(args):
+
+    logger.info("Generating containerfile")
+
+    manifest = ReleaseManifest(**read_yaml(args.manifest))
+
+    generate_containerfile(
+        manifest, args.template_dir.resolve(), args.output_dir.resolve()
+    )
+
+    logger.info("Finished generating containerfile")
+
+
+def register_containerfile_parser(subparsers):
+
+    parser = subparsers.add_parser("containerfile")
+    containerfile_subparsers = parser.add_subparsers(required=True)
+
+    generate_parser = containerfile_subparsers.add_parser("generate")
+
+    generate_parser.add_argument(
+        "--manifest",
+        type=Path,
+        help="Path to the repo's deployment manifest",
+    )
+
+    generate_parser.add_argument(
+        "--template_dir",
+        type=Path,
+        help="Directory with container templates",
+    )
+
+    generate_parser.add_argument(
+        "--output_dir",
+        type=Path,
+        default=Path(os.getcwd()),
+        help="Directory to output generated files",
+    )
+
+    generate_parser.set_defaults(func=containerfile_generate_cli)
+
+
+def register(subparsers):
+
+    register_containerfile_parser(subparsers)

ichec_cicd-0.1.12/src/ichec_cicd/container/container.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+from ..manifest import ReleaseManifest
+from ..packaging import generate_django_containerfile, generate_fastapi_containerfile
+
+_GENERATORS = {
+    "django": generate_django_containerfile,
+    "fastapi": generate_fastapi_containerfile,
+}
+
+
+def generate_containerfile(
+    manifest: ReleaseManifest,
+    template_dir: Path,
+    output_dir: Path,
+    generators: dict | None = None,
+):
+
+    if not generators:
+        generators = _GENERATORS
+
+    if manifest.project.type not in generators:
+        raise RuntimeError(
+            f"Generator not found for supplied project type: {manifest.project.type}"
+        )
+
+    generators[manifest.project.type](template_dir, manifest, output_dir)

ichec_cicd-0.1.12/src/ichec_cicd/main_cli.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+"""
+This is the entrypoint for the `iccicd` application.
+
+CLI arguments and handlers for individual modules are
+registered here and defined in the respective `cli` submodules.
+
+`launch_common()` is common code run before all handlers.
+"""
+
+import argparse
+import logging
+
+from ichec_platform_core import logging_utils, runtime
+
+import ichec_cicd.ci.cli
+import ichec_cicd.container.cli
+import ichec_cicd.packaging.cli
+import ichec_cicd.repo.cli
+
+logger = logging.getLogger(__name__)
+
+
+def launch_common(args) -> None:
+    """
+    Run this code before all CLI handlers. It sets up a global
+    runtime context to store runtime configuration, such as the
+    'dry run' flag and sets up logging.
+    """
+    runtime.ctx.set_is_dry_run(args.dry_run)
+    logging_utils.setup_default_logger()
+
+
+def main_cli():
+    """
+    This method is associated with the `ichec_cicd` application in the
+    project's pyproject.toml, i.e. it will be run when `ichec_cicd` is
+    called in a terminal.
+    """
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dry_run",
+        type=int,
+        default=0,
+        help="Dry run script - 0 can modify, 1 can read, 2 no modify - no read",
+    )
+    subparsers = parser.add_subparsers(required=True)
+
+    ichec_cicd.packaging.cli.register(subparsers)
+    ichec_cicd.repo.cli.register(subparsers)
+    ichec_cicd.ci.cli.register(subparsers)
+    ichec_cicd.container.cli.register(subparsers)
+
+    args = parser.parse_args()
+    launch_common(args)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main_cli()

ichec_cicd-0.1.12/src/ichec_cicd/manifest/__init__.py

@@ -0,0 +1 @@
+from .manifest import *  # NOQA