c2cciutils 1.8.0.dev63__py3-none-any.whl → 1.8.0.dev68__py3-none-any.whl

c2cciutils/publish.py

@@ -1,451 +0,0 @@
-"""The publishing functions."""
-
-import argparse
-import datetime
-import glob
-import os
-import pickle  # nosec
-import re
-import subprocess  # nosec
-import sys
-import uuid
-from typing import Optional
-
-import ruamel.yaml
-import tomllib
-from google.auth.transport.requests import Request
-from google.oauth2.credentials import Credentials
-from google_auth_oauthlib.flow import InstalledAppFlow
-from googleapiclient.discovery import build
-
-import c2cciutils.configuration
-
-
-class GoogleCalendar:
-    """Interact with the Google Calendar API."""
-
-    # pylint: disable=too-many-instance-attributes
-    def __init__(self) -> None:
-        """Initialize."""
-        self.scopes = ["https://www.googleapis.com/auth/calendar"]  # in fact it is better to hard-code this
-        self.credentials_pickle_file = os.environ.get("TMP_CREDS_FILE", f"/tmp/{uuid.uuid4()}.pickle")  # noqa: S108 # nosec
-        self.credentials_json_file = os.environ.get(
-            "GOOGLE_CREDS_JSON_FILE", "~/google-credentials-c2cibot.json"
-        )  # used to refresh the refresh_token or to initialize the credentials the first time
-        self.calendar_id = os.environ.get(
-            "GOOGLE_CALENDAR_ID", c2cciutils.gopass("gs/ci/google_calendar/calendarId")
-        )
-        self.token = os.environ.get("GOOGLE_TOKEN", c2cciutils.gopass("gs/ci/google_calendar/token"))
-        self.token_uri = os.environ.get(
-            "GOOGLE_TOKEN_URI", c2cciutils.gopass("gs/ci/google_calendar/token_uri")
-        )
-        self.refresh_token = os.environ.get(
-            "GOOGLE_REFRESH_TOKEN",
-            c2cciutils.gopass("gs/ci/google_calendar/refresh_token"),
-        )
-        self.client_id = os.environ.get(
-            "GOOGLE_CLIENT_ID", c2cciutils.gopass("gs/ci/google_calendar/client_id")
-        )
-        self.client_secret = os.environ.get(
-            "GOOGLE_CLIENT_SECRET",
-            c2cciutils.gopass("gs/ci/google_calendar/client_secret"),
-        )
-
-        self.creds: Credentials = self.init_calendar_service()
-        self._update_creds()
-        self.service = build("calendar", "v3", credentials=self.creds)
-
-    def init_calendar_service(self) -> Credentials:  # type: ignore
-        """Initialize the calendar service."""
-        # The file token pickle stores the user's access and refresh tokens, and is
-        # created automatically when the authorization flow completes for the first
-        # time.
-        if os.path.exists(self.credentials_pickle_file):
-            with open(self.credentials_pickle_file, "rb") as token:
-                creds = pickle.load(token)  # noqa: S301 # nosec
-        # If there are no (valid) credentials available, let the user log in.
-        if not creds or not creds.valid:  # pylint: disable=possibly-used-before-assignment
-            if creds and creds.expired and creds.refresh_token:
-                creds.refresh(Request())  # type: ignore
-            else:
-                if self.token:
-                    creds = Credentials(  # type: ignore
-                        self.token,
-                        refresh_token=self.refresh_token,
-                        token_uri=self.token_uri,
-                        client_id=self.client_id,
-                        client_secret=self.client_secret,
-                        scopes=self.scopes,
-                    )
-                else:
-                    flow = InstalledAppFlow.from_client_secrets_file(self.credentials_json_file, self.scopes)
-                    creds = flow.run_local_server(port=0)
-                    self.refresh_token = creds
-
-            # Save the credentials for the next run
-            with open(self.credentials_pickle_file, "wb") as token:
-                pickle.dump(creds, token)
-
-    def _update_creds(self) -> None:
-        """Update the credentials."""
-        self.client_id = self.creds.client_id
-        self.client_secret = self.creds.client_secret
-        self.token = self.creds.token
-        self.token_uri = self.creds.token_uri
-        self.refresh_token = self.creds.refresh_token
-
-    def print_all_calendars(self) -> None:
-        """Print all calendar events."""
-        # list all the calendars that the user has access to.
-        # used to debug credentials
-        print("Getting list of calendars")
-        calendars_result = self.service.calendarList().list().execute()  # pylint: disable=no-member
-
-        calendars = calendars_result.get("items", [])
-
-        if not calendars:
-            print("::error::No calendars found.")
-        for calendar in calendars:
-            summary = calendar["summary"]
-            event_id = calendar["id"]
-            primary = "Primary" if calendar.get("primary") else ""
-            print(f"{summary}\t{event_id}\t{primary}")
-
-    def print_latest_events(self, time_min: Optional[datetime.datetime] = None) -> None:
-        """
-        Print latest events.
-
-        Arguments:
-            time_min: The time to be considered.
-
-        """
-        now = datetime.datetime.utcnow()
-        if not time_min:
-            time_min = datetime.datetime.utcnow() - datetime.timedelta(days=30)
-        events_result = (
-            self.service.events()  # pylint: disable=no-member
-            .list(
-                calendarId=self.calendar_id,
-                timeMin=time_min.isoformat() + "Z",
-                timeMax=now.isoformat() + "Z",
-                singleEvents=True,
-                orderBy="startTime",
-            )
-            .execute()
-        )
-        events = events_result.get("items", [])
-
-        if not events:
-            print("::error::No upcoming events found.")
-        for event in events:
-            start = event["start"].get("dateTime", event["start"].get("date"))
-            print(start, event["summary"])
-
-    def create_event(
-        self,
-        summary: str = f"dummy/image:{datetime.datetime.now().isoformat()}",
-        description: str = "description",
-    ) -> None:
-        """
-        Create a calendar event.
-
-        Arguments:
-            summary: The event summary
-            description: The event description
-
-        """
-        now = datetime.datetime.now()
-        start = now.isoformat()
-        end = (now + datetime.timedelta(minutes=15)).isoformat()
-        body = {
-            "summary": summary,
-            "description": description,
-            "start": {"dateTime": start, "timeZone": "Europe/Zurich"},
-            "end": {"dateTime": end, "timeZone": "Europe/Zurich"},
-        }
-
-        event_result = self.service.events().insert(calendarId=self.calendar_id, body=body).execute()  # pylint: disable=no-member
-        print(f"Created event with id: {event_result['id']}")
-
-    def save_credentials_to_gopass(self) -> None:
-        """Save the calendar credentials to gopass."""
-        objects_to_save = {
-            "gs/ci/google_calendar/calendarId": self.calendar_id,
-            "gs/ci/google_calendar/token": self.token,
-            "gs/ci/google_calendar/token_uri": self.token_uri,
-            "gs/ci/google_calendar/refresh_token": self.refresh_token,
-            "gs/ci/google_calendar/client_id": self.client_id,
-            "gs/ci/google_calendar/client_secret": self.client_secret,
-        }
-        for key, secret in objects_to_save.items():
-            assert secret is not None
-            c2cciutils.gopass_put(secret, key)
-
-    def __del__(self) -> None:
-        """Delete the credentials file."""
-        if os.path.exists(self.credentials_pickle_file):
-            os.remove(self.credentials_pickle_file)
-
-
-def main_calendar() -> None:
-    """Run the calendar main function."""
-    parser = argparse.ArgumentParser(
-        description="Interact with google API for the Docker publishing calendar"
-    )
-    parser.add_argument(
-        "--refresh-gopass-credentials",
-        action="store_true",
-        help="Refresh the credentials in gopass using google API",
-    )
-    parser.add_argument(
-        "--show-events-since",
-        help="show the calendar events since a date in 'YYYY-mm-dd' format",
-        type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),
-    )
-    parser.add_argument(
-        "--create-test-event",
-        action="store_true",
-        help="Create a dummy event to check that the calendar settings are correct",
-    )
-    args = parser.parse_args()
-
-    if args.show_events_since or args.refresh_gopass_credentials or args.create_test_event:
-        google_calendar = GoogleCalendar()
-    else:
-        parser.print_help()
-
-    if args.show_events_since:
-        google_calendar.print_latest_events(  # pylint: disable=possibly-used-before-assignment
-            args.show_events_since
-        )
-
-    if args.refresh_gopass_credentials:
-        google_calendar.save_credentials_to_gopass()
-
-    if args.create_test_event:
-        google_calendar.create_event()
-
-
-def pip(
-    package: c2cciutils.configuration.PublishPypiPackage, version: str, version_type: str, publish: bool
-) -> bool:
-    """
-    Publish to pypi.
-
-    Arguments:
-        version: The version that will be published
-        version_type: Describe the kind of release we do: rebuild (specified using --type), version_tag,
-                    version_branch, feature_branch, feature_tag (for pull request)
-        publish: If False only check the package
-        package: The package configuration
-
-    """
-    print(f"::group::{'Publishing' if publish else 'Checking'} '{package.get('path')}' to pypi")
-    sys.stdout.flush()
-    sys.stderr.flush()
-
-    try:
-        env = {}
-        env["VERSION"] = version
-        env["VERSION_TYPE"] = version_type
-        full_repo = c2cciutils.get_repository()
-        full_repo_split = full_repo.split("/")
-        master_branch, _ = c2cciutils.get_master_branch(full_repo_split)
-        is_master = master_branch == version
-        env["IS_MASTER"] = "TRUE" if is_master else "FALSE"
-
-        cwd = os.path.abspath(package.get("path", "."))
-
-        dist = os.path.join(cwd, "dist")
-        if not os.path.exists(dist):
-            os.mkdir(dist)
-        if os.path.exists(os.path.join(cwd, "setup.py")):
-            cmd = ["python3", "./setup.py", "egg_info", "--no-date"]
-            cmd += (
-                ["--tag-build=dev" + datetime.datetime.now().strftime("%Y%m%d%H%M%S")]
-                if version_type in ("version_branch", "rebuild")
-                else []
-            )
-            cmd.append("bdist_wheel")
-        else:
-            if not os.path.exists(dist):
-                os.mkdir(dist)
-            cmd = ["pip", "wheel", "--no-deps", "--wheel-dir=dist", "."]
-            if os.path.exists(os.path.join(cwd, "pyproject.toml")):
-                use_poetry = False
-                if "build_command" not in package:
-                    with open(os.path.join(cwd, "pyproject.toml"), "rb") as project_file:
-                        pyproject = tomllib.load(project_file)
-                    re_splitter = re.compile(r"[<>=]+")
-                    for requirement in pyproject.get("build-system", {}).get("requires", []):
-                        requirement_split = re_splitter.split(requirement)
-                        if requirement_split[0] in ("poetry", "poetry-core"):
-                            use_poetry = True
-                            break
-                    subprocess.run(
-                        ["pip", "install", *pyproject.get("build-system", {}).get("requires", [])], check=True
-                    )
-                if use_poetry:
-                    freeze = subprocess.run(["pip", "freeze"], check=True, stdout=subprocess.PIPE)
-                    for freeze_line in freeze.stdout.decode("utf-8").split("\n"):
-                        if freeze_line.startswith("poetry-") or freeze_line.startswith("poetry="):
-                            print(freeze_line)
-                    env_bash = " ".join([f"{key}={value}" for key, value in env.items()])
-                    print(f"Run in {cwd}: {env_bash} poetry build")
-                    sys.stdout.flush()
-                    sys.stderr.flush()
-                    subprocess.run(["poetry", "build"], cwd=cwd, env={**os.environ, **env}, check=True)
-                    cmd = []
-        if cmd:
-            cmd = package.get("build_command", cmd)
-            subprocess.check_call(cmd, cwd=cwd, env=env)
-        cmd = ["twine"]
-        cmd += ["upload", "--verbose", "--disable-progress-bar"] if publish else ["check"]
-        cmd += glob.glob(os.path.join(cwd, "dist/*.whl"))
-        cmd += glob.glob(os.path.join(cwd, "dist/*.tar.gz"))
-        subprocess.check_call(cmd)
-        print("::endgroup::")
-    except subprocess.CalledProcessError as exception:
-        print(f"Error: {exception}")
-        print("::endgroup::")
-        print("::error::With error")
-        return False
-    return True
-
-
-def docker(
-    config: c2cciutils.configuration.PublishDockerRepository,
-    name: str,
-    image_config: c2cciutils.configuration.PublishDockerImage,
-    tag_src: str,
-    dst_tags: list[str],
-    images_full: list[str],
-) -> bool:
-    """
-    Publish to a Docker registry.
-
-    config is like:
-        server: # The server fqdn
-
-    image_config is like:
-        name: # The image name
-
-    Arguments:
-        config: The publishing config
-        name: The repository name, just used to print messages
-        image_config: The image config
-        tag_src: The source tag (usually latest)
-        dst_tags: Publish using the provided tags
-        images_full: The list of published images (with tag), used to build the dispatch event
-
-    """
-    print(
-        f"::group::Publishing {image_config['name']} to the server {name} using the tags {', '.join(dst_tags)}"
-    )
-    sys.stdout.flush()
-    sys.stderr.flush()
-
-    try:
-        new_images_full = []
-        if "server" in config:
-            for tag in dst_tags:
-                subprocess.run(
-                    [
-                        "docker",
-                        "tag",
-                        f"{image_config['name']}:{tag_src}",
-                        f"{config['server']}/{image_config['name']}:{tag}",
-                    ],
-                    check=True,
-                )
-                new_images_full.append(f"{config['server']}/{image_config['name']}:{tag}")
-        else:
-            for tag in dst_tags:
-                if tag_src != tag:
-                    subprocess.run(
-                        [
-                            "docker",
-                            "tag",
-                            f"{image_config['name']}:{tag_src}",
-                            f"{image_config['name']}:{tag}",
-                        ],
-                        check=True,
-                    )
-                new_images_full.append(f"{image_config['name']}:{tag}")
-
-        for image in new_images_full:
-            subprocess.run(["docker", "push", image], check=True)
-        images_full += new_images_full
-
-        print("::endgroup::")
-    except subprocess.CalledProcessError as exception:
-        print(f"Error: {exception}")
-        print("::endgroup::")
-        print("::error::With error")
-        return False
-    return True
-
-
-def helm(folder: str, version: str, owner: str, repo: str, commit_sha: str, token: str) -> bool:
-    """
-    Publish to pypi.
-
-    Arguments:
-        folder: The folder to be published
-        version: The version that will be published
-        owner: The GitHub repository owner
-        repo: The GitHub repository name
-        commit_sha: The sha of the current commit
-        token: The GitHub token
-
-    """
-    print(f"::group::Publishing Helm chart from '{folder}' to GitHub release")
-    sys.stdout.flush()
-    sys.stderr.flush()
-
-    try:
-        yaml_ = ruamel.yaml.YAML()
-        with open(os.path.join(folder, "Chart.yaml"), encoding="utf-8") as open_file:
-            chart = yaml_.load(open_file)
-        chart["version"] = version
-        with open(os.path.join(folder, "Chart.yaml"), "w", encoding="utf-8") as open_file:
-            yaml_.dump(chart, open_file)
-        for index, dependency in enumerate(chart.get("dependencies", [])):
-            if dependency["repository"].startswith("https://"):
-                subprocess.run(["helm", "repo", "add", str(index), dependency["repository"]], check=True)
-
-        subprocess.run(["cr", "package", folder], check=True)
-        subprocess.run(
-            [
-                "cr",
-                "upload",
-                f"--owner={owner}",
-                f"--git-repo={repo}",
-                f"--commit={commit_sha}",
-                "--release-name-template={{ .Version }}",
-                f"--token={token}",
-            ],
-            check=True,
-        )
-        if not os.path.exists(".cr-index"):
-            os.mkdir(".cr-index")
-        subprocess.run(
-            [
-                "cr",
-                "index",
-                f"--owner={owner}",
-                f"--git-repo={repo}",
-                f"--charts-repo=https://{owner}.github.io/{repo}",
-                "--push",
-                "--release-name-template={{ .Version }}",
-                f"--token={token}",
-            ],
-            check=True,
-        )
-        print("::endgroup::")
-    except subprocess.CalledProcessError as exception:
-        print(f"Error: {exception}")
-        print("::endgroup::")
-        print("::error::With error")
-        return False
-    return True

c2cciutils/schema-applications.json

@@ -1,50 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "https://raw.githubusercontent.com/camptocamp/tilecloud-chain/master/tilecloud_chain/schema.json",
-  "title": "Applications configuration",
-  "description": "All the applications configuration",
-  "type": "object",
-  "definitions": {},
-  "additionalProperties": {
-    "title": "Application configuration",
-    "description": "An application configuration",
-    "type": "object",
-    "properties": {
-      "url-pattern": {
-        "title": "URL pattern",
-        "description": "URL pattern, to be used for files that didn't come from GitHub release, available arguments: {version}",
-        "type": "string"
-      },
-      "type": {
-        "title": "The type of file",
-        "description": "The type of file",
-        "type": "string",
-        "enum": ["tar"]
-      },
-      "get-file-name": {
-        "title": "The filename to get",
-        "description": "The name of the file to get in the GitHub release",
-        "type": "string"
-      },
-      "to-file-name": {
-        "title": "The created tile name",
-        "description": "The name of the final tile we will create",
-        "type": "string"
-      },
-      "tar-file-name": {
-        "title": "The tile name to get in the tar file",
-        "type": "string"
-      },
-      "finish-commands": {
-        "title": "The commands to run after the tile creation",
-        "type": "array",
-        "items": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          }
-        }
-      }
-    }
-  }
-}

c2cciutils/scripts/clean.py

@@ -1,103 +0,0 @@
-#!/usr/bin/env python3
-
-"""The clean main function."""
-
-import argparse
-import json
-import os
-import sys
-from typing import cast
-
-import requests
-import yaml
-
-import c2cciutils
-
-
-def clean(image: str, tag: str, token: str) -> None:
-    """
-    Delete an image from Docker hub.
-
-    Arguments:
-        image: The image name that should be deleted (<organization>/<name>)
-        tag: The tag that should be deleted
-        token: The token used to be authenticated on Docker hub
-
-    """
-    print(f"Delete image '{image}:{tag}'.")
-
-    response = requests.head(
-        f"https://hub.docker.com/v2/repositories/{image}/tags/{tag}/",
-        headers={"Authorization": "JWT " + token},
-        timeout=int(os.environ.get("C2CCIUTILS_TIMEOUT", "30")),
-    )
-    if response.status_code == 404:
-        return
-    if not response.ok:
-        print(f"Error checking image '{image}:{tag}' status.")
-        print(response.text)
-        sys.exit(1)
-
-    response = requests.delete(
-        f"https://hub.docker.com/v2/repositories/{image}/tags/{tag}/",
-        headers={"Authorization": "JWT " + token},
-        timeout=int(os.environ.get("C2CCIUTILS_TIMEOUT", "30")),
-    )
-    if not response.ok:
-        print("::error::Error on deleting tag: " + tag)
-        print(response.text)
-        sys.exit(1)
-
-
-def main() -> None:
-    """Run the main function."""
-    parser = argparse.ArgumentParser(
-        description=(
-            "Clean the Docker images on Docker Hub for the branch we delete "
-            "(get from the GitHub event information)."
-        )
-    )
-    parser.parse_args()
-
-    username = (
-        os.environ["DOCKERHUB_USERNAME"]
-        if "DOCKERHUB_USERNAME" in os.environ
-        else c2cciutils.gopass("gs/ci/dockerhub/username")
-    )
-    password = (
-        os.environ["DOCKERHUB_PASSWORD"]
-        if "DOCKERHUB_PASSWORD" in os.environ
-        else c2cciutils.gopass("gs/ci/dockerhub/password")
-    )
-    token = requests.post(
-        "https://hub.docker.com/v2/users/login/",
-        headers={"Content-Type": "application/json"},
-        data=json.dumps(
-            {
-                "username": username,
-                "password": password,
-            }
-        ),
-        timeout=int(os.environ.get("C2CCIUTILS_TIMEOUT", "30")),
-    ).json()["token"]
-
-    with open(os.environ["GITHUB_EVENT_PATH"], encoding="utf-8") as event_file:
-        event = json.loads(event_file.read())
-        print(yaml.dump(event))
-        ref = str(event["number"]) if "pull_request" in event else event["ref"]
-
-    ref = ref.replace("/", "_")
-
-    config = c2cciutils.get_config()
-
-    docker_config = cast(
-        c2cciutils.configuration.PublishDockerConfig,
-        config.get("publish", {}).get("docker", {}) if config.get("publish", {}).get("docker", False) else {},
-    )
-    for image in docker_config.get("images", []):
-        for tag in image.get("tags", []):
-            clean(image["name"], tag.format(version=ref), token)
-
-
-if __name__ == "__main__":
-    main()

c2cciutils/scripts/docker_versions_gen.py

@@ -1,33 +0,0 @@
-import argparse
-
-import yaml
-
-import c2cciutils.lib.docker
-
-
-def main() -> None:
-    """Dump the actual versions of packages in image to file ci/dpkg-versions.yaml."""
-    argparser = argparse.ArgumentParser(
-        description="Dump the actual versions of packages in image to file ci/dpkg-versions.yaml."
-    )
-    argparser.add_argument("--distribution", help="The default distribution code to be used")
-    argparser.add_argument("--release", help="The default release version to be used")
-    argparser.add_argument("images", help="The image to check", nargs="+")
-    args = argparser.parse_args()
-
-    versions_config, _ = c2cciutils.lib.docker.get_versions_config()
-    for image in args.images:
-        _, versions_image = c2cciutils.lib.docker.get_dpkg_packages_versions(
-            image,
-            default_distribution=args.distribution,
-            default_release=args.release,
-        )
-        versions_config[image] = {k: str(v) for k, v in versions_image.items()}
-
-    with open("ci/dpkg-versions.yaml", "w", encoding="utf-8") as versions_file:
-        versions_file.write("# See repository list: https://repology.org/repositories/statistics\n\n")
-        versions_file.write(yaml.dump(versions_config, Dumper=yaml.SafeDumper, default_flow_style=False))
-
-
-if __name__ == "__main__":
-    main()

c2cciutils/scripts/pin_pipenv.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-
-import argparse
-import configparser
-import json
-
-
-def main() -> None:
-    """Run the command."""
-    parser = argparse.ArgumentParser(
-        description="""Output packages with versions from Pipfile.lock in Pipfile format (similar to pip freeze).
-Useful to pin all the dependency in the Pipfile, on stabilization branch to be able to upgrade one package that have a security issue."""
-    )
-    parser.add_argument("--packages", action="store_true", help="Output only the packages section")
-    parser.add_argument("--dev-packages", action="store_true", help="Output only the dev-packages section")
-    parser.add_argument("--pipfile", default="Pipfile", help="The base Pipfile filename")
-    args = parser.parse_args()
-
-    packages = {}
-    dev_packages = {}
-
-    with open(args.pipfile + ".lock", encoding="utf-8") as pipfilelock_file:
-        pipfilelock = json.loads(pipfilelock_file.read())
-        for pkg, pkg_config in pipfilelock["default"].items():
-            packages[pkg] = pkg_config["version"]
-        for pkg, pkg_config in pipfilelock["develop"].items():
-            dev_packages[pkg] = pkg_config["version"]
-
-    config = configparser.ConfigParser()
-    config.read(args.pipfile)
-
-    if args.packages or not args.packages and not args.dev_packages:
-        print("[packages]")
-        print("# Lock dependencies")
-        for pkg, version in packages.items():
-            if pkg not in config["packages"] and f'"{pkg}"' not in config["packages"]:
-                quote = '"' if "." in pkg else ""
-                print(f'{quote}{pkg}{quote} = "{version}"')
-
-    if args.packages and args.dev_packages or not args.packages and not args.dev_packages:
-        print()
-
-    if args.dev_packages or not args.packages and not args.dev_packages:
-        print("[dev-packages]")
-        print("# Lock dependencies")
-        for pkg, version in dev_packages.items():
-            if pkg not in config["dev-packages"] and f'"{pkg}"' not in config["dev-packages"]:
-                quote = '"' if "." in pkg else ""
-                print(f'{quote}{pkg}{quote} = "{version}"')
-
-
-if __name__ == "__main__":
-    main()