generic-gitlab-cicd 0.3.2__py3-none-any.whl

generic_ci/dependencies.py

@@ -0,0 +1,186 @@
+"""Temporary uv environments with workspace ownership and immutable candidates."""
+import contextlib
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tomllib
+
+import tomlkit
+from packaging.requirements import Requirement
+
+from .config import allowed_url
+from .models import relative
+
+
+def normalize(name):
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def resolve_candidates(environment, hosts, projects):
+    single = [environment.get("CI_DEPENDENCY_" + key, "") for key in ("REPO", "REF", "PACKAGE")]
+    raw = environment.get("CI_DEPENDENCY_OVERRIDES", "[]") or "[]"
+    filename = environment.get("CI_DEPENDENCY_FILE", "")
+    decoded = json.loads(raw)
+    if not isinstance(decoded, list):
+        raise ValueError("CI_DEPENDENCY_OVERRIDES must be a JSON array")
+    if sum([bool(any(single)), bool(decoded), bool(filename)]) > 1:
+        raise ValueError("provide only one override input: single fields, JSON, or file")
+    if filename:
+        decoded = json.loads(Path(filename).read_text())
+    if any(single):
+        if not all(single):
+            raise ValueError("provide repository, ref and package together")
+        decoded = [dict(zip(("repository", "ref", "package"), single),
+                        subdirectory=environment.get("CI_DEPENDENCY_SUBDIRECTORY", ""))]
+    if not isinstance(decoded, list):
+        raise ValueError("dependency file must contain a JSON array")
+    seen, result = set(), []
+    for item in decoded:
+        if not isinstance(item, dict) or set(item) - {"repository", "repo", "ref", "package", "subdirectory", "projects"}:
+            raise ValueError("override fields: repository, ref, package, subdirectory, projects")
+        if "repo" in item and "repository" in item:
+            raise ValueError("use repository or legacy repo, not both")
+        repo, ref, name = item.get("repository", item.get("repo", "")), item.get("ref", ""), item.get("package", "")
+        if not all(isinstance(v, str) for v in (repo, ref, name)):
+            raise ValueError("override repository/ref/package must be strings")
+        allowed_url(repo, hosts)
+        if not re.fullmatch(r"(?:@[A-Za-z0-9._-]+/)?[A-Za-z0-9][A-Za-z0-9._-]*", name):
+            raise ValueError("invalid override distribution name")
+        if normalize(name) in seen:
+            raise ValueError(f"duplicate override package {name}")
+        seen.add(normalize(name))
+        if not ref or ref.startswith("-") or re.search(r"[\s~^:?*\[\\]", ref):
+            raise ValueError("invalid dependency ref")
+        sub = item.get("subdirectory", "")
+        if sub:
+            relative(sub)
+        scope = item.get("projects", list(projects))
+        if not isinstance(scope, list) or not scope or any(p not in projects for p in scope):
+            raise ValueError("override projects must name existing projects")
+        if re.fullmatch(r"[a-fA-F0-9]{40}", ref):
+            sha = ref.lower()
+        else:
+            wanted = [ref] if ref.startswith("refs/") else ["refs/heads/" + ref, "refs/tags/" + ref]
+            output = subprocess.check_output(["git", "ls-remote", "--exit-code", repo, *wanted, *[v + "^{}" for v in wanted]], text=True)
+            matches = dict(line.split()[::-1] for line in output.splitlines())
+            found = [v for v in wanted if v in matches]
+            if len(found) != 1:
+                raise ValueError("ambiguous/missing dependency ref; use refs/heads/... or refs/tags/...")
+            sha = matches.get(found[0] + "^{}", matches[found[0]])
+        result.append({"package": name, "repository": repo, "requested_ref": ref, "commit": sha,
+                       "subdirectory": sub, "projects": scope})
+    return result
+
+
+def workspace(directory, repository):
+    directory, repository = Path(directory).resolve(), Path(repository).resolve()
+    directory.relative_to(repository)
+    root = directory
+    for parent in (directory, *directory.parents):
+        if not parent.is_relative_to(repository):
+            break
+        manifest = parent / "pyproject.toml"
+        if manifest.is_file():
+            data = tomllib.loads(manifest.read_text())
+            settings = data.get("tool", {}).get("uv", {}).get("workspace")
+            if settings:
+                members = {p.resolve() for pattern in settings.get("members", []) for p in parent.glob(pattern)}
+                excluded = {p.resolve() for pattern in settings.get("exclude", []) for p in parent.glob(pattern)}
+                if directory == parent or directory in members - excluded:
+                    root = parent
+                    break
+    return root
+
+
+def validate_indexes(root, hosts):
+    data = tomllib.loads((root / "pyproject.toml").read_text())
+    uv = data.get("tool", {}).get("uv", {})
+    for index in uv.get("index", []):
+        for key in ("url", "publish-url"):
+            if key in index:
+                allowed_url(index[key], hosts)
+    for key in ("PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "UV_INDEX_URL", "UV_DEFAULT_INDEX", "UV_EXTRA_INDEX_URL"):
+        for url in os.environ.get(key, "").split():
+            allowed_url(url, hosts)
+    for dependency in data.get("project", {}).get("dependencies", []):
+        url = Requirement(dependency).url
+        if url and not url.startswith("file:"):
+            allowed_url(url.removeprefix("git+"), hosts)
+    return data
+
+
+@contextlib.contextmanager
+def prepared(directory, repository, policy, candidates, project, hosts, record_dir):
+    """Keep temporary metadata active until caller completes; always restore it."""
+    if policy["manager"] == "none":
+        yield {"manager": "none"}
+        return
+    root = workspace(directory, repository)
+    manifest, lock = root / "pyproject.toml", root / "uv.lock"
+    if not lock.is_file():
+        raise ValueError(f"commit workspace lockfile {lock}")
+    validate_indexes(root, hosts)
+    original, original_lock = manifest.read_bytes(), lock.read_bytes()
+    selected = [r for r in candidates if project in r["projects"]]
+    groups = policy.get("groups")
+    options = [] if groups is None else ["--no-default-groups"]
+    if groups == "all" or (groups is not None and "*" in groups):
+        options += ["--all-groups"]
+    else:
+        for group in groups or []:
+            options += ["--group", group]
+    if policy["extras"] == "all":
+        options += ["--all-extras"]
+    else:
+        for extra in policy["extras"]:
+            options += ["--extra", extra]
+    if Path(directory).resolve() != root:
+        member = tomllib.loads((Path(directory) / "pyproject.toml").read_text())["project"]["name"]
+        options += ["--package", member]
+    environment = {**os.environ, "UV_PYTHON_DOWNLOADS": "never", "UV_PROJECT_ENVIRONMENT": str(root / ".venv")}
+    python_options = ["--python", policy["python"]] if policy.get("python") else []
+    try:
+        data = tomlkit.parse(original.decode())
+        uv = data.setdefault("tool", {}).setdefault("uv", {})
+        if selected:
+            names = {normalize(r["package"]) for r in selected}
+            previous = uv.get("override-dependencies", [])
+            if any(not isinstance(v, str) for v in previous):
+                raise ValueError("scoped uv overrides require an explicit adapter; string overrides supported")
+            uv["override-dependencies"] = [v for v in previous if normalize(Requirement(v).name) not in names] + [
+                f"{r['package']} @ git+{r['repository']}@{r['commit']}" + (f"#subdirectory={r['subdirectory']}" if r["subdirectory"] else "") for r in selected]
+            for key in list(uv.get("sources", {})):
+                if normalize(key) in names:
+                    del uv["sources"][key]
+            manifest.write_text(tomlkit.dumps(data))
+        upgrade = policy["upgrade"]
+        upgrade_options = ["--upgrade"] if upgrade == "all" else []
+        if isinstance(upgrade, list):
+            for package in upgrade:
+                upgrade_options += ["--upgrade-package", package]
+        if selected or upgrade != "none":
+            subprocess.run(["uv", "lock", *upgrade_options, *python_options], cwd=root, env=environment, check=True)
+        subprocess.run(["uv", "sync", "--locked", *options, *python_options], cwd=root, env=environment, check=True)
+        interpreter = root / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
+        actual = subprocess.check_output([str(interpreter), "-c", "import platform; print(platform.python_version())"], text=True).strip()
+        if policy.get("python") and not (actual == policy["python"] or actual.startswith(policy["python"] + ".")):
+            raise ValueError(f"requested Python {policy['python']}; got {actual}")
+        for candidate in selected:
+            code = "import importlib.metadata as m; print(m.distribution(__import__('sys').argv[1]).read_text('direct_url.json') or '{}')"
+            source = json.loads(subprocess.check_output([str(interpreter), "-c", code, candidate["package"]], text=True))
+            if source.get("vcs_info", {}).get("commit_id") != candidate["commit"] or source.get("url", "").rstrip("/") != candidate["repository"].rstrip("/"):
+                raise ValueError(f"installed {candidate['package']} does not match candidate repository/commit")
+        record_dir = Path(record_dir)
+        record_dir.mkdir(parents=True, exist_ok=True)
+        (record_dir / "uv.lock").write_bytes(lock.read_bytes())
+        record = {"manager": "uv", "python": actual, "workspace": root.relative_to(repository).as_posix(),
+                  "lock_sha256": hashlib.sha256(lock.read_bytes()).hexdigest(), "candidates": selected,
+                  "policy": policy, "interpreter": str(interpreter)}
+        yield record
+    finally:
+        manifest.write_bytes(original)
+        lock.write_bytes(original_lock)

generic_ci/models.py

@@ -0,0 +1,189 @@
+"""Toolkit vocabulary. Native job execution fields retain their GitLab meanings."""
+from typing import Annotated, Any, Literal
+from pathlib import PurePosixPath
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+Name = Annotated[str, Field(pattern=r"^[a-z][a-z0-9-]{0,39}$")]
+
+
+class Model(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True, strict=True,
+                              alias_generator=lambda value: value.replace("_", "-"))
+
+
+def relative(value: str) -> str:
+    if not value or "\\" in value or "$" in value or ":" in value:
+        raise ValueError("use a literal POSIX relative path")
+    path = PurePosixPath(value)
+    if path.is_absolute() or ".." in path.parts:
+        raise ValueError("path must remain inside the repository")
+    return value
+
+
+class RootPath(Model):
+    repository: str = Field(description="Explicit path relative to the repository root.")
+    _path = field_validator("repository")(relative)
+
+
+class Dependencies(Model):
+    manager: Literal["uv", "none"] = "uv"
+    upgrade: Literal["none", "all"] | list[str] = Field(default="none", description="Upgrade policy within project constraints; changes are temporary.")
+    groups: list[str] = Field(default_factory=lambda: ["*"], description="Dependency groups; * selects all, [] excludes dev groups.")
+    extras: list[str] = Field(default_factory=list)
+    python: str | None = Field(default=None, description="Required interpreter version, verified without downloading Python.")
+
+
+class NativeJob(Model):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True, strict=True, alias_generator=None)
+    image: str | None = None
+    tags: list[str] | None = None
+    services: list[Any] | None = None
+    parallel: dict[str, Any] | None = Field(default=None, description="Native parallel:matrix; compiled into explicit jobs for receipt identity.")
+    rules: list[dict[str, Any]] | None = None
+    timeout: str | None = None
+    cache: dict[str, Any] | None = None
+    resource_group: str | None = None
+    variables: dict[str, str] = Field(default_factory=dict)
+    allow_failure: bool = False
+
+    @field_validator("variables")
+    @classmethod
+    def reserved(cls, value):
+        if any(k.startswith(("CI_", "TOOLKIT_")) or k in {"PYTHONPATH", "PYTHONHOME"} for k in value):
+            raise ValueError("CI_*, TOOLKIT_* and Python bootstrap variables are reserved")
+        return value
+
+
+class Check(Model):
+    runtime: str = Field(default="default", description="Platform runtime profile; selects tools and shell, independently of runner tags.")
+    script: list[str] | None = Field(default=None, min_length=1, description="Commands after dependency setup; omit only when inheriting a preset check.")
+    dependencies: Dependencies | None = None
+    gitlab: NativeJob = Field(default_factory=NativeJob)
+    needs: list[str] = Field(default_factory=list, description="Public step references: step or project.step.")
+    junit: list[str] = Field(default_factory=list, description="Project-relative report paths.")
+    _paths = field_validator("junit")(lambda paths: [relative(p) for p in paths])
+
+
+class Step(Check):
+    script: list[str] = Field(min_length=1)
+    outputs: dict[Name, str] = Field(default_factory=dict, description="Named files/directories passed to dependent steps and builds.")
+    _output_paths = field_validator("outputs")(lambda paths: {k: relative(v) for k, v in paths.items()})
+
+
+class Package(Model):
+    directory: str = Field(default=".", description="Package build directory relative to project.path; useful for generated SDKs.")
+    index: str | None = Field(default=None, description="Named tool.uv.index in pyproject; required to publish.")
+    needs: list[str] = Field(default_factory=list)
+    checks: list[str] | None = None
+    _directory = field_validator("directory")(relative)
+
+
+class Container(Model):
+    dockerfile: str = "Dockerfile"
+    context: str | RootPath = "."
+    repository: str | None = None
+    platform: str = "linux/amd64"
+    target: str | None = Field(default=None, description="Optional Dockerfile build stage.")
+    build_args: dict[str, str] = Field(default_factory=dict)
+    secrets: dict[str, str] = Field(default_factory=dict, description="BuildKit secret ID to file-type CI variable name.")
+    needs: list[str] = Field(default_factory=list)
+    checks: list[str] | None = None
+    dependency_bundle: bool = Field(default=False, description="Dockerfile explicitly consumes named ci-dependencies context (see recipe).")
+    _dockerfile = field_validator("dockerfile")(relative)
+
+    @field_validator("context")
+    @classmethod
+    def path(cls, value):
+        return relative(value) if isinstance(value, str) else value
+
+
+class ImageBinding(Model):
+    repository: list[str] = Field(min_length=1, description="Path in chart values for image repository.")
+    digest: list[str] = Field(min_length=1, description="Path in chart values for image sha256 digest.")
+
+
+class Deployment(Model):
+    target: str
+    chart: str | None = None
+    chart_version: str | None = None
+    values: list[str] = Field(default_factory=list, description="Project-relative values files, applied in order.")
+    checks: list[str] | None = Field(default=None, description="Required source checks; omitted inherits all enabled project checks, [] opts out.")
+    before: list[str] = Field(default_factory=list, description="Fresh commands after approval and before rollout, e.g. migrations.")
+    after: list[str] = Field(default_factory=list, description="Fresh commands after rollout; failure marks deployment failed.")
+    images: dict[str, ImageBinding] = Field(default_factory=dict, description="Container project to chart values binding; stock chart defaults to own container.")
+    preview: bool = False
+    auto_stop_in: str = "2 days"
+    when: Literal["manual", "on_success"] = "manual"
+    _values = field_validator("values")(lambda paths: [relative(p) for p in paths])
+
+
+class Release(Model):
+    tag: str = Field(default="v{version}", description="Existing tag pattern; exactly one {version} placeholder.")
+    version_file: str = "pyproject.toml"
+    bump: bool = True
+    needs: list[str] = Field(default_factory=list, description="Projects whose publication must complete first.")
+    notes: str = "CHANGELOG.md"
+    gitlab_release: bool = True
+    _version = field_validator("version_file", "notes")(relative)
+
+    @field_validator("tag")
+    @classmethod
+    def pattern(cls, value):
+        if value.count("{version}") != 1 or "{" in value.replace("{version}", ""):
+            raise ValueError("tag must contain exactly one {version}")
+        return value
+
+
+class Project(Model):
+    path: str = "."
+    preset: Literal["generic", "python-package", "python-service"] = "generic"
+    dependencies: Dependencies | None = None
+    checks: dict[Name, Check | Literal[False]] = Field(default_factory=dict)
+    steps: dict[Name, Step] = Field(default_factory=dict)
+    package: Package | None = None
+    container: Container | None = None
+    deploy: dict[Name, Deployment] = Field(default_factory=dict)
+    release: Release | None = None
+    depends_on: list[str] = Field(default_factory=list, description="Project dependencies used for transitive changed-only selection.")
+    watch: list[str] = Field(default_factory=list, description="Additional repository-relative changed paths/globs.")
+    _path = field_validator("path")(relative)
+
+
+class Pipeline(Model):
+    schema_version: Literal[1] = 1
+    projects: dict[Name, Project] = Field(min_length=1)
+
+
+class Runtime(Model):
+    image: str | None = Field(default=None, description="Approved image with generic-gitlab-ci at the compiler version plus required tools.")
+    tags: list[str] = Field(default_factory=list)
+    shell: Literal["sh", "powershell"] = "sh"
+
+
+class Target(Model):
+    namespace: str
+    release_prefix: str = "app"
+    url: str = ""
+    production: bool = True
+    kubeconfig_variable: str = "KUBECONFIG"
+
+
+class Platform(Model):
+    schema_version: Literal[1] = 1
+    runtimes: dict[str, Runtime] = Field(description="Must include default; optional build, helm, release runtime overrides.")
+    registry: str
+    preview_registry: str
+    chart: str
+    chart_version: str
+    targets: dict[str, Target] = Field(default_factory=dict)
+    allowed_hosts: list[str] = Field(min_length=1, description="Internal Git, registry, index and chart hosts; no implicit public fallback.")
+    max_jobs: int = Field(default=150, ge=1, le=1000)
+    artifact_retention: str = "30 days"
+    mandatory_checks: list[str] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def default_runtime(self):
+        if "default" not in self.runtimes:
+            raise ValueError("runtimes.default is required")
+        return self