gitlab-compliance 2.2.0__tar.gz → 2.3.0__tar.gz

{gitlab_compliance-2.2.0 → gitlab_compliance-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitlab-compliance
-Version: 2.2.0
+Version: 2.3.0
 Summary: BDD compliance testing for GitLab CI/CD pipelines and project settings — by MaturityBuilder
 License-Expression: Apache-2.0
 License-File: LICENSE

{gitlab_compliance-2.2.0 → gitlab_compliance-2.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gitlab-compliance"
-version = "2.2.0"
+version = "2.3.0"
 dynamic = [ "readme" ]
 license = "Apache-2.0"
 requires-python = ">=3.12"

{gitlab_compliance-2.2.0 → gitlab_compliance-2.3.0}/src/compliance/behave_support/environment.py

@@ -6,6 +6,15 @@ from src.compliance.model import load_pipeline_entities
 from src.compliance.release_cache import ReleaseMetadataCache
 
 
+def _resolve_external_flag(value: str | None) -> bool | None:
+    normalized = str(value or "auto").strip().lower()
+    if normalized in {"true", "yes", "on", "1"}:
+        return True
+    if normalized in {"false", "no", "off", "0"}:
+        return False
+    return None
+
+
 def before_all(context):
     context.compliance_entities = {}
     context.stash = []
@@ -42,6 +51,9 @@ def before_scenario(context, scenario):
             == "true",
             load_api_entities=context.config.userdata.get("load_api_entities", "true")
             == "true",
+            resolve_external_includes=_resolve_external_flag(
+                context.config.userdata.get("resolve_external_includes", "auto")
+            ),
             cache=getattr(context, "release_cache", None),
         )
 

gitlab_compliance-2.3.0/src/compliance/include_fetch.py

@@ -0,0 +1,340 @@
+"""Fetch GitLab CI YAML from remote and project include entries."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from src.compliance.include_versions import resolve_include_project_path
+from src.modules.common import load_yml_documents
+
+GITLAB_TEMPLATE_BASE = (
+    "https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates"
+)
+
+
+@dataclass(frozen=True)
+class FetchedInclude:
+    """YAML content retrieved from an external include."""
+
+    config_label: str
+    yaml_text: str
+
+
+@dataclass(frozen=True)
+class IncludeFetchFailure:
+    """Reason an include could not be fetched."""
+
+    reason: str
+    detail: str = ""
+
+
+@dataclass
+class IncludeFetchCache:
+    """In-memory cache for fetched include YAML."""
+
+    entries: dict[str, FetchedInclude | IncludeFetchFailure] = field(
+        default_factory=dict
+    )
+
+
+@dataclass(frozen=True)
+class ExternalIncludeContext:
+    """Origin metadata for YAML loaded from remote or project includes."""
+
+    kind: str
+    visit_key: str
+    base_url: str = ""
+    project_path: str = ""
+    ref: str = ""
+
+
+def _cache_key(include_type: str, *parts: str) -> str:
+    return f"{include_type}:" + ":".join(parts)
+
+
+def _gitlab_client(gitlab_url: str, token: str):
+    import gitlab
+
+    client = gitlab.Gitlab(gitlab_url, private_token=token)
+    client.auth()
+    return client
+
+
+def _normalize_project_files(file_value: Any) -> list[str]:
+    if file_value is None or file_value == "":
+        return [".gitlab-ci.yml"]
+    if isinstance(file_value, str):
+        return [file_value]
+    if isinstance(file_value, list):
+        return [str(item) for item in file_value if item]
+    return [str(file_value)]
+
+
+def remote_url_allows_token(url: str, gitlab_url: str | None) -> bool:
+    """Return True when ``url`` is on the same host as the GitLab instance."""
+    if not gitlab_url:
+        return False
+    remote_host = (urlparse(url).netloc or "").lower()
+    gitlab_host = (urlparse(gitlab_url).netloc or "").lower()
+    if not remote_host or not gitlab_host:
+        return False
+    if remote_host == gitlab_host:
+        return True
+    return remote_host == f"www.{gitlab_host}" or gitlab_host == f"www.{remote_host}"
+
+
+def _remote_headers(
+    url: str, token: str | None, gitlab_url: str | None
+) -> dict[str, str]:
+    if token and remote_url_allows_token(url, gitlab_url):
+        return {"PRIVATE-TOKEN": token}
+    return {}
+
+
+def remote_directory_url(url: str) -> str:
+    """Return the directory URL for resolving nested local includes."""
+    parsed = urlparse(url)
+    path = parsed.path or "/"
+    if not path.endswith("/"):
+        path = path.rsplit("/", 1)[0] + "/"
+    return parsed._replace(path=path, params="", query="", fragment="").geturl()
+
+
+def _fetch_remote(
+    url: str,
+    token: str | None,
+    gitlab_url: str | None = None,
+) -> FetchedInclude | IncludeFetchFailure:
+    headers = _remote_headers(url, token, gitlab_url)
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+    except requests.RequestException as exc:
+        return IncludeFetchFailure(reason="fetch_failed", detail=str(exc))
+    text = response.text
+    if not text.strip():
+        return IncludeFetchFailure(reason="fetch_failed", detail="empty response")
+    try:
+        load_yml_documents(text)
+    except Exception as exc:
+        return IncludeFetchFailure(reason="fetch_failed", detail=f"invalid YAML: {exc}")
+    return FetchedInclude(config_label=url, yaml_text=text)
+
+
+def _fetch_project_files(
+    *,
+    project_path: str,
+    ref: str,
+    files: list[str],
+    gitlab_url: str,
+    token: str,
+) -> FetchedInclude | IncludeFetchFailure:
+    try:
+        gl = _gitlab_client(gitlab_url, token)
+        project = gl.projects.get(project_path)
+    except Exception as exc:
+        return IncludeFetchFailure(reason="fetch_failed", detail=str(exc))
+
+    documents: list[dict] = []
+    label_parts: list[str] = []
+    for file_path in files:
+        try:
+            raw = project.files.raw(file_path=file_path, ref=ref or "main")
+        except Exception as exc:
+            return IncludeFetchFailure(
+                reason="fetch_failed",
+                detail=f"{file_path}@{ref or 'main'}: {exc}",
+            )
+        text = raw.decode("utf-8") if isinstance(raw, bytes) else str(raw)
+        try:
+            parsed_docs = load_yml_documents(text)
+        except Exception as exc:
+            return IncludeFetchFailure(
+                reason="fetch_failed",
+                detail=f"{file_path}: invalid YAML: {exc}",
+            )
+        documents.extend(doc for doc in parsed_docs if isinstance(doc, dict))
+        label_parts.append(file_path)
+
+    if not documents:
+        return IncludeFetchFailure(reason="fetch_failed", detail="no YAML documents")
+
+    import yaml
+
+    merged = yaml.safe_dump_all(documents, sort_keys=False)
+    label = f"project:{project_path}@{ref or 'main'}:" + ",".join(label_parts)
+    return FetchedInclude(config_label=label, yaml_text=merged)
+
+
+def _fetch_template(template_name: str) -> FetchedInclude | IncludeFetchFailure:
+    name = str(template_name).strip()
+    if not name:
+        return IncludeFetchFailure(
+            reason="unsupported_type", detail="empty template name"
+        )
+    url = f"{GITLAB_TEMPLATE_BASE}/{name}"
+    return _fetch_remote(url, token=None, gitlab_url=None)
+
+
+def fetch_include_content(
+    parsed: dict[str, Any],
+    *,
+    gitlab_url: str,
+    token: str | None,
+    cache: IncludeFetchCache | None = None,
+    allow_template: bool = False,
+) -> FetchedInclude | IncludeFetchFailure:
+    """Fetch YAML for a parsed include entry (remote or project)."""
+    include_type = parsed.get("include_type", "")
+    cache = cache or IncludeFetchCache()
+
+    if include_type == "remote":
+        url = str(parsed.get("project", "")).strip()
+        key = _cache_key("remote", url)
+        if key in cache.entries:
+            return cache.entries[key]
+        result = _fetch_remote(url, token, gitlab_url=gitlab_url)
+        cache.entries[key] = result
+        return result
+
+    if include_type == "project":
+        if not token:
+            return IncludeFetchFailure(reason="no_token")
+        project_path = resolve_include_project_path(
+            "project", str(parsed.get("project", ""))
+        )
+        if not project_path:
+            return IncludeFetchFailure(
+                reason="fetch_failed", detail="could not resolve project path"
+            )
+        ref = str(parsed.get("version", "") or parsed.get("ref", "") or "main")
+        files = _normalize_project_files(parsed.get("file"))
+        key = _cache_key("project", project_path, ref, *files)
+        if key in cache.entries:
+            return cache.entries[key]
+        result = _fetch_project_files(
+            project_path=project_path,
+            ref=ref,
+            files=files,
+            gitlab_url=gitlab_url,
+            token=token,
+        )
+        cache.entries[key] = result
+        return result
+
+    if include_type == "template" and allow_template:
+        template_name = str(parsed.get("project", "")).strip()
+        key = _cache_key("template", template_name)
+        if key in cache.entries:
+            return cache.entries[key]
+        result = _fetch_template(template_name)
+        cache.entries[key] = result
+        return result
+
+    if include_type in {"component", "template"}:
+        return IncludeFetchFailure(reason="unsupported_type")
+
+    return IncludeFetchFailure(reason="unsupported_type", detail=include_type)
+
+
+def context_for_fetched(
+    parsed: dict[str, Any], fetched: FetchedInclude
+) -> ExternalIncludeContext:
+    """Build origin context used to resolve nested locals inside fetched YAML."""
+    include_type = parsed.get("include_type", "")
+    if include_type == "remote":
+        return ExternalIncludeContext(
+            kind="remote",
+            visit_key=fetched.config_label,
+            base_url=remote_directory_url(fetched.config_label),
+        )
+    if include_type == "project":
+        project_path = resolve_include_project_path(
+            "project", str(parsed.get("project", ""))
+        ) or str(parsed.get("project", ""))
+        ref = str(parsed.get("version", "") or parsed.get("ref", "") or "main")
+        return ExternalIncludeContext(
+            kind="project",
+            visit_key=fetched.config_label,
+            project_path=project_path,
+            ref=ref or "main",
+        )
+    return ExternalIncludeContext(kind=include_type, visit_key=fetched.config_label)
+
+
+def resolve_nested_local_from_context(
+    local_path: str,
+    context: ExternalIncludeContext,
+    *,
+    gitlab_url: str,
+    token: str | None,
+    cache: IncludeFetchCache,
+) -> FetchedInclude | IncludeFetchFailure:
+    """Resolve a nested ``local:`` path relative to an external include origin."""
+    normalized = str(local_path).lstrip("/")
+    if context.kind == "remote":
+        url = urljoin(context.base_url, normalized)
+        key = _cache_key("remote", url)
+        if key in cache.entries:
+            return cache.entries[key]
+        result = _fetch_remote(url, token, gitlab_url=gitlab_url)
+        cache.entries[key] = result
+        return result
+
+    if context.kind == "project":
+        if not token:
+            return IncludeFetchFailure(reason="no_token")
+        parsed = {
+            "include_type": "project",
+            "project": context.project_path,
+            "version": context.ref,
+            "file": normalized,
+        }
+        return fetch_include_content(
+            parsed,
+            gitlab_url=gitlab_url,
+            token=token,
+            cache=cache,
+        )
+
+    return IncludeFetchFailure(
+        reason="unsupported_type", detail=f"nested local from {context.kind}"
+    )
+
+
+def include_reference_label(parsed: dict[str, Any]) -> str:
+    """Human-readable label for an include entry."""
+    include_type = parsed.get("include_type", "unknown")
+    if include_type == "local":
+        return str(parsed.get("project", ""))
+    if include_type == "remote":
+        return str(parsed.get("project", ""))
+    if include_type == "project":
+        file_part = parsed.get("file") or ".gitlab-ci.yml"
+        ref = parsed.get("version") or "main"
+        return f"{parsed.get('project', '')} ({file_part} @ {ref})"
+    if include_type == "component":
+        return f"{parsed.get('project', '')}@{parsed.get('version', '')}"
+    if include_type == "template":
+        return str(parsed.get("project", ""))
+    return str(parsed.get("project", ""))
+
+
+def default_gitlab_url(gitlab_url: str | None = None) -> str:
+    if gitlab_url:
+        return gitlab_url.rstrip("/")
+    for env_key in ("CI_SERVER_URL", "GITLAB_URL"):
+        value = os.getenv(env_key)
+        if value:
+            return value.rstrip("/")
+    return "https://gitlab.com"
+
+
+def is_public_gitlab_host(url: str) -> bool:
+    parsed = urlparse(url)
+    return parsed.netloc in {"gitlab.com", "www.gitlab.com"}

gitlab_compliance-2.3.0/src/compliance/include_resolution.py

@@ -0,0 +1,116 @@
+"""Include resolution reporting for compliance and shell-check runs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from src.compliance.include_fetch import include_reference_label
+
+REASON_MESSAGES = {
+    "no_token": "missing authentication",  # nosec B105 - user-facing reason label
+    "fetch_failed": "fetch failed",
+    "unsupported_type": "unsupported include type",
+    "depth_exceeded": "max include depth exceeded",
+    "include_nested_disabled": "nested includes disabled",
+    "external_resolution_disabled": "external include resolution disabled",
+    "path_rejected": "local include path rejected",
+    "not_found": "include file not found",
+}
+
+
+@dataclass(frozen=True)
+class UnresolvedInclude:
+    """An include entry that was not merged into the pipeline stash."""
+
+    include_type: str
+    reference: str
+    reason: str
+    source_file: str = ""
+    line: int = 0
+    detail: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "include_type": self.include_type,
+            "reference": self.reference,
+            "reason": self.reason,
+            "source_file": self.source_file,
+            "line": self.line,
+            "detail": self.detail,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> UnresolvedInclude:
+        return cls(
+            include_type=str(payload.get("include_type", "")),
+            reference=str(payload.get("reference", "")),
+            reason=str(payload.get("reason", "")),
+            source_file=str(payload.get("source_file", "")),
+            line=int(payload.get("line") or 0),
+            detail=str(payload.get("detail", "")),
+        )
+
+
+@dataclass
+class IncludeResolutionReport:
+    """Aggregated include coverage information for a pipeline load."""
+
+    unresolved: list[UnresolvedInclude] = field(default_factory=list)
+
+    def extend(self, items: list[UnresolvedInclude | dict[str, Any]]) -> None:
+        for item in items:
+            if isinstance(item, UnresolvedInclude):
+                self.unresolved.append(item)
+            else:
+                self.unresolved.append(UnresolvedInclude.from_dict(item))
+
+    def to_dicts(self) -> list[dict[str, Any]]:
+        return [item.to_dict() for item in self.unresolved]
+
+    def has_gaps(self) -> bool:
+        return bool(self.unresolved)
+
+
+def record_unresolved(
+    bucket: list[UnresolvedInclude],
+    parsed: dict[str, Any],
+    *,
+    reason: str,
+    detail: str = "",
+) -> None:
+    bucket.append(
+        UnresolvedInclude(
+            include_type=str(parsed.get("include_type", "")),
+            reference=include_reference_label(parsed),
+            reason=reason,
+            source_file=str(parsed.get("source_file", "")),
+            line=int(parsed.get("line") or 0),
+            detail=detail,
+        )
+    )
+
+
+def format_include_warning(report: IncludeResolutionReport) -> str:
+    """Format unresolved includes for console warnings."""
+    if not report.unresolved:
+        return ""
+
+    lines = [
+        "Some include entries were not resolved; scripts from those files were "
+        + "not validated. Pass --token (or set GITLAB_TOKEN) and enable external "
+        + "include resolution to fetch project and remote includes.",
+        "",
+    ]
+    for item in report.unresolved:
+        reason = REASON_MESSAGES.get(item.reason, item.reason)
+        detail = f" ({item.detail})" if item.detail else ""
+        location = ""
+        if item.source_file:
+            location = f" in {item.source_file}"
+            if item.line:
+                location += f":{item.line}"
+        lines.append(
+            f"- [{item.include_type}] {item.reference}{location}: {reason}{detail}"
+        )
+    return "\n".join(lines)

{gitlab_compliance-2.2.0 → gitlab_compliance-2.3.0}/src/compliance/model.py

@@ -110,7 +110,10 @@ def load_yaml_entities(
     pipeline_file: str,
     include_nested: bool = True,
     max_include_depth: int | None = None,
-) -> dict[str, list[dict]]:
+    resolve_external_includes: bool | None = None,
+    gitlab_url: str | None = None,
+    token: str | None = None,
+) -> tuple[dict[str, list[dict]], list[dict]]:
     if not os.path.exists(pipeline_file):
         raise FileNotFoundError(f"Pipeline file not found: {pipeline_file}")
 
@@ -121,6 +124,9 @@ def load_yaml_entities(
         max_include_depth=max_include_depth,
         include_scripts=True,
         resolve_job_composition=True,
+        resolve_external_includes=resolve_external_includes,
+        gitlab_url=gitlab_url,
+        token=token,
     )
 
     line_index = pipeline_data.get("line_index") or {}
@@ -170,7 +176,7 @@ def load_yaml_entities(
             for index, image in enumerate(pipeline_data["container_images"], 1)
         ],
     }
-    return entities
+    return entities, list(pipeline_data.get("unresolved_includes") or [])
 
 
 def _job_values(job: dict) -> dict:
@@ -205,6 +211,9 @@ def load_merged_job_scripts(
     pipeline_file: str,
     include_nested: bool = True,
     max_include_depth: int | None = None,
+    resolve_external_includes: bool | None = None,
+    gitlab_url: str | None = None,
+    token: str | None = None,
 ) -> list[dict]:
     """Return jobs with composed effective scripts for shell validation."""
     pipeline_data = collect_pipeline_data(
@@ -214,6 +223,9 @@ def load_merged_job_scripts(
         max_include_depth=max_include_depth,
         include_scripts=True,
         resolve_job_composition=True,
+        resolve_external_includes=resolve_external_includes,
+        gitlab_url=gitlab_url,
+        token=token,
     )
     return list(pipeline_data.get("merged_jobs") or pipeline_data.get("jobs") or [])
 
@@ -229,16 +241,21 @@ def load_pipeline_entities(
     enrich_includes: bool = True,
     enrich_images: bool = True,
     load_api_entities: bool = True,
+    resolve_external_includes: bool | None = None,
     cache: "ReleaseMetadataCache | None" = None,
 ) -> dict[str, list[dict]]:
     from src.compliance.api_config import resolve_group, resolve_project, resolve_token
     from src.compliance.release_cache import ReleaseMetadataCache
 
-    entities = load_yaml_entities(
+    entities, unresolved_includes = load_yaml_entities(
         pipeline_file,
         include_nested=include_nested,
         max_include_depth=max_include_depth,
+        resolve_external_includes=resolve_external_includes,
+        gitlab_url=gitlab_url,
+        token=token,
     )
+    entities["unresolved_includes"] = unresolved_includes
 
     userdata = {
         "project": project or "",

{gitlab_compliance-2.2.0 → gitlab_compliance-2.3.0}/src/compliance/models.py

@@ -27,3 +27,4 @@ class ComplianceResult:
     failed: int = 0
     skipped: int = 0
     scenario_results: list[ScenarioResult] = field(default_factory=list)
+    unresolved_includes: list[dict] = field(default_factory=list)