gitlab-compliance 2.0.0__py3-none-any.whl

src/compliance/include_versions.py

@@ -0,0 +1,250 @@
+"""Include version validation and GitLab release metadata for compliance checks."""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+import semver
+
+from src.compliance.release_cache import ReleaseMetadataCache
+from src.modules.release import parse_gitlab_date
+
+
+def is_valid_semver_version(version: str) -> bool:
+    if not version or not str(version).strip():
+        return False
+    try:
+        return semver.Version.is_valid(str(version).strip())
+    except Exception:
+        return False
+
+
+def resolve_include_project_path(include_type: str, project: str) -> str | None:
+    if include_type not in {"project", "component"}:
+        return None
+
+    path = str(project or "").strip()
+    if not path:
+        return None
+
+    if path.startswith("http://") or path.startswith("https://"):
+        path = urlparse(path).path.lstrip("/")
+    elif "/" in path and "." in path.split("/", 1)[0]:
+        path = "/".join(path.split("/")[1:])
+
+    if path.endswith(".git"):
+        path = path[:-4]
+
+    return path or None
+
+
+def days_between(start: datetime, end: datetime) -> int:
+    """Whole days from start to end (end may be before start)."""
+    start_utc = start.astimezone(timezone.utc)
+    end_utc = end.astimezone(timezone.utc)
+    return (end_utc.date() - start_utc.date()).days
+
+
+def compute_release_lag_days(
+    pinned_date: datetime | None, latest_date: datetime | None
+) -> int | None:
+    if pinned_date is None or latest_date is None:
+        return None
+    return days_between(pinned_date, latest_date)
+
+
+def compute_latest_release_age_days(
+    latest_date: datetime | None, *, now: datetime | None = None
+) -> int | None:
+    if latest_date is None:
+        return None
+    reference = now or datetime.now(timezone.utc)
+    return days_between(latest_date, reference)
+
+
+def _tag_committed_date(tag: Any) -> datetime | None:
+    commit = getattr(tag, "commit", None) or {}
+    if isinstance(commit, dict):
+        committed_date = commit.get("committed_date")
+    else:
+        committed_date = getattr(commit, "committed_date", None)
+    if not committed_date:
+        return None
+    try:
+        return parse_gitlab_date(str(committed_date))
+    except ValueError:
+        return None
+
+
+def fetch_semver_tag_dates(
+    gl: Any,
+    project_path: str,
+    *,
+    gitlab_url: str | None = None,
+    cache: ReleaseMetadataCache | None = None,
+) -> dict[str, datetime]:
+    """Map semver tag names to commit dates from a single tags.list call."""
+    if cache is not None and gitlab_url:
+        cached = cache.get_gitlab_tag_dates(gitlab_url, project_path)
+        if cached is not None:
+            return cached
+
+    project = gl.projects.get(project_path)
+    tag_dates: dict[str, datetime] = {}
+    for tag in project.tags.list(all=True):
+        if not is_valid_semver_version(tag.name):
+            continue
+        committed_date = _tag_committed_date(tag)
+        if committed_date is not None:
+            tag_dates[tag.name] = committed_date
+
+    if cache is not None and gitlab_url:
+        cache.set_gitlab_tag_dates(gitlab_url, project_path, tag_dates)
+    return tag_dates
+
+
+def _semver_tags_descending(tag_names: list[str]) -> list[str]:
+    valid = [name for name in tag_names if is_valid_semver_version(name)]
+    return sorted(valid, key=lambda value: semver.Version.parse(value), reverse=True)
+
+
+def compute_version_tag_rank(version: str, tags_desc: list[str]) -> int | None:
+    try:
+        return tags_desc.index(version) + 1
+    except ValueError:
+        return None
+
+
+def version_within_latest_tags(version: str, tags_desc: list[str], count: int) -> bool:
+    rank = compute_version_tag_rank(version, tags_desc)
+    if rank is None:
+        return False
+    return rank <= count
+
+
+def _latest_semver_tag(tag_names: list[str]) -> str | None:
+    valid = [name for name in tag_names if is_valid_semver_version(name)]
+    if not valid:
+        return None
+    return max(valid, key=lambda value: semver.Version.parse(value))
+
+
+def fetch_latest_semver_version(gl: Any, project_path: str) -> str | None:
+    tag_dates = fetch_semver_tag_dates(gl, project_path)
+    return _latest_semver_tag(list(tag_dates.keys()))
+
+
+def include_update_available(current_version: str, latest_version: str) -> bool:
+    if not latest_version:
+        return False
+    current = str(current_version or "").strip()
+    if not is_valid_semver_version(current):
+        return is_valid_semver_version(latest_version)
+    if not is_valid_semver_version(latest_version):
+        return False
+    return semver.Version.parse(latest_version) > semver.Version.parse(current_version)
+
+
+def enrich_include_release_metadata(
+    include: dict,
+    *,
+    gitlab_url: str | None,
+    token: str,
+    gl: Any | None = None,
+    cache: ReleaseMetadataCache | None = None,
+) -> dict:
+    enriched = dict(include)
+    include_type = enriched.get("include_type", "")
+    version = str(enriched.get("version", "")).strip()
+    enriched["valid_version"] = is_valid_semver_version(version)
+    enriched["latest_version"] = ""
+    enriched["update_available"] = False
+    enriched["release_metadata_resolved"] = False
+    enriched["version_released_at"] = ""
+    enriched["latest_version_released_at"] = ""
+    enriched["latest_release_age_days"] = None
+    enriched["release_lag_days"] = None
+    enriched["version_tag_rank"] = None
+    enriched["semver_tag_count"] = 0
+
+    project_path = resolve_include_project_path(
+        include_type, enriched.get("project", "")
+    )
+    if not project_path:
+        return enriched
+
+    gitlab_url = (
+        gitlab_url
+        or os.getenv("CI_SERVER_URL")
+        or os.getenv("GITLAB_URL")
+        or "https://gitlab.com"
+    )
+
+    try:
+        if gl is None:
+            import gitlab
+
+            gl = gitlab.Gitlab(gitlab_url, private_token=token)
+            gl.auth()
+        tag_dates = fetch_semver_tag_dates(
+            gl, project_path, gitlab_url=gitlab_url, cache=cache
+        )
+    except Exception:
+        return enriched
+
+    latest_version = _latest_semver_tag(list(tag_dates.keys()))
+    if not latest_version:
+        return enriched
+
+    tags_desc = _semver_tags_descending(list(tag_dates.keys()))
+    pinned_date = tag_dates.get(version) if enriched["valid_version"] else None
+    latest_date = tag_dates.get(latest_version)
+
+    enriched["latest_version"] = latest_version
+    enriched["update_available"] = include_update_available(version, latest_version)
+    enriched["release_metadata_resolved"] = True
+    enriched["semver_tag_count"] = len(tags_desc)
+    if enriched["valid_version"]:
+        enriched["version_tag_rank"] = compute_version_tag_rank(version, tags_desc)
+
+    if pinned_date is not None:
+        enriched["version_released_at"] = pinned_date.isoformat()
+    if latest_date is not None:
+        enriched["latest_version_released_at"] = latest_date.isoformat()
+        enriched["latest_release_age_days"] = compute_latest_release_age_days(
+            latest_date
+        )
+
+    if (
+        enriched["update_available"]
+        and pinned_date is not None
+        and latest_date is not None
+    ):
+        enriched["release_lag_days"] = compute_release_lag_days(
+            pinned_date, latest_date
+        )
+
+    return enriched
+
+
+def enrich_includes_with_releases(
+    includes: list[dict],
+    *,
+    gitlab_url: str | None,
+    token: str,
+    gl: Any | None = None,
+    cache: ReleaseMetadataCache | None = None,
+) -> list[dict]:
+    return [
+        enrich_include_release_metadata(
+            include,
+            gitlab_url=gitlab_url,
+            token=token,
+            gl=gl,
+            cache=cache,
+        )
+        for include in includes
+    ]

src/compliance/metadata.py

@@ -0,0 +1,245 @@
+"""Parse Conftest-style # METADATA annotations from compliance policy files."""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+METADATA_MARKER = "# METADATA"
+
+
+@dataclass
+class PolicyAnnotation:
+    policy_id: str
+    title: str
+    description: str = ""
+    scope: str = ""
+    feature_file: str = ""
+    line: int = 0
+    feature_name: str = ""
+    scenario_name: str = ""
+    custom: dict = field(default_factory=dict)
+
+
+@dataclass
+class FeaturePolicies:
+    feature_file: str
+    feature_name: str
+    annotation: PolicyAnnotation | None = None
+    scenarios: list[PolicyAnnotation] = field(default_factory=list)
+
+
+@dataclass
+class PolicyCatalog:
+    features: list[FeaturePolicies] = field(default_factory=list)
+
+    def lookup_scenario(
+        self, feature_file: str, scenario_name: str
+    ) -> PolicyAnnotation | None:
+        normalized = os.path.realpath(feature_file)
+        for feature in self.features:
+            if os.path.realpath(feature.feature_file) != normalized:
+                continue
+            for scenario in feature.scenarios:
+                if scenario.scenario_name == scenario_name:
+                    return scenario
+        return None
+
+
+def _slug(value: str) -> str:
+    slug = re.sub(r"[^a-zA-Z0-9]+", "-", value).strip("-").upper()
+    return slug or "POLICY"
+
+
+def _auto_feature_id(feature_file: str) -> str:
+    return f"GLCI-{_slug(Path(feature_file).stem)}"
+
+
+def _auto_scenario_id(feature_id: str, index: int) -> str:
+    return f"{feature_id}-{index:03d}"
+
+
+def _quote_yaml_scalar(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _prepare_metadata_yaml_line(line: str) -> str:
+    if line.startswith("  "):
+        return line
+    if ":" not in line:
+        return line
+    key, sep, rest = line.partition(":")
+    value = rest.strip()
+    if not value or value in ("|", ">"):
+        return line
+    if (value.startswith('"') and value.endswith('"')) or (
+        value.startswith("'") and value.endswith("'")
+    ):
+        return line
+    if ":" in value:
+        return f"{key}{sep} {_quote_yaml_scalar(value)}"
+    return line
+
+
+def _parse_metadata_yaml(yaml_lines: list[str]) -> dict:
+    if not yaml_lines:
+        return {}
+    prepared = [_prepare_metadata_yaml_line(line) for line in yaml_lines]
+    parsed = yaml.safe_load("\n".join(prepared))
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def _collect_metadata_block(lines: list[str], start_index: int) -> tuple[dict, int]:
+    yaml_lines: list[str] = []
+    index = start_index + 1
+    while index < len(lines):
+        line = lines[index]
+        if line.startswith("# "):
+            yaml_lines.append(line[2:])
+            index += 1
+            continue
+        break
+    return _parse_metadata_yaml(yaml_lines), index
+
+
+def _annotation_from_raw(
+    raw: dict,
+    *,
+    scope: str,
+    feature_file: str,
+    line: int,
+    feature_name: str,
+    scenario_name: str = "",
+    default_title: str,
+    default_id: str,
+) -> PolicyAnnotation:
+    custom = raw.get("custom", {})
+    if not isinstance(custom, dict):
+        custom = {}
+
+    policy_id = str(custom.get("id") or raw.get("id") or default_id)
+    title = str(raw.get("title") or default_title)
+    description = str(raw.get("description") or "")
+
+    return PolicyAnnotation(
+        policy_id=policy_id,
+        title=title,
+        description=description,
+        scope=scope,
+        feature_file=feature_file,
+        line=line,
+        feature_name=feature_name,
+        scenario_name=scenario_name,
+        custom=custom,
+    )
+
+
+def parse_feature_policies(feature_file: str) -> FeaturePolicies:
+    feature_path = os.path.realpath(feature_file)
+    with open(feature_path, encoding="utf-8") as handle:
+        lines = handle.read().splitlines()
+
+    pending_metadata: dict = {}
+    feature_name = Path(feature_path).stem
+    feature_annotation: PolicyAnnotation | None = None
+    scenarios: list[PolicyAnnotation] = []
+    scenario_index = 0
+
+    index = 0
+    while index < len(lines):
+        line = lines[index]
+        if line.startswith(METADATA_MARKER):
+            pending_metadata, index = _collect_metadata_block(lines, index)
+            continue
+
+        if line.startswith("Feature:"):
+            feature_name = line.split(":", 1)[1].strip() or feature_name
+            feature_id = _auto_feature_id(feature_path)
+            feature_annotation = _annotation_from_raw(
+                pending_metadata,
+                scope="feature",
+                feature_file=feature_path,
+                line=index + 1,
+                feature_name=feature_name,
+                default_title=feature_name,
+                default_id=feature_id,
+            )
+            pending_metadata = {}
+            index += 1
+            continue
+
+        stripped = line.strip()
+        if stripped.startswith("Scenario:"):
+            scenario_name = stripped.split(":", 1)[1].strip()
+            scenario_index += 1
+            feature_id = (
+                feature_annotation.policy_id
+                if feature_annotation
+                else _auto_feature_id(feature_path)
+            )
+            scenarios.append(
+                _annotation_from_raw(
+                    pending_metadata,
+                    scope="scenario",
+                    feature_file=feature_path,
+                    line=index + 1,
+                    feature_name=feature_name,
+                    scenario_name=scenario_name,
+                    default_title=scenario_name,
+                    default_id=_auto_scenario_id(feature_id, scenario_index),
+                )
+            )
+            pending_metadata = {}
+            index += 1
+            continue
+
+        index += 1
+
+    if feature_annotation is None:
+        feature_id = _auto_feature_id(feature_path)
+        feature_annotation = PolicyAnnotation(
+            policy_id=feature_id,
+            title=feature_name,
+            scope="feature",
+            feature_file=feature_path,
+            line=1,
+            feature_name=feature_name,
+        )
+
+    return FeaturePolicies(
+        feature_file=feature_path,
+        feature_name=feature_name,
+        annotation=feature_annotation,
+        scenarios=scenarios,
+    )
+
+
+def build_policy_catalog(features_dir: str) -> PolicyCatalog:
+    catalog = PolicyCatalog()
+    if not os.path.isdir(features_dir):
+        raise FileNotFoundError(f"Features directory not found: {features_dir}")
+
+    feature_files: list[str] = []
+    for root, _dirs, files in os.walk(features_dir):
+        for filename in sorted(files):
+            if filename.endswith(".feature"):
+                feature_files.append(os.path.join(root, filename))
+
+    for feature_file in feature_files:
+        catalog.features.append(parse_feature_policies(feature_file))
+
+    return catalog
+
+
+def collect_policy_index(catalog: PolicyCatalog) -> list[PolicyAnnotation]:
+    policies: list[PolicyAnnotation] = []
+    for feature in catalog.features:
+        if feature.annotation:
+            policies.append(feature.annotation)
+        policies.extend(feature.scenarios)
+    return policies