pyossmtool 0.0.1__py3-none-any.whl

pyossmtool/policy/folder_breadth.py

@@ -0,0 +1,306 @@
+"""Folder breadth policy gate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from pyossmtool.ignore import EffectiveIgnores, ignores_from_env
+
+
+@dataclass(frozen=True, slots=True)
+class DirBreadthViolation:
+    path: str
+    count: int
+    max_allowed: int
+
+
+@dataclass(frozen=True, slots=True)
+class FolderBreadthReport:
+    max_allowed: int
+    scan_roots: tuple[str, ...]
+    extensions: tuple[str, ...]
+    leaf_dirs_scanned: int
+    leaf_dirs_over_max: int
+    worst_leaf_dir: str
+    worst_leaf_count: int
+    violations: tuple[DirBreadthViolation, ...]
+
+    def report_metadata(self) -> dict[str, int | str]:
+        return {
+            "leaf_dirs_scanned": self.leaf_dirs_scanned,
+            "worst_leaf_dir": self.worst_leaf_dir,
+            "worst_leaf_count": self.worst_leaf_count,
+        }
+
+
+SKIP_DIR_NAMES = frozenset({"__pycache__"})
+
+
+def load_allowlist(path: Path) -> set[str]:
+    if not path.is_file():
+        return set()
+    entries: set[str] = set()
+    for line in path.read_text(encoding="utf-8").splitlines():
+        stripped = line.split("#", 1)[0].strip()
+        if stripped:
+            entries.add(stripped.rstrip("/"))
+    return entries
+
+
+def settings_from_config(
+    config: dict[str, Any],
+) -> tuple[int, tuple[str, ...], tuple[str, ...], bool]:
+    max_allowed = int(config.get("max_allowed", 12))
+    scan_roots = tuple(str(item) for item in config.get("scan_roots", ["src/"]))
+    extensions_raw = config.get("extensions", [".py", ".md", ".yaml", ".sh"])
+    extensions: list[str] = []
+    for part in extensions_raw:
+        stripped = str(part).strip()
+        if not stripped:
+            continue
+        extensions.append(stripped if stripped.startswith(".") else f".{stripped}")
+    strict = bool(config.get("strict", True))
+    return max_allowed, scan_roots, tuple(extensions), strict
+
+
+def iter_scan_directories(base: Path) -> list[Path]:
+    if not base.is_dir():
+        return []
+    directories = [base]
+    for path in sorted(base.rglob("*")):
+        if path.is_dir() and path.name not in SKIP_DIR_NAMES:
+            directories.append(path)
+    return directories
+
+
+def count_direct_files(directory: Path, extensions: tuple[str, ...]) -> int:
+    count = 0
+    for child in directory.iterdir():
+        if not child.is_file():
+            continue
+        if child.name == "__init__.py":
+            continue
+        if child.suffix in extensions:
+            count += 1
+    return count
+
+
+def is_allowlisted(rel_posix: str, allowlist: set[str]) -> bool:
+    normalized = rel_posix.rstrip("/")
+    if normalized in allowlist:
+        return True
+    return any(normalized.startswith(f"{entry}/") for entry in allowlist)
+
+
+def scan_folder_breadth(
+    root: Path,
+    *,
+    max_allowed: int,
+    scan_roots: tuple[str, ...],
+    extensions: tuple[str, ...],
+    allowlist: set[str],
+    ignores: EffectiveIgnores | None = None,
+) -> FolderBreadthReport:
+    violations: list[DirBreadthViolation] = []
+    dirs_scanned = 0
+    worst_path = ""
+    worst_count = 0
+
+    for scan_root in scan_roots:
+        scanned, worst_path, worst_count, root_violations = _scan_root_breadth(
+            root,
+            scan_root,
+            max_allowed=max_allowed,
+            extensions=extensions,
+            allowlist=allowlist,
+            ignores=ignores,
+            worst_path=worst_path,
+            worst_count=worst_count,
+        )
+        dirs_scanned += scanned
+        violations.extend(root_violations)
+
+    violations.sort(key=lambda item: (-item.count, item.path))
+    return FolderBreadthReport(
+        max_allowed=max_allowed,
+        scan_roots=scan_roots,
+        extensions=extensions,
+        leaf_dirs_scanned=dirs_scanned,
+        leaf_dirs_over_max=len(violations),
+        worst_leaf_dir=worst_path,
+        worst_leaf_count=worst_count,
+        violations=tuple(violations),
+    )
+
+
+def _scan_root_breadth(
+    root: Path,
+    scan_root: str,
+    *,
+    max_allowed: int,
+    extensions: tuple[str, ...],
+    allowlist: set[str],
+    ignores: EffectiveIgnores | None,
+    worst_path: str,
+    worst_count: int,
+) -> tuple[int, str, int, list[DirBreadthViolation]]:
+    violations: list[DirBreadthViolation] = []
+    dirs_scanned = 0
+    base = root / scan_root
+    for directory in iter_scan_directories(base):
+        outcome = _directory_breadth_outcome(
+            directory,
+            root,
+            max_allowed=max_allowed,
+            extensions=extensions,
+            allowlist=allowlist,
+            ignores=ignores,
+        )
+        if outcome is None:
+            continue
+        violation, file_count, rel = outcome
+        dirs_scanned += 1
+        if file_count > worst_count:
+            worst_count = file_count
+            worst_path = rel
+        if violation is not None:
+            violations.append(violation)
+    return dirs_scanned, worst_path, worst_count, violations
+
+
+def _directory_breadth_outcome(
+    directory: Path,
+    root: Path,
+    *,
+    max_allowed: int,
+    extensions: tuple[str, ...],
+    allowlist: set[str],
+    ignores: EffectiveIgnores | None,
+) -> tuple[DirBreadthViolation | None, int, str] | None:
+    rel = directory.relative_to(root).as_posix()
+    if _should_skip_directory(rel, allowlist, ignores):
+        return None
+    file_count = count_direct_files(directory, extensions)
+    if file_count == 0:
+        return None
+    violation = _breadth_violation(rel, file_count, max_allowed)
+    return violation, file_count, rel
+
+
+def _should_skip_directory(rel: str, allowlist: set[str], ignores: EffectiveIgnores | None) -> bool:
+    if is_allowlisted(rel, allowlist):
+        return True
+    return bool(ignores and ignores.is_ignored(rel))
+
+
+def _breadth_violation(rel: str, file_count: int, max_allowed: int) -> DirBreadthViolation | None:
+    if file_count <= max_allowed:
+        return None
+    return DirBreadthViolation(path=rel, count=file_count, max_allowed=max_allowed)
+
+
+def findings_from_report(report: FolderBreadthReport) -> list[dict[str, Any]]:
+    findings: list[dict[str, Any]] = []
+    for violation in report.violations:
+        findings.append(
+            {
+                "rule_id": "folder-breadth",
+                "severity": "error",
+                "message": (f"{violation.path} has {violation.count} sibling files (max {violation.max_allowed})"),
+                "location": {"file": violation.path},
+            }
+        )
+    return findings
+
+
+def run_gate(
+    *,
+    root: Path,
+    config: dict[str, Any],
+    allowlist_path: Path | None,
+    report_path: Path | None,
+    strict: bool | None = None,
+) -> int:
+    max_allowed, scan_roots, extensions, config_strict = settings_from_config(config)
+    enforcing = config_strict if strict is None else strict
+    report = scan_folder_breadth(
+        root,
+        max_allowed=max_allowed,
+        scan_roots=scan_roots,
+        extensions=extensions,
+        allowlist=load_allowlist(allowlist_path or Path()),
+        ignores=ignores_from_env(root),
+    )
+    _write_folder_breadth_report(report, report_path)
+    return _folder_breadth_exit(report, enforcing)
+
+
+def _write_folder_breadth_report(report: FolderBreadthReport, report_path: Path | None) -> None:
+    if not report_path:
+        return
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {"findings": findings_from_report(report), **report.report_metadata()}
+    report_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+
+def _folder_breadth_exit(report: FolderBreadthReport, enforcing: bool) -> int:
+    if not enforcing or report.leaf_dirs_over_max == 0:
+        return 0
+    for violation in report.violations:
+        print(
+            f"FAIL folder-breadth: {violation.path} has {violation.count} files (max {violation.max_allowed})",
+            file=sys.stderr,
+        )
+    return 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_folder_breadth_args(argv)
+    root = args.root.resolve()
+    config = _load_gate_mapping(args.config)
+    return run_gate(
+        root=root,
+        config=config,
+        allowlist_path=_resolve_config_allowlist(root, config),
+        report_path=args.report,
+        strict=False if args.advisory else args.strict,
+    )
+
+
+def _parse_folder_breadth_args(argv: list[str] | None):
+    parser = argparse.ArgumentParser(description="Folder breadth gate.")
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    parser.add_argument("--config", type=Path, required=True)
+    parser.add_argument("--report", type=Path, default=None)
+    parser.add_argument("--strict", action="store_true", default=None)
+    parser.add_argument("--advisory", action="store_true", default=False)
+    return parser.parse_args(argv)
+
+
+def _load_gate_mapping(config_path: Path) -> dict[str, Any]:
+    import yaml
+
+    config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+    if not isinstance(config, dict):
+        msg = f"Gate config must be a mapping: {config_path}"
+        raise ValueError(msg)
+    return config
+
+
+def _resolve_config_allowlist(root: Path, config: dict[str, Any]) -> Path | None:
+    allowlist_file = config.get("allowlist_file")
+    if not allowlist_file:
+        return None
+    allowlist_path = Path(str(allowlist_file))
+    if not allowlist_path.is_absolute():
+        allowlist_path = root / allowlist_path
+    return allowlist_path
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyossmtool/registry.py

@@ -0,0 +1,72 @@
+"""Load and validate catalog definitions."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from pyossmtool.models import CheckDef, ProjectConfig, SuiteDef, ToolDef
+
+BUNDLE_ROOT = Path(__file__).resolve().parent / "bundle"
+PROJECT_CATALOG = ".pyossmtool/catalog"
+PROJECT_SUITES = ".pyossmtool/suites"
+
+
+class Registry:
+    def __init__(self, root: Path | None = None, project_root: Path | None = None) -> None:
+        self.root = root or BUNDLE_ROOT
+        self.project_root = (project_root or Path.cwd()).resolve()
+        self.catalog_root = self.root / "catalog"
+        self.suites_root = self.root / "suites"
+        self.tools: dict[str, ToolDef] = {}
+        self.checks: dict[str, CheckDef] = {}
+        self.suites: dict[str, SuiteDef] = {}
+        self._load()
+
+    def _load_yaml_dir(self, directory: Path) -> dict[str, dict]:
+        items: dict[str, dict] = {}
+        if not directory.exists():
+            return items
+        for path in sorted(directory.glob("*.yaml")):
+            with path.open(encoding="utf-8") as handle:
+                data = yaml.safe_load(handle) or {}
+            items[data["id"]] = data
+        return items
+
+    def _load(self) -> None:
+        self._load_catalog_tree(self.catalog_root, self.suites_root)
+        project_catalog = self.project_root / PROJECT_CATALOG
+        self._load_catalog_tree(project_catalog, self.project_root / PROJECT_SUITES)
+
+    def _load_catalog_tree(self, catalog_root: Path, suites_root: Path) -> None:
+        for tool_id, data in self._load_yaml_dir(catalog_root / "tools").items():
+            self.tools[tool_id] = ToolDef.model_validate(data)
+        for check_id, data in self._load_yaml_dir(catalog_root / "checks").items():
+            self.checks[check_id] = CheckDef.model_validate(data)
+        for suite_id, data in self._load_yaml_dir(suites_root).items():
+            self.suites[suite_id] = SuiteDef.model_validate(data)
+
+    def get_tool(self, tool_id: str) -> ToolDef:
+        if tool_id not in self.tools:
+            raise KeyError(f"Unknown tool: {tool_id}")
+        return self.tools[tool_id]
+
+    def get_check(self, check_id: str) -> CheckDef:
+        if check_id not in self.checks:
+            raise KeyError(f"Unknown check: {check_id}")
+        return self.checks[check_id]
+
+    def get_suite(self, suite_id: str) -> SuiteDef:
+        if suite_id not in self.suites:
+            raise KeyError(f"Unknown suite: {suite_id}")
+        return self.suites[suite_id]
+
+    def load_project_config(self, project_root: Path | None = None) -> ProjectConfig | None:
+        root = project_root or self.project_root
+        config_path = root / "pyossmtool.yaml"
+        if not config_path.exists():
+            return None
+        with config_path.open(encoding="utf-8") as handle:
+            data = yaml.safe_load(handle) or {}
+        return ProjectConfig.model_validate(data)

pyossmtool/reporter.py

@@ -0,0 +1,87 @@
+"""Write normalized failure reports."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from pathlib import Path
+
+from pyossmtool.models import (
+    CheckDef,
+    FailureArtifacts,
+    FailureReport,
+    Finding,
+    Remediation,
+    ToolDef,
+    summarize_findings,
+)
+
+
+class Reporter:
+    def __init__(self, project_root: Path | None = None) -> None:
+        self.project_root = (project_root or Path.cwd()).resolve()
+        self.failures_dir = self.project_root / "reports" / "failures"
+
+    def write_failure(
+        self,
+        *,
+        check: CheckDef,
+        tool: ToolDef,
+        suite_id: str | None,
+        target: str,
+        started_at: datetime,
+        duration_ms: int,
+        findings: list[Finding],
+        stdout: str,
+        stderr: str,
+    ) -> Path:
+        self.failures_dir.mkdir(parents=True, exist_ok=True)
+        timestamp = started_at.strftime("%Y%m%dT%H%M%SZ")
+        report_dir = self.failures_dir / f"{check.id}-{timestamp}"
+        report_dir.mkdir(parents=True, exist_ok=True)
+
+        raw_stdout_path = report_dir / "raw.stdout.txt"
+        raw_stderr_path = report_dir / "raw.stderr.txt"
+        raw_stdout_path.write_text(stdout, encoding="utf-8")
+        raw_stderr_path.write_text(stderr, encoding="utf-8")
+
+        cov_target = check.policy.coverage_source if check.policy and check.policy.coverage_source else "src/"
+        remediation = Remediation(
+            docs_url=tool.documentation_url,
+            suggested_commands=[
+                cmd.format(target=target, cov=cov_target, binary=tool.binary)
+                for cmd in check.remediation.get("suggested_commands", [])
+            ],
+        )
+        if not remediation.suggested_commands:
+            remediation.suggested_commands = self._fallback_commands(check, tool, target)
+
+        report = FailureReport(
+            check_id=check.id,
+            tool_id=tool.id,
+            suite_id=suite_id,
+            target=target,
+            started_at=started_at,
+            duration_ms=duration_ms,
+            summary=summarize_findings(findings),
+            findings=findings,
+            remediation=remediation,
+            artifacts=FailureArtifacts(
+                raw_stdout=str(raw_stdout_path.relative_to(self.project_root)),
+                raw_stderr=str(raw_stderr_path.relative_to(self.project_root)),
+            ),
+        )
+
+        report_path = report_dir / "report.json"
+        report_path.write_text(
+            json.dumps(report.model_dump(mode="json"), indent=2) + "\n",
+            encoding="utf-8",
+        )
+        return report_path
+
+    def _fallback_commands(self, check: CheckDef, tool: ToolDef, target: str) -> list[str]:
+        if check.tool == "script" and check.script:
+            if check.script.startswith("bundled:"):
+                return [f"pyossmtool run --check {check.id} --target {target}"]
+            return [f"./{check.script}"]
+        return [f"{tool.binary} {target}"]

pyossmtool/resolver.py

@@ -0,0 +1,146 @@
+"""Hybrid binary resolution for quality tools."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import tomllib
+from pathlib import Path
+
+from pyossmtool.models import EnvMode, ToolDef
+
+
+class BinaryResolver:
+    def __init__(self, project_root: Path | None = None) -> None:
+        self.project_root = (project_root or Path.cwd()).resolve()
+        self.cache_root = Path.home() / ".cache" / "pyossmtool" / "tools"
+        self.local_tools_root = self.project_root / ".pyossmtool" / "tools"
+
+    def resolve_env_mode(self, mode: EnvMode) -> EnvMode:
+        if mode != EnvMode.AUTO:
+            return mode
+        if self._project_venv_exists():
+            return EnvMode.PROJECT
+        return EnvMode.MANAGED
+
+    def _project_venv_exists(self) -> bool:
+        return (self.project_root / "pyproject.toml").exists() and (
+            (self.project_root / ".venv").exists() or (self.project_root / "venv").exists()
+        )
+
+    def _venv_bin_dirs(self) -> list[Path]:
+        candidates = [
+            self.project_root / ".pyossmtool" / "tools" / "venv" / "bin",
+            self.project_root / ".pyossmtool" / "tools" / "npm" / "node_modules" / ".bin",
+            self.project_root / ".venv" / "bin",
+            self.project_root / "venv" / "bin",
+            self.local_tools_root / "bin",
+            self.cache_root / "bin",
+        ]
+        return [path for path in candidates if path.is_dir()]
+
+    def _tool_in_project_deps(self, tool: ToolDef) -> bool:
+        pyproject = self.project_root / "pyproject.toml"
+        if not pyproject.exists() or tool.install.package is None:
+            return False
+        package_name = tool.install.package.split(">=")[0].split("==")[0].strip()
+        return package_name in self._project_dependency_text(pyproject)
+
+    def _project_dependency_text(self, pyproject: Path) -> str:
+        text = pyproject.read_text(encoding="utf-8")
+        try:
+            data = tomllib.loads(text)
+        except tomllib.TOMLDecodeError:
+            return text
+        deps = list(data.get("project", {}).get("dependencies", []))
+        for group in data.get("dependency-groups", {}).values():
+            if isinstance(group, list):
+                deps.extend(group)
+        return " ".join(deps)
+
+    def resolve(self, tool: ToolDef, mode: EnvMode) -> Path:
+        resolved_mode = self.resolve_env_mode(mode)
+        for candidate in self._resolution_candidates(tool, resolved_mode):
+            if candidate is not None:
+                return candidate
+        raise FileNotFoundError(
+            f"Could not resolve binary '{tool.binary}' for tool '{tool.id}' (mode={resolved_mode.value})"
+        )
+
+    def _resolution_candidates(self, tool: ToolDef, mode: EnvMode):
+        yield self._resolve_for_mode(tool, mode)
+        yield self._resolve_managed(tool)
+        yield self._resolve_which(tool.binary)
+        if tool.install.method.value == "system":
+            yield self._resolve_system(tool.binary)
+
+    def _resolve_for_mode(self, tool: ToolDef, mode: EnvMode) -> Path | None:
+        if mode == EnvMode.PROJECT:
+            return self._resolve_project_paths(tool)
+        if mode == EnvMode.AUTO and self._tool_in_project_deps(tool):
+            return self._resolve_from_path(tool.binary)
+        return None
+
+    def _resolve_project_paths(self, tool: ToolDef) -> Path | None:
+        binary = self._resolve_from_path(tool.binary)
+        if binary is not None:
+            return binary
+        if tool.install.method.value == "npm":
+            return self._resolve_npm(tool.binary)
+        return None
+
+    def _resolve_which(self, binary_name: str) -> Path | None:
+        found = shutil.which(binary_name)
+        return Path(found) if found else None
+
+    def _resolve_from_path(self, binary_name: str) -> Path | None:
+        for bin_dir in self._venv_bin_dirs():
+            candidate = bin_dir / binary_name
+            if candidate.exists():
+                return candidate
+        found = shutil.which(binary_name)
+        if found:
+            return Path(found)
+        return None
+
+    def _resolve_npm(self, binary_name: str) -> Path | None:
+        managed = self.project_root / ".pyossmtool" / "tools" / "npm" / "node_modules" / ".bin" / binary_name
+        if managed.exists():
+            return managed
+        local = self.project_root / "node_modules" / ".bin" / binary_name
+        if local.exists():
+            return local
+        return self._resolve_from_path(binary_name)
+
+    def _resolve_managed(self, tool: ToolDef) -> Path | None:
+        managed_venv_bin = self.project_root / ".pyossmtool" / "tools" / "venv" / "bin"
+        if managed_venv_bin.is_dir():
+            candidate = managed_venv_bin / tool.binary
+            if candidate.exists():
+                return candidate
+        if tool.install.method.value == "npm":
+            npm_bin = self._resolve_npm(tool.binary)
+            if npm_bin is not None:
+                return npm_bin
+        for bin_dir in [self.local_tools_root / "bin", self.cache_root / "bin"]:
+            candidate = bin_dir / tool.binary
+            if candidate.exists():
+                return candidate
+        return None
+
+    def _resolve_system(self, binary_name: str) -> Path | None:
+        found = shutil.which(binary_name)
+        if found:
+            return Path(found)
+        trunk_glob = Path.home() / ".cache" / "trunk" / "repos"
+        if trunk_glob.exists():
+            matches = sorted(trunk_glob.glob(f"*/tools/{binary_name}"))
+            if matches:
+                return matches[-1]
+        return None
+
+    def prepend_managed_path(self, env: dict[str, str] | None = None) -> dict[str, str]:
+        merged = dict(env or os.environ)
+        paths = [str(path) for path in self._venv_bin_dirs()]
+        merged["PATH"] = os.pathsep.join(paths + [merged.get("PATH", "")])
+        return merged