pyossmtool 0.0.1__py3-none-any.whl

pyossmtool/ignore_materialize.py

@@ -0,0 +1,184 @@
+"""Per-tool ignore pattern materialization driven by ToolDef.ignore."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import yaml
+
+from pyossmtool.ignore import EffectiveIgnores, ToolIgnoreMaterial, _all_patterns, _merge_yaml_config
+from pyossmtool.models import ExtendFrom, IgnoreKind, ToolDef, ToolIgnoreSpec
+
+
+def materialize_for_tool(
+    tool: ToolDef,
+    ignores: EffectiveIgnores,
+    *,
+    project_root: Path,
+    check_id: str,
+    base_config_path: Path | None = None,
+) -> ToolIgnoreMaterial:
+    patterns = _all_patterns(ignores)
+    if not patterns or tool.ignore is None:
+        return ToolIgnoreMaterial()
+
+    cache_dir = project_root / ".pyossmtool" / "cache" / "ignores" / check_id
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return _materialize(tool.ignore, patterns, project_root, cache_dir, base_config_path)
+
+
+def _materialize(
+    spec: ToolIgnoreSpec,
+    patterns: list[str],
+    project_root: Path,
+    cache_dir: Path,
+    base_config_path: Path | None,
+) -> ToolIgnoreMaterial:
+    if spec.kind == IgnoreKind.FLAG_PAIRS:
+        return _flag_pairs(spec, patterns)
+    if spec.kind == IgnoreKind.PREFIX:
+        return _prefix_flags(spec, patterns)
+    if spec.kind == IgnoreKind.SKIP_FILE:
+        return _skip_file(spec, patterns, cache_dir)
+    if spec.kind == IgnoreKind.CONFIG_OVERLAY:
+        return _config_overlay(spec, patterns, project_root, cache_dir, base_config_path)
+    return ToolIgnoreMaterial()
+
+
+def _flag_pairs(spec: ToolIgnoreSpec, patterns: list[str]) -> ToolIgnoreMaterial:
+    if not spec.flag:
+        return ToolIgnoreMaterial()
+    argv: list[str] = []
+    for pattern in patterns:
+        argv.extend([spec.flag, pattern])
+    return ToolIgnoreMaterial(argv=argv, post_subcommand=spec.post_subcommand)
+
+
+def _prefix_flags(spec: ToolIgnoreSpec, patterns: list[str]) -> ToolIgnoreMaterial:
+    if not spec.prefix:
+        return ToolIgnoreMaterial()
+    return ToolIgnoreMaterial(
+        argv=[f"{spec.prefix}{pattern}" for pattern in patterns],
+        post_subcommand=spec.post_subcommand,
+    )
+
+
+def _skip_file(spec: ToolIgnoreSpec, patterns: list[str], cache_dir: Path) -> ToolIgnoreMaterial:
+    filename = spec.file or "ignores.skip"
+    skip_file = cache_dir / filename
+    skip_file.write_text("\n".join(patterns) + "\n", encoding="utf-8")
+    flag = spec.flag or "--skip"
+    return ToolIgnoreMaterial(
+        argv=[flag, str(skip_file)],
+        skip_file=skip_file,
+        post_subcommand=spec.post_subcommand,
+    )
+
+
+def _config_overlay(
+    spec: ToolIgnoreSpec,
+    patterns: list[str],
+    project_root: Path,
+    cache_dir: Path,
+    base_config_path: Path | None,
+) -> ToolIgnoreMaterial:
+    filename = spec.file or f"ignores.{spec.format or 'txt'}"
+    config_path = cache_dir / filename
+    body = _overlay_body(spec, patterns, project_root, base_config_path)
+    if body is None:
+        return ToolIgnoreMaterial()
+    config_path.write_text(body, encoding="utf-8")
+    flag = spec.flag or "--config"
+    return ToolIgnoreMaterial(
+        argv=[flag, str(config_path)],
+        config_path=config_path,
+        post_subcommand=spec.post_subcommand,
+    )
+
+
+def _overlay_body(
+    spec: ToolIgnoreSpec,
+    patterns: list[str],
+    project_root: Path,
+    base_config_path: Path | None,
+) -> str | None:
+    if spec.format == "toml":
+        return _toml_overlay(spec, patterns, project_root, base_config_path)
+    if spec.format == "yaml":
+        return _yaml_overlay(spec, patterns, base_config_path)
+    if spec.format == "json":
+        return _json_overlay(spec, patterns)
+    return None
+
+
+def _toml_overlay(
+    spec: ToolIgnoreSpec,
+    patterns: list[str],
+    project_root: Path,
+    base_config_path: Path | None,
+) -> str:
+    lines: list[str] = []
+    extend_line = _toml_extend_line(spec, project_root, base_config_path)
+    if extend_line:
+        lines.append(extend_line)
+    if spec.toml_table:
+        lines.append(f"[{spec.toml_table}]")
+    key = spec.key or "exclude"
+    lines.append(f"{key} = [")
+    lines.extend(f'  "{pattern}",' for pattern in patterns)
+    lines.append("]")
+    return "\n".join(lines) + "\n"
+
+
+def _toml_extend_line(
+    spec: ToolIgnoreSpec,
+    project_root: Path,
+    base_config_path: Path | None,
+) -> str | None:
+    extend_key = spec.extend_key or "extend"
+    if spec.extend_from == ExtendFrom.BASE_CONFIG:
+        return _extend_base_only(extend_key, base_config_path)
+    if spec.extend_from == ExtendFrom.BASE_OR_PYPROJECT:
+        return _extend_base_or_pyproject(extend_key, project_root, base_config_path)
+    if spec.extend_from == ExtendFrom.PYPROJECT:
+        return _extend_pyproject_list(extend_key, project_root)
+    return None
+
+
+def _extend_base_only(extend_key: str, base_config_path: Path | None) -> str | None:
+    if base_config_path and base_config_path.exists():
+        return f'{extend_key} = "{base_config_path}"'
+    return None
+
+
+def _extend_base_or_pyproject(extend_key: str, project_root: Path, base_config_path: Path | None) -> str | None:
+    base = _extend_base_only(extend_key, base_config_path)
+    if base:
+        return base
+    pyproject = project_root / "pyproject.toml"
+    if pyproject.exists():
+        return f'{extend_key} = "{pyproject}"'
+    return None
+
+
+def _extend_pyproject_list(extend_key: str, project_root: Path) -> str | None:
+    pyproject = project_root / "pyproject.toml"
+    if pyproject.exists():
+        return f'{extend_key} = ["{pyproject}"]'
+    return None
+
+
+def _yaml_overlay(spec: ToolIgnoreSpec, patterns: list[str], base_config_path: Path | None) -> str:
+    key = spec.key or "exclude"
+    if spec.key_style == "block":
+        overlay = {key: "\n".join(patterns) + ("\n" if patterns else "")}
+    else:
+        overlay = {key: patterns}
+    merged = _merge_yaml_config(base_config_path, overlay)
+    return yaml.safe_dump(merged, sort_keys=False)
+
+
+def _json_overlay(spec: ToolIgnoreSpec, patterns: list[str]) -> str:
+    key = spec.key or "ignore"
+    return json.dumps({key: patterns}, indent=2) + "\n"

pyossmtool/installer.py

@@ -0,0 +1,119 @@
+"""Install quality tool dependencies into managed environments."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from pyossmtool.models import EnvMode, InstallMethod, SuiteDef, ToolDef
+from pyossmtool.registry import Registry
+from pyossmtool.resolver import BinaryResolver
+
+
+class Installer:
+    def __init__(self, registry: Registry, project_root: Path | None = None) -> None:
+        self.registry = registry
+        self.project_root = (project_root or Path.cwd()).resolve()
+        self.resolver = BinaryResolver(self.project_root)
+        self.managed_venv = self.project_root / ".pyossmtool" / "tools" / "venv"
+
+    def tools_for_suite(self, suite: SuiteDef) -> list[ToolDef]:
+        tool_ids: set[str] = set()
+        for check_ref in suite.checks:
+            check = self.registry.get_check(check_ref.id)
+            tool_ids.add(check.tool)
+        return [self.registry.get_tool(tool_id) for tool_id in sorted(tool_ids)]
+
+    def install_suite(self, suite_id: str) -> None:
+        suite = self.registry.get_suite(suite_id)
+        for tool in self.tools_for_suite(suite):
+            self.install_tool(tool)
+
+    def install_tool(self, tool: ToolDef) -> None:
+        method = tool.install.method
+        handlers = {
+            InstallMethod.SKIP: lambda _tool: None,
+            InstallMethod.SYSTEM: self._warn_missing_system_tool,
+            InstallMethod.PIP: self._install_pip_tool,
+            InstallMethod.NPM: self._install_npm_tool,
+        }
+        handler = handlers.get(method)
+        if handler is None:
+            raise RuntimeError(f"Unsupported install method: {method}")
+        handler(tool)
+
+    def _install_npm_tool(self, tool: ToolDef) -> None:
+        if self._binary_available(tool):
+            return
+        self._install_npm(tool)
+        self._require_binary(tool)
+
+    def _warn_missing_system_tool(self, tool: ToolDef) -> None:
+        if self.resolver._resolve_system(tool.binary) is not None:
+            return
+        print(
+            f"WARN: system tool '{tool.binary}' not found; install manually to enable related checks",
+            file=sys.stderr,
+        )
+
+    def _install_pip_tool(self, tool: ToolDef) -> None:
+        self._ensure_managed_venv()
+        self._install_pip(tool)
+        self._require_binary(tool)
+
+    def _require_binary(self, tool: ToolDef) -> None:
+        if self._binary_available(tool):
+            return
+        raise RuntimeError(f"Failed to install tool '{tool.id}' ({tool.binary})")
+
+    def _binary_available(self, tool: ToolDef) -> bool:
+        try:
+            self.resolver.resolve(tool, mode=EnvMode.MANAGED)
+            return True
+        except FileNotFoundError:
+            return False
+
+    def _managed_python(self) -> str:
+        if sys.version_info < (3, 14):
+            return sys.executable
+        for candidate in ("python3.13", "python3.12", "python3.11"):
+            found = shutil.which(candidate)
+            if found:
+                return found
+        return sys.executable
+
+    def _ensure_managed_venv(self) -> None:
+        python_bin = self.managed_venv / "bin" / "python"
+        if python_bin.exists():
+            version = subprocess.run(
+                [str(python_bin), "--version"],
+                capture_output=True,
+                text=True,
+                check=True,
+            ).stdout
+            if "3.14" in version:
+                shutil.rmtree(self.managed_venv)
+        if (self.managed_venv / "bin" / "python").exists():
+            return
+        self.managed_venv.parent.mkdir(parents=True, exist_ok=True)
+        subprocess.run([self._managed_python(), "-m", "venv", str(self.managed_venv)], check=True)
+
+    def _install_pip(self, tool: ToolDef) -> None:
+        pip = self.managed_venv / "bin" / "pip"
+        packages = (tool.install.package or tool.id).split()
+        subprocess.run([str(pip), "install", "--upgrade", *packages], check=True)
+
+    def _install_npm(self, tool: ToolDef) -> None:
+        package = tool.install.package or tool.id
+        npm_root = self.project_root / ".pyossmtool" / "tools" / "npm"
+        npm_root.mkdir(parents=True, exist_ok=True)
+        package_json = npm_root / "package.json"
+        if not package_json.exists():
+            package_json.write_text('{"name":"pyossmtool-tools","private":true}\n', encoding="utf-8")
+        subprocess.run(
+            ["npm", "install", "--no-save", "--prefix", str(npm_root), package],
+            check=True,
+            cwd=npm_root,
+        )

pyossmtool/models.py

@@ -0,0 +1,279 @@
+"""Pydantic models for tools, checks, suites, and failure reports."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+SCHEMA_VERSION = "1.0"
+
+
+class InstallMethod(StrEnum):
+    PIP = "pip"
+    NPM = "npm"
+    SYSTEM = "system"
+    SKIP = "skip"
+
+
+class EnvMode(StrEnum):
+    MANAGED = "managed"
+    PROJECT = "project"
+    AUTO = "auto"
+
+
+class Severity(StrEnum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+
+class InstallSpec(BaseModel):
+    method: InstallMethod
+    package: str | None = None
+    version: str | None = None
+
+
+class ToolConfigSpec(BaseModel):
+    """Declarative config discovery for a tool (repo-native vs bundled)."""
+
+    flag: str | None = None
+    pass_flag: bool | None = None
+    bundled: str | None = None
+    bundled_path: str | None = None
+    repo_files: list[str] = Field(default_factory=list)
+    repo_dirs: list[str] = Field(default_factory=list)
+    pyproject: list[str] = Field(default_factory=list)
+    use_pyproject_as_path: bool = False
+
+    def should_pass_flag(self) -> bool:
+        if self.pass_flag is not None:
+            return self.pass_flag
+        return bool(self.flag)
+
+
+class IgnoreKind(StrEnum):
+    FLAG_PAIRS = "flag_pairs"
+    PREFIX = "prefix"
+    SKIP_FILE = "skip_file"
+    CONFIG_OVERLAY = "config_overlay"
+
+
+class ExtendFrom(StrEnum):
+    NONE = "none"
+    BASE_CONFIG = "base_config"
+    BASE_OR_PYPROJECT = "base_or_pyproject"
+    PYPROJECT = "pyproject"
+
+
+class ToolIgnoreSpec(BaseModel):
+    """How this tool consumes unified ignore patterns."""
+
+    kind: IgnoreKind
+    flag: str | None = None
+    prefix: str | None = None
+    post_subcommand: bool = False
+    format: str | None = None
+    file: str | None = None
+    key: str | None = None
+    key_style: str = "list"
+    extend_from: ExtendFrom = ExtendFrom.NONE
+    extend_key: str | None = None
+    toml_table: str | None = None
+
+
+class ToolDef(BaseModel):
+    id: str
+    name: str
+    description: str
+    install: InstallSpec
+    binary: str
+    documentation_url: str | None = None
+    files: list[str] = Field(default_factory=list)
+    config: ToolConfigSpec | None = None
+    ignore: ToolIgnoreSpec | None = None
+
+
+class SuccessCriteria(BaseModel):
+    exit_codes: list[int] = Field(default_factory=lambda: [0])
+
+
+class CheckPolicy(BaseModel):
+    max_complexity_rank: str | None = None
+    max_duplication_percent: float | None = None
+    min_coverage_percent: float | None = None
+    coverage_source: str | None = None
+    min_confidence: int | None = None
+
+
+class ConfigMode(StrEnum):
+    AUTO = "auto"
+    BUNDLED = "bundled"
+    PATHS = "paths"
+
+
+class CheckMode(StrEnum):
+    CHECK = "check"
+    FORMAT = "format"
+
+
+class ConfigSpec(BaseModel):
+    mode: ConfigMode = ConfigMode.AUTO
+    paths: dict[str, str] = Field(default_factory=dict)
+
+
+class IgnoreSpec(BaseModel):
+    """Repo-wide ignore rules merged across suite, project, and per-check layers."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    ignore_profile: list[str] = Field(default_factory=list, alias="ignore-profile")
+    ignore_paths: list[str] = Field(default_factory=list, alias="ignore-paths")
+
+
+class GateConfigSpec(BaseModel):
+    """Declarative policy-config discovery for script gates."""
+
+    bundled: str | None = None
+    project_file: str | None = None
+    allowlist_bundled: str | None = None
+
+
+class CheckDef(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    id: str
+    tool: str
+    name: str
+    description: str
+    argv: list[str] = Field(default_factory=list)
+    parser: str
+    mode: CheckMode = CheckMode.CHECK
+    script: str | None = None
+    include: list[str] = Field(default_factory=list)
+    config: GateConfigSpec | None = None
+    success: SuccessCriteria = Field(default_factory=SuccessCriteria)
+    policy: CheckPolicy | None = None
+    remediation: dict[str, Any] = Field(default_factory=dict)
+    output_file: str | None = None
+    ignore_profile: list[str] = Field(default_factory=list, alias="ignore-profile")
+    ignore_paths: list[str] = Field(default_factory=list, alias="ignore-paths")
+
+    @model_validator(mode="after")
+    def validate_script_gate(self) -> CheckDef:
+        if self.tool == "script" and not self.script:
+            raise ValueError(f"Check '{self.id}' uses tool 'script' but has no script path")
+        return self
+
+
+class SuiteCheckRef(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    id: str
+    target: str | None = None
+    ignore_profile: list[str] = Field(default_factory=list, alias="ignore-profile")
+    ignore_paths: list[str] = Field(default_factory=list, alias="ignore-paths")
+
+
+class SuiteDef(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    id: str
+    name: str
+    description: str
+    env: EnvMode = EnvMode.AUTO
+    checks: list[SuiteCheckRef]
+    target: str = "."
+    ignore_profile: list[str] = Field(default_factory=list, alias="ignore-profile")
+    ignore_paths: list[str] = Field(default_factory=list, alias="ignore-paths")
+
+
+class ProjectConfig(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    suite: str
+    env: EnvMode = EnvMode.AUTO
+    target: str = "."
+    checks: list[SuiteCheckRef] = Field(default_factory=list)
+    configs: ConfigSpec = Field(default_factory=ConfigSpec)
+    ignore_profile: list[str] = Field(default_factory=list, alias="ignore-profile")
+    ignore_paths: list[str] = Field(default_factory=list, alias="ignore-paths")
+
+
+class Location(BaseModel):
+    file: str
+    line: int | None = None
+    column: int | None = None
+    end_line: int | None = None
+    end_column: int | None = None
+
+
+class FixHint(BaseModel):
+    description: str | None = None
+    command: str | None = None
+
+
+class Finding(BaseModel):
+    rule_id: str
+    severity: Severity
+    message: str
+    location: Location | None = None
+    snippet: str | None = None
+    fix: FixHint | None = None
+
+
+class FailureSummary(BaseModel):
+    finding_count: int
+    by_severity: dict[str, int] = Field(default_factory=dict)
+
+
+class Remediation(BaseModel):
+    docs_url: str | None = None
+    suggested_commands: list[str] = Field(default_factory=list)
+
+
+class FailureArtifacts(BaseModel):
+    raw_stdout: str | None = None
+    raw_stderr: str | None = None
+
+
+class FailureReport(BaseModel):
+    schema_version: str = SCHEMA_VERSION
+    check_id: str
+    tool_id: str
+    status: str = "failed"
+    suite_id: str | None = None
+    target: str
+    started_at: datetime
+    duration_ms: int
+    summary: FailureSummary
+    findings: list[Finding]
+    remediation: Remediation = Field(default_factory=Remediation)
+    artifacts: FailureArtifacts = Field(default_factory=FailureArtifacts)
+
+
+class CheckResult(BaseModel):
+    check_id: str
+    passed: bool
+    report_path: str | None = None
+    error: str | None = None
+
+
+class SuiteResult(BaseModel):
+    suite_id: str
+    passed: bool
+    results: list[CheckResult] = Field(default_factory=list)
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+def summarize_findings(findings: list[Finding]) -> FailureSummary:
+    by_severity: dict[str, int] = {}
+    for finding in findings:
+        key = finding.severity.value
+        by_severity[key] = by_severity.get(key, 0) + 1
+    return FailureSummary(finding_count=len(findings), by_severity=by_severity)

pyossmtool/parsers/__init__.py

@@ -0,0 +1,76 @@
+"""Parse native tool output into normalized Finding objects."""
+
+from __future__ import annotations
+
+from pyossmtool.models import CheckDef, Finding
+from pyossmtool.parsers.analysis import (
+    BanditParser,
+    DeadcodeParser,
+    JscpdParser,
+    PydepsCyclesParser,
+    PytestParser,
+    RadonCcParser,
+    RadonMiParser,
+    SemgrepParser,
+    TyParser,
+    VultureParser,
+)
+from pyossmtool.parsers.base import REGISTRY
+from pyossmtool.parsers.format_text import MdformatParser, YamlfmtParser
+from pyossmtool.parsers.gates import CliTextParser, GateJsonParser, NoopParser, ScriptTextParser
+from pyossmtool.parsers.prose import (
+    CodespellParser,
+    GitleaksParser,
+    HadolintParser,
+    MarkdownlintParser,
+    MutmutParser,
+    SourceryParser,
+    YamllintParser,
+)
+from pyossmtool.parsers.ruff import RuffFormatParser, RuffJsonParser
+from pyossmtool.parsers.shell import ShellcheckParser, ShfmtDiffParser
+
+# Keep registered classes referenced for static dead-code analysis.
+_PARSER_CLASSES = (
+    BanditParser,
+    CliTextParser,
+    CodespellParser,
+    DeadcodeParser,
+    GateJsonParser,
+    GitleaksParser,
+    HadolintParser,
+    JscpdParser,
+    MarkdownlintParser,
+    MdformatParser,
+    MutmutParser,
+    NoopParser,
+    PydepsCyclesParser,
+    PytestParser,
+    RadonCcParser,
+    RadonMiParser,
+    RuffFormatParser,
+    RuffJsonParser,
+    ScriptTextParser,
+    SemgrepParser,
+    ShellcheckParser,
+    ShfmtDiffParser,
+    SourceryParser,
+    TyParser,
+    VultureParser,
+    YamllintParser,
+    YamlfmtParser,
+)
+
+
+def parse_output(check: CheckDef, stdout: str, stderr: str) -> list[Finding]:
+    try:
+        parser_cls = REGISTRY[check.parser]
+    except KeyError as exc:
+        raise ValueError(f"Unknown parser: {check.parser}") from exc
+    parser = parser_cls()
+    if parser_cls.needs_check:
+        return parser.parse(stdout, stderr, check=check)
+    return parser.parse(stdout, stderr)
+
+
+__all__ = ["REGISTRY", "parse_output", "_PARSER_CLASSES"]