pyossmtool 0.0.1__py3-none-any.whl

pyossmtool/parsers/prose.py

@@ -0,0 +1,193 @@
+"""Prose and misc linter parsers."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, ClassVar
+
+from pyossmtool.models import CheckDef, Finding, Location, Severity
+from pyossmtool.parsers.base import Parser, register
+from pyossmtool.parsers.gates import CliTextParser
+from pyossmtool.parsers.patterns import FallbackTextParser, LineRegexParser
+
+
+def _load_json_or_none(payload_text: str) -> Any:
+    try:
+        return json.loads(payload_text)
+    except json.JSONDecodeError:
+        return None
+
+
+@register
+class GitleaksParser(Parser):
+    id = "gitleaks_json"
+
+    def parse(self, stdout: str, stderr: str = "", *, check: CheckDef | None = None) -> list[Finding]:
+        payload_text = stdout.strip() or stderr.strip()
+        if not payload_text:
+            return []
+        try:
+            payload = json.loads(payload_text)
+        except json.JSONDecodeError:
+            return CliTextParser().parse(stdout, stderr)
+        return [self.map_item(item) for item in payload]
+
+    def map_item(self, item: dict) -> Finding:
+        return Finding(
+            rule_id=item.get("RuleID", "gitleaks"),
+            severity=Severity.ERROR,
+            message=f"Secret detected: {item.get('Description', 'potential leak')}",
+            location=Location(
+                file=item.get("File", ""),
+                line=item.get("StartLine"),
+                end_line=item.get("EndLine"),
+            ),
+        )
+
+
+@register
+class CodespellParser(LineRegexParser):
+    id = "codespell_text"
+    pattern: ClassVar[re.Pattern[str]] = re.compile(
+        r"^(?P<file>[^:]+):(?P<line>\d+):\s*(?P<wrong>[^=]+)\s*==>\s*(?P<right>[^\s]+)"
+    )
+
+    def map_match(self, match: re.Match[str]) -> Finding:
+        return Finding(
+            rule_id="spelling",
+            severity=Severity.ERROR,
+            message=f"{match.group('wrong').strip()} -> {match.group('right').strip()}",
+            location=Location(file=match.group("file"), line=int(match.group("line"))),
+        )
+
+
+@register
+class MarkdownlintParser(Parser):
+    id = "markdownlint_json"
+
+    def parse(self, stdout: str, stderr: str = "", *, check: CheckDef | None = None) -> list[Finding]:
+        payload_text = stdout.strip() or stderr.strip()
+        if not payload_text:
+            return []
+        payload = _load_json_or_none(payload_text)
+        if payload is None:
+            return CliTextParser().parse(stdout, stderr)
+        return self._from_payload(payload)
+
+    def _from_payload(self, payload: Any) -> list[Finding]:
+        if isinstance(payload, list):
+            return self._list_findings(payload)
+        if isinstance(payload, dict):
+            return self._dict_findings(payload)
+        return []
+
+    def _list_findings(self, payload: list) -> list[Finding]:
+        findings: list[Finding] = []
+        for item in payload:
+            if isinstance(item, dict):
+                file_path = item.get("fileName", item.get("file", ""))
+                findings.append(self._item_finding(item, file_path))
+        return findings
+
+    def _dict_findings(self, payload: dict) -> list[Finding]:
+        findings: list[Finding] = []
+        for file_path, items in payload.items():
+            for item in items:
+                findings.append(self._item_finding(item, file_path))
+        return findings
+
+    def _item_finding(self, item: dict, file_path: str) -> Finding:
+        rule_names = item.get("ruleNames")
+        rule_id = rule_names[0] if rule_names else "markdownlint"
+        return Finding(
+            rule_id=rule_id,
+            severity=Severity.ERROR,
+            message=item.get("ruleDescription", item.get("ruleInformation", "markdownlint finding")),
+            location=Location(
+                file=file_path,
+                line=item.get("lineNumber"),
+                column=item.get("columnNumber"),
+            ),
+        )
+
+
+@register
+class YamllintParser(LineRegexParser):
+    id = "yamllint_text"
+    pattern: ClassVar[re.Pattern[str]] = re.compile(
+        r"^(?P<file>[^:]+):(?P<line>\d+):(?P<col>\d+):\s*"
+        r"\[(?P<severity>error|warning)\]\s*(?P<message>.+)$"
+    )
+
+    def map_match(self, match: re.Match[str]) -> Finding | None:
+        if match.group("severity") != "error":
+            return None
+        return Finding(
+            rule_id="yamllint",
+            severity=Severity.ERROR,
+            message=match.group("message").strip(),
+            location=Location(
+                file=match.group("file"),
+                line=int(match.group("line")),
+                column=int(match.group("col")),
+            ),
+        )
+
+
+@register
+class HadolintParser(Parser):
+    id = "hadolint_json"
+
+    def parse(self, stdout: str, stderr: str = "", *, check: CheckDef | None = None) -> list[Finding]:
+        payload_text = stdout.strip() or stderr.strip()
+        if not payload_text:
+            return []
+        payload = _load_json_or_none(payload_text)
+        if payload is None:
+            return CliTextParser().parse(stdout, stderr)
+        return [self.map_item(item) for item in payload]
+
+    def map_item(self, item: dict) -> Finding:
+        return Finding(
+            rule_id=item.get("code", "hadolint"),
+            severity=Severity.ERROR if item.get("level") == "error" else Severity.WARNING,
+            message=item.get("message", "hadolint finding"),
+            location=Location(file=item.get("file", ""), line=item.get("line")),
+        )
+
+
+@register
+class MutmutParser(FallbackTextParser):
+    id = "mutmut_text"
+
+    def parse(self, stdout: str, stderr: str = "", *, check: CheckDef | None = None) -> list[Finding]:
+        text = (stdout or stderr).strip()
+        if not text or self._all_passed(text):
+            return []
+        return CliTextParser().parse(stdout, stderr)
+
+    def _all_passed(self, text: str) -> bool:
+        lowered = text.lower()
+        return "passed" in lowered and "failed" not in lowered
+
+
+@register
+class SourceryParser(FallbackTextParser):
+    id = "sourcery_text"
+
+    def parse(self, stdout: str, stderr: str = "", *, check: CheckDef | None = None) -> list[Finding]:
+        text = (stdout or stderr).strip()
+        if not text:
+            return []
+        return [
+            Finding(rule_id="sourcery", severity=Severity.WARNING, message=message)
+            for line in text.splitlines()
+            if (message := self._message(line)) is not None
+        ]
+
+    def _message(self, line: str) -> str | None:
+        stripped = line.strip()
+        if not stripped or stripped.lower().startswith("reviewing"):
+            return None
+        return stripped

pyossmtool/parsers/ruff.py

@@ -0,0 +1,103 @@
+"""Ruff parsers."""
+
+from __future__ import annotations
+
+from pyossmtool.models import CheckDef, Finding, FixHint, Location, Severity
+from pyossmtool.parsers.base import register
+from pyossmtool.parsers.common import strip_ansi
+from pyossmtool.parsers.patterns import DiffTextParser, JsonListParser
+
+
+@register
+class RuffJsonParser(JsonListParser):
+    id = "ruff_json"
+
+    def map_item(self, item: dict) -> Finding:
+        filename = item.get("filename")
+        location = None
+        if filename:
+            location = Location(
+                file=filename,
+                line=item.get("location", {}).get("row"),
+                column=item.get("location", {}).get("column"),
+                end_line=item.get("end_location", {}).get("row"),
+                end_column=item.get("end_location", {}).get("column"),
+            )
+        code = item.get("code", "ruff")
+        return Finding(
+            rule_id=code,
+            severity=Severity.ERROR if str(code).startswith("E") else Severity.WARNING,
+            message=item.get("message", "ruff finding"),
+            location=location,
+            snippet=item.get("message"),
+            fix=FixHint(
+                description="Apply ruff auto-fix",
+                command=f"ruff check --fix {filename}",
+            )
+            if filename
+            else None,
+        )
+
+
+@register
+class RuffFormatParser(DiffTextParser):
+    id = "ruff_format_text"
+
+    def parse(self, stdout: str, stderr: str = "", *, check: CheckDef | None = None) -> list[Finding]:
+        text = strip_ansi(stdout or stderr)
+        if not text.strip():
+            return []
+        findings, _current_file = self._scan_lines(text.splitlines())
+        if findings:
+            return findings
+        if "reformatted" not in text.lower():
+            return []
+        return [
+            Finding(
+                rule_id="format",
+                severity=Severity.ERROR,
+                message="Formatting differences detected",
+                snippet=text[:500],
+                fix=FixHint(description="Run ruff format", command="ruff format"),
+            )
+        ]
+
+    def _scan_lines(self, lines: list[str]) -> tuple[list[Finding], str | None]:
+        findings: list[Finding] = []
+        current_file: str | None = None
+        for line in lines:
+            current_file = self._file_from_line(line, current_file)
+            finding = self._finding_from_line(line, current_file)
+            if finding is not None:
+                findings.append(finding)
+                continue
+            self._append_diff(findings, line, current_file)
+        return findings, current_file
+
+    def _append_diff(self, findings: list[Finding], line: str, current_file: str | None) -> None:
+        if not line.startswith(("---", "+++", "@@")):
+            return
+        if not findings or not current_file:
+            return
+        findings[-1].snippet = (findings[-1].snippet or "") + line + "\n"
+
+    def _file_from_line(self, line: str, current_file: str | None) -> str | None:
+        if line.endswith(".py") or line.endswith(".py would be reformatted"):
+            return strip_ansi(line.split()[0].rstrip(":"))
+        if line.startswith("Would reformat:"):
+            return line.split(":", 1)[1].strip()
+        return current_file
+
+    def _finding_from_line(self, line: str, current_file: str | None) -> Finding | None:
+        if not line.startswith("Would reformat:"):
+            return None
+        return Finding(
+            rule_id="format",
+            severity=Severity.ERROR,
+            message="File would be reformatted",
+            location=Location(file=current_file) if current_file else None,
+            fix=FixHint(
+                description="Run ruff format",
+                command=f"ruff format {current_file}" if current_file else "ruff format",
+            ),
+        )

pyossmtool/parsers/shell.py

@@ -0,0 +1,83 @@
+"""Shell tool parsers (shellcheck, shfmt)."""
+
+from __future__ import annotations
+
+from pyossmtool.models import CheckDef, Finding, FixHint, Location, Severity
+from pyossmtool.parsers.base import register
+from pyossmtool.parsers.common import strip_ansi
+from pyossmtool.parsers.patterns import DiffTextParser, JsonListParser
+
+
+@register
+class ShellcheckParser(JsonListParser):
+    id = "shellcheck_json"
+
+    def map_item(self, item: dict) -> Finding:
+        level = item.get("level", "warning")
+        severity = Severity.ERROR if level in {"error", "warning"} else Severity.INFO
+        return Finding(
+            rule_id=f"SC{item.get('code', 0)}",
+            severity=severity,
+            message=item.get("message", "shellcheck finding"),
+            location=Location(
+                file=item.get("file", ""),
+                line=item.get("line"),
+                column=item.get("column"),
+                end_line=item.get("endLine"),
+                end_column=item.get("endColumn"),
+            ),
+        )
+
+
+@register
+class ShfmtDiffParser(DiffTextParser):
+    id = "shfmt_diff"
+
+    def parse(self, stdout: str, stderr: str = "", *, check: CheckDef | None = None) -> list[Finding]:
+        text = strip_ansi(stdout or stderr)
+        if not text.strip():
+            return []
+        findings = self._collect_findings(text.splitlines())
+        if findings:
+            return findings
+        return [
+            Finding(
+                rule_id="format",
+                severity=Severity.ERROR,
+                message="Shell formatting differences detected",
+                snippet=text[:500],
+            )
+        ]
+
+    def _collect_findings(self, lines: list[str]) -> list[Finding]:
+        findings: list[Finding] = []
+        current_file: str | None = None
+        for line in lines:
+            current_file = self._file_from_diff_line(line, current_file)
+            finding = self._diff_finding(line, current_file)
+            if finding is not None:
+                findings.append(finding)
+        return findings
+
+    def _file_from_diff_line(self, line: str, current_file: str | None) -> str | None:
+        if not line.startswith(("--- ", "+++ ")):
+            return current_file
+        marker = line[4:].split("\t")[0].strip()
+        if marker and marker != "/dev/null":
+            return marker
+        return current_file
+
+    def _diff_finding(self, line: str, current_file: str | None) -> Finding | None:
+        if line.startswith("@@") or not line.startswith(("-", "+")):
+            return None
+        return Finding(
+            rule_id="format",
+            severity=Severity.ERROR,
+            message="Shell formatting difference",
+            location=Location(file=current_file) if current_file else None,
+            snippet=line[:200],
+            fix=FixHint(
+                description="Run shfmt -w",
+                command=f"shfmt -w {current_file}" if current_file else "shfmt -w",
+            ),
+        )

pyossmtool/policy/acronym_allowlist.py

@@ -0,0 +1,322 @@
+"""Documented-acronym policy gate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from pyossmtool.ignore import EffectiveIgnores, ignores_from_env
+
+ACRONYM_RE = re.compile(r"\b[A-Z]{2,}\b")
+FENCED_CODE_BLOCK_RE = re.compile(r"(`{3,})[\s\S]*?\1", re.DOTALL)
+INLINE_CODE_RE = re.compile(r"`[^`]+`")
+
+BUILTIN_EXEMPT: frozenset[str] = frozenset(
+    {
+        "AA",
+        "AGENTS",
+        "AMC",
+        "AND",
+        "API",
+        "ASGI",
+        "AST",
+        "AVOIDANCE",
+        "BIGCO",
+        "BLUE",
+        "BMO",
+        "BUMPY",
+        "CALM",
+        "CHEAP",
+        "CI",
+        "CLI",
+        "CM",
+        "CORS",
+        "CPU",
+        "CRUD",
+        "CSRF",
+        "CSS",
+        "CSV",
+        "DD",
+        "DEAR",
+        "DELETE",
+        "DO",
+        "DTO",
+        "DX",
+        "ECHO",
+        "EPSILON",
+        "ESG",
+        "FAIL",
+        "FAILED",
+        "FIFO",
+        "FLAT",
+        "FK",
+        "GB",
+        "GET",
+        "GICS",
+        "GPL",
+        "GREEN",
+        "HEAD",
+        "HEREDOC",
+        "HIGH",
+        "HOW",
+        "HTML",
+        "HTTP",
+        "HTTPS",
+        "IA",
+        "IDE",
+        "ID",
+        "II",
+        "III",
+        "INFY",
+        "IO",
+        "IP",
+        "IPO",
+        "ISO",
+        "IST",
+        "IT",
+        "JSON",
+        "JSONB",
+        "JSCPD",
+        "JWT",
+        "KNOWLEDGE",
+        "LOC",
+        "LOW",
+        "MA",
+        "MB",
+        "MEDIUM",
+        "MID",
+        "MINT",
+        "MM",
+        "MOTHERSON",
+        "MOTHERSUMI",
+        "NO",
+        "NOT",
+        "NOVA",
+        "NULL",
+        "OK",
+        "ON",
+        "OOM",
+        "OPAL",
+        "OPTIONS",
+        "PARTIAL",
+        "PASS",
+        "PATCH",
+        "PATH",
+        "PDF",
+        "PEM",
+        "PG",
+        "PINE",
+        "PK",
+        "POST",
+        "PUT",
+        "QUILL",
+        "RAM",
+        "README",
+        "RED",
+        "RELIANCE",
+        "REST",
+        "ROSE",
+        "RUN",
+        "SCOPE",
+        "SHA",
+        "SIGINT",
+        "SIGTERM",
+        "SKILL",
+        "SLO",
+        "SMALLB",
+        "SMOOTH",
+        "SPA",
+        "SQL",
+        "SSG",
+        "SSR",
+        "SSH",
+        "SVG",
+        "SURVIVAL",
+        "TBD",
+        "TEMPORAL",
+        "TEXT",
+        "TIMESTAMPTZ",
+        "TODO",
+        "TS",
+        "UPDATE",
+        "UP",
+        "URL",
+        "UTC",
+        "UUID",
+        "VM",
+        "VPN",
+        "WCAG",
+        "WHAT",
+        "WILD",
+        "WORKDIR",
+        "WSL",
+        "XML",
+        "YAML",
+        "YYYY",
+        "ZIP",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class AcronymViolation:
+    path: str
+    line: int
+    token: str
+
+
+def strip_markdown_code(text: str) -> str:
+    without_fences = FENCED_CODE_BLOCK_RE.sub("", text)
+    return INLINE_CODE_RE.sub("", without_fences)
+
+
+def load_allowlist(path: Path) -> set[str]:
+    if not path.is_file():
+        return set()
+    raw = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(raw, dict):
+        msg = f"acronym allowlist must be a YAML mapping: {path}"
+        raise ValueError(msg)
+    return {str(key) for key in raw}
+
+
+def find_violations_in_line(line: str, *, allowlisted: set[str]) -> list[str]:
+    tokens: list[str] = []
+    for match in ACRONYM_RE.finditer(line):
+        token = match.group(0)
+        if token in BUILTIN_EXEMPT or token in allowlisted:
+            continue
+        tokens.append(token)
+    return tokens
+
+
+def find_violations_in_text(text: str, *, path: str, allowlisted: set[str]) -> list[AcronymViolation]:
+    prose = strip_markdown_code(text)
+    violations: list[AcronymViolation] = []
+    for line_no, line in enumerate(prose.splitlines(), start=1):
+        for token in find_violations_in_line(line, allowlisted=allowlisted):
+            violations.append(AcronymViolation(path=path, line=line_no, token=token))
+    return violations
+
+
+def iter_markdown_files(scan_roots: tuple[str, ...], repo_root: Path) -> list[Path]:
+    files: list[Path] = []
+    for root in scan_roots:
+        candidate = repo_root / root
+        if candidate.is_file():
+            files.append(candidate)
+            continue
+        if candidate.is_dir():
+            files.extend(sorted(candidate.rglob("*.md")))
+    return files
+
+
+def scan_paths(
+    *,
+    repo_root: Path,
+    allowlist_path: Path,
+    scan_roots: tuple[str, ...],
+    ignores: EffectiveIgnores | None = None,
+) -> list[AcronymViolation]:
+    allowlisted = load_allowlist(allowlist_path)
+    violations: list[AcronymViolation] = []
+    for file_path in iter_markdown_files(scan_roots, repo_root):
+        rel = file_path.relative_to(repo_root).as_posix()
+        if ignores and ignores.is_ignored(rel):
+            continue
+        text = file_path.read_text(encoding="utf-8")
+        violations.extend(find_violations_in_text(text, path=rel, allowlisted=allowlisted))
+    return violations
+
+
+def findings_from_violations(violations: list[AcronymViolation]) -> list[dict[str, Any]]:
+    findings: list[dict[str, Any]] = []
+    for item in violations:
+        findings.append(
+            {
+                "rule_id": "undocumented-acronym",
+                "severity": "error",
+                "message": f"undocumented acronym {item.token!r}",
+                "location": {"file": item.path, "line": item.line},
+            }
+        )
+    return findings
+
+
+def settings_from_config(config: dict[str, Any]) -> tuple[tuple[str, ...], Path | None]:
+    scan_roots = tuple(str(item) for item in config.get("scan_roots", ["docs/", "AGENTS.md"]))
+    allowlist_file = config.get("allowlist_file")
+    allowlist_path = Path(str(allowlist_file)) if allowlist_file else None
+    return scan_roots, allowlist_path
+
+
+def run_gate(
+    *,
+    root: Path,
+    config: dict[str, Any],
+    report_path: Path | None,
+) -> int:
+    scan_roots, allowlist_path = settings_from_config(config)
+    allowlist_path = _require_allowlist_path(root, allowlist_path)
+    violations = scan_paths(
+        repo_root=root,
+        allowlist_path=allowlist_path,
+        scan_roots=scan_roots,
+        ignores=ignores_from_env(root),
+    )
+    _write_acronym_report(violations, report_path)
+    return _acronym_exit(violations)
+
+
+def _require_allowlist_path(root: Path, allowlist_path: Path | None) -> Path:
+    if allowlist_path is None:
+        msg = "acronym allowlist_file is required in gate config"
+        raise ValueError(msg)
+    if not allowlist_path.is_absolute():
+        return root / allowlist_path
+    return allowlist_path
+
+
+def _write_acronym_report(violations, report_path: Path | None) -> None:
+    if not report_path:
+        return
+    report_path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {"findings": findings_from_violations(violations)}
+    report_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+
+def _acronym_exit(violations) -> int:
+    if not violations:
+        return 0
+    for item in violations:
+        print(
+            f"FAIL acronym: {item.path}:{item.line}: undocumented acronym {item.token!r}",
+            file=sys.stderr,
+        )
+    return 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Documented-acronym gate.")
+    parser.add_argument("--root", type=Path, default=Path.cwd())
+    parser.add_argument("--config", type=Path, required=True)
+    parser.add_argument("--report", type=Path, default=None)
+    args = parser.parse_args(argv)
+
+    root = args.root.resolve()
+    config = yaml.safe_load(args.config.read_text(encoding="utf-8")) or {}
+    if not isinstance(config, dict):
+        msg = f"Gate config must be a mapping: {args.config}"
+        raise ValueError(msg)
+    return run_gate(root=root, config=config, report_path=args.report)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())