mxready 1.0.0__py3-none-any.whl

mxready/scanning/indexer.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from array import array
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from mxready.errors import MxReadyError
+from mxready.models import ScanCoverage
+
+_ALLOWED_EXTENSIONS = {
+    ".c",
+    ".cc",
+    ".cfg",
+    ".cmake",
+    ".cpp",
+    ".cu",
+    ".cuh",
+    ".dockerfile",
+    ".h",
+    ".hpp",
+    ".md",
+    ".mk",
+    ".py",
+    ".sh",
+    ".toml",
+    ".txt",
+    ".yaml",
+    ".yml",
+}
+# Makefile 三种常见拼写都要收：CUDA 侧大量项目（含 cuda-samples）用 Make 构建，
+# cu-bridge 的官方入口 make_maca 本身就是 make 包装器，漏掉它等于漏掉整条构建链。
+_ALLOWED_FILENAMES = {
+    "CMakeLists.txt",
+    "Dockerfile",
+    "GNUmakefile",
+    "Makefile",
+    "makefile",
+}
+_EXCLUDED_DIRECTORIES = {
+    ".git",
+    ".venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "node_modules",
+    "target",
+    "vendor",
+    "venv",
+}
+_HIDDEN_CACHE_DIRECTORIES = {
+    ".cache",
+    ".hypothesis",
+    ".mypy_cache",
+    ".nox",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".tox",
+}
+# 测试运行器与基准框架的临时目录：内容是夹具的运行时副本，不属于仓库内容，
+# 每次运行还会漂移（pytest 的 `--basetemp=.pytest-temp` 即属此类）。
+# 扫描它们只会产生随时间变化、无法复核的噪音命中，因此整目录排除。
+_SCRATCH_DIRECTORY_PREFIXES = (".benchmarks", ".pytest")
+
+
+@dataclass(frozen=True, slots=True)
+class IndexedFile:
+    relative_path: str
+    size: int
+    sha256: str
+    text: str
+    line_offsets: array[int] = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        offsets = array("I", [0])
+        offsets.extend(match.end() for match in re.finditer("\n", self.text))
+        object.__setattr__(self, "line_offsets", offsets)
+
+
+@dataclass(frozen=True, slots=True)
+class IndexWarning:
+    code: str
+    relative_path: str
+    message: str
+
+
+@dataclass(frozen=True, slots=True)
+class FileIndex:
+    files: tuple[IndexedFile, ...]
+    warnings: tuple[IndexWarning, ...]
+    coverage: ScanCoverage | None = None
+
+
+def build_file_index(
+    root: Path,
+    *,
+    max_files: int = 10_000,
+    max_file_bytes: int = 1_048_576,
+) -> FileIndex:
+    """Build a deterministic text-only index without following symlinks."""
+    if max_files <= 0 or max_file_bytes <= 0:
+        raise ValueError("file index limits must be positive")
+
+    root = Path(root)
+    if not root.is_dir():
+        raise MxReadyError(
+            "SCAN_INTERNAL_ERROR",
+            "待扫描的仓库目录不存在或不可读取。",
+        )
+
+    indexed_files: list[IndexedFile] = []
+    warnings: list[IndexWarning] = []
+    candidate_count = 0
+    skipped: Counter[str] = Counter()
+
+    for path, relative_path in _walk_candidate_files(root, warnings, skipped):
+        candidate_count += 1
+        if candidate_count > max_files:
+            raise MxReadyError(
+                "TOO_MANY_FILES",
+                f"可分析源码文件超过 {max_files} 个限制。",
+            )
+
+        try:
+            size = path.stat(follow_symlinks=False).st_size
+        except OSError:
+            warnings.append(
+                IndexWarning(
+                    code="FILE_READ_ERROR",
+                    relative_path=relative_path,
+                    message="无法读取该文件的元数据，已跳过。",
+                )
+            )
+            continue
+
+        if size > max_file_bytes:
+            warnings.append(
+                IndexWarning(
+                    code="FILE_TOO_LARGE",
+                    relative_path=relative_path,
+                    message=f"文件超过 {max_file_bytes} 字节限制，已跳过。",
+                )
+            )
+            continue
+
+        try:
+            content = path.read_bytes()
+        except OSError:
+            warnings.append(
+                IndexWarning(
+                    code="FILE_READ_ERROR",
+                    relative_path=relative_path,
+                    message="无法安全读取该文件，已跳过。",
+                )
+            )
+            continue
+
+        if len(content) > max_file_bytes:
+            warnings.append(
+                IndexWarning(
+                    code="FILE_TOO_LARGE",
+                    relative_path=relative_path,
+                    message=f"文件超过 {max_file_bytes} 字节限制，已跳过。",
+                )
+            )
+            continue
+        if _looks_binary(content):
+            warnings.append(
+                IndexWarning(
+                    code="BINARY_FILE_SKIPPED",
+                    relative_path=relative_path,
+                    message="文件包含二进制内容，已跳过。",
+                )
+            )
+            continue
+
+        text = _decode_text(content)
+        if text is None:
+            warnings.append(
+                IndexWarning(
+                    code="UNSUPPORTED_TEXT_ENCODING",
+                    relative_path=relative_path,
+                    message="文件不是 UTF-8 或 GB18030 文本，已跳过。",
+                )
+            )
+            continue
+
+        indexed_files.append(
+            IndexedFile(
+                relative_path=relative_path,
+                size=len(content),
+                sha256=hashlib.sha256(content).hexdigest(),
+                text=text,
+            )
+        )
+
+    skipped.update(w.code for w in warnings if w.code != "DIRECTORY_READ_ERROR")
+    return FileIndex(
+        coverage=ScanCoverage(
+            analyzed_files=len(indexed_files),
+            analyzed_bytes=sum(f.size for f in indexed_files),
+            files_by_type=dict(
+                sorted(
+                    Counter(
+                        Path(f.relative_path).suffix.lower() or Path(f.relative_path).name
+                        for f in indexed_files
+                    ).items()
+                )
+            ),
+            excluded_directories=skipped.pop("EXCLUDED_DIRECTORY", 0),
+            skipped_by_reason=dict(sorted(skipped.items())),
+        ),
+        files=tuple(sorted(indexed_files, key=lambda item: item.relative_path)),
+        warnings=tuple(sorted(warnings, key=lambda item: (item.relative_path, item.code))),
+    )
+
+
+def _walk_candidate_files(
+    root: Path,
+    warnings: list[IndexWarning],
+    skipped: Counter[str],
+):
+    pending = [root]
+    while pending:
+        directory = pending.pop()
+        try:
+            with os.scandir(directory) as entries:
+                ordered_entries = sorted(entries, key=lambda item: item.name)
+        except OSError:
+            relative_path = _relative_path(directory, root)
+            warnings.append(
+                IndexWarning(
+                    code="DIRECTORY_READ_ERROR",
+                    relative_path=relative_path,
+                    message="无法读取该目录，已跳过。",
+                )
+            )
+            continue
+
+        child_directories: list[Path] = []
+        for entry in ordered_entries:
+            try:
+                if entry.is_symlink():
+                    skipped["SYMLINK"] += 1
+                    continue
+                if entry.is_dir(follow_symlinks=False):
+                    if not _is_excluded_directory(entry.name):
+                        child_directories.append(Path(entry.path))
+                    else:
+                        skipped["EXCLUDED_DIRECTORY"] += 1
+                    continue
+                if not entry.is_file(follow_symlinks=False):
+                    continue
+            except OSError:
+                skipped["ENTRY_READ_ERROR"] += 1
+                continue
+
+            path = Path(entry.path)
+            if _is_allowed_file(path):
+                yield path, _relative_path(path, root)
+            else:
+                skipped["UNSUPPORTED_FILE_TYPE"] += 1
+
+        pending.extend(reversed(child_directories))
+
+
+def _is_excluded_directory(name: str) -> bool:
+    return (
+        name in _EXCLUDED_DIRECTORIES
+        or name in _HIDDEN_CACHE_DIRECTORIES
+        or name.startswith(_SCRATCH_DIRECTORY_PREFIXES)
+        or name.startswith(".")
+        and "cache" in name.casefold()
+    )
+
+
+def _is_allowed_file(path: Path) -> bool:
+    return path.name in _ALLOWED_FILENAMES or path.suffix.casefold() in _ALLOWED_EXTENSIONS
+
+
+def _relative_path(path: Path, root: Path) -> str:
+    relative = path.relative_to(root).as_posix()
+    return relative or "."
+
+
+def _looks_binary(content: bytes) -> bool:
+    return b"\x00" in content
+
+
+def _decode_text(content: bytes) -> str | None:
+    for encoding in ("utf-8-sig", "gb18030"):
+        try:
+            return content.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    return None

mxready/scanning/path_context.py

@@ -0,0 +1,174 @@
+"""按路径判定文件角色：非交付路径上的命中只作提示。
+
+静态规则看不到"这段代码是交付产物还是验证材料"。仓库的 `tests/`、`examples/`、
+`docs/` 里常常放着 `.cu` 夹具、构建脚本与代码片段——它们的写法与真实 kernel 一样，
+但**不是**迁移对象。按原级别上报会让报告整体失真：本仓库自扫描一度出现 22 条
+blocker，全部出自测试夹具，真实用户第一次运行就会看到"工具判定自己是阻塞状态"。
+
+处理原则：
+
+1. 不排除这些文件。仍然索引、仍然评估、仍然出现在报告里——"看不见"比"级别低"
+   更危险，漏掉一条真实风险比看到一条弱提示代价大得多。
+2. 只把**交付路径之外**的命中封顶为 `info`，不改变交付路径的任何结论。
+3. 降级必须留痕。与工具链画像（`scanning/toolchain.py`）同一套约定：
+   `Finding.message` 末尾追加原因，并往 `analysis_warnings` 写一条可追溯记录，
+   含角色、规则 ID 与文件数。
+
+路径判定按**路径段精确匹配**，不做子串匹配：`contest/`、`attestation/`、
+`latest/` 都不是测试目录。
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from mxready.models import AnalysisWarning, Finding, Severity
+
+PATH_CONTEXT_WARNING_CODE = "PATH_CONTEXT_DEMOTION"
+
+
+class PathRole(StrEnum):
+    SOURCE = "source"
+    TEST = "test"
+    EXAMPLE = "example"
+    DOC = "doc"
+    BENCHMARK = "benchmark"
+
+
+# 判定顺序即优先级：`docs/examples/x.cu` 归为示例，而不是文档。
+_ROLE_SEGMENTS: tuple[tuple[PathRole, frozenset[str]], ...] = (
+    (
+        PathRole.TEST,
+        frozenset(
+            {
+                "__tests__",
+                "fixture",
+                "fixtures",
+                "test",
+                "test-data",
+                "test_data",
+                "testdata",
+                "tests",
+                "testing",
+            }
+        ),
+    ),
+    (
+        PathRole.BENCHMARK,
+        frozenset({"bench", "benches", "benchmark", "benchmarks"}),
+    ),
+    (
+        PathRole.EXAMPLE,
+        frozenset({"demo", "demos", "example", "examples", "sample", "samples"}),
+    ),
+    (
+        PathRole.DOC,
+        frozenset({"doc", "docs", "documentation"}),
+    ),
+)
+
+# 只对这两档降级：info 本来就是提示，再写一遍原因只是噪音。
+_DEMOTABLE = frozenset({Severity.BLOCKER, Severity.WARNING})
+
+_ROLE_LABELS = {
+    PathRole.TEST: "测试/夹具",
+    PathRole.BENCHMARK: "基准测试",
+    PathRole.EXAMPLE: "示例",
+    PathRole.DOC: "文档",
+}
+
+_ROLE_NOTES = {
+    PathRole.TEST: (
+        "（该命中位于测试或夹具路径：规则命中的是测试代码本身，通常不代表交付产物的"
+        "迁移风险，已从 {declared} 降为提示；若该夹具描述的正是交付行为，请另行复核。）"
+    ),
+    PathRole.BENCHMARK: (
+        "（该命中位于基准测试路径：基准代码一般不随交付构建，已从 {declared} 降为提示。）"
+    ),
+    PathRole.EXAMPLE: (
+        "（该命中位于示例路径：示例代码一般不随交付构建，已从 {declared} 降为提示。）"
+    ),
+    PathRole.DOC: ("（该命中位于文档路径：命中的是文档中的代码片段，已从 {declared} 降为提示。）"),
+}
+
+_SEVERITY_LABELS = {
+    Severity.BLOCKER: "blocker",
+    Severity.WARNING: "warning",
+}
+
+
+def classify_path(relative_path: str) -> PathRole:
+    """按路径段给出文件角色；文件名本身只有 `test_*` / `*_test` 参与判定。"""
+    segments = [segment.casefold() for segment in relative_path.split("/") if segment]
+    if not segments:
+        return PathRole.SOURCE
+
+    directories = segments[:-1]
+    for role, names in _ROLE_SEGMENTS:
+        if any(segment in names for segment in directories):
+            return role
+
+    if _is_test_filename(segments[-1]):
+        return PathRole.TEST
+    return PathRole.SOURCE
+
+
+def is_demoted(relative_path: str) -> bool:
+    return classify_path(relative_path) is not PathRole.SOURCE
+
+
+def apply_path_context(
+    findings: list[Finding],
+) -> tuple[list[Finding], list[AnalysisWarning]]:
+    """把非交付路径上的 blocker/warning 封顶为 info，并返回需要写入报告的告警。"""
+    adjusted: list[Finding] = []
+    demoted: dict[PathRole, dict[str, set[str]]] = {}
+
+    for finding in findings:
+        role = classify_path(finding.relative_path)
+        if role is PathRole.SOURCE or finding.severity not in _DEMOTABLE:
+            adjusted.append(finding)
+            continue
+        demoted.setdefault(role, {}).setdefault(finding.rule_id, set()).add(finding.relative_path)
+        adjusted.append(
+            finding.model_copy(
+                update={
+                    "severity": Severity.INFO,
+                    "message": _append_note(finding.message, role, finding.severity),
+                }
+            )
+        )
+
+    if not demoted:
+        return adjusted, []
+
+    warnings = [
+        AnalysisWarning(
+            code=PATH_CONTEXT_WARNING_CODE,
+            relative_path=None,
+            message=(
+                f"{_ROLE_LABELS[role]}路径上的 {_describe_rules(rules)} 已封顶为提示，"
+                "不参与静态状态判定；文件仍被扫描，结论可逐条复核。"
+            ),
+        )
+        for role, rules in sorted(demoted.items(), key=lambda item: item[0].value)
+    ]
+    return adjusted, warnings
+
+
+def _is_test_filename(name: str) -> bool:
+    stem = name.rsplit(".", 1)[0]
+    return stem.startswith("test_") or stem.endswith(("_test", "-test"))
+
+
+def _append_note(message: str, role: PathRole, declared: Severity) -> str:
+    note = _ROLE_NOTES[role].format(declared=_SEVERITY_LABELS[declared])
+    if note in message:
+        return message
+    return f"{message}{note}"
+
+
+def _describe_rules(rules: dict[str, set[str]]) -> str:
+    return "；".join(
+        f"{rule_id}（{len(paths)} 个文件）" for rule_id, paths in sorted(rules.items())
+    )

mxready/scanning/python_source.py

@@ -0,0 +1,67 @@
+"""识别 Python 文档字符串占用的精确字符区间。
+
+静态规则按词法匹配，分不清"可执行代码"与"文档里对代码的描述"。本仓库自身就有
+反例：`scanning/toolchain.py` 的模块文档字符串写了
+`subprocess.run(["nvcc", ...])` 作为反例说明，于是被 `MXR-TOOLCHAIN-001`
+判成 blocker——一条描述规则的文字被当成了违反规则的行为。
+
+这里的判定刻意收得很窄，只处理**文档字符串**：
+
+- 只有模块、类、函数的首条语句、且是字符串常量，才算文档字符串；
+- 普通字符串字面量**不**豁免。`os.system("nvcc main.cu")` 命令就在字符串里，
+  豁免它会制造真实的漏报，而漏报比多报危险得多；
+- 解析失败（Python 2 源码、生成代码、语法错误）一律返回空区间，即不豁免。
+
+因此这是"去掉文档噪音"，不是"忽略字符串"。
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+
+
+def docstring_spans(text: str) -> tuple[tuple[int, int], ...]:
+    """返回半开字符偏移区间，与正则 match.span() 使用同一坐标。"""
+    try:
+        tree = ast.parse(text)
+    except (SyntaxError, ValueError):
+        return ()
+
+    line_starts = [0, *(match.end() for match in re.finditer(r"\r\n?|\n", text))]
+
+    def offset(row: int, byte_column: int) -> int:
+        start = line_starts[row - 1]
+        end = line_starts[row] if row < len(line_starts) else len(text)
+        # AST columns count UTF-8 bytes; regex offsets count Unicode characters.
+        return start + len(text[start:end].encode("utf-8")[:byte_column].decode("utf-8"))
+
+    spans: list[tuple[int, int]] = []
+    for node in ast.walk(tree):
+        if not isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        if not node.body:
+            continue
+        first = node.body[0]
+        if not isinstance(first, ast.Expr):
+            continue
+        value = first.value
+        if not isinstance(value, ast.Constant) or not isinstance(value.value, str):
+            continue
+        if value.end_lineno is not None and value.end_col_offset is not None:
+            spans.append(
+                (
+                    offset(value.lineno, value.col_offset),
+                    offset(value.end_lineno, value.end_col_offset),
+                )
+            )
+    return tuple(spans)
+
+
+def is_inside_docstring(
+    spans: tuple[tuple[int, int], ...],
+    offset_start: int,
+    offset_end: int,
+) -> bool:
+    """只有命中的字符范围完整落在文档字符串内才豁免。"""
+    return any(start <= offset_start and offset_end <= end for start, end in spans)