mxready 1.0.0__py3-none-any.whl

mxready/scanning/rule_loader.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+from typing import Any, Literal
+from urllib.parse import urlsplit
+
+import yaml
+from pydantic import Field, ValidationError, field_validator, model_validator
+
+from mxready.errors import MxReadyError
+from mxready.models import Severity, SourceReference, StrictModel
+
+_RULE_ID = re.compile(r"MXR-[A-Z]+-[0-9]{3}")
+_CATEGORY = re.compile(r"[a-z][a-z0-9-]*")
+_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")
+_FACT_NAME = re.compile(r"[A-Za-z][A-Za-z0-9_]*")
+_RULE_FILE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\.ya?ml")
+_REGEX_FLAGS = {
+    "ASCII": re.ASCII,
+    "DOTALL": re.DOTALL,
+    "IGNORECASE": re.IGNORECASE,
+    "MULTILINE": re.MULTILINE,
+    "VERBOSE": re.VERBOSE,
+}
+_MAX_RULE_FILE_BYTES = 1_048_576
+_RULE_UPDATED_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+
+
+class RulePattern(StrictModel):
+    type: Literal["regex", "dependency", "fact"]
+    expression: str | None = None
+    flags: list[str] = Field(default_factory=list)
+    name: str | None = None
+    equals: bool | str | None = None
+
+    @field_validator("flags")
+    @classmethod
+    def flags_are_supported_and_unique(cls, value: list[str]) -> list[str]:
+        if len(value) != len(set(value)) or any(flag not in _REGEX_FLAGS for flag in value):
+            raise ValueError("regex flags must be supported and unique")
+        return value
+
+    @model_validator(mode="after")
+    def pattern_contract_matches_type(self) -> RulePattern:
+        if self.type == "regex":
+            if (
+                not isinstance(self.expression, str)
+                or not self.expression.strip()
+                or len(self.expression) > 1_000
+                or self.name is not None
+                or self.equals is not None
+            ):
+                raise ValueError("invalid regex pattern contract")
+            try:
+                re.compile(self.expression, regex_flags(self.flags))
+            except re.error as error:
+                raise ValueError("invalid regex expression") from error
+            return self
+
+        if self.type == "dependency":
+            if (
+                not isinstance(self.name, str)
+                or _NAME.fullmatch(self.name) is None
+                or self.expression is not None
+                or self.flags
+                or self.equals is not None
+            ):
+                raise ValueError("invalid dependency pattern contract")
+            return self
+
+        if (
+            not isinstance(self.name, str)
+            or _FACT_NAME.fullmatch(self.name) is None
+            or self.equals is None
+            or self.expression is not None
+            or self.flags
+        ):
+            raise ValueError("invalid fact pattern contract")
+        return self
+
+
+class RuleDefinition(StrictModel):
+    id: str
+    version: int = Field(ge=1)
+    title: str
+    category: str
+    severity: Severity
+    file_globs: list[str] = Field(min_length=1)
+    patterns: list[RulePattern] = Field(min_length=1)
+    message: str
+    recommendation: str
+    references: list[SourceReference] = Field(min_length=1)
+    updated: str | None = None
+    confidence: Literal["documented", "needs-review"] = "documented"
+
+    @field_validator("updated")
+    @classmethod
+    def updated_has_iso_date(cls, value: str | None) -> str | None:
+        if value is not None and _RULE_UPDATED_DATE.fullmatch(value) is None:
+            raise ValueError("updated must use YYYY-MM-DD")
+        return value
+
+    @field_validator("id")
+    @classmethod
+    def id_has_public_format(cls, value: str) -> str:
+        if _RULE_ID.fullmatch(value) is None:
+            raise ValueError("invalid rule id")
+        return value
+
+    @field_validator("category")
+    @classmethod
+    def category_has_stable_format(cls, value: str) -> str:
+        if _CATEGORY.fullmatch(value) is None:
+            raise ValueError("invalid rule category")
+        return value
+
+    @field_validator("title", "message", "recommendation")
+    @classmethod
+    def prose_is_non_empty(cls, value: str) -> str:
+        stripped = value.strip()
+        if not stripped:
+            raise ValueError("rule prose must not be empty")
+        return stripped
+
+    @field_validator("file_globs")
+    @classmethod
+    def file_globs_are_relative_and_unique(cls, value: list[str]) -> list[str]:
+        if len(value) != len(set(value)):
+            raise ValueError("file globs must be unique")
+        for pattern in value:
+            if (
+                not pattern
+                or len(pattern) > 200
+                or pattern.startswith(("/", "\\"))
+                or "\\" in pattern
+                or ".." in pattern.split("/")
+            ):
+                raise ValueError("file globs must be safe relative patterns")
+        return value
+
+    @field_validator("references")
+    @classmethod
+    def references_are_https(
+        cls,
+        value: list[SourceReference],
+    ) -> list[SourceReference]:
+        for reference in value:
+            try:
+                parsed = urlsplit(reference.url)
+                port = parsed.port
+            except ValueError as error:
+                raise ValueError("invalid reference URL") from error
+            if (
+                not reference.title.strip()
+                or not reference.url.startswith("https://")
+                or parsed.scheme != "https"
+                or not parsed.hostname
+                or parsed.username is not None
+                or parsed.password is not None
+                or port is not None
+            ):
+                raise ValueError("rule references must use public HTTPS URLs")
+        return value
+
+
+class RuleCatalog(StrictModel):
+    version: str
+    rules: list[RuleDefinition]
+    extra_version: str | None = None
+
+    @property
+    def digest(self) -> str:
+        document = self.model_dump(mode="json")
+        document["rules"] = sorted(document["rules"], key=lambda rule: rule["id"])
+        payload = json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        return "sha256:" + hashlib.sha256(payload.encode()).hexdigest()
+
+
+class _RuleManifest(StrictModel):
+    schema_version: Literal["1.0"]
+    ruleset_version: str
+    rule_files: list[str] = Field(min_length=1)
+
+    @field_validator("ruleset_version")
+    @classmethod
+    def version_is_non_empty(cls, value: str) -> str:
+        stripped = value.strip()
+        if not stripped or len(stripped) > 100:
+            raise ValueError("invalid ruleset version")
+        return stripped
+
+    @field_validator("rule_files")
+    @classmethod
+    def files_are_safe_and_unique(cls, value: list[str]) -> list[str]:
+        if len(value) != len(set(value)) or any(
+            _RULE_FILE.fullmatch(item) is None for item in value
+        ):
+            raise ValueError("invalid rule file list")
+        return value
+
+
+def load_rule_catalog(
+    directory: Path,
+    extra_directory: Path | None = None,
+) -> RuleCatalog:
+    """Load a complete, strictly validated, versioned YAML rule catalog.
+
+    `extra_directory` 可选：合并另一目录中的规则集（自定义规则扩展）。
+    """
+    try:
+        catalog = _load_catalog(Path(directory))
+        if extra_directory is None:
+            return catalog
+        return _merge_catalogs(catalog, _load_catalog(Path(extra_directory)))
+    except MxReadyError:
+        raise
+    except (
+        OSError,
+        TypeError,
+        UnicodeError,
+        ValidationError,
+        ValueError,
+        yaml.YAMLError,
+    ) as error:
+        raise _ruleset_invalid() from error
+
+
+def _load_catalog(directory: Path) -> RuleCatalog:
+    manifest_document = _load_yaml(directory / "manifest.yml")
+    manifest = _RuleManifest.model_validate(manifest_document)
+
+    rules: list[RuleDefinition] = []
+    seen_ids: set[str] = set()
+    for filename in manifest.rule_files:
+        document = _load_yaml(directory / filename)
+        if not isinstance(document, list) or not document:
+            raise _ruleset_invalid()
+        for rule_document in document:
+            rule = RuleDefinition.model_validate(rule_document)
+            if rule.id in seen_ids:
+                raise _ruleset_invalid()
+            seen_ids.add(rule.id)
+            rules.append(rule)
+
+    if not rules:
+        raise _ruleset_invalid()
+    return RuleCatalog(version=manifest.ruleset_version, rules=rules)
+
+
+def _merge_catalogs(base: RuleCatalog, extra: RuleCatalog) -> RuleCatalog:
+    merged = {rule.id: rule for rule in base.rules}
+    for rule in extra.rules:
+        existing = merged.get(rule.id)
+        if existing is not None and existing != rule:
+            raise _ruleset_invalid()
+        merged[rule.id] = rule
+    return RuleCatalog(
+        version=base.version, rules=list(merged.values()), extra_version=extra.version
+    )
+
+
+def regex_flags(flags: list[str]) -> re.RegexFlag:
+    resolved = re.NOFLAG
+    for flag in flags:
+        resolved |= _REGEX_FLAGS[flag]
+    return resolved
+
+
+def _load_yaml(path: Path) -> Any:
+    if path.stat().st_size > _MAX_RULE_FILE_BYTES:
+        raise ValueError("rule file is too large")
+    return yaml.safe_load(path.read_text(encoding="utf-8-sig"))
+
+
+def _ruleset_invalid() -> MxReadyError:
+    return MxReadyError(
+        "RULESET_INVALID",
+        "规则集无法加载或未通过校验，请检查规则文件。",
+    )

mxready/scanning/toolchain.py

@@ -0,0 +1,100 @@
+"""工具链画像：接入 cu-bridge 之后，部分"未迁移"前提下成立的命中不再是阻塞项。
+
+静态命中本身没有错——`NVCC = nvcc` 确实还在文件里。错的是严重级别所依赖的前提：
+"项目尚未接入 MXMACA 兼容层"。cu-bridge 的 `make_maca` / `cucc` 会在构建期接管
+这些调用，此时该写法是**预期状态**，继续报 blocker 等于把已迁移项目判成未迁移。
+
+降级必须留痕，否则就变成用规则把结论藏起来。因此每次降级都会：
+1. 把严重级别下调一级，并在 message 末尾注明原因；
+2. 往 `analysis_warnings` 里写一条可追溯记录（含规则 ID 与文件数）。
+
+只降一级、不降到 info：cu-bridge 接管的是构建入口，无法证明每个 nvcc 调用都走了
+包装器——脚本里 `subprocess.run(["nvcc", ...])` 仍然绕过它，那属于真问题，降成
+info 会放跑。
+
+同理，降级只作用于**声明式构建文件**（Makefile / `*.mk` / CMakeLists.txt / `*.cmake`）。
+这些文件里的 `NVCC = nvcc` 是构建设置，由 cu-bridge 的 conf.json 在编译期改写；
+而 `.py` / `.sh` 里的 nvcc 是运行时命令，是否走了包装器只能人工判断，保持 blocker。
+"""
+
+from __future__ import annotations
+
+from mxready.models import AnalysisWarning, Finding, Severity
+from mxready.scanning.facts import ProjectFacts
+
+CU_BRIDGE_FLAG = "uses_cu_bridge_toolchain"
+CU_BRIDGE_WARNING_CODE = "TOOLCHAIN_CU_BRIDGE_DOWNGRADE"
+
+# 规则 ID -> 降级后的级别。只有"前提是未迁移"的规则才该进这张表；
+# 描述源码语义（PTX、线程束宽度）的规则与构建方式无关，不适用。
+_CU_BRIDGE_DOWNGRADES: dict[str, Severity] = {
+    "MXR-TOOLCHAIN-001": Severity.WARNING,
+}
+
+# 只在这些文件里认定"构建入口已被接管"。命令式脚本不在内。
+_BUILD_DEFINITION_NAMES = {"cmakelists.txt", "gnumakefile", "makefile"}
+_BUILD_DEFINITION_SUFFIXES = (".cmake", ".mk")
+
+_DOWNGRADE_NOTE = (
+    "（已检测到 cu-bridge 工具链接入：该构建入口通常由 make_maca/cucc 接管，"
+    "此项从 blocker 降为 warning，请确认是否有脚本绕过包装器。）"
+)
+
+
+def apply_toolchain_profile(
+    findings: list[Finding],
+    facts: ProjectFacts,
+) -> tuple[list[Finding], list[AnalysisWarning]]:
+    """按工具链画像调整严重级别，返回新列表与需要记录到报告的告警。"""
+    if not facts.flags.get(CU_BRIDGE_FLAG):
+        return findings, []
+
+    adjusted: list[Finding] = []
+    downgraded_rules: dict[str, set[str]] = {}
+    for finding in findings:
+        target = _CU_BRIDGE_DOWNGRADES.get(finding.rule_id)
+        if (
+            target is None
+            or finding.severity is not Severity.BLOCKER
+            or not _is_build_definition(finding.relative_path)
+        ):
+            adjusted.append(finding)
+            continue
+        downgraded_rules.setdefault(finding.rule_id, set()).add(finding.relative_path)
+        adjusted.append(
+            finding.model_copy(
+                update={
+                    "severity": target,
+                    "message": _append_note(finding.message),
+                }
+            )
+        )
+
+    if not downgraded_rules:
+        return adjusted, []
+
+    details = "；".join(
+        f"{rule_id}（{len(paths)} 个文件）" for rule_id, paths in sorted(downgraded_rules.items())
+    )
+    return adjusted, [
+        AnalysisWarning(
+            code=CU_BRIDGE_WARNING_CODE,
+            relative_path=None,
+            message=(
+                f"检测到 cu-bridge 工具链接入，已将 {details} 从 blocker 降为 warning。"
+                "降级只覆盖声明式构建文件；命令式脚本（.py / .sh）中的 nvcc 调用仍按 "
+                "blocker 报告，因为它可能绕过 make_maca 的 PATH 注入。"
+            ),
+        )
+    ]
+
+
+def _is_build_definition(relative_path: str) -> bool:
+    name = relative_path.rsplit("/", 1)[-1].casefold()
+    return name in _BUILD_DEFINITION_NAMES or name.endswith(_BUILD_DEFINITION_SUFFIXES)
+
+
+def _append_note(message: str) -> str:
+    if _DOWNGRADE_NOTE in message:
+        return message
+    return f"{message}{_DOWNGRADE_NOTE}"

mxready/services/__init__.py

@@ -0,0 +1,3 @@
+from mxready.services.scans import ScanService
+
+__all__ = ["ScanService"]

mxready/services/scans.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
+from contextlib import contextmanager
+from datetime import UTC, datetime
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from threading import Event, Thread
+from uuid import UUID, uuid4
+
+from mxready.config import Settings
+from mxready.errors import MxReadyError
+from mxready.models import ScanJob, ScanReport, ScanStatus
+from mxready.persistence.sqlite import SQLiteStore
+from mxready.repository.git_client import GitClient
+from mxready.repository.identity import parse_repository_url, validate_git_ref
+from mxready.scanning.analyzer import ScanAnalyzer
+from mxready.verification.validation import (
+    validate_verification_upload,
+    verification_status_at,
+)
+
+logger = logging.getLogger(__name__)
+
+_STAGE_MESSAGES = {
+    ScanStatus.CLONING: "正在安全获取仓库",
+    ScanStatus.INDEXING: "正在建立源码索引",
+    ScanStatus.ANALYZING: "正在执行兼容性规则",
+    ScanStatus.COMPLETED: "扫描完成",
+    ScanStatus.FAILED: "扫描失败",
+}
+
+
+class ScanService:
+    def __init__(
+        self,
+        store: SQLiteStore,
+        git_client: GitClient,
+        analyzer: ScanAnalyzer,
+        settings: Settings,
+        providers: Mapping[str, str] | None = None,
+    ) -> None:
+        self.store = store
+        self.git_client = git_client
+        self.analyzer = analyzer
+        self.settings = settings
+        self.providers = providers
+
+    def create_scan(
+        self,
+        repo_url: str,
+        requested_ref: str | None,
+    ) -> ScanJob:
+        identity = parse_repository_url(repo_url, self.providers)
+        reference = validate_git_ref(requested_ref)
+        canonical_url = identity.clone_url.removesuffix(".git")
+        return self.store.create_job(
+            canonical_url,
+            reference,
+            max_active=self.settings.max_concurrent_scans,
+        )
+
+    def get_scan(self, scan_id: UUID) -> ScanJob:
+        job = self.store.get_job(scan_id)
+        if job is None:
+            raise MxReadyError(
+                "SCAN_NOT_FOUND",
+                "The requested scan does not exist.",
+            )
+        return job
+
+    def get_report(self, scan_id: UUID) -> ScanReport:
+        job = self.get_scan(scan_id)
+        if job.status is not ScanStatus.COMPLETED:
+            raise MxReadyError(
+                "SCAN_NOT_COMPLETED",
+                "The requested scan has not completed successfully.",
+            )
+        report = self.store.get_report(scan_id)
+        if report is None:
+            raise MxReadyError(
+                "SCAN_INTERNAL_ERROR",
+                "The completed scan report could not be loaded.",
+            )
+        run = self.store.get_verification(scan_id)
+        if run is not None:
+            report = report.model_copy(
+                update={
+                    "verification_status": verification_status_at(run, now=datetime.now(UTC)),
+                }
+            )
+        return report
+
+    def run_scan(self, scan_id: UUID) -> None:
+        job = self.get_scan(scan_id)
+        token = str(uuid4())
+        if not self.store.claim_job(scan_id, token, lease_seconds=self.settings.task_lease_seconds):
+            return
+        with self._heartbeat(scan_id, token):
+            self._run_claimed_scan(job, token)
+
+    @contextmanager
+    def _heartbeat(self, scan_id: UUID, token: str):
+        stopped = Event()
+
+        def renew() -> None:
+            while not stopped.wait(self.settings.task_lease_seconds / 3):
+                try:
+                    if not self.store.renew_lease(
+                        scan_id,
+                        token,
+                        lease_seconds=self.settings.task_lease_seconds,
+                    ):
+                        return
+                except Exception:
+                    logger.exception("scan lease renewal failed", extra={"scan_id": str(scan_id)})
+                    return
+
+        thread = Thread(target=renew, name=f"scan-heartbeat-{scan_id}", daemon=True)
+        thread.start()
+        try:
+            yield
+        finally:
+            stopped.set()
+            thread.join()
+
+    def _run_claimed_scan(self, job: ScanJob, token: str) -> None:
+        scan_id = job.id
+
+        logger.info("scan started", extra={"scan_id": str(scan_id)})
+        try:
+            self.store.update_job(
+                scan_id,
+                status=ScanStatus.CLONING,
+                stage_message=_STAGE_MESSAGES[ScanStatus.CLONING],
+                worker_token=token,
+            )
+            identity = parse_repository_url(job.repo_url, self.providers)
+
+            with TemporaryDirectory(
+                prefix=f"mxready-{scan_id}-",
+                dir=self.settings.temp_dir,
+            ) as temporary_directory:
+                repository_root = Path(temporary_directory) / "repository"
+                commit = self.git_client.clone(
+                    identity,
+                    job.requested_ref,
+                    repository_root,
+                )
+                logger.info(
+                    "repository cloned",
+                    extra={"scan_id": str(scan_id), "commit": commit},
+                )
+
+                def persist_stage(status: ScanStatus) -> None:
+                    self.store.update_job(
+                        scan_id,
+                        status=status,
+                        stage_message=_STAGE_MESSAGES[status],
+                        resolved_commit=commit,
+                        worker_token=token,
+                    )
+
+                report = self.analyzer.analyze(
+                    repository_root,
+                    repository_url=job.repo_url,
+                    commit=commit,
+                    scan_id=scan_id,
+                    stage_callback=persist_stage,
+                )
+                self.store.save_report(report, worker_token=token)
+                logger.info(
+                    "scan analysis completed",
+                    extra={"scan_id": str(scan_id)},
+                )
+
+            self.store.update_job(
+                scan_id,
+                status=ScanStatus.COMPLETED,
+                stage_message=_STAGE_MESSAGES[ScanStatus.COMPLETED],
+                resolved_commit=commit,
+                worker_token=token,
+            )
+        except MxReadyError as error:
+            if error.code != "SCAN_LEASE_LOST":
+                self._mark_failed(scan_id, error.code, error.message, token)
+        except Exception:
+            logger.exception(
+                "Unexpected scan failure for scan_id=%s",
+                scan_id,
+            )
+            self._mark_failed(
+                scan_id,
+                "SCAN_INTERNAL_ERROR",
+                "The scan failed unexpectedly. Start a new scan and try again.",
+                token,
+            )
+
+    def attach_verification(
+        self,
+        scan_id: UUID,
+        payload: bytes,
+        *,
+        now: datetime | None = None,
+    ) -> ScanReport:
+        report = self.get_report(scan_id)
+        validated = validate_verification_upload(
+            report,
+            payload,
+            now=now or datetime.now(UTC),
+        )
+        updated_report = report.model_copy(update={"verification_status": validated.status})
+        self.store.save_verification_and_report(
+            scan_id,
+            validated.run,
+            updated_report,
+        )
+        return updated_report
+
+    def _mark_failed(
+        self,
+        scan_id: UUID,
+        code: str,
+        message: str,
+        token: str,
+    ) -> None:
+        try:
+            self.store.update_job(
+                scan_id,
+                status=ScanStatus.FAILED,
+                stage_message=_STAGE_MESSAGES[ScanStatus.FAILED],
+                failure_code=code,
+                failure_message=message,
+                worker_token=token,
+            )
+        except MxReadyError as error:
+            if error.code != "SCAN_LEASE_LOST":
+                raise

mxready/services/worker.py

@@ -0,0 +1,38 @@
+"""Bounded consumers of the durable SQLite scan queue."""
+
+import logging
+from threading import Event, Thread
+
+from mxready.services.scans import ScanService
+
+logger = logging.getLogger(__name__)
+
+
+class ScanWorker:
+    def __init__(self, service: ScanService) -> None:
+        self.service = service
+        self.stopped = Event()
+        self.threads: list[Thread] = []
+
+    def start(self) -> None:
+        for index in range(self.service.settings.scan_worker_threads):
+            thread = Thread(target=self._consume, name=f"scan-worker-{index}", daemon=True)
+            self.threads.append(thread)
+            thread.start()
+
+    def stop(self) -> None:
+        self.stopped.set()
+        for thread in self.threads:
+            thread.join()
+
+    def _consume(self) -> None:
+        while not self.stopped.is_set():
+            try:
+                self.service.store.recover_expired_jobs()
+                for scan_id in self.service.store.queued_job_ids():
+                    if self.stopped.is_set():
+                        break
+                    self.service.run_scan(scan_id)
+            except Exception:
+                logger.exception("scan queue consumer failed")
+            self.stopped.wait(0.5)

mxready/verification/__init__.py

@@ -0,0 +1,11 @@
+from mxready.verification.bundle import build_verification_bundle
+from mxready.verification.validation import (
+    ValidatedVerification,
+    validate_verification_upload,
+)
+
+__all__ = [
+    "ValidatedVerification",
+    "build_verification_bundle",
+    "validate_verification_upload",
+]