mxready 1.0.0__py3-none-any.whl

mxready/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

mxready/api/__init__.py

@@ -0,0 +1,4 @@
+from mxready.api.rules import router as rules_router
+from mxready.api.scans import router as scans_router
+
+__all__ = ["rules_router", "scans_router"]

mxready/api/middleware.py

@@ -0,0 +1,149 @@
+"""进程内 API 防护中间件：滑动窗口限流 + 请求体上限（默认关闭，按需开启）。"""
+
+import time
+from collections import defaultdict, deque
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.responses import JSONResponse
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+
+class _SlidingWindowLimiter:
+    """滑动窗口日志限流。桶按 IP 建立且条目过期后并不自动消失，
+    因此周期性清扫空桶与过期桶，避免公网环境下大量一次性 IP 造成内存无界增长。"""
+
+    def __init__(
+        self,
+        limit: int,
+        window_seconds: float = 60.0,
+        *,
+        sweep_interval: int = 64,
+    ) -> None:
+        self._limit = limit
+        self._window = window_seconds
+        self._hits: dict[str, deque[float]] = defaultdict(deque)
+        self._sweep_interval = max(1, sweep_interval)
+        self._calls_since_sweep = 0
+
+    def allow(self, key: str) -> bool:
+        now = time.monotonic()
+        self._calls_since_sweep += 1
+        if self._calls_since_sweep >= self._sweep_interval:
+            self._calls_since_sweep = 0
+            self._sweep(now)
+        window = self._hits[key]
+        while window and now - window[0] >= self._window:
+            window.popleft()
+        if len(window) >= self._limit:
+            return False
+        window.append(now)
+        return True
+
+    def _sweep(self, now: float) -> None:
+        expired = [
+            key
+            for key, window in self._hits.items()
+            if not window or now - window[-1] >= self._window
+        ]
+        for key in expired:
+            del self._hits[key]
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """按客户端 IP 限制写操作频率（创建扫描 / 上传验证 / 保存清单）。"""
+
+    def __init__(self, app, *, limit: int, window_seconds: float = 60.0) -> None:
+        super().__init__(app)
+        self._limiter = _SlidingWindowLimiter(limit, window_seconds)
+
+    async def dispatch(self, request, call_next):
+        path = request.url.path.rstrip("/")
+        is_write = (
+            request.method == "POST"
+            and (
+                path == "/api/scans"
+                or path.endswith("/verification-runs")
+                or (path.startswith("/api/scans/") and path.endswith("/reviews"))
+            )
+        ) or (
+            request.method == "PATCH" and path.startswith("/api/scans/") and "/checklist/" in path
+        )
+        if is_write:
+            client_ip = request.client.host if request.client else "unknown"
+            if not self._limiter.allow(client_ip):
+                return JSONResponse(
+                    status_code=429,
+                    content={
+                        "error": {
+                            "code": "RATE_LIMITED",
+                            "message": "请求过于频繁，请稍后再试。",
+                            "details": {},
+                        }
+                    },
+                )
+        return await call_next(request)
+
+
+class RequestBodyLimitMiddleware:
+    """同时校验 Content-Length 和实际接收字节数。"""
+
+    def __init__(self, app: ASGIApp, *, max_bytes: int) -> None:
+        self.app = app
+        self._max_bytes = max_bytes
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        path = scope.get("path", "").rstrip("/")
+        # 验证结果上传走端点自身的流式分块校验（UPLOAD_TOO_LARGE 语义），
+        # 全局请求体上限在这里放行，避免错误码契约被静默替换。
+        if scope.get("method") == "POST" and path.endswith("/verification-runs"):
+            await self.app(scope, receive, send)
+            return
+
+        headers = {key.lower(): value for key, value in scope.get("headers", [])}
+        declared = headers.get(b"content-length")
+        if declared is not None:
+            try:
+                if int(declared.decode("ascii")) > self._max_bytes:
+                    await self._reject(scope, receive, send)
+                    return
+            except (UnicodeDecodeError, ValueError):
+                pass
+
+        received = 0
+        buffered: deque[Message] = deque()
+        while True:
+            message = await receive()
+            if message["type"] != "http.request":
+                buffered.append(message)
+                break
+            received += len(message.get("body", b""))
+            if received > self._max_bytes:
+                await self._reject(scope, receive, send)
+                return
+            buffered.append(message)
+            if not message.get("more_body", False):
+                break
+
+        async def replay_receive() -> Message:
+            if buffered:
+                return buffered.popleft()
+            return await receive()
+
+        await self.app(scope, replay_receive, send)
+
+    async def _reject(self, scope: Scope, receive: Receive, send: Send) -> None:
+        response = JSONResponse(
+            status_code=413,
+            content={
+                "error": {
+                    "code": "REQUEST_TOO_LARGE",
+                    "message": f"请求体不能超过 {self._max_bytes} 字节。",
+                    "details": {},
+                }
+            },
+        )
+        await response(scope, receive, send)

mxready/api/rules.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter, Request
+
+from mxready.scanning.rule_loader import RuleCatalog
+
+router = APIRouter(prefix="/rules", tags=["rules"])
+
+
+@router.get("", response_model=RuleCatalog)
+def get_rules(request: Request) -> RuleCatalog:
+    return request.app.state.rule_catalog

mxready/api/scans.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import re
+from uuid import UUID
+
+from fastapi import APIRouter, Request, status
+from fastapi.responses import Response
+from pydantic import Field, HttpUrl
+from starlette.concurrency import run_in_threadpool
+
+from mxready.errors import MxReadyError
+from mxready.models import ScanJob, ScanReport, StrictModel
+from mxready.reporting.badge import render_badge
+from mxready.reporting.markdown import render_markdown
+from mxready.reporting.sarif import render_sarif
+from mxready.verification.bundle import build_verification_bundle
+from mxready.verification.validation import MAX_VERIFICATION_UPLOAD_BYTES
+
+router = APIRouter(prefix="/scans", tags=["scans"])
+
+
+class CreateScanRequest(StrictModel):
+    repo_url: HttpUrl
+    ref: str | None = Field(default=None, max_length=200)
+
+
+@router.post("", response_model=ScanJob, status_code=status.HTTP_202_ACCEPTED)
+def create_scan(
+    payload: CreateScanRequest,
+    request: Request,
+) -> ScanJob:
+    service = request.app.state.scan_service
+    job = service.create_scan(str(payload.repo_url), payload.ref)
+    return job
+
+
+@router.get("/{scan_id}", response_model=ScanJob)
+def get_scan(scan_id: UUID, request: Request) -> ScanJob:
+    return request.app.state.scan_service.get_scan(scan_id)
+
+
+@router.get("/{scan_id}/report", response_model=ScanReport)
+def get_report(scan_id: UUID, request: Request) -> ScanReport:
+    return request.app.state.scan_service.get_report(scan_id)
+
+
+@router.get("/{scan_id}/report.md")
+def download_markdown_report(scan_id: UUID, request: Request) -> Response:
+    report = request.app.state.scan_service.get_report(scan_id)
+    return Response(
+        content=render_markdown(report),
+        media_type="text/markdown",
+        headers=_attachment_headers(report, "md"),
+    )
+
+
+@router.get("/{scan_id}/report.sarif")
+def download_sarif_report(scan_id: UUID, request: Request) -> Response:
+    report = request.app.state.scan_service.get_report(scan_id)
+    return Response(
+        content=render_sarif(report),
+        media_type="application/sarif+json",
+        headers=_attachment_headers(report, "sarif"),
+    )
+
+
+@router.get("/{scan_id}/report.json")
+def download_json_report(scan_id: UUID, request: Request) -> Response:
+    report = request.app.state.scan_service.get_report(scan_id)
+    return Response(
+        content=report.model_dump_json(indent=2),
+        media_type="application/json",
+        headers=_attachment_headers(report, "json"),
+    )
+
+
+@router.get("/{scan_id}/badge.svg")
+def get_badge(scan_id: UUID, request: Request) -> Response:
+    report = request.app.state.scan_service.get_report(scan_id)
+    return Response(
+        content=render_badge(report),
+        media_type="image/svg+xml",
+        headers={
+            "Cache-Control": "no-store",
+            "X-Content-Type-Options": "nosniff",
+        },
+    )
+
+
+@router.get("/{scan_id}/verification-bundle")
+def download_verification_bundle(scan_id: UUID, request: Request) -> Response:
+    report = request.app.state.scan_service.get_report(scan_id)
+    return Response(
+        content=build_verification_bundle(report),
+        media_type="application/zip",
+        headers=_attachment_headers(
+            report,
+            "zip",
+            label="mxready-verification",
+        ),
+    )
+
+
+@router.post("/{scan_id}/verification-runs", response_model=ScanReport)
+async def upload_verification_run(
+    scan_id: UUID,
+    request: Request,
+) -> ScanReport:
+    content_type = request.headers.get("content-type", "").partition(";")[0].strip().casefold()
+    if content_type != "application/json":
+        raise MxReadyError(
+            "VERIFICATION_SCHEMA_INVALID",
+            "Verification results must use application/json.",
+        )
+    payload = await _read_limited_body(request)
+    return await run_in_threadpool(
+        request.app.state.scan_service.attach_verification,
+        scan_id,
+        payload,
+    )
+
+
+async def _read_limited_body(request: Request) -> bytes:
+    declared_size = request.headers.get("content-length")
+    if declared_size is not None:
+        try:
+            if int(declared_size) > MAX_VERIFICATION_UPLOAD_BYTES:
+                raise MxReadyError(
+                    "UPLOAD_TOO_LARGE",
+                    "Verification results must not exceed 1 MiB.",
+                )
+        except ValueError:
+            pass
+
+    body = bytearray()
+    async for chunk in request.stream():
+        remaining = MAX_VERIFICATION_UPLOAD_BYTES + 1 - len(body)
+        body.extend(chunk[: max(0, remaining)])
+        if len(body) > MAX_VERIFICATION_UPLOAD_BYTES or len(chunk) > remaining:
+            raise MxReadyError(
+                "UPLOAD_TOO_LARGE",
+                "Verification results must not exceed 1 MiB.",
+            )
+    return bytes(body)
+
+
+def _attachment_headers(
+    report: ScanReport,
+    extension: str,
+    *,
+    label: str = "mxready",
+) -> dict[str, str]:
+    safe_repository_name = re.sub(
+        r"[^A-Za-z0-9_-]+",
+        "-",
+        report.repository.name,
+    ).strip("-_")[:64]
+    safe_repository_name = safe_repository_name or "repository"
+    filename = f"{safe_repository_name}-{report.repository.commit[:12]}-{label}.{extension}"
+    return {
+        "Content-Disposition": f'attachment; filename="{filename}"',
+        "Cache-Control": "no-store",
+        "X-Content-Type-Options": "nosniff",
+    }

mxready/api/workspace.py

@@ -0,0 +1,156 @@
+"""History and user annotations for a single trusted MXReady deployment."""
+
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from uuid import UUID
+
+from fastapi import APIRouter, Query, Request
+from pydantic import StrictBool
+
+from mxready.errors import MxReadyError
+from mxready.models import (
+    FindingReview,
+    FindingReviewInput,
+    FindingReviewPage,
+    ScanPage,
+    ScanReport,
+    ScanStatus,
+    StrictModel,
+    VerificationCheck,
+    VerificationCommand,
+    VerificationDetail,
+    VerificationOutput,
+    VerificationPage,
+    VerificationRun,
+    VerificationSummary,
+)
+from mxready.verification.validation import VERIFICATION_MAX_AGE, verification_status_at
+from mxready_runner.redact import sanitize_output
+
+router = APIRouter(prefix="/scans", tags=["workspace"])
+
+
+class ChecklistUpdate(StrictModel):
+    completed: StrictBool
+
+
+@router.get("", response_model=ScanPage)
+def list_scans(
+    request: Request,
+    q: str = Query(default="", max_length=200),
+    status: ScanStatus | None = None,
+    limit: int = Query(default=20, ge=1, le=50),
+    offset: int = Query(default=0, ge=0, le=1_000_000),
+) -> ScanPage:
+    return request.app.state.store.list_jobs(
+        query=q.strip(), status=status, limit=limit, offset=offset
+    )
+
+
+def _summary(
+    run_id: int, uploaded: datetime | None, run: VerificationRun, latest: bool
+) -> VerificationSummary:
+    return VerificationSummary(
+        id=run_id,
+        uploaded_at=uploaded,
+        started_at=run.started_at,
+        finished_at=run.finished_at,
+        expires_at=run.finished_at + VERIFICATION_MAX_AGE,
+        status=verification_status_at(run, now=datetime.now(UTC)),
+        overall_status=run.overall_status,
+        repository_commit=run.repository_commit,
+        environment_fingerprint=run.environment_fingerprint,
+        runner_version=run.runner_version,
+        check_count=len(run.checks),
+        command_count=len(run.commands),
+        is_latest=latest,
+    )
+
+
+@router.get("/{scan_id}/verification-runs", response_model=VerificationPage)
+def list_verifications(
+    scan_id: UUID,
+    request: Request,
+    limit: int = Query(default=10, ge=1, le=50),
+    offset: int = Query(default=0, ge=0, le=1_000_000),
+) -> VerificationPage:
+    request.app.state.scan_service.get_scan(scan_id)
+    rows, total, latest = request.app.state.store.verification_history(
+        scan_id, limit=limit, offset=offset
+    )
+    return VerificationPage(
+        items=[_summary(i, uploaded, run, i == latest) for i, uploaded, run in rows],
+        total=total,
+        limit=limit,
+        offset=offset,
+    )
+
+
+@router.get("/{scan_id}/verification-runs/{run_id}", response_model=VerificationDetail)
+def verification_detail(scan_id: UUID, run_id: int, request: Request) -> VerificationDetail:
+    request.app.state.scan_service.get_scan(scan_id)
+    record = request.app.state.store.verification_detail(scan_id, run_id)
+    if record is None:
+        raise MxReadyError("VERIFICATION_NOT_FOUND", "The verification record does not exist.")
+    uploaded, run, latest = record
+
+    # Do not expose command arguments (which may contain credentials). Logs remain bounded
+    # and are sanitized again at this new read boundary, including legacy records.
+    def outputs(
+        items: Sequence[VerificationCheck | VerificationCommand],
+    ) -> list[VerificationOutput]:
+        return [
+            VerificationOutput(
+                id=item.id,
+                status=item.status,
+                return_code=item.return_code,
+                duration_ms=item.duration_ms,
+                stdout=sanitize_output(item.stdout),
+                stderr=sanitize_output(item.stderr),
+            )
+            for item in items
+        ]
+
+    return VerificationDetail(
+        summary=_summary(run_id, uploaded, run, latest),
+        checks=outputs(run.checks),
+        commands=outputs(run.commands),
+    )
+
+
+@router.patch("/{scan_id}/checklist/{rule_id}", response_model=ScanReport)
+def update_checklist(
+    scan_id: UUID, rule_id: str, payload: ChecklistUpdate, request: Request
+) -> ScanReport:
+    service = request.app.state.scan_service
+    report = service.get_report(scan_id)
+    if not any(item.rule_id == rule_id for item in report.migration_checklist):
+        raise MxReadyError("CHECKLIST_ITEM_NOT_FOUND", "The checklist item does not exist.")
+    request.app.state.store.set_checklist_item(scan_id, rule_id, payload.completed)
+    return service.get_report(scan_id)
+
+
+@router.post("/{scan_id}/reviews", response_model=FindingReview)
+def append_review(scan_id: UUID, payload: FindingReviewInput, request: Request) -> FindingReview:
+    report = request.app.state.scan_service.get_report(scan_id)
+    if not any(
+        f.rule_id == payload.rule_id and f.relative_path == payload.relative_path
+        for f in report.findings
+    ):
+        raise MxReadyError("FINDING_NOT_FOUND", "The finding does not exist in this report.")
+    return request.app.state.store.append_review(scan_id, payload)
+
+
+@router.get("/{scan_id}/reviews", response_model=FindingReviewPage)
+def list_reviews(
+    scan_id: UUID,
+    request: Request,
+    rule_id: str = Query(min_length=1, max_length=100),
+    relative_path: str = Query(min_length=1, max_length=4096),
+    limit: int = Query(default=20, ge=1, le=50),
+    offset: int = Query(default=0, ge=0, le=1_000_000),
+) -> FindingReviewPage:
+    request.app.state.scan_service.get_report(scan_id)
+    return request.app.state.store.review_history(
+        scan_id, rule_id, relative_path, limit=limit, offset=offset
+    )

mxready/app.py

@@ -0,0 +1,179 @@
+import logging
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import FileResponse, JSONResponse, Response
+from fastapi.staticfiles import StaticFiles
+from starlette.concurrency import run_in_threadpool
+
+from mxready import __version__
+from mxready.api import rules_router, scans_router
+from mxready.api.middleware import (
+    RateLimitMiddleware,
+    RequestBodyLimitMiddleware,
+)
+from mxready.api.workspace import router as workspace_router
+from mxready.config import Settings
+from mxready.errors import MxReadyError
+from mxready.logging_setup import setup_logging
+from mxready.persistence.sqlite import SQLiteStore
+from mxready.repository.git_client import GitClient
+from mxready.repository.identity import build_providers
+from mxready.scanning.analyzer import ScanAnalyzer
+from mxready.scanning.rule_loader import load_rule_catalog
+from mxready.services.scans import ScanService
+from mxready.services.worker import ScanWorker
+
+_ERROR_STATUS_CODES = {
+    "INVALID_REPOSITORY_URL": 400,
+    "UNSUPPORTED_REPOSITORY_HOST": 400,
+    "INVALID_GIT_REF": 400,
+    "SCAN_NOT_FOUND": 404,
+    "VERIFICATION_NOT_FOUND": 404,
+    "CHECKLIST_ITEM_NOT_FOUND": 404,
+    "FINDING_NOT_FOUND": 404,
+    "SCAN_NOT_COMPLETED": 409,
+    "VERIFICATION_COMMIT_MISMATCH": 409,
+    "UPLOAD_TOO_LARGE": 413,
+    "VERIFICATION_SCHEMA_INVALID": 422,
+    "SCAN_LIMIT_REACHED": 429,
+    "RATE_LIMITED": 429,
+}
+
+logger = logging.getLogger(__name__)
+
+
+def create_app(settings: Settings | None = None, *, start_worker: bool = True) -> FastAPI:
+    resolved = settings or Settings.from_env()
+    setup_logging(resolved.log_level)
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        resolved.data_dir.mkdir(parents=True, exist_ok=True)
+        resolved.temp_dir.mkdir(parents=True, exist_ok=True)
+        app.state.settings = resolved
+        store = SQLiteStore(resolved.data_dir / "mxready.db")
+        store.initialize()
+        if resolved.scan_retention_days > 0:
+            pruned = store.prune_old_scans(resolved.scan_retention_days)
+            if pruned:
+                logger.info(
+                    "pruned old scan records",
+                    extra={"count": pruned, "retention_days": resolved.scan_retention_days},
+                )
+        app.state.store = store
+        providers = build_providers(resolved.allowed_hosts)
+        catalog = load_rule_catalog(
+            resolved.rules_dir,
+            extra_directory=resolved.extra_rules_dir,
+        )
+        app.state.rule_catalog = catalog
+        app.state.scan_service = ScanService(
+            store,
+            GitClient(),
+            ScanAnalyzer(
+                catalog,
+                providers=providers,
+                enable_prefilter=resolved.prefilter_enabled,
+            ),
+            resolved,
+            providers=providers,
+        )
+        worker = ScanWorker(app.state.scan_service)
+        if start_worker:
+            worker.start()
+        try:
+            yield
+        finally:
+            if start_worker:
+                await run_in_threadpool(worker.stop)
+
+    app = FastAPI(title="MXReady", version=__version__, lifespan=lifespan)
+    if resolved.rate_limit_enabled:
+        app.add_middleware(RateLimitMiddleware, limit=resolved.rate_limit_per_minute)
+    app.add_middleware(
+        RequestBodyLimitMiddleware,
+        max_bytes=resolved.max_request_bytes,
+    )
+    app.include_router(scans_router, prefix="/api")
+    app.include_router(workspace_router, prefix="/api")
+    app.include_router(rules_router, prefix="/api")
+
+    @app.get("/api/health")
+    def health() -> dict[str, str]:
+        return {"status": "ok", "version": __version__}
+
+    @app.exception_handler(MxReadyError)
+    async def handle_mxready_error(
+        request: Request,
+        error: MxReadyError,
+    ) -> JSONResponse:
+        del request
+        return JSONResponse(
+            status_code=_ERROR_STATUS_CODES.get(error.code, 500),
+            content={
+                "error": {
+                    "code": error.code,
+                    "message": error.message,
+                    "details": error.details,
+                }
+            },
+        )
+
+    @app.exception_handler(RequestValidationError)
+    async def handle_request_validation(
+        request: Request,
+        error: RequestValidationError,
+    ) -> JSONResponse:
+        del request, error
+        return JSONResponse(
+            status_code=400,
+            content={
+                "error": {
+                    "code": "INVALID_REQUEST",
+                    "message": "The request body or path parameters are invalid.",
+                    "details": {},
+                }
+            },
+        )
+
+    _mount_built_frontend(app, resolved.frontend_dist)
+    return app
+
+
+class _ImmutableStaticFiles(StaticFiles):
+    async def get_response(self, path: str, scope) -> Response:
+        response = await super().get_response(path, scope)
+        if response.status_code < 400:
+            response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
+            response.headers["X-Content-Type-Options"] = "nosniff"
+        return response
+
+
+def _mount_built_frontend(app: FastAPI, frontend_dist: Path) -> None:
+    index_path = frontend_dist / "index.html"
+    if not index_path.is_file():
+        return
+
+    assets_path = frontend_dist / "assets"
+    if assets_path.is_dir():
+        app.mount(
+            "/assets",
+            _ImmutableStaticFiles(directory=assets_path),
+            name="frontend-assets",
+        )
+
+    @app.get("/{full_path:path}", include_in_schema=False)
+    async def serve_spa(full_path: str) -> FileResponse:
+        if full_path == "api" or full_path.startswith("api/"):
+            raise HTTPException(status_code=404, detail="Not found")
+        return FileResponse(
+            index_path,
+            media_type="text/html",
+            headers={
+                "Cache-Control": "no-store",
+                "X-Content-Type-Options": "nosniff",
+            },
+        )

mxready/cli/__init__.py

@@ -0,0 +1 @@
+"""Developer and release utilities for MXReady."""