mcplint-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcplint/__about__.py +1 -0
- mcplint/__init__.py +3 -0
- mcplint/benchmark/__init__.py +0 -0
- mcplint/benchmark/dataset.py +34 -0
- mcplint/benchmark/providers/__init__.py +0 -0
- mcplint/benchmark/providers/anthropic_provider.py +92 -0
- mcplint/benchmark/providers/base.py +26 -0
- mcplint/benchmark/providers/factory.py +51 -0
- mcplint/benchmark/providers/fake.py +26 -0
- mcplint/benchmark/providers/openai_provider.py +25 -0
- mcplint/benchmark/runner.py +43 -0
- mcplint/benchmark/scorer.py +147 -0
- mcplint/cli/__init__.py +0 -0
- mcplint/cli/commands/__init__.py +0 -0
- mcplint/cli/commands/benchmark_cmd.py +88 -0
- mcplint/cli/commands/compare_cmd.py +139 -0
- mcplint/cli/commands/fix_cmd.py +60 -0
- mcplint/cli/commands/inspect_cmd.py +44 -0
- mcplint/cli/commands/rules_cmd.py +30 -0
- mcplint/cli/commands/scan_cmd.py +113 -0
- mcplint/cli/commands/snapshot_cmd.py +33 -0
- mcplint/cli/main.py +49 -0
- mcplint/compare/__init__.py +0 -0
- mcplint/compare/differ.py +148 -0
- mcplint/config/__init__.py +0 -0
- mcplint/config/loader.py +36 -0
- mcplint/config/schema.py +40 -0
- mcplint/core/__init__.py +0 -0
- mcplint/core/engine.py +40 -0
- mcplint/core/registry.py +52 -0
- mcplint/core/rules/__init__.py +0 -0
- mcplint/core/rules/ambiguity.py +157 -0
- mcplint/core/rules/ambiguity_rules.py +109 -0
- mcplint/core/rules/base.py +39 -0
- mcplint/core/rules/builtin.py +45 -0
- mcplint/core/rules/completeness_rules.py +217 -0
- mcplint/core/rules/description_rules.py +110 -0
- mcplint/core/rules/safety_rules.py +139 -0
- mcplint/core/rules/schema_rules.py +101 -0
- mcplint/core/score.py +132 -0
- mcplint/fix/__init__.py +0 -0
- mcplint/fix/suggest.py +183 -0
- mcplint/mcp_client/__init__.py +0 -0
- mcplint/mcp_client/canonical.py +25 -0
- mcplint/mcp_client/persistence.py +17 -0
- mcplint/mcp_client/session.py +80 -0
- mcplint/mcp_client/stdio.py +17 -0
- mcplint/models/__init__.py +0 -0
- mcplint/models/benchmark.py +67 -0
- mcplint/models/common.py +23 -0
- mcplint/models/comparison.py +53 -0
- mcplint/models/contracts.py +39 -0
- mcplint/models/findings.py +44 -0
- mcplint/models/fixes.py +13 -0
- mcplint/models/score.py +25 -0
- mcplint/models/snapshot.py +27 -0
- mcplint/py.typed +0 -0
- mcplint/reporters/__init__.py +0 -0
- mcplint/reporters/benchmark_terminal.py +46 -0
- mcplint/reporters/comparison_terminal.py +61 -0
- mcplint/reporters/fix_markdown.py +34 -0
- mcplint/reporters/html.py +71 -0
- mcplint/reporters/json_reporter.py +9 -0
- mcplint/reporters/sarif.py +77 -0
- mcplint/reporters/templates/report.html.j2 +176 -0
- mcplint/reporters/terminal.py +52 -0
- mcplint_cli-0.1.0.dist-info/METADATA +392 -0
- mcplint_cli-0.1.0.dist-info/RECORD +71 -0
- mcplint_cli-0.1.0.dist-info/WHEEL +4 -0
- mcplint_cli-0.1.0.dist-info/entry_points.txt +2 -0
- mcplint_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Finding, rule metadata, and the aggregate lint report."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from enum import StrEnum
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, Field
|
|
9
|
+
|
|
10
|
+
from mcplint.models.common import ArtifactMetadata
|
|
11
|
+
from mcplint.models.contracts import SourceLocation
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Severity(StrEnum):
|
|
15
|
+
ERROR = "error"
|
|
16
|
+
WARNING = "warning"
|
|
17
|
+
INFO = "info"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Finding(BaseModel):
|
|
21
|
+
rule_id: str
|
|
22
|
+
severity: Severity
|
|
23
|
+
message: str
|
|
24
|
+
evidence: str
|
|
25
|
+
location: SourceLocation
|
|
26
|
+
remediation: str
|
|
27
|
+
confidence: float = Field(ge=0.0, le=1.0)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class RuleMetadata(BaseModel):
|
|
31
|
+
id: str
|
|
32
|
+
title: str
|
|
33
|
+
description: str
|
|
34
|
+
default_severity: Severity
|
|
35
|
+
tags: list[str] = Field(default_factory=list)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class LintReport(BaseModel):
|
|
39
|
+
metadata: ArtifactMetadata
|
|
40
|
+
server_name: str
|
|
41
|
+
findings: list[Finding]
|
|
42
|
+
|
|
43
|
+
def count_by_severity(self) -> dict[Severity, int]:
|
|
44
|
+
return dict(Counter(finding.severity for finding in self.findings))
|
mcplint/models/fixes.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""The deterministic (or optionally LLM-assisted) rewrite suggestion model."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class RewriteSuggestion(BaseModel):
|
|
9
|
+
tool_name: str
|
|
10
|
+
proposed_description: str
|
|
11
|
+
resolved_rule_ids: list[str] = Field(default_factory=list)
|
|
12
|
+
explanation: str
|
|
13
|
+
confidence: float = Field(ge=0.0, le=1.0)
|
mcplint/models/score.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""The explainable 0-100 overall score. Not a scientifically validated metric —
|
|
2
|
+
a weighted, documented heuristic meant to make regressions visible at a glance.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, Field
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ScoreDeduction(BaseModel):
|
|
11
|
+
category: str
|
|
12
|
+
points_lost: float
|
|
13
|
+
finding_count: int
|
|
14
|
+
explanation: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ScoreBreakdown(BaseModel):
|
|
18
|
+
total_score: int = Field(ge=0, le=100)
|
|
19
|
+
deductions: list[ScoreDeduction] = Field(default_factory=list)
|
|
20
|
+
benchmark_accuracy: float | None = None
|
|
21
|
+
disclaimer: str = (
|
|
22
|
+
"This score is an explainable heuristic, not a scientifically validated "
|
|
23
|
+
"or universal quality metric. Use the deductions below to see exactly "
|
|
24
|
+
"why points were lost."
|
|
25
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""The canonical, persistable representation of one MCP server's tool contracts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Literal
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel
|
|
8
|
+
|
|
9
|
+
from mcplint.models.common import ArtifactMetadata
|
|
10
|
+
from mcplint.models.contracts import ToolContract
|
|
11
|
+
|
|
12
|
+
TransportKind = Literal["stdio", "http"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class MCPServerSnapshot(BaseModel):
|
|
16
|
+
metadata: ArtifactMetadata
|
|
17
|
+
server_name: str
|
|
18
|
+
server_version: str | None
|
|
19
|
+
transport: TransportKind
|
|
20
|
+
command: str | None
|
|
21
|
+
tools: list[ToolContract]
|
|
22
|
+
|
|
23
|
+
def get_tool(self, name: str) -> ToolContract | None:
|
|
24
|
+
for tool in self.tools:
|
|
25
|
+
if tool.name == name:
|
|
26
|
+
return tool
|
|
27
|
+
return None
|
mcplint/py.typed
ADDED
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Render a BenchmarkResult as Rich-formatted terminal text."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from io import StringIO
|
|
6
|
+
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
|
|
10
|
+
from mcplint.models.benchmark import BenchmarkResult
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def render_benchmark_terminal(result: BenchmarkResult) -> str:
|
|
14
|
+
buffer = StringIO()
|
|
15
|
+
console = Console(file=buffer, width=140, record=True)
|
|
16
|
+
|
|
17
|
+
console.print(
|
|
18
|
+
f"[bold]{result.dataset_name}[/bold] via {result.provider}/{result.model} "
|
|
19
|
+
f"({result.runs_per_case} runs/case, {len(result.trials)} trials total)"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
metrics = Table(title="Metrics")
|
|
23
|
+
metrics.add_column("Metric")
|
|
24
|
+
metrics.add_column("Value", justify="right")
|
|
25
|
+
metrics.add_row("Exact tool-selection accuracy", f"{result.exact_tool_selection_accuracy:.2%}")
|
|
26
|
+
metrics.add_row("Valid-argument rate", f"{result.valid_argument_rate:.2%}")
|
|
27
|
+
metrics.add_row("Required-argument accuracy", f"{result.required_argument_accuracy:.2%}")
|
|
28
|
+
metrics.add_row(
|
|
29
|
+
"Forbidden-tool invocation rate", f"{result.forbidden_tool_invocation_rate:.2%}"
|
|
30
|
+
)
|
|
31
|
+
metrics.add_row("No-tool rate", f"{result.no_tool_rate:.2%}")
|
|
32
|
+
metrics.add_row("Mean latency", f"{result.mean_latency_ms:.1f} ms")
|
|
33
|
+
metrics.add_row("P95 latency", f"{result.p95_latency_ms:.1f} ms")
|
|
34
|
+
if result.total_estimated_cost is not None:
|
|
35
|
+
metrics.add_row("Total estimated cost", f"${result.total_estimated_cost:.4f}")
|
|
36
|
+
console.print(metrics)
|
|
37
|
+
|
|
38
|
+
per_case = Table(title="Per-case results")
|
|
39
|
+
per_case.add_column("Case")
|
|
40
|
+
per_case.add_column("Pass rate", justify="right")
|
|
41
|
+
per_case.add_column("Stability", justify="right")
|
|
42
|
+
for case_id, pass_rate in result.per_case_pass_rate.items():
|
|
43
|
+
per_case.add_row(case_id, f"{pass_rate:.2%}", f"{result.stability.get(case_id, 0.0):.2%}")
|
|
44
|
+
console.print(per_case)
|
|
45
|
+
|
|
46
|
+
return buffer.getvalue()
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Render a ComparisonReport as Rich-formatted terminal text."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from io import StringIO
|
|
6
|
+
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
|
|
9
|
+
from mcplint.models.comparison import ComparisonReport
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def render_comparison_terminal(report: ComparisonReport) -> str:
|
|
13
|
+
buffer = StringIO()
|
|
14
|
+
console = Console(file=buffer, width=140, record=True)
|
|
15
|
+
|
|
16
|
+
console.print(
|
|
17
|
+
f"[bold]{report.baseline_server_name}[/bold] -> [bold]{report.candidate_server_name}[/bold]"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
if report.added_tools:
|
|
21
|
+
console.print(f"[green]Added tools:[/green] {', '.join(report.added_tools)}")
|
|
22
|
+
if report.removed_tools:
|
|
23
|
+
console.print(f"[red]Removed tools:[/red] {', '.join(report.removed_tools)}")
|
|
24
|
+
if report.description_changes:
|
|
25
|
+
console.print(f"Description changes: {len(report.description_changes)}")
|
|
26
|
+
if report.schema_changes:
|
|
27
|
+
console.print(f"Schema changes: {len(report.schema_changes)}")
|
|
28
|
+
|
|
29
|
+
console.print(
|
|
30
|
+
f"Findings: [red]{len(report.new_findings)} new[/red], "
|
|
31
|
+
f"[green]{len(report.resolved_findings)} resolved[/green]"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
if report.ambiguity_score_changes:
|
|
35
|
+
console.print(f"Ambiguity score changes: {len(report.ambiguity_score_changes)}")
|
|
36
|
+
for change in report.ambiguity_score_changes:
|
|
37
|
+
direction = "up" if change.after > change.before else "down"
|
|
38
|
+
console.print(
|
|
39
|
+
f" {change.tool_a} <-> {change.tool_b}: "
|
|
40
|
+
f"{change.before:.2f} -> {change.after:.2f} ({direction})"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
if report.benchmark_dataset_name is not None:
|
|
44
|
+
console.print(f"\n[bold]Benchmark: {report.benchmark_dataset_name}[/bold]")
|
|
45
|
+
console.print(
|
|
46
|
+
f" Accuracy: {report.baseline_accuracy:.2%} -> {report.candidate_accuracy:.2%} "
|
|
47
|
+
f"(delta {report.benchmark_accuracy_delta:+.2%})"
|
|
48
|
+
)
|
|
49
|
+
if report.regressions_by_case:
|
|
50
|
+
console.print(" [red]Regressions:[/red]")
|
|
51
|
+
for case_id, description in report.regressions_by_case.items():
|
|
52
|
+
console.print(f" {case_id}: {description}")
|
|
53
|
+
if report.min_accuracy_delta_threshold is not None:
|
|
54
|
+
verdict = "PASS" if report.passes_ci_threshold else "FAIL"
|
|
55
|
+
style = "green" if report.passes_ci_threshold else "bold red"
|
|
56
|
+
console.print(
|
|
57
|
+
f" CI threshold (--min-accuracy-delta "
|
|
58
|
+
f"{report.min_accuracy_delta_threshold:+.2%}): [{style}]{verdict}[/{style}]"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
return buffer.getvalue()
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Render RewriteSuggestions as a Markdown patch report — never applied automatically."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from mcplint.models.fixes import RewriteSuggestion
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def render_fix_markdown(server_name: str, suggestions: list[RewriteSuggestion]) -> str:
|
|
9
|
+
lines = [f"# MCPLint fix suggestions for `{server_name}`", ""]
|
|
10
|
+
|
|
11
|
+
if not suggestions:
|
|
12
|
+
lines.append("No actionable suggestions — nothing to fix.")
|
|
13
|
+
return "\n".join(lines) + "\n"
|
|
14
|
+
|
|
15
|
+
lines.append(
|
|
16
|
+
"These are proposed changes only. MCPLint never overwrites source files "
|
|
17
|
+
"automatically — review and apply them by hand."
|
|
18
|
+
)
|
|
19
|
+
lines.append("")
|
|
20
|
+
|
|
21
|
+
for suggestion in suggestions:
|
|
22
|
+
lines.append(f"## `{suggestion.tool_name}`")
|
|
23
|
+
lines.append("")
|
|
24
|
+
lines.append(f"**Confidence:** {suggestion.confidence:.2f}")
|
|
25
|
+
lines.append(f"**Resolves:** {', '.join(f'`{r}`' for r in suggestion.resolved_rule_ids)}")
|
|
26
|
+
lines.append("")
|
|
27
|
+
lines.append("**Proposed description:**")
|
|
28
|
+
lines.append("")
|
|
29
|
+
lines.append(f"> {suggestion.proposed_description}")
|
|
30
|
+
lines.append("")
|
|
31
|
+
lines.append(f"**Why:** {suggestion.explanation}")
|
|
32
|
+
lines.append("")
|
|
33
|
+
|
|
34
|
+
return "\n".join(lines) + "\n"
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Render a standalone HTML report — no backend, embedded CSS, no external requests."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib import resources
|
|
6
|
+
|
|
7
|
+
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
|
8
|
+
|
|
9
|
+
from mcplint.core.rules.ambiguity import (
|
|
10
|
+
DEFAULT_AMBIGUITY_THRESHOLD,
|
|
11
|
+
AmbiguityPairResult,
|
|
12
|
+
compute_ambiguity,
|
|
13
|
+
)
|
|
14
|
+
from mcplint.core.score import compute_score
|
|
15
|
+
from mcplint.models.benchmark import BenchmarkResult
|
|
16
|
+
from mcplint.models.comparison import ComparisonReport
|
|
17
|
+
from mcplint.models.findings import LintReport
|
|
18
|
+
from mcplint.models.fixes import RewriteSuggestion
|
|
19
|
+
from mcplint.models.snapshot import MCPServerSnapshot
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _score_class(total_score: int) -> str:
|
|
23
|
+
if total_score >= 80:
|
|
24
|
+
return "good"
|
|
25
|
+
if total_score >= 50:
|
|
26
|
+
return "ok"
|
|
27
|
+
return "bad"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _ambiguity_pairs(snapshot: MCPServerSnapshot, threshold: float) -> list[AmbiguityPairResult]:
|
|
31
|
+
pairs: list[AmbiguityPairResult] = []
|
|
32
|
+
tools = sorted(snapshot.tools, key=lambda t: t.name)
|
|
33
|
+
for index, tool_a in enumerate(tools):
|
|
34
|
+
for tool_b in tools[index + 1 :]:
|
|
35
|
+
result = compute_ambiguity(tool_a, tool_b)
|
|
36
|
+
if result.score >= threshold:
|
|
37
|
+
pairs.append(result)
|
|
38
|
+
return pairs
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def render_html_report(
|
|
42
|
+
snapshot: MCPServerSnapshot,
|
|
43
|
+
report: LintReport,
|
|
44
|
+
*,
|
|
45
|
+
benchmark_result: BenchmarkResult | None = None,
|
|
46
|
+
comparison: ComparisonReport | None = None,
|
|
47
|
+
suggestions: list[RewriteSuggestion] | None = None,
|
|
48
|
+
ambiguity_threshold: float = DEFAULT_AMBIGUITY_THRESHOLD,
|
|
49
|
+
) -> str:
|
|
50
|
+
template_dir = resources.files("mcplint.reporters") / "templates"
|
|
51
|
+
env = Environment(
|
|
52
|
+
loader=FileSystemLoader(str(template_dir)),
|
|
53
|
+
autoescape=select_autoescape(enabled_extensions=("html", "j2")),
|
|
54
|
+
)
|
|
55
|
+
template = env.get_template("report.html.j2")
|
|
56
|
+
|
|
57
|
+
score = compute_score(report, benchmark_result)
|
|
58
|
+
|
|
59
|
+
return template.render(
|
|
60
|
+
server_name=snapshot.server_name,
|
|
61
|
+
generated_at=report.metadata.generated_at.isoformat(),
|
|
62
|
+
mcplint_version=report.metadata.mcplint_version,
|
|
63
|
+
score=score,
|
|
64
|
+
score_class=_score_class(score.total_score),
|
|
65
|
+
findings=report.findings,
|
|
66
|
+
tools=snapshot.tools,
|
|
67
|
+
ambiguity_pairs=_ambiguity_pairs(snapshot, ambiguity_threshold),
|
|
68
|
+
benchmark=benchmark_result,
|
|
69
|
+
comparison=comparison,
|
|
70
|
+
suggestions=suggestions or [],
|
|
71
|
+
)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Render a LintReport as SARIF 2.1.0 for GitHub code scanning / CI upload."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
from mcplint.__about__ import __version__ as MCPLINT_VERSION
|
|
8
|
+
from mcplint.core.registry import RuleRegistry
|
|
9
|
+
from mcplint.models.findings import Finding, LintReport, Severity
|
|
10
|
+
|
|
11
|
+
SARIF_SCHEMA_URI = (
|
|
12
|
+
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"
|
|
13
|
+
)
|
|
14
|
+
SARIF_VERSION = "2.1.0"
|
|
15
|
+
MCPLINT_INFORMATION_URI = "https://github.com/mcplint/mcplint"
|
|
16
|
+
|
|
17
|
+
_LEVEL_BY_SEVERITY = {Severity.ERROR: "error", Severity.WARNING: "warning", Severity.INFO: "note"}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _rule_descriptors() -> list[dict[str, object]]:
|
|
21
|
+
registry = RuleRegistry.with_builtin_rules()
|
|
22
|
+
descriptors: list[dict[str, object]] = []
|
|
23
|
+
for rule in registry.all():
|
|
24
|
+
metadata = rule.metadata()
|
|
25
|
+
descriptors.append(
|
|
26
|
+
{
|
|
27
|
+
"id": metadata.id,
|
|
28
|
+
"name": metadata.id,
|
|
29
|
+
"shortDescription": {"text": metadata.title},
|
|
30
|
+
"fullDescription": {"text": metadata.description},
|
|
31
|
+
"defaultConfiguration": {"level": _LEVEL_BY_SEVERITY[metadata.default_severity]},
|
|
32
|
+
"properties": {"tags": list(metadata.tags)},
|
|
33
|
+
}
|
|
34
|
+
)
|
|
35
|
+
return descriptors
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _result(finding: Finding) -> dict[str, object]:
|
|
39
|
+
return {
|
|
40
|
+
"ruleId": finding.rule_id,
|
|
41
|
+
"level": _LEVEL_BY_SEVERITY[finding.severity],
|
|
42
|
+
"message": {"text": finding.message},
|
|
43
|
+
"properties": {
|
|
44
|
+
"evidence": finding.evidence,
|
|
45
|
+
"remediation": finding.remediation,
|
|
46
|
+
"confidence": finding.confidence,
|
|
47
|
+
},
|
|
48
|
+
"locations": [
|
|
49
|
+
{
|
|
50
|
+
"physicalLocation": {
|
|
51
|
+
"artifactLocation": {"uri": f"mcp-tool:{finding.location.tool_name}"},
|
|
52
|
+
},
|
|
53
|
+
"logicalLocations": [{"fullyQualifiedName": finding.location.json_path}],
|
|
54
|
+
}
|
|
55
|
+
],
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def render_sarif(report: LintReport) -> str:
|
|
60
|
+
sarif_log = {
|
|
61
|
+
"$schema": SARIF_SCHEMA_URI,
|
|
62
|
+
"version": SARIF_VERSION,
|
|
63
|
+
"runs": [
|
|
64
|
+
{
|
|
65
|
+
"tool": {
|
|
66
|
+
"driver": {
|
|
67
|
+
"name": "mcplint",
|
|
68
|
+
"informationUri": MCPLINT_INFORMATION_URI,
|
|
69
|
+
"version": MCPLINT_VERSION,
|
|
70
|
+
"rules": _rule_descriptors(),
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
"results": [_result(finding) for finding in report.findings],
|
|
74
|
+
}
|
|
75
|
+
],
|
|
76
|
+
}
|
|
77
|
+
return json.dumps(sarif_log, indent=2)
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<title>MCPLint report — {{ server_name }}</title>
|
|
6
|
+
<style>
|
|
7
|
+
:root { color-scheme: light dark; }
|
|
8
|
+
body {
|
|
9
|
+
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
|
|
10
|
+
max-width: 960px;
|
|
11
|
+
margin: 2rem auto;
|
|
12
|
+
padding: 0 1.5rem;
|
|
13
|
+
line-height: 1.5;
|
|
14
|
+
color: #1a1a1a;
|
|
15
|
+
background: #ffffff;
|
|
16
|
+
}
|
|
17
|
+
@media (prefers-color-scheme: dark) {
|
|
18
|
+
body { color: #e6e6e6; background: #14161a; }
|
|
19
|
+
table { border-color: #333 !important; }
|
|
20
|
+
th, td { border-color: #333 !important; }
|
|
21
|
+
.card { background: #1c1f24 !important; border-color: #333 !important; }
|
|
22
|
+
code { background: #262a30 !important; }
|
|
23
|
+
}
|
|
24
|
+
h1 { font-size: 1.6rem; margin-bottom: 0.25rem; }
|
|
25
|
+
h2 { font-size: 1.2rem; margin-top: 2.5rem; border-bottom: 2px solid currentColor; padding-bottom: 0.25rem; }
|
|
26
|
+
.meta { opacity: 0.7; font-size: 0.9rem; margin-bottom: 1.5rem; }
|
|
27
|
+
.score {
|
|
28
|
+
display: inline-block;
|
|
29
|
+
font-size: 2.5rem;
|
|
30
|
+
font-weight: 700;
|
|
31
|
+
border-radius: 0.5rem;
|
|
32
|
+
padding: 0.5rem 1.25rem;
|
|
33
|
+
margin: 0.5rem 0;
|
|
34
|
+
}
|
|
35
|
+
.score.good { background: #d4f4dd; color: #1a7431; }
|
|
36
|
+
.score.ok { background: #fff2cc; color: #8a6d00; }
|
|
37
|
+
.score.bad { background: #fbdcdc; color: #a3231f; }
|
|
38
|
+
.disclaimer { font-size: 0.85rem; opacity: 0.7; font-style: italic; }
|
|
39
|
+
table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
|
|
40
|
+
th, td { border: 1px solid #ddd; padding: 0.5rem 0.75rem; text-align: left; font-size: 0.92rem; }
|
|
41
|
+
th { background: rgba(127,127,127,0.1); }
|
|
42
|
+
.sev-error { color: #a3231f; font-weight: 600; }
|
|
43
|
+
.sev-warning { color: #8a6d00; font-weight: 600; }
|
|
44
|
+
.sev-info { color: #2563a8; font-weight: 600; }
|
|
45
|
+
.card { background: #f7f7f8; border: 1px solid #e2e2e2; border-radius: 0.5rem; padding: 1rem; margin: 0.75rem 0; }
|
|
46
|
+
code { background: rgba(127,127,127,0.15); padding: 0.1rem 0.3rem; border-radius: 0.25rem; }
|
|
47
|
+
.empty { opacity: 0.6; font-style: italic; }
|
|
48
|
+
</style>
|
|
49
|
+
</head>
|
|
50
|
+
<body>
|
|
51
|
+
|
|
52
|
+
<h1>MCPLint report</h1>
|
|
53
|
+
<div class="meta">
|
|
54
|
+
Server: <code>{{ server_name }}</code> ·
|
|
55
|
+
Generated: {{ generated_at }} ·
|
|
56
|
+
mcplint {{ mcplint_version }}
|
|
57
|
+
</div>
|
|
58
|
+
|
|
59
|
+
<div class="score {{ score_class }}">{{ score.total_score }}/100</div>
|
|
60
|
+
<p class="disclaimer">{{ score.disclaimer }}</p>
|
|
61
|
+
|
|
62
|
+
{% if score.deductions %}
|
|
63
|
+
<table>
|
|
64
|
+
<tr><th>Category</th><th>Points lost</th><th>Why</th></tr>
|
|
65
|
+
{% for d in score.deductions %}
|
|
66
|
+
<tr><td>{{ d.category }}</td><td>-{{ "%.1f"|format(d.points_lost) }}</td><td>{{ d.explanation }}</td></tr>
|
|
67
|
+
{% endfor %}
|
|
68
|
+
</table>
|
|
69
|
+
{% endif %}
|
|
70
|
+
|
|
71
|
+
<h2>Findings by severity ({{ findings|length }} total)</h2>
|
|
72
|
+
{% if findings %}
|
|
73
|
+
<table>
|
|
74
|
+
<tr><th>Severity</th><th>Rule</th><th>Tool</th><th>Message</th><th>Confidence</th></tr>
|
|
75
|
+
{% for f in findings %}
|
|
76
|
+
<tr>
|
|
77
|
+
<td class="sev-{{ f.severity }}">{{ f.severity }}</td>
|
|
78
|
+
<td><code>{{ f.rule_id }}</code></td>
|
|
79
|
+
<td>{{ f.location.tool_name }}</td>
|
|
80
|
+
<td>{{ f.message }}</td>
|
|
81
|
+
<td>{{ "%.2f"|format(f.confidence) }}</td>
|
|
82
|
+
</tr>
|
|
83
|
+
{% endfor %}
|
|
84
|
+
</table>
|
|
85
|
+
{% else %}
|
|
86
|
+
<p class="empty">No findings.</p>
|
|
87
|
+
{% endif %}
|
|
88
|
+
|
|
89
|
+
<h2>Tool inventory ({{ tools|length }} tools)</h2>
|
|
90
|
+
<table>
|
|
91
|
+
<tr><th>Name</th><th>Description</th><th>Params</th><th>Destructive</th><th>Read-only</th></tr>
|
|
92
|
+
{% for t in tools %}
|
|
93
|
+
<tr>
|
|
94
|
+
<td><code>{{ t.name }}</code></td>
|
|
95
|
+
<td>{{ t.description or "—" }}</td>
|
|
96
|
+
<td>{{ t.parameters|length }}</td>
|
|
97
|
+
<td>{{ "yes" if t.annotations.destructive_hint else "" }}</td>
|
|
98
|
+
<td>{{ "yes" if t.annotations.read_only_hint else "" }}</td>
|
|
99
|
+
</tr>
|
|
100
|
+
{% endfor %}
|
|
101
|
+
</table>
|
|
102
|
+
|
|
103
|
+
<h2>Ambiguity pairs ({{ ambiguity_pairs|length }})</h2>
|
|
104
|
+
{% if ambiguity_pairs %}
|
|
105
|
+
<table>
|
|
106
|
+
<tr><th>Tool A</th><th>Tool B</th><th>Score</th><th>Evidence</th></tr>
|
|
107
|
+
{% for pair in ambiguity_pairs %}
|
|
108
|
+
<tr>
|
|
109
|
+
<td>{{ pair.tool_a }}</td>
|
|
110
|
+
<td>{{ pair.tool_b }}</td>
|
|
111
|
+
<td>{{ "%.2f"|format(pair.score) }}</td>
|
|
112
|
+
<td>
|
|
113
|
+
{% if pair.evidence.shared_verbs %}shared verbs: {{ pair.evidence.shared_verbs|join(", ") }}<br>{% endif %}
|
|
114
|
+
{% if pair.evidence.overlapping_parameters %}overlapping params: {{ pair.evidence.overlapping_parameters|join(", ") }}<br>{% endif %}
|
|
115
|
+
{% if pair.evidence.absent_exact_vs_search_distinction %}missing exact-vs-search distinction<br>{% endif %}
|
|
116
|
+
{% if pair.evidence.absent_one_vs_many_distinction %}missing one-vs-many distinction<br>{% endif %}
|
|
117
|
+
{% if pair.evidence.absent_read_vs_write_distinction %}missing read-vs-write distinction{% endif %}
|
|
118
|
+
</td>
|
|
119
|
+
</tr>
|
|
120
|
+
{% endfor %}
|
|
121
|
+
</table>
|
|
122
|
+
{% else %}
|
|
123
|
+
<p class="empty">No ambiguous pairs above the threshold.</p>
|
|
124
|
+
{% endif %}
|
|
125
|
+
|
|
126
|
+
{% if benchmark %}
|
|
127
|
+
<h2>Benchmark: {{ benchmark.dataset_name }}</h2>
|
|
128
|
+
<div class="card">
|
|
129
|
+
<p>{{ benchmark.provider }}/{{ benchmark.model }} · {{ benchmark.runs_per_case }} runs/case · {{ benchmark.trials|length }} trials</p>
|
|
130
|
+
<table>
|
|
131
|
+
<tr><th>Metric</th><th>Value</th></tr>
|
|
132
|
+
<tr><td>Exact tool-selection accuracy</td><td>{{ "%.1f"|format(benchmark.exact_tool_selection_accuracy * 100) }}%</td></tr>
|
|
133
|
+
<tr><td>Valid-argument rate</td><td>{{ "%.1f"|format(benchmark.valid_argument_rate * 100) }}%</td></tr>
|
|
134
|
+
<tr><td>Required-argument accuracy</td><td>{{ "%.1f"|format(benchmark.required_argument_accuracy * 100) }}%</td></tr>
|
|
135
|
+
<tr><td>Forbidden-tool invocation rate</td><td>{{ "%.1f"|format(benchmark.forbidden_tool_invocation_rate * 100) }}%</td></tr>
|
|
136
|
+
<tr><td>No-tool rate</td><td>{{ "%.1f"|format(benchmark.no_tool_rate * 100) }}%</td></tr>
|
|
137
|
+
<tr><td>Mean / P95 latency</td><td>{{ "%.0f"|format(benchmark.mean_latency_ms) }} / {{ "%.0f"|format(benchmark.p95_latency_ms) }} ms</td></tr>
|
|
138
|
+
{% if benchmark.total_estimated_cost is not none %}
|
|
139
|
+
<tr><td>Total estimated cost</td><td>${{ "%.4f"|format(benchmark.total_estimated_cost) }}</td></tr>
|
|
140
|
+
{% endif %}
|
|
141
|
+
</table>
|
|
142
|
+
</div>
|
|
143
|
+
{% endif %}
|
|
144
|
+
|
|
145
|
+
{% if comparison %}
|
|
146
|
+
<h2>Before & after: {{ comparison.baseline_server_name }} → {{ comparison.candidate_server_name }}</h2>
|
|
147
|
+
<div class="card">
|
|
148
|
+
<p>
|
|
149
|
+
Added tools: {{ comparison.added_tools|join(", ") if comparison.added_tools else "none" }}<br>
|
|
150
|
+
Removed tools: {{ comparison.removed_tools|join(", ") if comparison.removed_tools else "none" }}<br>
|
|
151
|
+
Findings: {{ comparison.new_findings|length }} new, {{ comparison.resolved_findings|length }} resolved
|
|
152
|
+
</p>
|
|
153
|
+
{% if comparison.benchmark_dataset_name %}
|
|
154
|
+
<p>
|
|
155
|
+
Benchmark accuracy: {{ "%.1f"|format((comparison.baseline_accuracy or 0) * 100) }}% →
|
|
156
|
+
{{ "%.1f"|format((comparison.candidate_accuracy or 0) * 100) }}%
|
|
157
|
+
(delta {{ "%+.1f"|format((comparison.benchmark_accuracy_delta or 0) * 100) }}%)
|
|
158
|
+
</p>
|
|
159
|
+
{% endif %}
|
|
160
|
+
</div>
|
|
161
|
+
{% endif %}
|
|
162
|
+
|
|
163
|
+
{% if suggestions %}
|
|
164
|
+
<h2>Remediation suggestions ({{ suggestions|length }})</h2>
|
|
165
|
+
{% for s in suggestions %}
|
|
166
|
+
<div class="card">
|
|
167
|
+
<strong><code>{{ s.tool_name }}</code></strong> — confidence {{ "%.2f"|format(s.confidence) }}<br>
|
|
168
|
+
Resolves: {{ s.resolved_rule_ids|join(", ") }}<br>
|
|
169
|
+
<blockquote>{{ s.proposed_description }}</blockquote>
|
|
170
|
+
<em>{{ s.explanation }}</em>
|
|
171
|
+
</div>
|
|
172
|
+
{% endfor %}
|
|
173
|
+
{% endif %}
|
|
174
|
+
|
|
175
|
+
</body>
|
|
176
|
+
</html>
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Render a LintReport as Rich-formatted terminal text."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from io import StringIO
|
|
6
|
+
|
|
7
|
+
from rich.console import Console
|
|
8
|
+
from rich.table import Table
|
|
9
|
+
|
|
10
|
+
from mcplint.core.score import compute_score
|
|
11
|
+
from mcplint.models.findings import LintReport, Severity
|
|
12
|
+
|
|
13
|
+
_SEVERITY_STYLE = {Severity.ERROR: "bold red", Severity.WARNING: "yellow", Severity.INFO: "cyan"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def render_terminal(report: LintReport) -> str:
|
|
17
|
+
buffer = StringIO()
|
|
18
|
+
console = Console(file=buffer, width=120, record=True)
|
|
19
|
+
|
|
20
|
+
counts = report.count_by_severity()
|
|
21
|
+
score = compute_score(report)
|
|
22
|
+
console.print(
|
|
23
|
+
f"[bold]{report.server_name}[/bold] — "
|
|
24
|
+
f"{len(report.findings)} finding(s) "
|
|
25
|
+
f"({counts.get(Severity.ERROR, 0)} error, "
|
|
26
|
+
f"{counts.get(Severity.WARNING, 0)} warning, "
|
|
27
|
+
f"{counts.get(Severity.INFO, 0)} info) — "
|
|
28
|
+
f"score: [bold]{score.total_score}/100[/bold]"
|
|
29
|
+
)
|
|
30
|
+
if score.deductions:
|
|
31
|
+
for deduction in score.deductions:
|
|
32
|
+
console.print(f" -{deduction.points_lost:.1f} {deduction.explanation}")
|
|
33
|
+
|
|
34
|
+
if report.findings:
|
|
35
|
+
table = Table()
|
|
36
|
+
table.add_column("Rule")
|
|
37
|
+
table.add_column("Severity")
|
|
38
|
+
table.add_column("Tool")
|
|
39
|
+
table.add_column("Message")
|
|
40
|
+
table.add_column("Confidence", justify="right")
|
|
41
|
+
for finding in report.findings:
|
|
42
|
+
style = _SEVERITY_STYLE[finding.severity]
|
|
43
|
+
table.add_row(
|
|
44
|
+
finding.rule_id,
|
|
45
|
+
f"[{style}]{finding.severity.value}[/{style}]",
|
|
46
|
+
finding.location.tool_name,
|
|
47
|
+
finding.message,
|
|
48
|
+
f"{finding.confidence:.2f}",
|
|
49
|
+
)
|
|
50
|
+
console.print(table)
|
|
51
|
+
|
|
52
|
+
return buffer.getvalue()
|