mcp-guardbench 0.1.0__tar.gz

mcp_guardbench-0.1.0/.dockerignore

@@ -0,0 +1,25 @@
+.git
+.venv
+.venv-*
+venv
+__pycache__
+*.py[cod]
+.mypy_cache
+.ruff_cache
+.pytest_cache
+.coverage
+htmlcov
+data
+*.db
+*.sqlite
+*.sqlite3
+.env
+.env.*
+!.env.example
+reports/*
+!reports/.gitkeep
+tests
+docs
+.idea
+.vscode
+.DS_Store

mcp_guardbench-0.1.0/.env.example

@@ -0,0 +1,41 @@
+# MCP-GuardBench configuration. Copy to `.env` and adjust.
+# Every value below is a LOCAL LAB default. Nothing here is a real credential.
+# The only "secrets" in this project are the synthetic markers
+# (TEST_SECRET_123, TEST_PRIVATE_RECORD, SIMULATED_EXTERNAL_DESTINATION).
+
+# --- Runtime mode -----------------------------------------------------------
+# true  = simplified local setup: API-key check disabled, tables auto-created.
+# false = API requires GUARDBENCH_API_KEY on every non-health request.
+GUARDBENCH_DEV_MODE=true
+GUARDBENCH_LOG_LEVEL=INFO
+GUARDBENCH_LOG_JSON=true
+
+# --- Database ---------------------------------------------------------------
+# Local default is SQLite. docker-compose overrides this with PostgreSQL.
+GUARDBENCH_DATABASE_URL=sqlite:///./data/guardbench.db
+
+# PostgreSQL container credentials (docker-compose only). Change for anything
+# beyond a throwaway local lab.
+POSTGRES_DB=guardbench
+POSTGRES_USER=guardbench
+POSTGRES_PASSWORD=change-me-local-lab-only
+
+# --- API --------------------------------------------------------------------
+GUARDBENCH_API_HOST=127.0.0.1
+GUARDBENCH_API_PORT=8000
+# Required when GUARDBENCH_DEV_MODE=false. Generate one, e.g.:
+#   python -c "import secrets; print(secrets.token_urlsafe(32))"
+GUARDBENCH_API_KEY=
+
+# --- Dashboard --------------------------------------------------------------
+GUARDBENCH_API_URL=http://127.0.0.1:8000
+GUARDBENCH_DASHBOARD_PORT=8501
+
+# --- Lab safety limits ------------------------------------------------------
+# Allowlisted directories. Test cases and fixtures outside these are rejected.
+GUARDBENCH_TEST_CASES_DIR=test_cases
+GUARDBENCH_REPORTS_DIR=reports
+# Bounded model-context size for tool responses (bytes).
+GUARDBENCH_MAX_RESPONSE_BYTES=4096
+# Maximum tool calls allowed per trace (runaway-loop guard).
+GUARDBENCH_MAX_CALLS_PER_TRACE=10

mcp_guardbench-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,38 @@
+name: Bug report
+description: Something in the lab itself is broken (not a missed/false detection; see the other template).
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Before filing: this is a **local security lab**. If a reference control missed an attack or
+        flagged a benign case, that's expected and tracked in [docs/limitations.md](../../docs/limitations.md),
+        not a bug; use the "Detection result" template instead.
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened
+      description: What you did, what you expected, what happened instead.
+    validations:
+      required: true
+  - type: textarea
+    id: repro
+    attributes:
+      label: Steps to reproduce
+      description: Exact commands (`guardbench ...`, `make ...`). A failing test is even better.
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant output
+      description: Paste error output here. Please double-check it doesn't contain a real secret or API key before posting. Synthetic markers like `TEST_SECRET_123` are fine.
+      render: shell
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: Output of `guardbench --version`, plus your OS and Python version.
+    validations:
+      required: true

mcp_guardbench-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Report a security vulnerability in the project itself
+    url: https://github.com/Manishmaurya89/mcp-guardbench/security/advisories/new
+    about: Do not open a public issue for this; see SECURITY.md.

mcp_guardbench-0.1.0/.github/ISSUE_TEMPLATE/detection_result.yml

@@ -0,0 +1,50 @@
+name: Detection result / false positive / false negative
+description: A reference control (or a corpus test case) behaved in a way you think is wrong.
+labels: [detection]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This is the most useful kind of issue for this project. Please include enough for someone to
+        reproduce your exact run. See [docs/metrics.md](../../docs/metrics.md) for how detection and
+        prevention are defined and verified.
+  - type: dropdown
+    id: kind
+    attributes:
+      label: What kind of result?
+      options:
+        - False negative (an attack case was not detected/blocked when it should have been)
+        - False positive (a benign case triggered a detection/block/approval)
+        - Something else looks wrong in a report or the dashboard
+    validations:
+      required: true
+  - type: input
+    id: case
+    attributes:
+      label: Test case and adapter
+      description: e.g. `TP-001` under `reference-static`
+    validations:
+      required: true
+  - type: textarea
+    id: command
+    attributes:
+      label: Command used
+      render: shell
+      description: e.g. `guardbench benchmark run --adapter reference-static --case-id TP-001`
+    validations:
+      required: true
+  - type: textarea
+    id: report
+    attributes:
+      label: Relevant report excerpt or trace
+      description: Output of `guardbench trace --latest --adapter ... --case ...`, or the relevant report rows.
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: why
+    attributes:
+      label: Why you think this is wrong
+      description: What you expected the control (or the case's `expected:` block) to say, and why.
+    validations:
+      required: true

mcp_guardbench-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,27 @@
+name: Feature request
+description: A new fixture, adapter, metric, or capability you'd like to see.
+labels: [enhancement]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Read [SECURITY.md](../../SECURITY.md) first if this involves anything that touches real
+        servers, real credentials, or real network access: this project only ever attacks its own
+        local, synthetic fixtures, and any request that would weaken that boundary will be declined.
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem does this solve?
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: What would you like to see?
+      description: A new fixture (docs/limitations.md and mcp_lab/servers/README.md describe the constraints), a new adapter (see docs/adapter-development.md), a new metric, something else?
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives you've considered

mcp_guardbench-0.1.0/.github/ISSUE_TEMPLATE/inspect_result.yml

@@ -0,0 +1,41 @@
+name: guardbench inspect result on a real server
+description: A false alarm, a miss, or a server that could not be inspected.
+labels: [inspect]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Reports from real servers are how the analyzer's rules get better. **Remove anything private
+        before posting:** tokens, internal URLs, file paths, and the `env`/`headers` of your MCP config.
+        `guardbench inspect --json` never prints configured env values or headers, but check anyway.
+  - type: dropdown
+    id: kind
+    attributes:
+      label: What happened?
+      options:
+        - False alarm (a finding on a server you believe is fine)
+        - Missed something (a hostile description or instruction that was not flagged)
+        - The server could not be inspected (error, timeout)
+    validations:
+      required: true
+  - type: input
+    id: server
+    attributes:
+      label: Server
+      description: Public package name and version, e.g. `@modelcontextprotocol/server-filesystem 2026.1.0`.
+    validations:
+      required: true
+  - type: textarea
+    id: output
+    attributes:
+      label: Relevant output
+      description: The finding lines (rule id, tool, evidence) or the error message.
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: why
+    attributes:
+      label: Why you think this is wrong
+    validations:
+      required: true

mcp_guardbench-0.1.0/.github/ISSUE_TEMPLATE/test_case_proposal.yml

@@ -0,0 +1,56 @@
+name: Propose a test case (new attack or benign pattern)
+description: An MCP attack, or a benign real-world pattern, that the benchmark should cover.
+labels: [test-case, good first issue]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This is the contribution that makes the benchmark most meaningful. Cases written by people who
+        did **not** write the reference controls are the only way to learn how they really do: see the
+        "hard cases" in [test_cases/README.md](../../test_cases/README.md#-hard-cases).
+
+        Keep it inside the lab's safety rules ([SECURITY.md](../../SECURITY.md)): describe the attack
+        against a *synthetic* fixture, use only the synthetic markers (`TEST_SECRET_123`,
+        `TEST_PRIVATE_RECORD`, `SIMULATED_EXTERNAL_DESTINATION`), and never include a working exploit
+        against a real server.
+  - type: dropdown
+    id: kind
+    attributes:
+      label: What kind of case?
+      options:
+        - Attack the current controls probably miss
+        - Attack in a category not covered yet (resources, prompts, sampling, auth, multi-server, ...)
+        - Benign real-world pattern that might cause false alarms
+    validations:
+      required: true
+  - type: textarea
+    id: idea
+    attributes:
+      label: The attack (or pattern) in one paragraph
+      description: What does the server advertise or return, and what would a gullible agent do?
+    validations:
+      required: true
+  - type: textarea
+    id: example
+    attributes:
+      label: Example tool metadata or response (synthetic)
+      description: The hostile description, schema, or response text. Mark hostile text with GUARDBENCH-SYNTHETIC-INJECTION.
+      render: json
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected result
+      description: Which category should a good control detect it as, and should it be blocked?
+    validations:
+      required: true
+  - type: textarea
+    id: source
+    attributes:
+      label: Where this comes from (optional)
+      description: A paper, advisory, blog post, or real server this pattern was seen in.
+  - type: checkboxes
+    id: pr
+    attributes:
+      label: Contribution
+      options:
+        - label: I'd like to open a pull request with the fixture and YAML myself.

mcp_guardbench-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## What this changes and why
+
+
+
+## Checklist
+
+- [ ] `make check` passes locally (lint + typecheck + tests)
+- [ ] `make test-postgres` passes, if this touches the database layer
+- [ ] New behavior has new tests (this project does not accept "tested locally, trust me" for
+      anything touching detection, prevention, redaction, or input validation)
+- [ ] Docs updated in the same PR if this changes a metric definition, a policy rule, an API route,
+      a CLI command, or a documented limitation
+- [ ] If this touches fixtures, adapters, or anything that could reach outside the local lab: I've
+      read [SECURITY.md](../SECURITY.md) and this stays inside the local-fixture safety boundary
+
+## Anything reviewers should look at closely
+

mcp_guardbench-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,92 @@
+# CI for a local security lab: everything here runs against this repository's own fixtures
+# and its own Docker Compose stack on the runner. Nothing in this workflow contacts an
+# external server.
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    name: lint, typecheck, test (SQLite, Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v10.2.0
+      - run: make check
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+      - run: make coverage
+        if: matrix.python-version == '3.12'
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+
+  postgres:
+    name: full suite against real PostgreSQL (embedded, no Docker)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v10.2.0
+      - run: make install EXTRAS=dev,dashboard,pgtest
+      - run: .venv/bin/python scripts/test_postgres.py -q
+
+  cisco-scanner:
+    name: benchmark the open-source Cisco MCP Scanner (YARA analyzer, offline)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v10.2.0
+      - run: make setup
+      - name: Install the pinned scanner in its own venv and run all four adapters
+        run: make cisco-demo
+      - name: The live adapter test must agree with the published results
+        run: .venv/bin/python -m pytest -q tests/integration/test_cisco_scanner_adapter.py
+        env:
+          GUARDBENCH_CISCO_MCP_SCANNER: .venv-scanners/bin/mcp-scanner
+
+  docker:
+    name: Docker image build + Compose smoke test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      # docker-compose.yml refuses to render without a password and API key, so create .env first.
+      - name: Generate local-lab-only credentials for this CI run
+        run: python3 scripts/bootstrap_env.py
+      - name: Validate docker-compose.yml statically
+        run: docker compose config --quiet
+      - name: Build the image
+        run: docker compose build
+      - name: Start postgres, api, dashboard
+        run: docker compose up -d --wait --wait-timeout 180
+      - name: Health checks
+        run: |
+          curl -sf http://127.0.0.1:8000/health
+          curl -sf http://127.0.0.1:8501/_stcore/health
+      # Run as the runner's own user (like `make docker-demo`) so the container can write into the
+      # bind-mounted reports/ directory, which the checkout owns.
+      - name: Seed and run the demo inside the isolated test-runner container
+        run: |
+          docker compose --profile tools run --rm --user "$(id -u):$(id -g)" test-runner seed-demo
+          docker compose --profile tools run --rm --user "$(id -u):$(id -g)" test-runner benchmark run \
+            --project demo --cases test_cases/ \
+            --adapter no-defense-baseline --adapter reference-static --adapter reference-runtime \
+            --output reports/ci-demo
+      - name: Container logs (on failure)
+        if: failure()
+        run: docker compose logs
+      - name: Tear down
+        if: always()
+        run: docker compose down -v

mcp_guardbench-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,50 @@
+# Publishes to PyPI using Trusted Publishing (OIDC) -- no API token is stored in this repo.
+# Triggers only when a GitHub Release is published, never on an ordinary push, so a release is
+# always a deliberate, reviewable action, not a side effect of merging code.
+#
+# One-time setup required on pypi.org before this can succeed (see README/CONTRIBUTING):
+#   pypi.org -> your account -> Publishing -> "Add a pending publisher"
+#     PyPI project name : mcp-guardbench
+#     Owner              : Manishmaurya89
+#     Repository name     : mcp-guardbench
+#     Workflow name        : publish.yml
+#     Environment (optional, recommended): pypi
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build sdist + wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v10.2.0
+      - run: uv build
+      - name: Verify the built distributions (same check PyPI runs)
+        run: uvx twine check dist/*
+      - uses: actions/upload-artifact@v5
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/mcp-guardbench
+    permissions:
+      id-token: write  # required for Trusted Publishing; no secret token used or needed
+    steps:
+      - uses: actions/download-artifact@v6
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

mcp_guardbench-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+.venv-*/
+venv/
+build/
+dist/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.coverage
+coverage.xml
+htmlcov/
+
+# Local environment (never commit real values)
+.env
+.env.*
+!.env.example
+
+# Local databases and generated output
+*.db
+*.sqlite
+*.sqlite3
+data/
+reports/*
+!reports/.gitkeep
+!reports/sample/
+!reports/sample/**
+
+# Editors / OS
+.idea/
+.vscode/
+.DS_Store
+
+# Personal notes (audits, interview prep) - never meant for the public repo
+AUDIT_REPORT.md

mcp_guardbench-0.1.0/CHANGELOG.md

@@ -0,0 +1,26 @@
+# Changelog
+
+All notable changes to this project are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); the project uses
+[Semantic Versioning](https://semver.org/) and is pre-1.0, so minor versions may change behavior.
+
+## [0.1.0] - 2026-10-02
+
+First public release.
+
+### Added
+
+- `guardbench inspect`: read-only check of the MCP servers configured in Claude Desktop, Claude Code,
+  Cursor, VS Code and Windsurf (stdio, Streamable HTTP, SSE; JSONC; `${VAR}` expansion). Requests
+  `tools/list` only, analyzes tool metadata and server instructions with the benchmark's rules, pins
+  fingerprints to catch rug pulls, masks configured secrets, escapes server output, and exits 1 for CI
+  when a finding reaches `--fail-on`. Validated against 7 official MCP reference servers.
+- `cisco-mcp-scanner` adapter: benchmarks Cisco AI Defense's open-source MCP Scanner (YARA analyzer,
+  offline) as a separate program; `make cisco-demo` installs it into its own virtual environment.
+- Hard test cases `TP-003` (tool poisoning written in Spanish) and `DF-002` (a secret that leaves
+  base64-encoded), and benign control `BN-003` (descriptions that legitimately reference sibling tools),
+  with three new fixtures. The reference controls were not changed to pass them.
+- Fixture ground truth recognizes base64 and hex encodings of the synthetic markers.
+- The benchmark lab: 11 local fixtures, 12 test cases, reference static and runtime controls, verified
+  detection and prevention metrics, reports (JSON, Markdown, CSV, HTML), REST API, read-only dashboard,
+  PostgreSQL and SQLite support, Docker Compose setup.

mcp_guardbench-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,55 @@
+# Contributor Covenant Code of Conduct
+
+## Our pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a
+harassment-free experience for everyone, regardless of age, body size, visible or invisible
+disability, ethnicity, sex characteristics, gender identity and expression, level of experience,
+education, socio-economic status, nationality, personal appearance, race, religion, or sexual
+identity and orientation.
+
+## Our standards
+
+Examples of behavior that contributes to a positive environment:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes, and learning from
+  the experience
+* Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior:
+
+* The use of sexualized language or imagery, and sexual attention or advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address, without their
+  explicit permission
+* Using this project to test, scan, or attack systems you do not own or are not explicitly
+  authorized to test; see [SECURITY.md](SECURITY.md)
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Enforcement responsibilities
+
+Project maintainers are responsible for clarifying and enforcing our standards of acceptable
+behavior and will take appropriate and fair corrective action in response to any behavior they
+deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all community spaces (issues, pull requests, discussions), and
+also applies when an individual is officially representing the community in public spaces.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project
+maintainers via a private channel (a GitHub Security Advisory, or a direct message to a maintainer
+if one is listed on the repository). All complaints will be reviewed and investigated promptly and
+fairly.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org),
+version 2.1, available at
+https://www.contributor-covenant.org/version/2/1/code_of_conduct.html.