skillscan-trace 0.1.0__tar.gz

skillscan_trace-0.1.0/.dockerignore

@@ -0,0 +1,17 @@
+.git
+.github
+__pycache__
+*.pyc
+*.pyo
+.pytest_cache
+.mypy_cache
+.ruff_cache
+dist/
+build/
+*.egg-info/
+trace-output/
+trace-cache/
+.env
+*.env
+tests/
+docs/

skillscan_trace-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,46 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: Lint, type-check & test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Format check (ruff)
+        if: matrix.python-version == '3.11'
+        run: ruff format --check .
+
+      - name: Lint (ruff)
+        if: matrix.python-version == '3.11'
+        run: ruff check .
+
+      - name: Type-check (mypy)
+        if: matrix.python-version == '3.11'
+        run: mypy skillscan_trace --ignore-missing-imports
+
+      - name: Run tests
+        run: pytest tests/ -q --tb=short
+        env:
+          # Tests use mock providers — no real API key needed
+          OPENAI_API_KEY: "sk-test-placeholder"

skillscan_trace-0.1.0/.github/workflows/codeql.yml

@@ -0,0 +1,29 @@
+name: CodeQL
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '21 4 * * 1'
+jobs:
+  analyze:
+    name: Analyze (python)
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [python]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

skillscan_trace-0.1.0/.github/workflows/release-pypi.yml

@@ -0,0 +1,51 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/skillscan-trace/
+    permissions:
+      id-token: write  # required for OIDC trusted publisher
+
+    steps:
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

skillscan_trace-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.env
+
+# Trace outputs
+traces/
+*.trace.json
+
+# Canary state (never commit)
+.canary_*/
+.domain_state.json
+
+# IDE
+.vscode/
+.idea/
+*.swp
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Test artifacts
+.pytest_cache/
+.coverage
+htmlcov/

skillscan_trace-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,71 @@
+# Contributing to skillscan-trace
+
+## Context
+
+skillscan-trace is the behavioral execution engine in the SkillScan family. It is a sibling project to [skillscan-security](https://github.com/kurtpayne/skillscan-security), which contains the static analyzer, ML classifier, and skill corpus.
+
+If you are picking up this project for the first time, read these documents in order:
+
+1. [`README.md`](./README.md) — what the tool does and why
+2. [`SPEC.md`](./SPEC.md) — the complete behavioral specification (authoritative)
+3. [`ARCHITECTURE.md`](./ARCHITECTURE.md) — system design and component overview
+4. [`IMPLEMENTATION_PLAN.md`](./IMPLEMENTATION_PLAN.md) — ordered build plan with acceptance criteria
+5. [`skillscan-security/docs/TRACE_RESEARCH.md`](https://github.com/kurtpayne/skillscan-security/blob/main/docs/TRACE_RESEARCH.md) — research notes on prior art, canary taxonomy, and domain allowlist design
+
+## Repository relationship
+
+The two repos share artifacts. When working on skillscan-trace, you may need to reference or update:
+
+| Artifact | Location | Notes |
+|---|---|---|
+| Canary taxonomy | `skillscan-security/docs/TRACE_RESEARCH.md` | Source of truth for what to detect |
+| Domain allowlist | `skillscan-security/trace/domains/verified.yml` | Bundled into skillscan-trace at build time |
+| Finding ID namespace | `skillscan-security/src/skillscan/rules/` | Finding IDs must not conflict |
+| Skill corpus | `skillscan-security/corpus/` | Used for integration tests and batch tracing |
+
+## Development setup
+
+```bash
+git clone https://github.com/kurtpayne/skillscan-trace
+cd skillscan-trace
+python3 -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+
+# Verify Ollama is running
+ollama serve &
+ollama pull qwen2.5:7b
+
+# Run tests
+pytest
+```
+
+## Implementation guidance
+
+The `IMPLEMENTATION_PLAN.md` defines 10 milestones in dependency order. Start with Milestone 1 (instrumented MCP server) — it is the novel component with no prior art and everything else depends on it.
+
+The MCP server should be testable standalone before the agent harness exists. Write a simple test client that sends tool calls directly to the server and verifies the interceptor behavior. This avoids needing a running model for unit tests.
+
+The agent harness uses the OpenAI-compatible API that Ollama exposes at `http://localhost:11434/v1`. The same harness code works with real OpenAI/Anthropic models by changing the `base_url`. Do not write model-specific integration code.
+
+## Testing
+
+Every interceptor must have unit tests. The integration test suite runs known-malicious and known-benign skills through the full pipeline. The skillscan-security corpus provides the test cases — clone it alongside this repo and set `SKILLSCAN_CORPUS_PATH` to point at it.
+
+```bash
+export SKILLSCAN_CORPUS_PATH=../skillscan-security/corpus
+pytest tests/integration/
+```
+
+## Design constraints
+
+**No GPU required.** The default path must work on a laptop with 8GB RAM and no GPU. Do not add GPU-only dependencies to the core package.
+
+**No API key required by default.** Ollama is the zero-friction path. API model support is a configuration option, not the default.
+
+**Low compliance model.** The default model (`qwen2.5:7b`) is chosen for its low refusal rate, not its safety properties. This is intentional — see SPEC.md Section 4.1 for the rationale.
+
+**Structured output first.** Every trace produces a machine-readable JSON report. Human-readable text is a formatting layer on top. Do not design the data model around the text output.
+
+## Versioning
+
+skillscan-trace follows semantic versioning. The v1.0 milestone is defined by the acceptance criteria in `IMPLEMENTATION_PLAN.md`. Breaking changes to the trace report schema require a major version bump.

skillscan_trace-0.1.0/Dockerfile

@@ -0,0 +1,64 @@
+# skillscan-trace Docker image
+#
+# Supports two modes:
+#
+#   Single trace (default):
+#     docker run --rm \
+#       -e OPENAI_API_KEY=$OPENAI_API_KEY \
+#       -v $(pwd)/my-skill.md:/skill.md:ro \
+#       skillscan/trace run /skill.md
+#
+#   Self-hosted server:
+#     docker run -p 8080:8080 \
+#       skillscan/trace serve
+#
+#   OpenRouter:
+#     docker run --rm \
+#       -e OPENROUTER_API_KEY=$OPENROUTER_API_KEY \
+#       -v $(pwd)/my-skill.md:/skill.md:ro \
+#       skillscan/trace run /skill.md --provider openrouter
+#
+#   Ollama (fully local, no API key):
+#     docker run --rm \
+#       --network host \
+#       -v $(pwd)/my-skill.md:/skill.md:ro \
+#       skillscan/trace run /skill.md --provider ollama
+
+FROM python:3.11-slim
+
+# Security: run as non-root
+RUN groupadd --gid 1001 skillscan && \
+    useradd --uid 1001 --gid skillscan --shell /bin/bash --create-home skillscan
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        git \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy project files
+COPY pyproject.toml README.md ./
+COPY skillscan_trace/ ./skillscan_trace/
+
+# Install with serve extras
+RUN pip install --no-cache-dir ".[serve]"
+
+# Create directories with correct ownership
+RUN mkdir -p /trace-output /trace-cache && \
+    chown -R skillscan:skillscan /app /trace-output /trace-cache
+
+USER skillscan
+
+# Default output and cache directories
+VOLUME ["/trace-output", "/trace-cache"]
+
+# Health check for serve mode
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8080/v1/health || exit 1
+
+# Default: show help. Override with `run /skill.md` or `serve`
+ENTRYPOINT ["skillscan-trace"]
+CMD ["--help"]

skillscan_trace-0.1.0/PKG-INFO

@@ -0,0 +1,200 @@
+Metadata-Version: 2.4
+Name: skillscan-trace
+Version: 0.1.0
+Summary: Behavioral execution engine for MCP-based AI agent skills
+Project-URL: Repository, https://github.com/kurtpayne/skillscan-trace
+Project-URL: Issues, https://github.com/kurtpayne/skillscan-trace/issues
+Project-URL: skillscan-security, https://github.com/kurtpayne/skillscan-security
+License: MIT
+Keywords: agents,ai,behavioral-analysis,mcp,security
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Security
+Requires-Python: >=3.11
+Requires-Dist: bashlex>=0.18
+Requires-Dist: click>=8.0
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: mcp>=1.0
+Requires-Dist: openai>=1.0
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: python-frontmatter>=1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.0
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: mypy>=1.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1; extra == 'dev'
+Provides-Extra: serve
+Requires-Dist: fastapi>=0.110; extra == 'serve'
+Requires-Dist: pydantic>=2.0; extra == 'serve'
+Requires-Dist: uvicorn[standard]>=0.29; extra == 'serve'
+Description-Content-Type: text/markdown
+
+# skillscan-trace
+
+[![CI](https://github.com/kurtpayne/skillscan-trace/actions/workflows/ci.yml/badge.svg)](https://github.com/kurtpayne/skillscan-trace/actions/workflows/ci.yml)
+[![CodeQL](https://github.com/kurtpayne/skillscan-trace/actions/workflows/codeql.yml/badge.svg)](https://github.com/kurtpayne/skillscan-trace/actions/workflows/codeql.yml)
+[![PyPI](https://img.shields.io/pypi/v/skillscan-trace.svg)](https://pypi.org/project/skillscan-trace/)
+[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](pyproject.toml)
+
+**Behavioral execution engine for MCP-based AI agent skills.**
+
+skillscan-trace runs a skill against a real language model inside an instrumented, isolated environment and records everything the model does: every file it reads, every network request it makes, every environment variable it accesses, every binary it probes. The output is a structured, machine-readable trace report.
+
+It is the dynamic analysis counterpart to [skillscan](https://github.com/kurtpayne/skillscan-security), which performs static analysis. Together they form two legs of the skillscan family:
+
+```
+skillscan family
+├── skillscan        — static analysis (pattern matching, ML classifier)
+├── skillscan-trace  — behavioral execution engine  ← this repo
+└── skillscan-lint   — schema and format validation (planned)
+```
+
+---
+
+## What it does
+
+A skill is a Markdown file that becomes a system prompt for an AI agent. Most skills are benign. Some are malicious — they instruct the agent to exfiltrate credentials, probe the filesystem, call attacker-controlled servers, or hijack the agent's behavior through prompt injection embedded in tool output.
+
+Static analysis catches the obvious cases. Behavioral analysis catches the rest: conditional payloads that only activate in certain environments, obfuscated instructions that decode at runtime, and prompt injection delivered through external content the skill fetches.
+
+skillscan-trace works by:
+
+1. Loading the skill's `SKILL.md` as the system prompt for a local language model
+2. Sending a realistic user prompt that exercises the skill's stated functionality
+3. Driving the model's tool-use loop through an **instrumented MCP server** that intercepts every tool call
+4. Checking each call against a canary taxonomy (credential files, wallet paths, ENV vars, binary probing, network destinations)
+5. Emitting a structured trace report (JSON + SARIF) with every observed behavior and any findings
+
+The model runs locally via [Ollama](https://ollama.com/) — no API key required, no cloud dependency. API providers (OpenAI, OpenRouter, Anthropic) are supported for users who prefer them or want access to more capable models.
+
+---
+
+## Status
+
+**v0.1.0 — core CLI complete.** Phases 1–5 are implemented and 144/144 tests pass. The tool is installable and usable today.
+
+See [`SPEC.md`](./SPEC.md) for the full behavioral specification.  
+See [`ARCHITECTURE.md`](./ARCHITECTURE.md) for the system design.  
+See [`IMPLEMENTATION_PLAN.md`](./IMPLEMENTATION_PLAN.md) for the build plan and phase status.
+
+---
+
+## Quick start
+
+```bash
+# Install from source
+git clone https://github.com/kurtpayne/skillscan-trace
+cd skillscan-trace
+pip install -e .
+
+# Run a trace with OpenAI
+export OPENAI_API_KEY=sk-...
+skillscan-trace run ./path/to/skill/
+
+# Run with OpenRouter (200+ models via one key)
+export OPENROUTER_API_KEY=sk-or-...
+skillscan-trace run ./skill/ --provider openrouter --model mistralai/mistral-7b-instruct
+
+# Run with a local Ollama model (no API key required)
+skillscan-trace run ./skill/ --provider ollama --model qwen2.5:7b
+
+# Run with explicit base URL (Azure, Mistral, etc.)
+skillscan-trace run ./skill/ --base-url https://api.mistral.ai/v1 --api-key $MISTRAL_KEY
+
+# Output formats
+skillscan-trace run ./skill/ --format sarif   # SARIF 2.1.0 for CI
+skillscan-trace run ./skill/ --format json    # native trace format
+skillscan-trace run ./skill/ --format text    # human-readable summary
+
+# Verify connectivity
+skillscan-trace check
+skillscan-trace check --provider openrouter
+skillscan-trace check --provider ollama
+```
+
+---
+
+## Skill format support
+
+skillscan-trace handles all skill formats found in the wild:
+
+| Format | Description | Example |
+|---|---|---|
+| Single `SKILL.md` | Standard Claude Code / MCP skill | `./my-skill/SKILL.md` |
+| Single `.md` file | Flat file, no directory | `./my-skill.md` |
+| Directory with `SKILL.md` | Standard with supporting files | `./my-skill/` |
+| Frontmatter + body | YAML frontmatter + Markdown body | `name:`, `allowed-tools:` |
+| Plain Markdown | No frontmatter | Any `.md` file |
+| Multi-file skill | Directory with multiple `.md` files | Loaded in alphabetical order |
+
+---
+
+## Output: Trace Report
+
+Every trace produces a JSON trace report and optionally a SARIF report.
+
+```json
+{
+  "schema_version": "1.0.0",
+  "trace_id": "trc_20260320_abc123",
+  "skill": {
+    "path": "./my-skill/SKILL.md",
+    "name": "git-helper",
+    "sha256": "a1b2c3..."
+  },
+  "model": {
+    "provider": "ollama",
+    "model": "qwen2.5:7b",
+    "version": "..."
+  },
+  "prompt": "Help me commit my changes",
+  "duration_ms": 4823,
+  "tool_calls": [...],
+  "findings": [...],
+  "summary": {
+    "total_tool_calls": 7,
+    "finding_count": 1,
+    "severity_max": "HIGH",
+    "clean": false
+  }
+}
+```
+
+---
+
+## Relationship to skillscan-security
+
+skillscan-trace is a sibling project to [skillscan-security](https://github.com/kurtpayne/skillscan-security). The two projects share:
+
+- **Finding schema**: The same finding IDs (`EXF-001`, `MAL-001`, `IOC-001`, etc.) and severity levels
+- **Canary taxonomy**: The same list of high-value target paths and ENV var names
+- **Domain allowlist**: `trace/domains/verified.yml` from skillscan-security is the source of truth; skillscan-trace consumes it
+- **Corpus feedback loop**: Traces that produce findings can be reviewed and added to the skillscan-security corpus as `sandbox_verified/` examples, improving the ML classifier's recall on behavioral patterns that static analysis misses
+
+---
+
+## Privacy & Key Handling
+
+Your API key goes directly to the LLM provider you chose. The canary server runs in-process on your machine. Nothing leaves your network except the LLM API calls you explicitly authorize. SkillScan has no server-side component in local mode.
+
+See [`PRIVACY.md`](./PRIVACY.md) for the full data flow explanation.
+
+---
+
+## Contributing
+
+See [`CONTRIBUTING.md`](./CONTRIBUTING.md).
+
+---
+
+## License
+
+MIT

skillscan_trace-0.1.0/PRIVACY.md

@@ -0,0 +1,95 @@
+# Privacy & Key Handling
+
+skillscan-trace is a local tool. **Your skill files, API keys, and trace results never leave your machine** except for the LLM API calls you explicitly authorize.
+
+---
+
+## What happens during a trace
+
+When you run `skillscan-trace run ./skill.md`, the following happens entirely on your machine:
+
+1. **Skill loading.** The skill file is read from your local filesystem. It is not uploaded anywhere.
+2. **Canary environment setup.** A temporary directory is created in-process with synthetic credential files, wallet paths, and environment variables. These are randomly generated per run and have no real value.
+3. **LLM API call.** The skill's content is sent to the LLM provider you configured (`--provider openai`, `--provider openrouter`, or `--provider ollama`). This is the only network request skillscan-trace makes. The request goes directly from your machine to the provider — it does not pass through any SkillScan server.
+4. **Canary server.** The instrumented MCP server runs in-process. It intercepts every tool call the model makes and checks it against the canary taxonomy. No subprocess is spawned; no data is written to disk except the trace report you explicitly request.
+5. **Report output.** The trace report is written to your local `./trace-output/` directory (or the path you specify with `--output-dir`). It is not uploaded anywhere.
+
+---
+
+## Your API key
+
+Your API key is passed directly to the LLM provider you chose. It is not:
+
+- Stored by skillscan-trace (it is read from your `.env` file, environment variable, or `--api-key` flag and used only for the duration of the run)
+- Transmitted to any SkillScan server (there is no SkillScan server in local mode)
+- Logged to disk (the trace report does not include your API key)
+- Shared with any third party other than the LLM provider you selected
+
+If you use `--provider openrouter`, your key goes to [OpenRouter](https://openrouter.ai). If you use `--provider openai`, it goes to [OpenAI](https://openai.com). If you use `--provider ollama`, no key is required and no network request is made to any external service.
+
+### Storing keys in .env
+
+skillscan-trace reads a `.env` file automatically from the current directory and all parent directories (nearest file wins). This is the recommended way to store API keys:
+
+```
+# .env  — add this file to .gitignore, never commit it
+OPENAI_API_KEY=sk-...
+OPENROUTER_API_KEY=sk-or-...
+ANTHROPIC_API_KEY=sk-ant-...   # only needed when --judge is enabled
+```
+
+Priority order (highest wins):
+1. `--api-key` CLI flag
+2. Shell environment variable (`export OPENAI_API_KEY=...`)
+3. `.env` file in the current or nearest ancestor directory
+4. `skillscan-trace.yaml` config file (API keys are **not** supported here — use `.env` instead)
+
+The `.env` file is loaded with `override=False`, meaning shell variables you have already exported always take precedence over `.env` values. If no `.env` file is found, skillscan-trace falls back to the shell environment silently.
+
+---
+
+## What is sent to the LLM provider
+
+The LLM API call contains:
+
+- The skill's `SKILL.md` content (as the system prompt)
+- A generated user message that exercises the skill's stated functionality
+- The canary tool definitions (the list of tools the model can call)
+
+The canary tool definitions describe the tool interface (names, parameter schemas) but do not contain any real credential values. The canary values (fake AWS keys, fake wallet seeds, etc.) are only present in the tool *responses* that the canary server returns when the model calls a tool — those responses stay on your machine and are never sent back to the LLM provider in a way that would expose them.
+
+---
+
+## Ollama (fully local)
+
+If you use `--provider ollama`, the entire trace runs on your machine with no external network requests. The model runs locally via [Ollama](https://ollama.com/). No API key is required. No data leaves your machine.
+
+---
+
+## Trace reports
+
+Trace reports (JSON, SARIF, text) are written to your local filesystem only. They contain:
+
+- The skill path and SHA-256 hash of the skill content
+- The model and provider used
+- All tool calls the model made (tool name, arguments, canary server response)
+- Any findings (rule ID, severity, description)
+- Token usage for the run
+
+They do not contain your API key, your system's real credential files, or any data from outside the canary environment.
+
+---
+
+## Summary
+
+| Data | Where it goes |
+|---|---|
+| Skill file content | LLM provider (as system prompt) |
+| API key | LLM provider only (never stored or logged) |
+| Canary tool definitions | LLM provider (tool schemas, no real values) |
+| Canary credential values | Your machine only (in-process canary server) |
+| Trace report | Your local filesystem only |
+| Your real credential files | Not accessed (canary uses synthetic files) |
+| `.env` file contents | Your machine only (never transmitted) |
+
+If you have questions or concerns about data handling, open an issue at [github.com/kurtpayne/skillscan-trace](https://github.com/kurtpayne/skillscan-trace/issues).