devguard-scanner 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. devguard/__init__.py +5 -0
  2. devguard/ai/__init__.py +0 -0
  3. devguard/ai/investigator.py +380 -0
  4. devguard/ai/prompts.py +113 -0
  5. devguard/ai/providers/__init__.py +0 -0
  6. devguard/ai/providers/anthropic.py +163 -0
  7. devguard/ai/providers/openai_compat.py +157 -0
  8. devguard/ai/registry.py +160 -0
  9. devguard/ai/run.py +119 -0
  10. devguard/ai/sdk.py +102 -0
  11. devguard/ai/testing.py +126 -0
  12. devguard/ai/tools.py +389 -0
  13. devguard/analyzers/__init__.py +0 -0
  14. devguard/analyzers/base.py +135 -0
  15. devguard/analyzers/carriers.py +345 -0
  16. devguard/analyzers/command.py +343 -0
  17. devguard/analyzers/content.py +240 -0
  18. devguard/analyzers/filesystem.py +142 -0
  19. devguard/analyzers/hidden.py +273 -0
  20. devguard/analyzers/imports.py +102 -0
  21. devguard/analyzers/jsonc.py +221 -0
  22. devguard/analyzers/surfaces/__init__.py +0 -0
  23. devguard/analyzers/surfaces/ai_assistants.py +318 -0
  24. devguard/analyzers/surfaces/autoload.py +142 -0
  25. devguard/analyzers/surfaces/common.py +242 -0
  26. devguard/analyzers/surfaces/deps.py +159 -0
  27. devguard/analyzers/surfaces/devcontainer.py +195 -0
  28. devguard/analyzers/surfaces/env.py +53 -0
  29. devguard/analyzers/surfaces/git.py +236 -0
  30. devguard/analyzers/surfaces/npm.py +304 -0
  31. devguard/analyzers/surfaces/python_pkg.py +138 -0
  32. devguard/analyzers/surfaces/vscode.py +515 -0
  33. devguard/cli.py +302 -0
  34. devguard/core/__init__.py +0 -0
  35. devguard/core/config.py +147 -0
  36. devguard/core/filetype.py +313 -0
  37. devguard/core/models.py +195 -0
  38. devguard/core/redact.py +72 -0
  39. devguard/core/safefs.py +280 -0
  40. devguard/core/scoring.py +60 -0
  41. devguard/core/service.py +211 -0
  42. devguard/correlate/__init__.py +0 -0
  43. devguard/correlate/chains.py +160 -0
  44. devguard/mcp/__init__.py +0 -0
  45. devguard/mcp/server.py +442 -0
  46. devguard/report/__init__.py +0 -0
  47. devguard/report/escape.py +31 -0
  48. devguard/report/markdown.py +242 -0
  49. devguard/report/sarif.py +96 -0
  50. devguard/report/terminal.py +47 -0
  51. devguard/report/writer.py +52 -0
  52. devguard/rules/__init__.py +0 -0
  53. devguard/rules/catalog.py +96 -0
  54. devguard/rules/content/ai_instructions.yaml +111 -0
  55. devguard/rules/content/cross_language.yaml +228 -0
  56. devguard/rules/content/docs.yaml +63 -0
  57. devguard/rules/content/javascript.yaml +171 -0
  58. devguard/rules/content/obfuscation.yaml +138 -0
  59. devguard/rules/content/python.yaml +144 -0
  60. devguard/rules/content/shell.yaml +109 -0
  61. devguard/rules/content/windows.yaml +95 -0
  62. devguard/rules/data/vscode_settings.yaml +81 -0
  63. devguard_scanner-0.1.0.dist-info/METADATA +84 -0
  64. devguard_scanner-0.1.0.dist-info/RECORD +67 -0
  65. devguard_scanner-0.1.0.dist-info/WHEEL +4 -0
  66. devguard_scanner-0.1.0.dist-info/entry_points.txt +7 -0
  67. devguard_scanner-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,157 @@
1
+ """Built-in providers: OpenAI and OpenAI-compatible Chat Completions APIs (extra: devguard-scanner[openai]).
2
+
3
+ `openai` the OpenAI API.
4
+ `openai-compatible` Ollama, vLLM, LM Studio, llama.cpp server, OpenRouter, company gateways, and other
5
+ services exposing the Chat Completions API. Requires `base_url`.
6
+
7
+ Profile settings:
8
+ model required
9
+ api_key resolved from secrets (local servers usually accept any value)
10
+ base_url endpoint; required for openai-compatible
11
+ temperature sent only if set (several models reject sampling parameters)
12
+ max_tokens_param "max_completion_tokens" (default for openai) or "max_tokens" (default for compatible)
13
+ tool_calling false for servers/models without function calling (core then uses prompted tools)
14
+ max_context_tokens context size used to bound the evidence payload (default 128000)
15
+ data_stays_local true for local servers; shown in the consent notice
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ from typing import Any
22
+
23
+ from devguard.ai.sdk import (
24
+ CompletionRequest,
25
+ CompletionResponse,
26
+ Message,
27
+ ProviderCapabilities,
28
+ ToolCall,
29
+ Usage,
30
+ error_response,
31
+ )
32
+
33
+
34
+ class OpenAICompatibleProvider:
35
+ def __init__(self, settings: dict[str, Any], *, name: str, default_tokens_param: str) -> None:
36
+ try:
37
+ import openai
38
+ except ImportError as exc: # pragma: no cover - optional extra
39
+ raise RuntimeError(
40
+ "The 'openai' package is not installed. Run: pip install 'devguard-scanner[openai]'"
41
+ ) from exc
42
+ self.name = name
43
+ self._openai = openai
44
+ if name == "openai-compatible" and not settings.get("base_url") and not settings.get("_client"):
45
+ raise ValueError("openai-compatible profiles require base_url")
46
+ self.client = settings.get("_client") or openai.OpenAI(
47
+ api_key=settings.get("api_key") or "not-needed",
48
+ base_url=settings.get("base_url"),
49
+ max_retries=int(settings.get("max_retries", 2)),
50
+ )
51
+ self.temperature = settings.get("temperature")
52
+ self.tokens_param = settings.get("max_tokens_param", default_tokens_param)
53
+ self.tool_calling = bool(settings.get("tool_calling", True))
54
+ self.context = int(settings.get("max_context_tokens", 128_000))
55
+ self.local = bool(settings.get("data_stays_local", False))
56
+
57
+ def capabilities(self) -> ProviderCapabilities:
58
+ return ProviderCapabilities(
59
+ tool_calling=self.tool_calling,
60
+ structured_output=False,
61
+ max_context_tokens=self.context,
62
+ data_stays_local=self.local,
63
+ )
64
+
65
+ def complete(self, request: CompletionRequest) -> CompletionResponse:
66
+ params: dict[str, Any] = {
67
+ "model": request.model,
68
+ "messages": _to_messages(request.messages),
69
+ self.tokens_param: request.max_output_tokens,
70
+ "timeout": request.timeout_s,
71
+ }
72
+ if request.tools and self.tool_calling:
73
+ params["tools"] = [
74
+ {
75
+ "type": "function",
76
+ "function": {"name": t.name, "description": t.description, "parameters": t.input_schema},
77
+ }
78
+ for t in request.tools
79
+ ]
80
+ if self.temperature is not None:
81
+ params["temperature"] = float(self.temperature)
82
+ o = self._openai
83
+ try:
84
+ resp = self.client.chat.completions.create(**params)
85
+ except o.RateLimitError as exc:
86
+ return error_response(request.model, f"rate limited: {exc}")
87
+ except o.APIStatusError as exc:
88
+ return error_response(request.model, f"API error {exc.status_code}: {exc.message}")
89
+ except o.APITimeoutError:
90
+ return error_response(request.model, "request timed out")
91
+ except o.APIConnectionError as exc:
92
+ return error_response(request.model, f"connection error: {exc}")
93
+ if not resp.choices:
94
+ return error_response(request.model, "empty response")
95
+ choice = resp.choices[0]
96
+ calls: list[ToolCall] = []
97
+ for c in choice.message.tool_calls or []:
98
+ try:
99
+ args = json.loads(c.function.arguments or "{}")
100
+ except json.JSONDecodeError:
101
+ args = {"_invalid_json": c.function.arguments}
102
+ calls.append(ToolCall(id=c.id, name=c.function.name, arguments=args if isinstance(args, dict) else {}))
103
+ finish = choice.finish_reason
104
+ stop = (
105
+ "tool_use"
106
+ if calls
107
+ else "max_tokens"
108
+ if finish == "length"
109
+ else "refused"
110
+ if finish == "content_filter"
111
+ else "final"
112
+ )
113
+ usage = Usage(
114
+ input_tokens=getattr(resp.usage, "prompt_tokens", None),
115
+ output_tokens=getattr(resp.usage, "completion_tokens", None),
116
+ )
117
+ return CompletionResponse(
118
+ text=choice.message.content,
119
+ tool_calls=calls,
120
+ stop_reason=stop, # type: ignore[arg-type]
121
+ usage=usage,
122
+ model=resp.model or request.model,
123
+ error="content filtered" if stop == "refused" else None,
124
+ )
125
+
126
+
127
+ def _to_messages(messages: list[Message]) -> list[dict[str, Any]]:
128
+ out: list[dict[str, Any]] = []
129
+ for m in messages:
130
+ if m.role == "tool":
131
+ out += [{"role": "tool", "tool_call_id": r.call_id, "content": r.content} for r in m.tool_results]
132
+ elif m.role == "assistant" and m.tool_calls:
133
+ out.append(
134
+ {
135
+ "role": "assistant",
136
+ "content": m.content or None,
137
+ "tool_calls": [
138
+ {
139
+ "id": c.id,
140
+ "type": "function",
141
+ "function": {"name": c.name, "arguments": json.dumps(c.arguments)},
142
+ }
143
+ for c in m.tool_calls
144
+ ],
145
+ }
146
+ )
147
+ else:
148
+ out.append({"role": m.role, "content": m.content})
149
+ return out
150
+
151
+
152
+ def make_provider(settings: dict[str, Any]) -> OpenAICompatibleProvider:
153
+ return OpenAICompatibleProvider(settings, name="openai-compatible", default_tokens_param="max_tokens")
154
+
155
+
156
+ def make_openai_provider(settings: dict[str, Any]) -> OpenAICompatibleProvider:
157
+ return OpenAICompatibleProvider(settings, name="openai", default_tokens_param="max_completion_tokens")
@@ -0,0 +1,160 @@
1
+ """Provider discovery and loading (PLG-002..006).
2
+
3
+ Providers load only from: built-in entry points, installed packages exposing the `devguard.ai_providers`
4
+ entry point, or a Python file named in user-level configuration. Paths inside the scan target are refused
5
+ (SAFE-017). Secrets are referenced as `env:VAR` and resolved at call time.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import importlib.util
12
+ import os
13
+ import sys
14
+ from dataclasses import dataclass, field
15
+ from importlib.metadata import entry_points
16
+ from pathlib import Path
17
+ from typing import Any
18
+
19
+ from pydantic import BaseModel, ConfigDict, Field
20
+
21
+ from devguard.ai.sdk import AIProvider
22
+ from devguard.core.config import is_within
23
+
24
+ ENTRY_POINT_GROUP = "devguard.ai_providers"
25
+
26
+
27
+ class ProviderError(Exception):
28
+ pass
29
+
30
+
31
+ class AIProfile(BaseModel):
32
+ model_config = ConfigDict(extra="allow")
33
+
34
+ provider: str
35
+ model: str = ""
36
+ plugin: str | None = None # "path/to/file.py:factory" when provider == "python"
37
+ secrets: dict[str, str] = Field(default_factory=dict)
38
+
39
+ def settings(self) -> dict[str, Any]:
40
+ return dict(self.model_extra or {})
41
+
42
+
43
+ class AIConfig(BaseModel):
44
+ model_config = ConfigDict(extra="forbid")
45
+
46
+ default_profile: str | None = None
47
+ max_steps: int = 8
48
+ max_tool_calls: int = 20
49
+ max_total_tokens: int = 400_000
50
+ max_cost_usd: float | None = 1.0
51
+ timeout_s: float = 120.0
52
+ max_output_tokens: int = 16_000
53
+ send_excerpts: bool = True
54
+ top_k: int = 25
55
+ allowed_plugins: list[str] | None = None
56
+ profiles: dict[str, AIProfile] = Field(default_factory=dict)
57
+
58
+
59
+ @dataclass
60
+ class LoadedProvider:
61
+ provider: AIProvider
62
+ profile_name: str
63
+ profile: AIProfile
64
+ origin: str # "entry point <dist> <version>" or "file <path> sha256:<..>"
65
+ settings: dict[str, Any] = field(default_factory=dict)
66
+
67
+
68
+ def resolve_secret(ref: str) -> str:
69
+ if ref.startswith("env:"):
70
+ var = ref[4:]
71
+ value = os.environ.get(var)
72
+ if not value:
73
+ raise ProviderError(f"environment variable {var} is not set")
74
+ return value
75
+ if ref.startswith("keychain:"):
76
+ try:
77
+ import keyring # optional
78
+ except ImportError as exc:
79
+ raise ProviderError("keychain: secrets need the optional 'keyring' package") from exc
80
+ value = keyring.get_password("devguard", ref[9:])
81
+ if not value:
82
+ raise ProviderError(f"no keychain entry devguard/{ref[9:]}")
83
+ return value
84
+ raise ProviderError("secrets must be referenced as env:VAR or keychain:NAME, never written inline")
85
+
86
+
87
+ def available_entry_points() -> dict[str, str]:
88
+ out: dict[str, str] = {}
89
+ for ep in entry_points(group=ENTRY_POINT_GROUP):
90
+ dist = getattr(ep, "dist", None)
91
+ origin = f"{dist.name} {dist.version}" if dist is not None else "unknown distribution"
92
+ out[ep.name] = origin
93
+ return out
94
+
95
+
96
+ def load_provider(cfg: AIConfig, profile_name: str | None, target: Path | None) -> LoadedProvider:
97
+ name = profile_name or cfg.default_profile
98
+ if not name:
99
+ raise ProviderError("no AI profile selected: pass --ai-profile or set ai.default_profile in your user config")
100
+ profile = cfg.profiles.get(name)
101
+ if profile is None:
102
+ raise ProviderError(f"unknown AI profile {name!r}; configured: {', '.join(sorted(cfg.profiles)) or 'none'}")
103
+ if cfg.allowed_plugins is not None and profile.provider not in cfg.allowed_plugins:
104
+ raise ProviderError(f"provider {profile.provider!r} is not in ai.allowed_plugins")
105
+ settings = profile.settings()
106
+ settings["model"] = profile.model
107
+ for key, ref in profile.secrets.items():
108
+ settings[key] = resolve_secret(ref)
109
+ if profile.provider == "python":
110
+ factory, origin = _load_file_plugin(profile.plugin, target)
111
+ else:
112
+ factory, origin = _load_entry_point(profile.provider)
113
+ try:
114
+ provider = factory(settings)
115
+ except Exception as exc:
116
+ raise ProviderError(f"provider {profile.provider!r} failed to initialize: {exc}") from exc
117
+ for attr in ("name", "capabilities", "complete"):
118
+ if not hasattr(provider, attr):
119
+ raise ProviderError(f"provider {profile.provider!r} does not implement AIProvider ({attr} missing)")
120
+ return LoadedProvider(
121
+ provider, name, profile, origin, {k: v for k, v in settings.items() if k not in profile.secrets}
122
+ )
123
+
124
+
125
+ def _load_entry_point(name: str) -> tuple[Any, str]:
126
+ for ep in entry_points(group=ENTRY_POINT_GROUP):
127
+ if ep.name == name:
128
+ dist = getattr(ep, "dist", None)
129
+ origin = f"entry point {name} ({dist.name} {dist.version})" if dist else f"entry point {name}"
130
+ try:
131
+ return ep.load(), origin
132
+ except Exception as exc:
133
+ raise ProviderError(f"cannot load provider {name!r}: {exc}") from exc
134
+ raise ProviderError(f"no provider named {name!r}. Installed: {', '.join(sorted(available_entry_points()))}")
135
+
136
+
137
+ def _load_file_plugin(spec: str | None, target: Path | None) -> tuple[Any, str]:
138
+ if not spec or ":" not in spec:
139
+ raise ProviderError('python profiles need plugin = "/path/to/file.py:factory"')
140
+ path_str, _, func = spec.rpartition(":")
141
+ path = Path(path_str).expanduser().resolve()
142
+ if target is not None and is_within(path, target):
143
+ raise ProviderError("refusing to load a provider plugin from inside the scan target")
144
+ if not path.is_file() or path.suffix != ".py":
145
+ raise ProviderError(f"plugin file not found: {path}")
146
+ digest = hashlib.sha256(path.read_bytes()).hexdigest()[:12]
147
+ mod_name = f"devguard_user_plugin_{digest}"
148
+ spec_obj = importlib.util.spec_from_file_location(mod_name, path)
149
+ if spec_obj is None or spec_obj.loader is None:
150
+ raise ProviderError(f"cannot import {path}")
151
+ module = importlib.util.module_from_spec(spec_obj)
152
+ sys.modules[mod_name] = module
153
+ try:
154
+ spec_obj.loader.exec_module(module) # user-configured plugin code, loaded like any installed package
155
+ except Exception as exc:
156
+ raise ProviderError(f"plugin {path} failed to import: {exc}") from exc
157
+ factory = getattr(module, func, None)
158
+ if not callable(factory):
159
+ raise ProviderError(f"{path} has no callable {func!r}")
160
+ return factory, f"file {path} sha256:{digest}"
devguard/ai/run.py ADDED
@@ -0,0 +1,119 @@
1
+ """AI stage orchestration: profile loading, consent (AI-003), response cache (AI-011), failure containment."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import sys
8
+ from collections.abc import Callable
9
+ from pathlib import Path
10
+
11
+ from devguard.ai.investigator import LoopInvestigator, build_evidence
12
+ from devguard.ai.registry import AIConfig, ProviderError, load_provider
13
+ from devguard.core.config import user_data_dir
14
+ from devguard.core.models import AIReport
15
+ from devguard.core.service import ScanOutcome
16
+
17
+
18
+ def _consent_file() -> Path:
19
+ return user_data_dir() / "ai-consent.json"
20
+
21
+
22
+ def _consent_key(profile: str, provider: str, model: str) -> str:
23
+ return f"{profile}|{provider}|{model}"
24
+
25
+
26
+ def has_consent(key: str) -> bool:
27
+ try:
28
+ return key in json.loads(_consent_file().read_text())
29
+ except (OSError, ValueError):
30
+ return False
31
+
32
+
33
+ def record_consent(key: str) -> None:
34
+ path = _consent_file()
35
+ path.parent.mkdir(parents=True, exist_ok=True)
36
+ try:
37
+ data = json.loads(path.read_text())
38
+ except (OSError, ValueError):
39
+ data = []
40
+ if key not in data:
41
+ data.append(key)
42
+ path.write_text(json.dumps(data, indent=1))
43
+
44
+
45
+ def run_ai(
46
+ outcome: ScanOutcome,
47
+ ai_settings: dict,
48
+ *,
49
+ profile: str | None,
50
+ tools_enabled: bool,
51
+ assume_consent: bool,
52
+ use_cache: bool = True,
53
+ ask: Callable[[str], bool] | None = None,
54
+ ) -> AIReport:
55
+ """Run the AI stage. Never raises: every failure becomes AIReport(status="failed"/"skipped")."""
56
+ try:
57
+ cfg = AIConfig.model_validate(ai_settings)
58
+ except Exception as exc:
59
+ return AIReport(status="failed", error=f"invalid [ai] configuration: {exc}")
60
+ target = Path(outcome.fs.root)
61
+ try:
62
+ loaded = load_provider(cfg, profile, target)
63
+ except ProviderError as exc:
64
+ return AIReport(status="failed", profile=profile or cfg.default_profile, error=str(exc))
65
+ caps = loaded.provider.capabilities()
66
+ key = _consent_key(loaded.profile_name, loaded.profile.provider, loaded.profile.model)
67
+ _, summary, _ = build_evidence(outcome.result, cfg, caps.max_context_tokens)
68
+ if not caps.data_stays_local and not assume_consent and not has_consent(key):
69
+ excerpts = "redacted code excerpts" if summary["excerpts_included"] else "no code excerpts"
70
+ tools = "read-only tool results (file contents on request, redacted)" if tools_enabled else "no tool access"
71
+ notice = (
72
+ f"AI investigation will send data to provider '{loaded.profile.provider}' "
73
+ f"(profile '{loaded.profile_name}', model '{loaded.profile.model or 'default'}'):\n"
74
+ f" - {summary['findings_sent']} finding(s) with {excerpts} (~{summary['bytes']} bytes)\n"
75
+ f" - {tools}\n"
76
+ "Secrets are redacted before sending. The provider's data-handling terms apply.\nProceed?"
77
+ )
78
+ if ask is None or not ask(notice):
79
+ return AIReport(
80
+ status="skipped",
81
+ profile=loaded.profile_name,
82
+ provider=loaded.profile.provider,
83
+ error="no consent to send data (use --yes-send-code in non-interactive runs)",
84
+ )
85
+ record_consent(key)
86
+ investigator = LoopInvestigator(loaded, cfg, tools_enabled=tools_enabled)
87
+ cache_path = None
88
+ if use_cache:
89
+ evidence, _, _ = build_evidence(outcome.result, cfg, caps.max_context_tokens)
90
+ cache_path = user_data_dir() / "ai-cache" / f"{investigator.cache_key(evidence)}.json"
91
+ if cache_path.is_file():
92
+ try:
93
+ cached = AIReport.model_validate_json(cache_path.read_text())
94
+ cached.usage = {**cached.usage, "cached": True}
95
+ return cached
96
+ except ValueError:
97
+ pass
98
+ try:
99
+ report = investigator.investigate(outcome)
100
+ except Exception as exc:
101
+ return AIReport(
102
+ status="failed",
103
+ profile=loaded.profile_name,
104
+ provider=loaded.profile.provider,
105
+ plugin=loaded.origin,
106
+ error=f"{type(exc).__name__}: {exc}",
107
+ )
108
+ if cache_path is not None and report.status == "completed":
109
+ cache_path.parent.mkdir(parents=True, exist_ok=True)
110
+ cache_path.write_text(report.model_dump_json())
111
+ return report
112
+
113
+
114
+ def interactive_ask(prompt: str) -> bool:
115
+ if not sys.stdin.isatty() or os.environ.get("CI"):
116
+ return False
117
+ sys.stderr.write(prompt + " [y/N] ")
118
+ sys.stderr.flush()
119
+ return sys.stdin.readline().strip().lower() in ("y", "yes")
devguard/ai/sdk.py ADDED
@@ -0,0 +1,102 @@
1
+ """DevGuard Provider Plugin API, version 1 (public and stable; see docs/plugins.md).
2
+
3
+ A provider plugin translates ONE completion request into a vendor API call and back. The agent loop,
4
+ tools, policy, redaction, budgets, and output validation all live in DevGuard's core, so a plugin cannot
5
+ widen what the AI can access.
6
+
7
+ Every plugin exposes a factory:
8
+
9
+ def make_provider(settings: dict[str, Any]) -> AIProvider
10
+
11
+ `settings` contains the profile's non-secret settings plus resolved secrets (e.g. settings["api_key"]).
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from dataclasses import dataclass, field
17
+ from typing import Any, Literal, Protocol, runtime_checkable
18
+
19
+ PLUGIN_API_VERSION = "1"
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class ToolSpec:
24
+ name: str
25
+ description: str
26
+ input_schema: dict[str, Any] # JSON Schema
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class ToolCall:
31
+ id: str
32
+ name: str
33
+ arguments: dict[str, Any]
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class ToolResult:
38
+ call_id: str
39
+ content: str # already capped and redacted by the core
40
+ is_error: bool = False
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class Message:
45
+ role: Literal["system", "user", "assistant", "tool"]
46
+ content: str = ""
47
+ tool_calls: list[ToolCall] = field(default_factory=list) # assistant turns
48
+ tool_results: list[ToolResult] = field(default_factory=list) # tool turns
49
+ # Opaque data a provider returned with this assistant turn (CompletionResponse.provider_data).
50
+ # The core passes it back unchanged so providers can replay their native content (e.g. thinking blocks).
51
+ provider_data: Any = None
52
+
53
+
54
+ @dataclass(frozen=True)
55
+ class CompletionRequest:
56
+ messages: list[Message]
57
+ tools: list[ToolSpec] # empty in no-tools mode
58
+ output_schema: dict[str, Any] | None # JSON Schema for the final answer
59
+ model: str
60
+ max_output_tokens: int
61
+ temperature: float | None # providers may ignore it (some models reject sampling parameters)
62
+ timeout_s: float
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class Usage:
67
+ input_tokens: int | None = None
68
+ output_tokens: int | None = None
69
+ cost_usd: float | None = None # optional; the core estimates from token prices otherwise
70
+
71
+
72
+ @dataclass(frozen=True)
73
+ class CompletionResponse:
74
+ text: str | None
75
+ tool_calls: list[ToolCall]
76
+ stop_reason: Literal["final", "tool_use", "max_tokens", "refused", "error"]
77
+ usage: Usage
78
+ model: str # model that actually answered (may differ after a provider-side fallback)
79
+ error: str | None = None
80
+ provider_data: Any = None
81
+
82
+
83
+ @dataclass(frozen=True)
84
+ class ProviderCapabilities:
85
+ tool_calling: bool
86
+ structured_output: bool
87
+ max_context_tokens: int
88
+ data_stays_local: bool
89
+
90
+
91
+ @runtime_checkable
92
+ class AIProvider(Protocol):
93
+ name: str
94
+
95
+ def capabilities(self) -> ProviderCapabilities: ...
96
+
97
+ def complete(self, request: CompletionRequest) -> CompletionResponse: ...
98
+
99
+
100
+ def error_response(model: str, message: str) -> CompletionResponse:
101
+ """Helper for plugins: report a failure instead of raising."""
102
+ return CompletionResponse(text=None, tool_calls=[], stop_reason="error", usage=Usage(), model=model, error=message)
devguard/ai/testing.py ADDED
@@ -0,0 +1,126 @@
1
+ """Contract checks for provider plugins (PLG-009).
2
+
3
+ Plugin authors can run these from pytest:
4
+
5
+ from devguard.ai.testing import run_contract
6
+ def test_my_provider():
7
+ report = run_contract(make_provider({"model": "...", "api_key": "..."}), model="...")
8
+ assert report.ok, report.failures
9
+
10
+ `devguard ai test --ai-profile NAME` runs the same checks against a configured profile. Only synthetic data
11
+ is sent; no repository content.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from dataclasses import dataclass, field
18
+
19
+ from devguard.ai.sdk import (
20
+ AIProvider,
21
+ CompletionRequest,
22
+ CompletionResponse,
23
+ Message,
24
+ ProviderCapabilities,
25
+ ToolResult,
26
+ ToolSpec,
27
+ )
28
+
29
+ _ECHO_TOOL = ToolSpec(
30
+ name="lookup_rule",
31
+ description="Look up a rule by id and return its title.",
32
+ input_schema={
33
+ "type": "object",
34
+ "properties": {"rule_id": {"type": "string"}},
35
+ "required": ["rule_id"],
36
+ "additionalProperties": False,
37
+ },
38
+ )
39
+
40
+
41
+ @dataclass
42
+ class ContractReport:
43
+ passed: list[str] = field(default_factory=list)
44
+ failures: list[str] = field(default_factory=list)
45
+ notes: list[str] = field(default_factory=list)
46
+
47
+ @property
48
+ def ok(self) -> bool:
49
+ return not self.failures
50
+
51
+
52
+ def _req(model: str, messages: list[Message], tools: list[ToolSpec] | None = None) -> CompletionRequest:
53
+ return CompletionRequest(
54
+ messages=messages,
55
+ tools=tools or [],
56
+ output_schema=None,
57
+ model=model,
58
+ max_output_tokens=2000,
59
+ temperature=None,
60
+ timeout_s=90,
61
+ )
62
+
63
+
64
+ def run_contract(provider: AIProvider, model: str) -> ContractReport:
65
+ r = ContractReport()
66
+ caps = provider.capabilities()
67
+ if not isinstance(caps, ProviderCapabilities):
68
+ r.failures.append("capabilities() must return ProviderCapabilities")
69
+ return r
70
+ r.passed.append("capabilities")
71
+
72
+ sys_msg = Message("system", "You are a test harness. Follow instructions exactly.")
73
+ resp = provider.complete(_req(model, [sys_msg, Message("user", 'Reply with the JSON object {"ok": true} only.')]))
74
+ if not isinstance(resp, CompletionResponse):
75
+ r.failures.append("complete() must return CompletionResponse")
76
+ return r
77
+ if resp.stop_reason == "error":
78
+ r.failures.append(f"plain completion failed: {resp.error}")
79
+ return r
80
+ try:
81
+ text = resp.text or ""
82
+ json.loads(text[text.find("{") : text.rfind("}") + 1])
83
+ r.passed.append("plain completion returns JSON when asked")
84
+ except ValueError:
85
+ r.notes.append("plain completion did not return clean JSON; the core will retry with feedback")
86
+
87
+ if caps.tool_calling:
88
+ msgs = [sys_msg, Message("user", "Call the lookup_rule tool with rule_id DG-TEST-001, then tell me the title.")]
89
+ first = provider.complete(_req(model, msgs, [_ECHO_TOOL]))
90
+ if not first.tool_calls:
91
+ r.failures.append(f"declared tool_calling but no tool call was returned (stop_reason={first.stop_reason})")
92
+ else:
93
+ call = first.tool_calls[0]
94
+ if call.name != "lookup_rule" or not isinstance(call.arguments, dict):
95
+ r.failures.append("tool call has wrong name or non-dict arguments")
96
+ msgs += [
97
+ Message("assistant", first.text or "", first.tool_calls, provider_data=first.provider_data),
98
+ Message("tool", tool_results=[ToolResult(c.id, "Title: Test rule") for c in first.tool_calls]),
99
+ ]
100
+ second = provider.complete(_req(model, msgs, [_ECHO_TOOL]))
101
+ if second.stop_reason == "error":
102
+ r.failures.append(f"tool-result round trip failed: {second.error}")
103
+ elif "test rule" not in (second.text or "").lower():
104
+ r.notes.append("model did not repeat the tool result; round trip technically succeeded")
105
+ r.passed.append("tool round trip")
106
+ else:
107
+ r.passed.append("tool round trip")
108
+ else:
109
+ r.notes.append("tool_calling=False: the core will use the prompted tool protocol")
110
+
111
+ bad = provider.complete(
112
+ CompletionRequest(
113
+ messages=[Message("user", "hi")],
114
+ tools=[],
115
+ output_schema=None,
116
+ model="devguard-nonexistent-model",
117
+ max_output_tokens=10,
118
+ temperature=None,
119
+ timeout_s=30,
120
+ )
121
+ )
122
+ if bad.stop_reason == "error":
123
+ r.passed.append("errors are returned, not raised")
124
+ else:
125
+ r.notes.append("unknown model did not produce an error (some gateways ignore the model name)")
126
+ return r