pytest-copilot 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pytest_copilot/__init__.py +133 -0
  2. pytest_copilot/copilot/__init__.py +47 -0
  3. pytest_copilot/copilot/_command_job.py +109 -0
  4. pytest_copilot/copilot/api.py +12 -0
  5. pytest_copilot/copilot/client.py +58 -0
  6. pytest_copilot/copilot/commands.py +265 -0
  7. pytest_copilot/copilot/config.py +133 -0
  8. pytest_copilot/copilot/contracts.py +222 -0
  9. pytest_copilot/copilot/controls.py +230 -0
  10. pytest_copilot/copilot/events.py +795 -0
  11. pytest_copilot/copilot/fixtures.py +289 -0
  12. pytest_copilot/copilot/personas.py +607 -0
  13. pytest_copilot/copilot/requests.py +350 -0
  14. pytest_copilot/copilot/result.py +309 -0
  15. pytest_copilot/copilot/runner.py +266 -0
  16. pytest_copilot/copilot/skills.py +104 -0
  17. pytest_copilot/copilot/task.py +546 -0
  18. pytest_copilot/core/__init__.py +77 -0
  19. pytest_copilot/core/errors.py +44 -0
  20. pytest_copilot/core/evals.py +521 -0
  21. pytest_copilot/core/plugin.py +403 -0
  22. pytest_copilot/core/prompt.py +167 -0
  23. pytest_copilot/core/result.py +543 -0
  24. pytest_copilot/core/serialization.py +128 -0
  25. pytest_copilot/core/skill.py +406 -0
  26. pytest_copilot/core/skill_evals.py +148 -0
  27. pytest_copilot/core/skill_grading.py +78 -0
  28. pytest_copilot/core/validation.py +83 -0
  29. pytest_copilot/execution/__init__.py +15 -0
  30. pytest_copilot/execution/cost.py +159 -0
  31. pytest_copilot/execution/rate_limiter.py +177 -0
  32. pytest_copilot/execution/servers.py +442 -0
  33. pytest_copilot/execution/skill_tools.py +228 -0
  34. pytest_copilot/fixtures/__init__.py +18 -0
  35. pytest_copilot/fixtures/factories.py +44 -0
  36. pytest_copilot/fixtures/iteration.py +16 -0
  37. pytest_copilot/onboarding.py +413 -0
  38. pytest_copilot/plugin.py +562 -0
  39. pytest_copilot/plugin_options.py +32 -0
  40. pytest_copilot/plugin_report.py +18 -0
  41. pytest_copilot/reporting/__init__.py +16 -0
  42. pytest_copilot/reporting/collector.py +203 -0
  43. pytest_copilot/reporting/generator.py +55 -0
  44. pytest_copilot/reporting/schema.py +5 -0
  45. pytest_copilot/testing/__init__.py +17 -0
  46. pytest_copilot/testing/banking.py +479 -0
  47. pytest_copilot/testing/banking_mcp.py +167 -0
  48. pytest_copilot/testing/todo.py +380 -0
  49. pytest_copilot/testing/todo_mcp.py +156 -0
  50. pytest_copilot/testing/types.py +31 -0
  51. pytest_copilot-2.0.0.dist-info/METADATA +177 -0
  52. pytest_copilot-2.0.0.dist-info/RECORD +55 -0
  53. pytest_copilot-2.0.0.dist-info/WHEEL +4 -0
  54. pytest_copilot-2.0.0.dist-info/entry_points.txt +5 -0
  55. pytest_copilot-2.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,83 @@
1
+ """Shared validation for plugin, custom-agent, and MCP configuration fields."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from pathlib import Path
7
+ from typing import Any
8
+ from urllib.parse import urlsplit
9
+
10
+
11
+ def _string_field(
12
+ data: dict[str, Any],
13
+ field: str,
14
+ source: Path,
15
+ *,
16
+ required: bool = False,
17
+ allow_empty: bool = False,
18
+ ) -> str:
19
+ """Read a string field, distinguishing optional absence from invalid content."""
20
+ if field not in data and not required:
21
+ return ""
22
+ value = data.get(field)
23
+ if not isinstance(value, str) or (not allow_empty and not value.strip()):
24
+ raise ValueError(f"{source}: '{field}' must be a non-empty string")
25
+ return value
26
+
27
+
28
+ def _validate_mcp_servers(raw: Any, source: Path, *, field: str) -> dict[str, dict[str, Any]]:
29
+ """Validate the shared server mapping contract for plugin and MCP config files."""
30
+ if not isinstance(raw, dict):
31
+ raise ValueError(f"{source}: '{field}' must be an object")
32
+ for name, config in raw.items():
33
+ label = f"MCP server '{name}'"
34
+ if not isinstance(name, str) or not name.strip() or not isinstance(config, dict):
35
+ raise ValueError(f"{source}: {label} must have a non-empty name and object config")
36
+ transport = config.get("type")
37
+ if "type" in config and transport not in ("stdio", "local", "http", "sse"):
38
+ raise ValueError(f"{source}: {label} has unsupported transport type {transport!r}")
39
+ has_command = "command" in config
40
+ has_url = "url" in config
41
+ if has_command == has_url:
42
+ raise ValueError(f"{source}: {label} requires exactly one of 'command' or 'url'")
43
+ if (
44
+ transport in ("http", "sse")
45
+ and not has_url
46
+ or (transport in ("stdio", "local") and not has_command)
47
+ ):
48
+ raise ValueError(f"{source}: {label} transport does not match command/url")
49
+ for key in ("command", "url", "cwd"):
50
+ if key in config:
51
+ _string_field(config, key, source, required=True)
52
+ if has_url:
53
+ try:
54
+ url = urlsplit(config["url"])
55
+ if url.scheme not in ("http", "https") or not url.hostname:
56
+ raise ValueError("expected an absolute HTTP(S) URL")
57
+ _ = url.port
58
+ except ValueError as exc:
59
+ raise ValueError(f"{source}: {label} invalid 'url': {exc}") from exc
60
+ for key in ("args", "tools"):
61
+ if key in config and (
62
+ not isinstance(config[key], list)
63
+ or any(not isinstance(item, str) for item in config[key])
64
+ ):
65
+ raise ValueError(f"{source}: {label} '{key}' must be a list of strings")
66
+ if "tools" in config and any(not tool.strip() for tool in config["tools"]):
67
+ raise ValueError(f"{source}: {label} 'tools' entries must be non-empty strings")
68
+ for key in ("env", "headers"):
69
+ if key in config and (
70
+ not isinstance(config[key], dict)
71
+ or any(
72
+ not isinstance(k, str) or not isinstance(v, str) for k, v in config[key].items()
73
+ )
74
+ ):
75
+ raise ValueError(f"{source}: {label} '{key}' must be a string-to-string object")
76
+ if "timeout" in config and (
77
+ isinstance(config["timeout"], bool)
78
+ or not isinstance(config["timeout"], (int, float))
79
+ or not math.isfinite(config["timeout"])
80
+ or config["timeout"] <= 0
81
+ ):
82
+ raise ValueError(f"{source}: {label} 'timeout' must be a positive number")
83
+ return dict(raw)
@@ -0,0 +1,15 @@
1
+ """Execution module - server management and utility functions."""
2
+
3
+ from pytest_copilot.execution.servers import (
4
+ MCPServer,
5
+ MCPServerProcess,
6
+ Wait,
7
+ WaitStrategy,
8
+ )
9
+
10
+ __all__ = [
11
+ "MCPServer",
12
+ "MCPServerProcess",
13
+ "Wait",
14
+ "WaitStrategy",
15
+ ]
@@ -0,0 +1,159 @@
1
+ """Token-based cost estimation for LLM calls.
2
+
3
+ Pricing lookup uses ``pricing.toml`` — a user-maintained file with per-million-token
4
+ pricing for models. Models without pricing return ``0.0``.
5
+
6
+ ``pricing.toml`` format::
7
+
8
+ # Per-million-token pricing. ``cache_read`` is optional (defaults to 0.0)
9
+ # and prices cached-prompt input tokens, which most vendors bill at a
10
+ # fraction of the normal input rate.
11
+ [models]
12
+ "claude-sonnet-4" = { input = 3.00, output = 15.00, cache_read = 0.30 }
13
+ "gpt-5.6-sol" = { input = 5.00, output = 30.00, cache_read = 0.50 }
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import logging
19
+ from dataclasses import dataclass, field
20
+ from pathlib import Path
21
+ from typing import Any
22
+
23
+ _logger = logging.getLogger(__name__)
24
+
25
+ # Models that callers asked about but had no pricing anywhere.
26
+ # Populated at runtime by :func:`estimate_cost`.
27
+ models_without_pricing: set[str] = set()
28
+
29
+ # ── User overrides (pricing.toml) ────────────────────────────────────────────
30
+
31
+ Pricing = tuple[float, float, float]
32
+ PricingTable = dict[str, Pricing]
33
+ PricingCacheKey = tuple[Path, int | None]
34
+
35
+
36
+ @dataclass(slots=True)
37
+ class _PricingCache:
38
+ """Pricing tables keyed by resolved file identity and modification time."""
39
+
40
+ tables: dict[PricingCacheKey, PricingTable] = field(default_factory=dict)
41
+
42
+ def load(self, current_dir: Path) -> PricingTable:
43
+ toml_path = _find_pricing_toml(current_dir)
44
+ cache_key = _pricing_cache_key(current_dir, toml_path)
45
+ cached = self.tables.get(cache_key)
46
+ if cached is not None:
47
+ return cached
48
+
49
+ overrides = _read_pricing_file(toml_path)
50
+ self.tables[cache_key] = overrides
51
+ return overrides
52
+
53
+ def clear(self) -> None:
54
+ """Discard all cached pricing tables."""
55
+ self.tables.clear()
56
+
57
+
58
+ _pricing_cache = _PricingCache()
59
+
60
+
61
+ def _load_user_overrides() -> PricingTable:
62
+ """Load per-million-token overrides from ``pricing.toml``.
63
+
64
+ Searches upward from cwd for the first ``pricing.toml`` found.
65
+ Returns an empty dict when no file exists.
66
+ """
67
+ current_dir = Path.cwd().resolve()
68
+ return _pricing_cache.load(current_dir)
69
+
70
+
71
+ def _read_pricing_file(toml_path: Path | None) -> PricingTable:
72
+ """Parse a pricing file into per-million-token rates."""
73
+ overrides: PricingTable = {}
74
+ if toml_path is None:
75
+ return overrides
76
+
77
+ import tomllib
78
+
79
+ try:
80
+ raw = tomllib.loads(toml_path.read_text(encoding="utf-8"))
81
+ models_section: dict[str, Any] = raw.get("models", {})
82
+ for key, value in models_section.items():
83
+ if isinstance(value, dict):
84
+ input_pm = float(value.get("input", 0))
85
+ output_pm = float(value.get("output", 0))
86
+ cache_read_pm = float(value.get("cache_read", 0))
87
+ overrides[key] = (input_pm, output_pm, cache_read_pm)
88
+ if overrides:
89
+ _logger.info(
90
+ "Loaded %d pricing override(s) from %s",
91
+ len(overrides),
92
+ toml_path,
93
+ )
94
+ except (
95
+ AttributeError,
96
+ OSError,
97
+ UnicodeDecodeError,
98
+ ValueError,
99
+ TypeError,
100
+ tomllib.TOMLDecodeError,
101
+ ):
102
+ _logger.warning("Failed to parse %s; ignoring", toml_path, exc_info=True)
103
+
104
+ return overrides
105
+
106
+
107
+ def _pricing_cache_key(current_dir: Path, toml_path: Path | None) -> PricingCacheKey:
108
+ """Build a cache key scoped to the active cwd and pricing file version."""
109
+ if toml_path is None:
110
+ return current_dir, None
111
+ stat = toml_path.stat()
112
+ return toml_path.resolve(), stat.st_mtime_ns
113
+
114
+
115
+ def _find_pricing_toml(current_dir: Path) -> Path | None:
116
+ """Walk upward from cwd looking for ``pricing.toml``."""
117
+ for parent in (current_dir, *current_dir.parents):
118
+ candidate = parent / "pricing.toml"
119
+ if candidate.is_file():
120
+ return candidate
121
+ return None
122
+
123
+
124
+ # ── Public API ───────────────────────────────────────────────────────────────
125
+
126
+
127
+ def unpriced_models(models: set[str]) -> list[str]:
128
+ """Find missing prices for this execution's usage, independent of earlier suites."""
129
+ return sorted(models - _load_user_overrides().keys())
130
+
131
+
132
+ def estimate_cost(
133
+ model: str,
134
+ input_tokens: int,
135
+ output_tokens: int,
136
+ cache_read_tokens: int = 0,
137
+ ) -> float:
138
+ """Return estimated USD cost for a single LLM call.
139
+
140
+ Checks user overrides (``pricing.toml``). ``cache_read_tokens`` are priced
141
+ at the model's optional ``cache_read`` rate (``0.0`` when unset, so they add
142
+ nothing unless a rate is configured).
143
+ Returns ``0.0`` and records the model in :data:`models_without_pricing`
144
+ when no pricing is found.
145
+ """
146
+ if input_tokens == 0 and output_tokens == 0 and cache_read_tokens == 0:
147
+ return 0.0
148
+
149
+ overrides = _load_user_overrides()
150
+ pricing = overrides.get(model)
151
+ if pricing is not None:
152
+ return (
153
+ input_tokens * pricing[0] + output_tokens * pricing[1] + cache_read_tokens * pricing[2]
154
+ ) / 1_000_000
155
+
156
+ # No pricing found
157
+ models_without_pricing.add(model)
158
+ _logger.debug("No pricing data for model %r; cost will be 0", model)
159
+ return 0.0
@@ -0,0 +1,177 @@
1
+ """Async rate limiter for LLM API calls.
2
+
3
+ Provides sliding-window rate limiting for requests per minute (rpm) and
4
+ tokens per minute (tpm). Rate limiters are shared across all engine instances
5
+ using the same model, so concurrent tests respect deployment limits.
6
+
7
+ Usage:
8
+ limiter = get_rate_limiter("copilot/gpt-5.6-sol", rpm=10, tpm=10000)
9
+ await limiter.acquire() # Waits if rate limit would be exceeded
10
+ # ... make API call ...
11
+ limiter.record_tokens(1500) # Track token usage for tpm enforcement
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import asyncio
17
+ import logging
18
+ import time
19
+ from collections import deque
20
+
21
+ _logger = logging.getLogger(__name__)
22
+
23
+ # Global registry of rate limiters, keyed by model string.
24
+ # Shared across all EvalEngine instances so concurrent tests
25
+ # respect the same deployment's rate limits.
26
+ _rate_limiters: dict[str, RateLimiter] = {}
27
+
28
+
29
+ def get_rate_limiter(
30
+ model: str,
31
+ *,
32
+ rpm: int | None = None,
33
+ tpm: int | None = None,
34
+ ) -> RateLimiter:
35
+ """Get or create a rate limiter for the given model.
36
+
37
+ If a limiter already exists for this model, updates it with the most
38
+ restrictive limits (minimum of old and new values).
39
+
40
+ Args:
41
+ model: Model identifier string (e.g. "copilot/gpt-5.6-sol").
42
+ rpm: Requests per minute limit.
43
+ tpm: Tokens per minute limit.
44
+
45
+ Returns:
46
+ Shared RateLimiter instance for this model.
47
+ """
48
+ if model not in _rate_limiters:
49
+ _rate_limiters[model] = RateLimiter(rpm=rpm, tpm=tpm)
50
+ else:
51
+ limiter = _rate_limiters[model]
52
+ # Take the most restrictive limit
53
+ if rpm is not None:
54
+ limiter.rpm = min(limiter.rpm, rpm) if limiter.rpm is not None else rpm
55
+ if tpm is not None:
56
+ limiter.tpm = min(limiter.tpm, tpm) if limiter.tpm is not None else tpm
57
+ return _rate_limiters[model]
58
+
59
+
60
+ def reset_rate_limiters() -> None:
61
+ """Reset all rate limiters. Called between test sessions."""
62
+ _rate_limiters.clear()
63
+
64
+
65
+ class RateLimiter:
66
+ """Sliding-window rate limiter for rpm and tpm.
67
+
68
+ Uses a 60-second sliding window to track request times and token usage.
69
+ When limits would be exceeded, ``acquire()`` awaits until enough capacity
70
+ is available.
71
+
72
+ This rate limiter operates at the engine.run() level — each call to
73
+ ``acquire()`` represents one agent run (which may internally make multiple
74
+ LLM requests across turns). For rpm, this means the limit applies to
75
+ agent runs, not individual LLM calls. This is a pragmatic trade-off:
76
+ true per-LLM-call rate limiting would require wrapping the Copilot SDK's
77
+ model execution path, which is significantly more complex.
78
+
79
+ For tpm, token usage is recorded after each run via ``record_tokens()``,
80
+ and the limiter checks cumulative usage in the sliding window before
81
+ allowing the next run.
82
+ """
83
+
84
+ WINDOW_SECONDS: float = 60.0
85
+
86
+ def __init__(self, rpm: int | None = None, tpm: int | None = None) -> None:
87
+ self.rpm = rpm
88
+ self.tpm = tpm
89
+ self._request_times: deque[float] = deque()
90
+ self._token_records: deque[tuple[float, int]] = deque()
91
+ self._lock = asyncio.Lock()
92
+
93
+ @property
94
+ def has_limits(self) -> bool:
95
+ """Whether any limits are configured."""
96
+ return self.rpm is not None or self.tpm is not None
97
+
98
+ async def acquire(self) -> None:
99
+ """Wait if necessary to stay within rate limits.
100
+
101
+ For rpm: checks if we've made too many requests in the last 60 seconds.
102
+ For tpm: checks if token usage in the last 60 seconds exceeds the limit.
103
+
104
+ If either limit is exceeded, sleeps until the oldest entry in the
105
+ sliding window expires, freeing capacity.
106
+ """
107
+ if not self.has_limits:
108
+ return
109
+
110
+ async with self._lock:
111
+ await self._enforce_rpm()
112
+ await self._enforce_tpm()
113
+
114
+ def record_tokens(self, tokens: int) -> None:
115
+ """Record token usage after a run completes.
116
+
117
+ Args:
118
+ tokens: Total tokens consumed (input + output).
119
+ """
120
+ if self.tpm is not None and tokens > 0:
121
+ self._token_records.append((time.monotonic(), tokens))
122
+
123
+ async def _enforce_rpm(self) -> None:
124
+ """Enforce requests-per-minute limit."""
125
+ if self.rpm is None:
126
+ return
127
+
128
+ now = time.monotonic()
129
+ self._prune_old_entries(self._request_times, now)
130
+
131
+ if len(self._request_times) >= self.rpm:
132
+ wait_seconds = self._request_times[0] + self.WINDOW_SECONDS - now
133
+ if wait_seconds > 0:
134
+ _logger.info(
135
+ "Rate limit (rpm=%d): waiting %.1fs before next request",
136
+ self.rpm,
137
+ wait_seconds,
138
+ )
139
+ await asyncio.sleep(wait_seconds)
140
+ now = time.monotonic()
141
+ self._prune_old_entries(self._request_times, now)
142
+
143
+ self._request_times.append(now)
144
+
145
+ async def _enforce_tpm(self) -> None:
146
+ """Enforce tokens-per-minute limit."""
147
+ if self.tpm is None:
148
+ return
149
+
150
+ now = time.monotonic()
151
+ self._prune_old_token_entries(now)
152
+
153
+ total_tokens = sum(tokens for _, tokens in self._token_records)
154
+ if total_tokens >= self.tpm:
155
+ # Wait until the oldest record falls out of the window
156
+ wait_seconds = self._token_records[0][0] + self.WINDOW_SECONDS - now
157
+ if wait_seconds > 0:
158
+ _logger.info(
159
+ "Rate limit (tpm=%d): waiting %.1fs (current window: %d tokens)",
160
+ self.tpm,
161
+ wait_seconds,
162
+ total_tokens,
163
+ )
164
+ await asyncio.sleep(wait_seconds)
165
+ self._prune_old_token_entries(time.monotonic())
166
+
167
+ def _prune_old_entries(self, times: deque[float], now: float) -> None:
168
+ """Remove entries older than the sliding window."""
169
+ cutoff = now - self.WINDOW_SECONDS
170
+ while times and times[0] < cutoff:
171
+ times.popleft()
172
+
173
+ def _prune_old_token_entries(self, now: float) -> None:
174
+ """Remove token records older than the sliding window."""
175
+ cutoff = now - self.WINDOW_SECONDS
176
+ while self._token_records and self._token_records[0][0] < cutoff:
177
+ self._token_records.popleft()