misterdev 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- misterdev/__init__.py +3 -0
- misterdev/agent.py +2166 -0
- misterdev/agent_helpers.py +194 -0
- misterdev/analyzers/__init__.py +0 -0
- misterdev/analyzers/project_analyzer/__init__.py +246 -0
- misterdev/analyzers/project_analyzer/detection.py +146 -0
- misterdev/analyzers/project_analyzer/merge.py +137 -0
- misterdev/analyzers/project_analyzer/overview.py +312 -0
- misterdev/analyzers/project_analyzer/prompts.py +83 -0
- misterdev/cli.py +370 -0
- misterdev/config.py +521 -0
- misterdev/core/__init__.py +0 -0
- misterdev/core/audit.py +88 -0
- misterdev/core/config.py +10 -0
- misterdev/core/context/__init__.py +0 -0
- misterdev/core/context/change_tracker.py +218 -0
- misterdev/core/context/contracts/__init__.py +63 -0
- misterdev/core/context/contracts/_log.py +9 -0
- misterdev/core/context/contracts/_text.py +12 -0
- misterdev/core/context/contracts/extraction.py +30 -0
- misterdev/core/context/contracts/python_generic.py +42 -0
- misterdev/core/context/contracts/registry.py +127 -0
- misterdev/core/context/contracts/rust_line.py +225 -0
- misterdev/core/context/contracts/rust_tree_sitter.py +141 -0
- misterdev/core/context/lsp.py +174 -0
- misterdev/core/context/scratchpad.py +111 -0
- misterdev/core/context/topography/__init__.py +43 -0
- misterdev/core/context/topography/_log.py +10 -0
- misterdev/core/context/topography/cache.py +91 -0
- misterdev/core/context/topography/engine.py +172 -0
- misterdev/core/context/topography/graph.py +675 -0
- misterdev/core/context/topography/nodes.py +55 -0
- misterdev/core/context/topography/parsers.py +95 -0
- misterdev/core/context/topography/syntax.py +54 -0
- misterdev/core/economics/__init__.py +0 -0
- misterdev/core/economics/context_budget.py +175 -0
- misterdev/core/economics/embeddings.py +232 -0
- misterdev/core/economics/free_models.py +108 -0
- misterdev/core/economics/llm_cache.py +105 -0
- misterdev/core/economics/model_catalog.py +79 -0
- misterdev/core/economics/model_ledger.py +331 -0
- misterdev/core/economics/model_selector.py +281 -0
- misterdev/core/execution/__init__.py +0 -0
- misterdev/core/execution/bounded.py +50 -0
- misterdev/core/execution/container.py +221 -0
- misterdev/core/execution/error_classifier.py +366 -0
- misterdev/core/execution/error_resolver.py +201 -0
- misterdev/core/execution/governance.py +283 -0
- misterdev/core/execution/outcomes.py +50 -0
- misterdev/core/execution/progress.py +120 -0
- misterdev/core/execution/project.py +231 -0
- misterdev/core/execution/registry.py +97 -0
- misterdev/core/execution/runtime.py +279 -0
- misterdev/core/gitcmd.py +39 -0
- misterdev/core/integration/__init__.py +0 -0
- misterdev/core/integration/mcp.py +368 -0
- misterdev/core/integration/mcp_gather.py +186 -0
- misterdev/core/models.py +35 -0
- misterdev/core/modes.py +184 -0
- misterdev/core/planning/__init__.py +0 -0
- misterdev/core/planning/advisor.py +89 -0
- misterdev/core/planning/assessment.py +135 -0
- misterdev/core/planning/decomposer.py +387 -0
- misterdev/core/planning/metacognition.py +103 -0
- misterdev/core/planning/sovereign.py +308 -0
- misterdev/core/planning/targets.py +201 -0
- misterdev/core/reporting/__init__.py +0 -0
- misterdev/core/reporting/report.py +377 -0
- misterdev/core/reporting/report_view.py +151 -0
- misterdev/core/task.py +163 -0
- misterdev/core/verification/__init__.py +0 -0
- misterdev/core/verification/claim_verifier.py +210 -0
- misterdev/core/verification/critic.py +324 -0
- misterdev/core/verification/gatekeeper/__init__.py +631 -0
- misterdev/core/verification/gatekeeper/constants.py +138 -0
- misterdev/core/verification/gatekeeper/helpers.py +28 -0
- misterdev/core/verification/goal_check.py +219 -0
- misterdev/core/verification/independent.py +68 -0
- misterdev/core/verification/mutation_gate.py +221 -0
- misterdev/core/verification/preflight.py +95 -0
- misterdev/core/verification/spec_tests.py +175 -0
- misterdev/core/verification/validator.py +495 -0
- misterdev/core/verification/vision_verify.py +185 -0
- misterdev/core/verification/web_verify.py +408 -0
- misterdev/environments/__init__.py +0 -0
- misterdev/environments/base_env.py +18 -0
- misterdev/environments/container_env.py +87 -0
- misterdev/environments/venv_env.py +42 -0
- misterdev/llm/__init__.py +0 -0
- misterdev/llm/client/__init__.py +152 -0
- misterdev/llm/client/base.py +382 -0
- misterdev/llm/client/edits.py +70 -0
- misterdev/llm/client/embeddings.py +121 -0
- misterdev/llm/client/errors.py +134 -0
- misterdev/llm/client/providers.py +535 -0
- misterdev/llm/client/response.py +24 -0
- misterdev/llm/prompt_manager.py +82 -0
- misterdev/llm/responses/__init__.py +34 -0
- misterdev/llm/responses/apply.py +131 -0
- misterdev/llm/responses/json_extract.py +80 -0
- misterdev/llm/responses/models.py +43 -0
- misterdev/llm/responses/parsing.py +494 -0
- misterdev/logging_setup.py +20 -0
- misterdev/mcp_server.py +208 -0
- misterdev/nl_cli.py +149 -0
- misterdev/plugins.py +115 -0
- misterdev/py.typed +0 -0
- misterdev/task_executors/__init__.py +0 -0
- misterdev/task_executors/base_executor.py +10 -0
- misterdev/task_executors/markdown_plan_executor/__init__.py +90 -0
- misterdev/task_executors/markdown_plan_executor/commands_mixin.py +82 -0
- misterdev/task_executors/markdown_plan_executor/context_mixin.py +221 -0
- misterdev/task_executors/markdown_plan_executor/critic_spec_mixin.py +174 -0
- misterdev/task_executors/markdown_plan_executor/edits_mixin.py +251 -0
- misterdev/task_executors/markdown_plan_executor/execute_mixin.py +727 -0
- misterdev/task_executors/markdown_plan_executor/gates_mixin.py +203 -0
- misterdev/task_executors/markdown_plan_executor/git_mixin.py +219 -0
- misterdev/task_executors/markdown_plan_executor/helpers.py +521 -0
- misterdev/task_executors/markdown_plan_executor/llm_mixin.py +238 -0
- misterdev/task_executors/markdown_plan_executor/results_mixin.py +23 -0
- misterdev/tools/__init__.py +19 -0
- misterdev/tools/base_tool.py +14 -0
- misterdev/tools/command.py +75 -0
- misterdev/tools/file_io.py +69 -0
- misterdev/tools/formatter.py +26 -0
- misterdev/tools/git_tool.py +90 -0
- misterdev/utils/__init__.py +0 -0
- misterdev/utils/file_utils.py +169 -0
- misterdev/utils/process.py +23 -0
- misterdev-0.2.0.dist-info/METADATA +326 -0
- misterdev-0.2.0.dist-info/RECORD +136 -0
- misterdev-0.2.0.dist-info/WHEEL +5 -0
- misterdev-0.2.0.dist-info/entry_points.txt +3 -0
- misterdev-0.2.0.dist-info/licenses/COMMERCIAL_LICENSE.md +34 -0
- misterdev-0.2.0.dist-info/licenses/LICENSE +661 -0
- misterdev-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
# Patterns that indicate incomplete or debug code
|
|
4
|
+
BANNED_MARKERS = ("todo!", "FIXME", "HACK", "XXX", "placeholder", "dummy")
|
|
5
|
+
|
|
6
|
+
# High-signal patterns: a bare substring match is enough to flag a file. Only
|
|
7
|
+
# DISTINCTIVE provider prefixes belong here — a short fragment like "sk_" would
|
|
8
|
+
# substring-match ordinary identifiers (task_, disk_) and block legitimate code,
|
|
9
|
+
# so Stripe/AWS use their full prefixes.
|
|
10
|
+
SECRET_PATTERNS = (
|
|
11
|
+
"PRIVATE KEY",
|
|
12
|
+
"BEGIN RSA",
|
|
13
|
+
"BEGIN EC",
|
|
14
|
+
"BEGIN DSA",
|
|
15
|
+
"sk_live_", # Stripe secret (live)
|
|
16
|
+
"sk_test_", # Stripe secret (test)
|
|
17
|
+
"ghp_", # GitHub PAT
|
|
18
|
+
"gho_", # GitHub OAuth
|
|
19
|
+
"ghs_", # GitHub server-to-server
|
|
20
|
+
"ghu_", # GitHub user-to-server
|
|
21
|
+
"ghr_", # GitHub refresh
|
|
22
|
+
"github_pat_", # GitHub fine-grained PAT
|
|
23
|
+
"glpat-", # GitLab PAT
|
|
24
|
+
"xoxb-", # Slack bot token
|
|
25
|
+
"xoxp-", # Slack user token
|
|
26
|
+
"AIza", # Google API key
|
|
27
|
+
"AKIA", # AWS access key id
|
|
28
|
+
"ASIA", # AWS temporary access key id
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# Patterns too short to match as a bare substring without colliding with ordinary
|
|
32
|
+
# tokens. "sk-" alone substring-matches kebab-case identifiers/URLs (disk-size,
|
|
33
|
+
# task-list, /task-…), so an OpenAI key is matched by a boundaried regex anchored
|
|
34
|
+
# at a word boundary (excludes di"sk-"/ta"sk-"). The run then qualifies EITHER by
|
|
35
|
+
# containing a digit (≥6 chars — catches typical high-entropy keys and short
|
|
36
|
+
# fakes) OR by being very long (≥40 chars — catches a rare all-letter real key,
|
|
37
|
+
# which is 48+ chars, without matching the short word-like "sk-spinner" CSS-class
|
|
38
|
+
# identifiers a human writes).
|
|
39
|
+
SECRET_REGEXES = (
|
|
40
|
+
re.compile(
|
|
41
|
+
r"\bsk-(?:(?=[A-Za-z0-9_-]*\d)[A-Za-z0-9_-]{6,}|[A-Za-z0-9_-]{40,})"
|
|
42
|
+
), # OpenAI
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
# Low-signal credential keys. These appear constantly in ordinary source
|
|
46
|
+
# (struct fields, function params, config keys), so they are only flagged when
|
|
47
|
+
# assigned a concrete quoted literal, not a variable/env reference.
|
|
48
|
+
ASSIGNMENT_SECRET_KEYS = (
|
|
49
|
+
"password",
|
|
50
|
+
"passwd",
|
|
51
|
+
"secret",
|
|
52
|
+
"api_key",
|
|
53
|
+
"apikey",
|
|
54
|
+
"access_key",
|
|
55
|
+
"token",
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# In a quoted credential VALUE these mark an environment/variable REFERENCE (the
|
|
59
|
+
# secret is read from elsewhere), so the literal is a reference, not a leak, and
|
|
60
|
+
# is skipped. Deliberately specific: a bare "env" substring test also skipped a
|
|
61
|
+
# real credential that merely contains those three letters (e.g. "Denver7x9").
|
|
62
|
+
ENV_REFERENCE_MARKERS = (
|
|
63
|
+
"os.environ",
|
|
64
|
+
"getenv",
|
|
65
|
+
"process.env",
|
|
66
|
+
"${",
|
|
67
|
+
"{{",
|
|
68
|
+
"%(",
|
|
69
|
+
"$env",
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# Extensions to skip during file scanning
|
|
73
|
+
SKIP_DIRS = frozenset(
|
|
74
|
+
{
|
|
75
|
+
".venv",
|
|
76
|
+
"venv",
|
|
77
|
+
".git",
|
|
78
|
+
"node_modules",
|
|
79
|
+
"__pycache__",
|
|
80
|
+
"target",
|
|
81
|
+
"build",
|
|
82
|
+
"dist",
|
|
83
|
+
".tox",
|
|
84
|
+
".mypy_cache",
|
|
85
|
+
".eggs",
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
CODE_EXTENSIONS = frozenset(
|
|
90
|
+
{
|
|
91
|
+
".py",
|
|
92
|
+
".js",
|
|
93
|
+
".ts",
|
|
94
|
+
".tsx",
|
|
95
|
+
".jsx",
|
|
96
|
+
".rs",
|
|
97
|
+
".go",
|
|
98
|
+
".java",
|
|
99
|
+
".c",
|
|
100
|
+
".cpp",
|
|
101
|
+
".h",
|
|
102
|
+
".rb",
|
|
103
|
+
".php",
|
|
104
|
+
".swift",
|
|
105
|
+
".kt",
|
|
106
|
+
".sh",
|
|
107
|
+
}
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# Secrets leak through config/env files just as readily as source — and a
|
|
111
|
+
# planted credential there is never "code", so the code-only scan (G5/G9) misses
|
|
112
|
+
# it. G6 therefore scans these in addition to CODE_EXTENSIONS. Banned-marker and
|
|
113
|
+
# debug-artifact scans deliberately stay code-only (a TODO in a YAML is fine).
|
|
114
|
+
SECRET_SCAN_EXTENSIONS = CODE_EXTENSIONS | frozenset(
|
|
115
|
+
{
|
|
116
|
+
".env",
|
|
117
|
+
".yaml",
|
|
118
|
+
".yml",
|
|
119
|
+
".json",
|
|
120
|
+
".toml",
|
|
121
|
+
".ini",
|
|
122
|
+
".cfg",
|
|
123
|
+
".conf",
|
|
124
|
+
".properties",
|
|
125
|
+
".xml",
|
|
126
|
+
".tfvars",
|
|
127
|
+
}
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
# Dotfiles whose whole name is the extension (``Path(".env").suffix == ""``), so
|
|
131
|
+
# they must be matched by name rather than suffix.
|
|
132
|
+
SECRET_SCAN_FILENAMES = frozenset({".env", ".envrc", ".netrc", ".pgpass"})
|
|
133
|
+
|
|
134
|
+
# File types where ``KEY=value`` is a literal config assignment (so an UNQUOTED
|
|
135
|
+
# value can be a real secret), as opposed to source code where an unquoted RHS
|
|
136
|
+
# is a variable reference. Used to gate the unquoted-secret heuristic so it never
|
|
137
|
+
# fires on .py/.rs/etc and blocks legitimate code.
|
|
138
|
+
ENV_LITERAL_EXTENSIONS = frozenset({".env", ".ini", ".properties", ".conf", ".cfg"})
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
# Cap how much of a single file the scanners read into memory. A generated
|
|
5
|
+
# project can contain a multi-GB file (data dumps, bundles); reading it whole to
|
|
6
|
+
# grep for secrets would exhaust the gate's memory. Secrets/markers of interest
|
|
7
|
+
# live near the top or in small config files, so the head is what matters.
|
|
8
|
+
_MAX_SCAN_CHARS = 2_000_000
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _path_in_scope(
|
|
12
|
+
path_str: str,
|
|
13
|
+
extensions: frozenset,
|
|
14
|
+
filenames: frozenset = frozenset(),
|
|
15
|
+
) -> bool:
|
|
16
|
+
"""True when ``path_str`` is in scope by file extension or exact name."""
|
|
17
|
+
p = Path(path_str)
|
|
18
|
+
return p.suffix in extensions or p.name in filenames
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _read_capped(path: Path, max_chars: int = _MAX_SCAN_CHARS) -> Optional[str]:
|
|
22
|
+
"""Read at most ``max_chars`` characters of ``path`` for scanning, or ``None``
|
|
23
|
+
on an OS error. Bounds memory so a huge file can't OOM the gate."""
|
|
24
|
+
try:
|
|
25
|
+
with path.open("r", encoding="utf-8", errors="replace") as fh:
|
|
26
|
+
return fh.read(max_chars)
|
|
27
|
+
except OSError:
|
|
28
|
+
return None
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Optional goal-completion check (LLM judge).
|
|
2
|
+
|
|
3
|
+
Green gates prove the code compiles, tests pass, and the diff is hygienic — but
|
|
4
|
+
not that the work actually accomplished what was asked. "Gates green != goal
|
|
5
|
+
met": a build can satisfy every mechanical gate while quietly omitting a feature
|
|
6
|
+
the goal called for. This check closes that gap. Given the goal text, the
|
|
7
|
+
acceptance criteria, and the cumulative diff/summary of the build, an LLM judge
|
|
8
|
+
returns a verdict {satisfied: bool, gaps: [str]}.
|
|
9
|
+
|
|
10
|
+
It mirrors :mod:`misterdev.core.verification.vision_verify`: best-effort and run
|
|
11
|
+
in a daemon worker thread with a hard timeout so a slow or unreachable model can
|
|
12
|
+
NEVER block the build. It is ADVISORY by default — the verdict's gaps are
|
|
13
|
+
recorded into the report and logged, and the build is NOT failed. It blocks only
|
|
14
|
+
when ``orchestrator.block_on_goal_gap`` is true (the caller's choice). Absent a
|
|
15
|
+
goal / criteria, absent an LLM client, an unparseable verdict, or a timeout is a
|
|
16
|
+
SKIP (no opinion), never a failure.
|
|
17
|
+
|
|
18
|
+
Like the vision gate, the signal IS a model judgment, so the judge is shown the
|
|
19
|
+
concrete evidence the build produced (the diff/summary) rather than asked to
|
|
20
|
+
self-report on intent.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from typing import Callable, List, Optional
|
|
24
|
+
|
|
25
|
+
from misterdev.core.execution.bounded import run_bounded
|
|
26
|
+
from misterdev.core.verification.independent import build_independent_call
|
|
27
|
+
from misterdev.llm.responses import (
|
|
28
|
+
extract_json_object as _extract_json_object,
|
|
29
|
+
)
|
|
30
|
+
from misterdev.logging_setup import setup_logger
|
|
31
|
+
|
|
32
|
+
logger = setup_logger(__name__)
|
|
33
|
+
|
|
34
|
+
# Outcome constants. SKIP means "no opinion" (no goal/criteria/client, an
|
|
35
|
+
# unparseable verdict, or a timeout) and must never be treated as a pass/fail
|
|
36
|
+
# signal by callers.
|
|
37
|
+
SKIP = "skip"
|
|
38
|
+
SATISFIED = "satisfied"
|
|
39
|
+
GAP = "gap"
|
|
40
|
+
|
|
41
|
+
_PROMPT = (
|
|
42
|
+
"You are a strict acceptance reviewer. Decide whether the WORK that was done "
|
|
43
|
+
"actually satisfies the stated GOAL and ACCEPTANCE CRITERIA. Judge only from "
|
|
44
|
+
"the evidence given; do not assume work that is not shown.\n\n"
|
|
45
|
+
"## Goal\n{goal}\n\n"
|
|
46
|
+
"## Acceptance Criteria\n{criteria}\n\n"
|
|
47
|
+
"## Work Done (diff / summary)\n{evidence}\n\n"
|
|
48
|
+
"Reply with a single JSON object on the first line and nothing else:\n"
|
|
49
|
+
'{{"satisfied": true|false, "gaps": ["short description of each unmet '
|
|
50
|
+
'requirement"]}}\n'
|
|
51
|
+
"If satisfied is true, gaps must be an empty list. Keep each gap to one "
|
|
52
|
+
"sentence."
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# Cap the evidence we feed the judge so a huge diff can't blow up the prompt or
|
|
56
|
+
# cost; the head of a diff carries the most signal (new files, signatures).
|
|
57
|
+
_MAX_EVIDENCE_CHARS = 16000
|
|
58
|
+
|
|
59
|
+
# A judge call takes the assembled prompt text and returns the model's text.
|
|
60
|
+
# Injected in tests; defaulted to the project client's generate_code path.
|
|
61
|
+
JudgeCall = Callable[[str], str]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class GoalVerdict:
|
|
65
|
+
"""Outcome of a goal-completion check.
|
|
66
|
+
|
|
67
|
+
``status`` is SKIP / SATISFIED / GAP. ``gaps`` lists the unmet requirements
|
|
68
|
+
(empty when satisfied or skipped). ``raw`` is the model's text (evidence);
|
|
69
|
+
``reason`` explains a SKIP.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(
|
|
73
|
+
self,
|
|
74
|
+
status: str,
|
|
75
|
+
gaps: Optional[List[str]] = None,
|
|
76
|
+
raw: str = "",
|
|
77
|
+
reason: str = "",
|
|
78
|
+
):
|
|
79
|
+
self.status = status
|
|
80
|
+
self.gaps = list(gaps or [])
|
|
81
|
+
self.raw = raw
|
|
82
|
+
self.reason = reason
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def satisfied(self) -> bool:
|
|
86
|
+
return self.status == SATISFIED
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def skipped(self) -> bool:
|
|
90
|
+
return self.status == SKIP
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def has_gap(self) -> bool:
|
|
94
|
+
return self.status == GAP
|
|
95
|
+
|
|
96
|
+
def __repr__(self) -> str:
|
|
97
|
+
return f"GoalVerdict(status={self.status!r}, gaps={len(self.gaps)})"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def run_goal_check(
|
|
101
|
+
goal: Optional[str],
|
|
102
|
+
criteria: Optional[str],
|
|
103
|
+
evidence: Optional[str],
|
|
104
|
+
judge_call: Optional[JudgeCall] = None,
|
|
105
|
+
llm_client=None,
|
|
106
|
+
judge_model: Optional[str] = None,
|
|
107
|
+
timeout: float = 60,
|
|
108
|
+
) -> GoalVerdict:
|
|
109
|
+
"""Judge whether ``evidence`` satisfies ``goal`` + ``criteria``.
|
|
110
|
+
|
|
111
|
+
The model call is performed by ``judge_call`` when supplied (the test seam);
|
|
112
|
+
otherwise one is built from ``llm_client`` if provided. ``judge_model``, when
|
|
113
|
+
given, routes the judgment through an INDEPENDENT model so it does not share
|
|
114
|
+
the generator's blind spots (its absence is logged — the same-model judge is
|
|
115
|
+
weaker). With neither a callable nor a client, the check SKIPs (no model).
|
|
116
|
+
SKIP also when there is no goal AND no criteria (no target to judge against),
|
|
117
|
+
on an unparseable verdict, on any judge error, or on the hard timeout (never
|
|
118
|
+
blocks). ``timeout`` is the hard ceiling for the whole run.
|
|
119
|
+
"""
|
|
120
|
+
if not (goal or "").strip() and not (criteria or "").strip():
|
|
121
|
+
return GoalVerdict(SKIP, reason="no goal or acceptance criteria")
|
|
122
|
+
|
|
123
|
+
call = judge_call or _default_judge_call(llm_client, judge_model)
|
|
124
|
+
if call is None:
|
|
125
|
+
return GoalVerdict(SKIP, reason="no LLM judge available")
|
|
126
|
+
|
|
127
|
+
prompt = _PROMPT.format(
|
|
128
|
+
goal=(goal or "(none given)").strip(),
|
|
129
|
+
criteria=(criteria or "(none given)").strip(),
|
|
130
|
+
evidence=(evidence or "(no diff/summary captured)").strip()[
|
|
131
|
+
:_MAX_EVIDENCE_CHARS
|
|
132
|
+
],
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def _work() -> GoalVerdict:
|
|
136
|
+
try:
|
|
137
|
+
return _parse_verdict(call(prompt) or "")
|
|
138
|
+
except Exception as e: # any model/IO failure is non-fatal -> skip
|
|
139
|
+
logger.debug(f"Goal-completion check unavailable: {e}")
|
|
140
|
+
return GoalVerdict(SKIP, reason=f"error: {e}")
|
|
141
|
+
|
|
142
|
+
return run_bounded(
|
|
143
|
+
_work, timeout, GoalVerdict(SKIP, reason="timed out"), "Goal-completion check"
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _parse_verdict(text: str) -> GoalVerdict:
|
|
148
|
+
"""Deterministically parse the judge's JSON verdict.
|
|
149
|
+
|
|
150
|
+
Tolerates surrounding prose and markdown fences by extracting the first
|
|
151
|
+
balanced ``{...}`` object. A missing/invalid object, or a missing
|
|
152
|
+
``satisfied`` boolean, is a SKIP (no opinion), not a failure. When
|
|
153
|
+
``satisfied`` is false, ``gaps`` is normalized to a list of non-empty
|
|
154
|
+
strings; an unsatisfied verdict with no listed gaps still records one
|
|
155
|
+
generic gap so the report never claims a gap exists without saying which.
|
|
156
|
+
"""
|
|
157
|
+
if not text or not text.strip():
|
|
158
|
+
return GoalVerdict(SKIP, raw=text, reason="empty verdict")
|
|
159
|
+
|
|
160
|
+
obj = _extract_json_object(text)
|
|
161
|
+
if obj is None or "satisfied" not in obj:
|
|
162
|
+
return GoalVerdict(SKIP, raw=text, reason="unparseable verdict")
|
|
163
|
+
|
|
164
|
+
satisfied = obj.get("satisfied")
|
|
165
|
+
if not isinstance(satisfied, bool):
|
|
166
|
+
return GoalVerdict(SKIP, raw=text, reason="non-boolean 'satisfied'")
|
|
167
|
+
|
|
168
|
+
if satisfied:
|
|
169
|
+
return GoalVerdict(SATISFIED, gaps=[], raw=text)
|
|
170
|
+
|
|
171
|
+
gaps_raw = obj.get("gaps")
|
|
172
|
+
gaps: List[str] = []
|
|
173
|
+
if isinstance(gaps_raw, list):
|
|
174
|
+
for g in gaps_raw:
|
|
175
|
+
if isinstance(g, str) and g.strip():
|
|
176
|
+
gaps.append(g.strip())
|
|
177
|
+
elif isinstance(gaps_raw, str) and gaps_raw.strip():
|
|
178
|
+
gaps.append(gaps_raw.strip())
|
|
179
|
+
if not gaps:
|
|
180
|
+
gaps = ["goal not satisfied (no specific gap reported)"]
|
|
181
|
+
return GoalVerdict(GAP, gaps=gaps, raw=text)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _default_judge_call(
|
|
185
|
+
llm_client, judge_model: Optional[str] = None
|
|
186
|
+
) -> Optional[JudgeCall]:
|
|
187
|
+
"""Build a judge call from the project's LLM client, or None if unusable.
|
|
188
|
+
|
|
189
|
+
Uses the client's ``generate_code(prompt, system)`` text interface. When
|
|
190
|
+
``judge_model`` is given and the client supports ``with_model``, the call is
|
|
191
|
+
routed through that INDEPENDENT model; otherwise it runs on the generator's
|
|
192
|
+
own model and the weaker independence is logged. Kept tolerant of client
|
|
193
|
+
shape so an absent/limited client degrades to SKIP rather than raising. No
|
|
194
|
+
network is touched until the returned callable is invoked inside the worker.
|
|
195
|
+
"""
|
|
196
|
+
system = (
|
|
197
|
+
"You are a precise acceptance reviewer. Output only the requested JSON "
|
|
198
|
+
"object. Do not invent work that is not shown in the evidence."
|
|
199
|
+
)
|
|
200
|
+
return build_independent_call(
|
|
201
|
+
llm_client, system, judge_model, "Goal-completion check"
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def build_evidence(diff: str = "", summary: str = "") -> str:
|
|
206
|
+
"""Compose the judge's evidence from a diff and/or a summary.
|
|
207
|
+
|
|
208
|
+
Either may be empty; the summary (what the build reports it did) is shown
|
|
209
|
+
first because it is the most compact statement of intent-realized, with the
|
|
210
|
+
raw diff following as the ground truth. Returned empty when both are empty,
|
|
211
|
+
which the caller treats as "no evidence" (the judge still runs but is told
|
|
212
|
+
so, and a satisfied verdict on no evidence is unlikely).
|
|
213
|
+
"""
|
|
214
|
+
parts = []
|
|
215
|
+
if summary and summary.strip():
|
|
216
|
+
parts.append(f"### Summary\n{summary.strip()}")
|
|
217
|
+
if diff and diff.strip():
|
|
218
|
+
parts.append(f"### Diff\n{diff.strip()}")
|
|
219
|
+
return "\n\n".join(parts)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Shared routing for an INDEPENDENT reviewer/judge model.
|
|
2
|
+
|
|
3
|
+
The orchestrator runs several LLM judgments that should not share the generating
|
|
4
|
+
model's blind spots: the edit-time adversarial critic, the post-build
|
|
5
|
+
goal-completion check, and the LLM acceptance judge. Each wants the same thing —
|
|
6
|
+
run ``generate_code`` on a configured independent model when the client can
|
|
7
|
+
switch to it, otherwise fall back to the generator's own model and make the
|
|
8
|
+
weaker independence visible rather than silent. This centralizes that so the
|
|
9
|
+
three call sites don't each re-implement it.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from typing import Callable, Optional
|
|
13
|
+
|
|
14
|
+
from misterdev.logging_setup import setup_logger
|
|
15
|
+
|
|
16
|
+
logger = setup_logger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def generate_independent(
|
|
20
|
+
llm_client, prompt: str, system: str = "", *, model: Optional[str] = None
|
|
21
|
+
) -> str:
|
|
22
|
+
"""Run ``generate_code`` on ``model`` when the client can switch to it.
|
|
23
|
+
|
|
24
|
+
Routes through the client's ``with_model`` context manager for a configured
|
|
25
|
+
independent model; otherwise (no model, or a client without ``with_model``)
|
|
26
|
+
runs on the client's own model. Returns the model's text (never None).
|
|
27
|
+
"""
|
|
28
|
+
if model and hasattr(llm_client, "with_model"):
|
|
29
|
+
with llm_client.with_model(model):
|
|
30
|
+
return llm_client.generate_code(prompt, system) or ""
|
|
31
|
+
return llm_client.generate_code(prompt, system) or ""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_independent_call(
|
|
35
|
+
llm_client, system: str, model: Optional[str], role: str
|
|
36
|
+
) -> Optional[Callable[[str], str]]:
|
|
37
|
+
"""Build a ``call(prompt) -> str`` bound to an independent model, or None.
|
|
38
|
+
|
|
39
|
+
Returns None when ``llm_client`` can't generate text (so the caller SKIPs).
|
|
40
|
+
Otherwise returns a closure that runs each prompt through ``model`` (when the
|
|
41
|
+
client supports ``with_model``) or the generator's own model. The
|
|
42
|
+
independence level is logged ONCE here, at build time — a warning when a
|
|
43
|
+
model is set but the client can't switch, an info when none is configured —
|
|
44
|
+
so the weaker case is never silently assumed. ``role`` names the judge for
|
|
45
|
+
the log. No network is touched until the returned closure is invoked.
|
|
46
|
+
"""
|
|
47
|
+
if llm_client is None or not hasattr(llm_client, "generate_code"):
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
can_switch = hasattr(llm_client, "with_model")
|
|
51
|
+
if model and not can_switch:
|
|
52
|
+
logger.warning(
|
|
53
|
+
f"{role}: an independent model is set but the client cannot switch "
|
|
54
|
+
f"models; running on the generator's own model (weaker independence)."
|
|
55
|
+
)
|
|
56
|
+
elif not model:
|
|
57
|
+
logger.info(
|
|
58
|
+
f"{role}: no independent model configured; running on the generator's "
|
|
59
|
+
f"own model (weaker independence — set one for a true second "
|
|
60
|
+
f"component)."
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
effective = model if (model and can_switch) else None
|
|
64
|
+
|
|
65
|
+
def _call(prompt: str) -> str:
|
|
66
|
+
return generate_independent(llm_client, prompt, system, model=effective)
|
|
67
|
+
|
|
68
|
+
return _call
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Optional mutation-score gate (generic, command-based, language-agnostic).
|
|
2
|
+
|
|
3
|
+
A passing test suite proves the tests *run* green, but not that they would
|
|
4
|
+
*catch a regression*: a suite with no meaningful assertions passes while a
|
|
5
|
+
mutation testing tool shows it kills almost no injected faults. This gate runs a
|
|
6
|
+
project-configured mutation command, parses a mutation score from its output,
|
|
7
|
+
and asserts it meets a floor.
|
|
8
|
+
|
|
9
|
+
It is deliberately tool-agnostic: it does NOT hardcode mutmut / cosmic-ray /
|
|
10
|
+
Stryker / cargo-mutants. The project supplies the command and the floor; this
|
|
11
|
+
module just runs it and robustly parses a percentage or fraction from common
|
|
12
|
+
output formats, skipping cleanly when the score is unparseable.
|
|
13
|
+
|
|
14
|
+
It mirrors :mod:`misterdev.core.execution.runtime`: strictly opt-in (off
|
|
15
|
+
unless ``mutation.command`` is configured and ``orchestrator.mutation_gate`` is
|
|
16
|
+
true), best-effort, and run in a daemon worker thread with a hard timeout so a
|
|
17
|
+
slow or hung mutation run can NEVER block the build. Absent config or a timeout
|
|
18
|
+
is a SKIP (no opinion); an unparseable score is ALSO a SKIP (we will not fail a
|
|
19
|
+
build on a number we could not read); only a parsed score strictly below the
|
|
20
|
+
floor is a RED.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
import subprocess
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Optional
|
|
27
|
+
|
|
28
|
+
from misterdev.core.execution.bounded import run_bounded
|
|
29
|
+
from misterdev.core.execution.outcomes import (
|
|
30
|
+
GREEN,
|
|
31
|
+
RED,
|
|
32
|
+
SKIP,
|
|
33
|
+
GateOutcome,
|
|
34
|
+
)
|
|
35
|
+
from misterdev.logging_setup import setup_logger
|
|
36
|
+
|
|
37
|
+
logger = setup_logger(__name__)
|
|
38
|
+
|
|
39
|
+
# Outcome constants. SKIP means "no opinion" (no config, unparseable score, or
|
|
40
|
+
# timeout) and must never be treated as a pass/fail signal by callers.
|
|
41
|
+
_MAX_EVIDENCE_CHARS = 16384
|
|
42
|
+
|
|
43
|
+
# Score patterns, tried in order. Each captures a number normalized to a 0..1
|
|
44
|
+
# fraction by :func:`_parse_score`. Ordered most-specific first so a labeled
|
|
45
|
+
# score ("mutation score: 82%") wins over a bare trailing percentage.
|
|
46
|
+
_SCORE_PATTERNS = (
|
|
47
|
+
# "mutation score: 82.5%" / "score = 82%" / "killed 82.5 %"
|
|
48
|
+
re.compile(
|
|
49
|
+
r"(?:mutation\s+)?score[^0-9]{0,12}([0-9]+(?:\.[0-9]+)?)\s*%",
|
|
50
|
+
re.IGNORECASE,
|
|
51
|
+
),
|
|
52
|
+
# "mutation score: 0.82" / "score = .82" (fraction, no percent sign)
|
|
53
|
+
re.compile(
|
|
54
|
+
r"(?:mutation\s+)?score[^0-9]{0,12}([01](?:\.[0-9]+)?|\.[0-9]+)\b",
|
|
55
|
+
re.IGNORECASE,
|
|
56
|
+
),
|
|
57
|
+
# "killed 41/50" / "41 / 50 mutants killed" (a ratio)
|
|
58
|
+
re.compile(r"([0-9]+)\s*/\s*([0-9]+)"),
|
|
59
|
+
# Fallback: a standalone percentage anywhere ("82.5%").
|
|
60
|
+
re.compile(r"([0-9]+(?:\.[0-9]+)?)\s*%"),
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class MutationResult(GateOutcome):
|
|
65
|
+
"""Outcome of a mutation-score gate run.
|
|
66
|
+
|
|
67
|
+
``status`` is SKIP/GREEN/RED; ``score`` is the parsed mutation score as a
|
|
68
|
+
0..1 fraction (None when unparseable/skipped); ``evidence`` is the captured
|
|
69
|
+
output (truncated); ``reason`` explains a SKIP/RED.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(
|
|
73
|
+
self,
|
|
74
|
+
status: str,
|
|
75
|
+
score: Optional[float] = None,
|
|
76
|
+
evidence: str = "",
|
|
77
|
+
reason: str = "",
|
|
78
|
+
):
|
|
79
|
+
super().__init__(status, reason)
|
|
80
|
+
self.score = score
|
|
81
|
+
self.evidence = evidence
|
|
82
|
+
|
|
83
|
+
def __repr__(self) -> str:
|
|
84
|
+
return f"MutationResult(status={self.status!r}, score={self.score!r})"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def run_mutation_gate(
|
|
88
|
+
project_root: Path,
|
|
89
|
+
mutation_config: Optional[dict],
|
|
90
|
+
runner=None,
|
|
91
|
+
) -> MutationResult:
|
|
92
|
+
"""Run the mutation gate described by ``mutation_config``.
|
|
93
|
+
|
|
94
|
+
``mutation_config`` keys:
|
|
95
|
+
- ``command`` (required): the mutation-testing command to run. No command
|
|
96
|
+
means the feature is off -> SKIP.
|
|
97
|
+
- ``min_score`` (optional, default 0.0): the floor the parsed score must
|
|
98
|
+
meet, accepted as a fraction (0.8) or a percentage (80). A score at or
|
|
99
|
+
above the floor is GREEN, strictly below is RED.
|
|
100
|
+
- ``timeout`` (optional, default 1800): hard ceiling for the whole run, in
|
|
101
|
+
seconds (mutation runs are slow). The outer join always returns.
|
|
102
|
+
|
|
103
|
+
Returns a :class:`MutationResult`. ``runner`` accepts the gate's container
|
|
104
|
+
seam ``(cmd, timeout) -> (ok, output)``; when None the command runs locally.
|
|
105
|
+
"""
|
|
106
|
+
if not mutation_config or not mutation_config.get("command"):
|
|
107
|
+
return MutationResult(SKIP, reason="no mutation config")
|
|
108
|
+
|
|
109
|
+
command = mutation_config["command"]
|
|
110
|
+
min_score = _normalize_floor(mutation_config.get("min_score", 0.0))
|
|
111
|
+
timeout = float(mutation_config.get("timeout", 1800))
|
|
112
|
+
|
|
113
|
+
def _work() -> MutationResult:
|
|
114
|
+
try:
|
|
115
|
+
return _mutation(project_root, command, min_score, timeout, runner)
|
|
116
|
+
except Exception as e: # any launch/IO failure is non-fatal -> skip
|
|
117
|
+
logger.debug(f"Mutation gate unavailable: {e}")
|
|
118
|
+
return MutationResult(SKIP, reason=f"error: {e}")
|
|
119
|
+
|
|
120
|
+
# A small margin over the inner timeout so a clean inner teardown is
|
|
121
|
+
# preferred, but the outer bound still guarantees we return.
|
|
122
|
+
return run_bounded(
|
|
123
|
+
_work, timeout + 5, MutationResult(SKIP, reason="timed out"), "Mutation gate"
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _mutation(
|
|
128
|
+
project_root: Path,
|
|
129
|
+
command: str,
|
|
130
|
+
min_score: float,
|
|
131
|
+
timeout: float,
|
|
132
|
+
runner,
|
|
133
|
+
) -> MutationResult:
|
|
134
|
+
"""Run the command, parse a score, and compare it to the floor.
|
|
135
|
+
|
|
136
|
+
A non-zero exit is not itself a failure: many mutation tools exit non-zero
|
|
137
|
+
when survivors exist yet still print a usable score, so we parse the output
|
|
138
|
+
regardless and only RED on a score below the floor. We SKIP (not RED) when no
|
|
139
|
+
score can be parsed, so the build is never failed on an unread number.
|
|
140
|
+
"""
|
|
141
|
+
if runner is not None:
|
|
142
|
+
ok, output = runner(command, int(timeout))
|
|
143
|
+
else:
|
|
144
|
+
try:
|
|
145
|
+
proc = subprocess.run(
|
|
146
|
+
command,
|
|
147
|
+
shell=True,
|
|
148
|
+
cwd=str(project_root),
|
|
149
|
+
capture_output=True,
|
|
150
|
+
text=True,
|
|
151
|
+
timeout=timeout,
|
|
152
|
+
)
|
|
153
|
+
except subprocess.TimeoutExpired:
|
|
154
|
+
return MutationResult(SKIP, reason="command timed out")
|
|
155
|
+
output = (proc.stdout or "") + (proc.stderr or "")
|
|
156
|
+
|
|
157
|
+
evidence = output[-_MAX_EVIDENCE_CHARS:]
|
|
158
|
+
score = parse_mutation_score(output)
|
|
159
|
+
if score is None:
|
|
160
|
+
return MutationResult(
|
|
161
|
+
SKIP, evidence=evidence, reason="could not parse a mutation score"
|
|
162
|
+
)
|
|
163
|
+
if score >= min_score:
|
|
164
|
+
return MutationResult(GREEN, score=score, evidence=evidence)
|
|
165
|
+
return MutationResult(
|
|
166
|
+
RED,
|
|
167
|
+
score=score,
|
|
168
|
+
evidence=evidence,
|
|
169
|
+
reason=f"mutation score {score:.2%} below floor {min_score:.2%}",
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def parse_mutation_score(output: str) -> Optional[float]:
|
|
174
|
+
"""Parse a mutation score from ``output`` as a 0..1 fraction, or None.
|
|
175
|
+
|
|
176
|
+
Handles labeled percentages ("mutation score: 82.5%"), labeled fractions
|
|
177
|
+
("score: 0.82"), kill ratios ("killed 41/50"), and a bare trailing
|
|
178
|
+
percentage. Returns None when nothing parseable is found OR the value is out
|
|
179
|
+
of the sane 0..1 range, so a stray number can't be misread as a score.
|
|
180
|
+
"""
|
|
181
|
+
if not output:
|
|
182
|
+
return None
|
|
183
|
+
for pattern in _SCORE_PATTERNS:
|
|
184
|
+
match = pattern.search(output)
|
|
185
|
+
if not match:
|
|
186
|
+
continue
|
|
187
|
+
groups = match.groups()
|
|
188
|
+
if len(groups) == 2: # ratio "killed/total"
|
|
189
|
+
killed = float(groups[0])
|
|
190
|
+
total = float(groups[1])
|
|
191
|
+
if total <= 0:
|
|
192
|
+
continue
|
|
193
|
+
value = killed / total
|
|
194
|
+
else:
|
|
195
|
+
raw = float(groups[0])
|
|
196
|
+
# A value > 1 is a percentage even without a sign (the percent
|
|
197
|
+
# patterns already matched the sign); normalize to a fraction.
|
|
198
|
+
value = raw / 100.0 if raw > 1 else raw
|
|
199
|
+
if 0.0 <= value <= 1.0:
|
|
200
|
+
return value
|
|
201
|
+
return None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _normalize_floor(value) -> float:
|
|
205
|
+
"""Accept a floor as a fraction (0.8) or a percentage (80) -> 0..1 fraction.
|
|
206
|
+
|
|
207
|
+
A value > 1 is read as a percentage. Out-of-range or non-numeric input
|
|
208
|
+
clamps to a 0.0 floor (effectively "any score passes") rather than raising,
|
|
209
|
+
so a misconfigured floor degrades to permissive instead of crashing.
|
|
210
|
+
"""
|
|
211
|
+
try:
|
|
212
|
+
f = float(value)
|
|
213
|
+
except (TypeError, ValueError):
|
|
214
|
+
return 0.0
|
|
215
|
+
if f > 1.0:
|
|
216
|
+
f = f / 100.0
|
|
217
|
+
if f < 0.0:
|
|
218
|
+
return 0.0
|
|
219
|
+
if f > 1.0:
|
|
220
|
+
return 1.0
|
|
221
|
+
return f
|