devagent-ai 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent/__init__.py +5 -0
- agent/llm.py +6 -0
- agent/loop.py +16 -0
- agent/memory.py +5 -0
- agent/prompts.py +3 -0
- agent/tools.py +6 -0
- devagent/__init__.py +3 -0
- devagent/__main__.py +4 -0
- devagent/artifacts.py +46 -0
- devagent/cli.py +164 -0
- devagent/config.py +72 -0
- devagent/discovery.py +504 -0
- devagent/evaluation.py +388 -0
- devagent/memory.py +51 -0
- devagent/models.py +251 -0
- devagent/orchestrator.py +887 -0
- devagent/providers.py +344 -0
- devagent/report.py +52 -0
- devagent/retrieval.py +478 -0
- devagent/safety.py +131 -0
- devagent/state_machine.py +45 -0
- devagent/tasking.py +83 -0
- devagent/workspace.py +199 -0
- devagent/worktree.py +151 -0
- devagent_ai-0.3.1.dist-info/METADATA +415 -0
- devagent_ai-0.3.1.dist-info/RECORD +31 -0
- devagent_ai-0.3.1.dist-info/WHEEL +5 -0
- devagent_ai-0.3.1.dist-info/entry_points.txt +2 -0
- devagent_ai-0.3.1.dist-info/licenses/LICENSE +21 -0
- devagent_ai-0.3.1.dist-info/licenses/NOTICE +10 -0
- devagent_ai-0.3.1.dist-info/top_level.txt +2 -0
devagent/retrieval.py
ADDED
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import re
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
from dataclasses import asdict, dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from devagent.models import RepositoryModel
|
|
10
|
+
from devagent.safety import SafetyError
|
|
11
|
+
from devagent.workspace import Workspace
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
_STOP_WORDS = {
|
|
15
|
+
"the", "and", "for", "with", "this", "that", "from", "when", "where", "into",
|
|
16
|
+
"add", "fix", "bug", "test", "tests", "application", "without", "changing",
|
|
17
|
+
}
|
|
18
|
+
_CONCEPT_ALIASES = {
|
|
19
|
+
"authentication": ("auth", "authenticate"),
|
|
20
|
+
"authorization": ("authz", "authorize"),
|
|
21
|
+
"configuration": ("config", "configure"),
|
|
22
|
+
"reconnection": ("reconnect",),
|
|
23
|
+
}
|
|
24
|
+
_SOURCE_EXTENSIONS = frozenset(
|
|
25
|
+
{
|
|
26
|
+
".py", ".pyi", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".go", ".rs",
|
|
27
|
+
".java", ".kt", ".kts", ".c", ".cc", ".cpp", ".h", ".hpp", ".cs", ".rb",
|
|
28
|
+
".php", ".swift", ".scala", ".sh", ".sql", ".vue", ".svelte",
|
|
29
|
+
}
|
|
30
|
+
)
|
|
31
|
+
_CONFIG_EXTENSIONS = frozenset({".toml", ".yaml", ".yml"})
|
|
32
|
+
_CONFIG_NAMES = frozenset(
|
|
33
|
+
{
|
|
34
|
+
"pyproject.toml", "requirements.txt", "setup.py", "setup.cfg", "tox.ini",
|
|
35
|
+
"pytest.ini", "package.json", "tsconfig.json", "cargo.toml", "go.mod", "go.sum",
|
|
36
|
+
"pom.xml", "build.gradle", "gradlew", "cmakelists.txt", "makefile", "meson.build",
|
|
37
|
+
"dockerfile", "jenkinsfile", ".gitlab-ci.yml", "azure-pipelines.yml",
|
|
38
|
+
"readme", "readme.md", "readme.rst", "contributing.md", "contributing.rst",
|
|
39
|
+
}
|
|
40
|
+
)
|
|
41
|
+
_GENERATED_NAMES = frozenset(
|
|
42
|
+
{
|
|
43
|
+
"package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml",
|
|
44
|
+
"poetry.lock", "uv.lock", "pipfile.lock", "cargo.lock",
|
|
45
|
+
}
|
|
46
|
+
)
|
|
47
|
+
_GENERATED_PARTS = frozenset(
|
|
48
|
+
{
|
|
49
|
+
"vendor", "vendors", "coverage", "htmlcov", ".tox", ".nox", ".pytest_cache",
|
|
50
|
+
".mypy_cache", ".ruff_cache", ".next", ".nuxt", "out", "site-packages",
|
|
51
|
+
}
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class RetrievalBudget:
|
|
57
|
+
max_files: int = 20
|
|
58
|
+
max_chars: int = 24_000
|
|
59
|
+
max_per_file_chars: int = 6_000
|
|
60
|
+
max_fallback_files: int = 6
|
|
61
|
+
small_repository_max_files: int = 20
|
|
62
|
+
inventory_max_files: int = 12_000
|
|
63
|
+
max_scan_chars: int = 12_000_000
|
|
64
|
+
max_relationship_files: int = 500
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _split_identifier(value: str) -> list[str]:
|
|
68
|
+
separated = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", value)
|
|
69
|
+
separated = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1 \2", separated)
|
|
70
|
+
return [token.casefold() for token in re.findall(r"[A-Za-z][A-Za-z0-9]*", separated)]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _normalized_forms(token: str) -> tuple[str, ...]:
|
|
74
|
+
token = token.casefold()
|
|
75
|
+
forms = [token]
|
|
76
|
+
forms.extend(_CONCEPT_ALIASES.get(token, ()))
|
|
77
|
+
if len(token) > 5 and token.endswith("ies"):
|
|
78
|
+
forms.append(token[:-3] + "y")
|
|
79
|
+
if len(token) > 5 and token.endswith("ing"):
|
|
80
|
+
forms.extend((token[:-3], token[:-3] + "e"))
|
|
81
|
+
if len(token) > 4 and token.endswith("ed"):
|
|
82
|
+
forms.extend((token[:-2], token[:-1]))
|
|
83
|
+
if len(token) > 6 and token.endswith("ation"):
|
|
84
|
+
forms.append(token[:-5] + "e")
|
|
85
|
+
if len(token) > 6 and token.endswith("sion"):
|
|
86
|
+
forms.append(token[:-4] + "de")
|
|
87
|
+
elif len(token) > 6 and token.endswith("tion"):
|
|
88
|
+
forms.append(token[:-3])
|
|
89
|
+
if len(token) > 4 and token.endswith("s") and not token.endswith(("ss", "is")):
|
|
90
|
+
forms.append(token[:-1])
|
|
91
|
+
return tuple(dict.fromkeys(form for form in forms if len(form) >= 3))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def task_terms(task: str) -> list[str]:
|
|
95
|
+
terms: list[str] = []
|
|
96
|
+
for token in _split_identifier(task):
|
|
97
|
+
if token in _STOP_WORDS:
|
|
98
|
+
continue
|
|
99
|
+
terms.extend(_normalized_forms(token))
|
|
100
|
+
return list(dict.fromkeys(terms))[:32]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _raw_task_terms(task: str) -> list[str]:
|
|
104
|
+
return list(
|
|
105
|
+
dict.fromkeys(
|
|
106
|
+
token
|
|
107
|
+
for token in _split_identifier(task)
|
|
108
|
+
if len(token) >= 3 and token not in _STOP_WORDS
|
|
109
|
+
)
|
|
110
|
+
)[:16]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _is_test_path(path: str) -> bool:
|
|
114
|
+
lowered = path.casefold()
|
|
115
|
+
parts = Path(lowered).parts
|
|
116
|
+
name = Path(lowered).name
|
|
117
|
+
return (
|
|
118
|
+
any(part in {"test", "tests", "spec", "specs", "__tests__"} for part in parts)
|
|
119
|
+
or name.startswith("test_")
|
|
120
|
+
or name.endswith("_test.py")
|
|
121
|
+
or ".test." in name
|
|
122
|
+
or ".spec." in name
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _file_kind(path: str) -> str | None:
|
|
127
|
+
candidate = Path(path)
|
|
128
|
+
lowered_parts = tuple(part.casefold() for part in candidate.parts)
|
|
129
|
+
name = candidate.name.casefold()
|
|
130
|
+
if any(part in _GENERATED_PARTS for part in lowered_parts):
|
|
131
|
+
return None
|
|
132
|
+
if name in _GENERATED_NAMES or candidate.suffix.casefold() == ".map" or ".min." in name:
|
|
133
|
+
return None
|
|
134
|
+
if candidate.suffix.casefold() in _SOURCE_EXTENSIONS:
|
|
135
|
+
return "test" if _is_test_path(path) else "source"
|
|
136
|
+
if name in _CONFIG_NAMES or candidate.suffix.casefold() in _CONFIG_EXTENSIONS:
|
|
137
|
+
return "documentation" if name.startswith(("readme", "contributing")) else "manifest"
|
|
138
|
+
if path.startswith(".github/workflows/") and candidate.suffix.casefold() in {".yml", ".yaml"}:
|
|
139
|
+
return "manifest"
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _inventory(workspace: Workspace, budget: RetrievalBudget) -> list[tuple[str, str, int]]:
|
|
144
|
+
inventory: list[tuple[str, str, int]] = []
|
|
145
|
+
for relative in workspace.list_files(limit=budget.inventory_max_files):
|
|
146
|
+
kind = _file_kind(relative)
|
|
147
|
+
if kind is None:
|
|
148
|
+
continue
|
|
149
|
+
try:
|
|
150
|
+
target = workspace.paths.resolve(relative, allow_missing=False)
|
|
151
|
+
size = target.stat().st_size
|
|
152
|
+
except (OSError, SafetyError):
|
|
153
|
+
continue
|
|
154
|
+
if size > 2_000_000:
|
|
155
|
+
continue
|
|
156
|
+
inventory.append((relative, kind, size))
|
|
157
|
+
return sorted(inventory)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _content_tokens(text: str) -> tuple[set[str], set[str]]:
|
|
161
|
+
raw = set(_split_identifier(text))
|
|
162
|
+
normalized = {form for token in raw for form in _normalized_forms(token)}
|
|
163
|
+
return raw, normalized
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _python_module_map(paths: list[str]) -> dict[str, str]:
|
|
167
|
+
modules: dict[str, str] = {}
|
|
168
|
+
for path in paths:
|
|
169
|
+
candidate = Path(path)
|
|
170
|
+
if candidate.suffix.casefold() not in {".py", ".pyi"}:
|
|
171
|
+
continue
|
|
172
|
+
parts = list(candidate.with_suffix("").parts)
|
|
173
|
+
if parts and parts[-1] == "__init__":
|
|
174
|
+
parts.pop()
|
|
175
|
+
if parts:
|
|
176
|
+
modules.setdefault(".".join(parts), path)
|
|
177
|
+
modules.setdefault(parts[-1], path)
|
|
178
|
+
return modules
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _python_import_relationships(
|
|
182
|
+
workspace: Workspace, paths: list[str], content_cache: dict[str, str], budget: RetrievalBudget
|
|
183
|
+
) -> list[dict[str, str]]:
|
|
184
|
+
module_map = _python_module_map(paths)
|
|
185
|
+
relationships: set[tuple[str, str, str]] = set()
|
|
186
|
+
parsed = 0
|
|
187
|
+
for path in paths:
|
|
188
|
+
if parsed >= budget.max_relationship_files or Path(path).suffix.casefold() not in {".py", ".pyi"}:
|
|
189
|
+
continue
|
|
190
|
+
try:
|
|
191
|
+
text = content_cache.get(path)
|
|
192
|
+
if text is None:
|
|
193
|
+
text = workspace.read_file(path, max_chars=80_000)
|
|
194
|
+
tree = ast.parse(text)
|
|
195
|
+
except (OSError, SyntaxError, UnicodeError, SafetyError):
|
|
196
|
+
continue
|
|
197
|
+
parsed += 1
|
|
198
|
+
imported: set[str] = set()
|
|
199
|
+
for node in ast.walk(tree):
|
|
200
|
+
if isinstance(node, ast.Import):
|
|
201
|
+
imported.update(alias.name for alias in node.names)
|
|
202
|
+
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
203
|
+
imported.add(node.module)
|
|
204
|
+
for module in imported:
|
|
205
|
+
target = module_map.get(module) or module_map.get(module.split(".")[0])
|
|
206
|
+
if not target or target == path:
|
|
207
|
+
continue
|
|
208
|
+
if _is_test_path(path) and not _is_test_path(target):
|
|
209
|
+
relationships.add((target, path, "python_import"))
|
|
210
|
+
elif _is_test_path(target) and not _is_test_path(path):
|
|
211
|
+
relationships.add((path, target, "python_import"))
|
|
212
|
+
return [
|
|
213
|
+
{"source": source, "test": test, "kind": kind}
|
|
214
|
+
for source, test, kind in sorted(relationships)
|
|
215
|
+
]
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _naming_relationships(paths: list[str]) -> list[dict[str, str]]:
|
|
219
|
+
path_set = set(paths)
|
|
220
|
+
relationships: set[tuple[str, str, str]] = set()
|
|
221
|
+
for test in paths:
|
|
222
|
+
candidate = Path(test)
|
|
223
|
+
if not _is_test_path(test) or candidate.suffix.casefold() not in _SOURCE_EXTENSIONS:
|
|
224
|
+
continue
|
|
225
|
+
stem = candidate.stem
|
|
226
|
+
if stem.startswith("test_"):
|
|
227
|
+
source_stem = stem[5:]
|
|
228
|
+
elif stem.endswith("_test"):
|
|
229
|
+
source_stem = stem[:-5]
|
|
230
|
+
else:
|
|
231
|
+
source_stem = stem.removesuffix(".test").removesuffix(".spec")
|
|
232
|
+
possible = [
|
|
233
|
+
candidate.with_name(source_stem + candidate.suffix).as_posix(),
|
|
234
|
+
(candidate.parent.parent / (source_stem + candidate.suffix)).as_posix(),
|
|
235
|
+
]
|
|
236
|
+
for source in possible:
|
|
237
|
+
if source in path_set and source != test and not _is_test_path(source):
|
|
238
|
+
relationships.add((source, test, "test_naming"))
|
|
239
|
+
return [
|
|
240
|
+
{"source": source, "test": test, "kind": kind}
|
|
241
|
+
for source, test, kind in sorted(relationships)
|
|
242
|
+
]
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _javascript_relationships(
|
|
246
|
+
paths: list[str], content_cache: dict[str, str]
|
|
247
|
+
) -> list[dict[str, str]]:
|
|
248
|
+
path_set = set(paths)
|
|
249
|
+
relationships: set[tuple[str, str, str]] = set()
|
|
250
|
+
extensions = (".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx")
|
|
251
|
+
pattern = re.compile(r"(?:from\s+|require\s*\(\s*)[\"'](\.[^\"']+)")
|
|
252
|
+
for path in paths:
|
|
253
|
+
if not _is_test_path(path) or Path(path).suffix.casefold() not in extensions:
|
|
254
|
+
continue
|
|
255
|
+
text = content_cache.get(path)
|
|
256
|
+
if text is None:
|
|
257
|
+
continue
|
|
258
|
+
for import_path in pattern.findall(text):
|
|
259
|
+
base = (Path(path).parent / import_path).as_posix()
|
|
260
|
+
candidates = [base, *(base + extension for extension in extensions)]
|
|
261
|
+
candidates.extend((Path(base) / ("index" + extension)).as_posix() for extension in extensions)
|
|
262
|
+
for source in candidates:
|
|
263
|
+
if source in path_set and not _is_test_path(source):
|
|
264
|
+
relationships.add((source, path, "relative_import"))
|
|
265
|
+
break
|
|
266
|
+
return [
|
|
267
|
+
{"source": source, "test": test, "kind": kind}
|
|
268
|
+
for source, test, kind in sorted(relationships)
|
|
269
|
+
]
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _relationships(
|
|
273
|
+
workspace: Workspace, paths: list[str], content_cache: dict[str, str], budget: RetrievalBudget
|
|
274
|
+
) -> list[dict[str, str]]:
|
|
275
|
+
combined = [
|
|
276
|
+
*_python_import_relationships(workspace, paths, content_cache, budget),
|
|
277
|
+
*_javascript_relationships(paths, content_cache),
|
|
278
|
+
*_naming_relationships(paths),
|
|
279
|
+
]
|
|
280
|
+
unique = {(item["source"], item["test"], item["kind"]): item for item in combined}
|
|
281
|
+
return [unique[key] for key in sorted(unique)]
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _component_for(path: str, repository: RepositoryModel) -> str:
|
|
285
|
+
candidates = [
|
|
286
|
+
component.path
|
|
287
|
+
for component in repository.components
|
|
288
|
+
if component.path == "." or path == component.path or path.startswith(component.path.rstrip("/") + "/")
|
|
289
|
+
]
|
|
290
|
+
return max(candidates, key=len, default=".")
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def retrieve_context(
|
|
294
|
+
workspace: Workspace,
|
|
295
|
+
repository: RepositoryModel,
|
|
296
|
+
task: str,
|
|
297
|
+
max_chars: int = 24_000,
|
|
298
|
+
*,
|
|
299
|
+
requires_tests: bool | None = None,
|
|
300
|
+
budget: RetrievalBudget | None = None,
|
|
301
|
+
) -> dict[str, object]:
|
|
302
|
+
configured = budget or RetrievalBudget(max_chars=max_chars)
|
|
303
|
+
if max_chars != configured.max_chars:
|
|
304
|
+
configured = RetrievalBudget(**{**asdict(configured), "max_chars": max_chars})
|
|
305
|
+
requires_tests = (
|
|
306
|
+
any(term in task.casefold() for term in ("test", "regression", "verify"))
|
|
307
|
+
if requires_tests is None
|
|
308
|
+
else requires_tests
|
|
309
|
+
)
|
|
310
|
+
inventory = _inventory(workspace, configured)
|
|
311
|
+
paths = [path for path, _, _ in inventory]
|
|
312
|
+
kinds = {path: kind for path, kind, _ in inventory}
|
|
313
|
+
sizes = {path: size for path, _, size in inventory}
|
|
314
|
+
raw_terms = _raw_task_terms(task)
|
|
315
|
+
terms = task_terms(task)
|
|
316
|
+
scores: defaultdict[str, int] = defaultdict(int)
|
|
317
|
+
lexical_scores: defaultdict[str, int] = defaultdict(int)
|
|
318
|
+
matches: list[str] = []
|
|
319
|
+
content_cache: dict[str, str] = {}
|
|
320
|
+
scanned_chars = 0
|
|
321
|
+
exact_lexical_matches = 0
|
|
322
|
+
normalized_lexical_matches = 0
|
|
323
|
+
|
|
324
|
+
for path in paths:
|
|
325
|
+
path_raw, path_normalized = _content_tokens(path)
|
|
326
|
+
exact_path = path_raw.intersection(raw_terms)
|
|
327
|
+
normalized_path = path_normalized.intersection(terms)
|
|
328
|
+
if exact_path:
|
|
329
|
+
lexical_scores[path] += 12 * len(exact_path)
|
|
330
|
+
exact_lexical_matches += len(exact_path)
|
|
331
|
+
if normalized_path:
|
|
332
|
+
lexical_scores[path] += 6 * len(normalized_path)
|
|
333
|
+
normalized_lexical_matches += len(normalized_path - exact_path)
|
|
334
|
+
if scanned_chars >= configured.max_scan_chars:
|
|
335
|
+
continue
|
|
336
|
+
allowance = min(200_000, configured.max_scan_chars - scanned_chars)
|
|
337
|
+
try:
|
|
338
|
+
text = workspace.read_file(path, max_chars=allowance)
|
|
339
|
+
except (OSError, UnicodeError, SafetyError):
|
|
340
|
+
continue
|
|
341
|
+
content_cache[path] = text
|
|
342
|
+
scanned_chars += min(len(text), allowance)
|
|
343
|
+
raw, normalized = _content_tokens(text)
|
|
344
|
+
exact = raw.intersection(raw_terms)
|
|
345
|
+
related = normalized.intersection(terms)
|
|
346
|
+
if exact:
|
|
347
|
+
lexical_scores[path] += 8 * len(exact)
|
|
348
|
+
exact_lexical_matches += len(exact)
|
|
349
|
+
matches.append(f"{path}: exact terms: {', '.join(sorted(exact))}")
|
|
350
|
+
if related:
|
|
351
|
+
lexical_scores[path] += 4 * len(related)
|
|
352
|
+
normalized_lexical_matches += len(related - exact)
|
|
353
|
+
if not exact:
|
|
354
|
+
matches.append(f"{path}: normalized terms: {', '.join(sorted(related))}")
|
|
355
|
+
|
|
356
|
+
for path in paths:
|
|
357
|
+
scores[path] = lexical_scores[path]
|
|
358
|
+
if kinds[path] == "test" and requires_tests:
|
|
359
|
+
scores[path] += 3
|
|
360
|
+
elif kinds[path] == "source":
|
|
361
|
+
scores[path] += 1
|
|
362
|
+
elif kinds[path] == "manifest":
|
|
363
|
+
scores[path] += 1
|
|
364
|
+
|
|
365
|
+
relationships = _relationships(workspace, paths, content_cache, configured)
|
|
366
|
+
lexical_paths = {path for path, score in lexical_scores.items() if score > 0}
|
|
367
|
+
for relationship in relationships:
|
|
368
|
+
source, test = relationship["source"], relationship["test"]
|
|
369
|
+
if source in lexical_paths:
|
|
370
|
+
scores[test] += 10
|
|
371
|
+
if test in lexical_paths:
|
|
372
|
+
scores[source] += 10
|
|
373
|
+
|
|
374
|
+
small_repository = (
|
|
375
|
+
len(inventory) <= configured.small_repository_max_files
|
|
376
|
+
and sum(sizes.values()) <= configured.max_chars
|
|
377
|
+
and all(size <= configured.max_per_file_chars for size in sizes.values())
|
|
378
|
+
)
|
|
379
|
+
fallback = "none"
|
|
380
|
+
fallback_paths: list[str] = []
|
|
381
|
+
if small_repository:
|
|
382
|
+
selected = paths[: configured.max_files]
|
|
383
|
+
fallback = "small-repository inventory"
|
|
384
|
+
else:
|
|
385
|
+
ranked_lexical = sorted(
|
|
386
|
+
(path for path in paths if lexical_scores[path] > 0),
|
|
387
|
+
key=lambda path: (-scores[path], path),
|
|
388
|
+
)
|
|
389
|
+
primary_limit = max(1, configured.max_files - configured.max_fallback_files)
|
|
390
|
+
selected = ranked_lexical[:primary_limit]
|
|
391
|
+
selected_kinds = {kinds[path] for path in selected}
|
|
392
|
+
low_coverage = (
|
|
393
|
+
not any(kinds[path] == "source" for path in selected)
|
|
394
|
+
or selected_kinds <= {"manifest", "documentation"}
|
|
395
|
+
or (requires_tests and not any(kinds[path] == "test" for path in selected))
|
|
396
|
+
or len(selected) < 2
|
|
397
|
+
)
|
|
398
|
+
if low_coverage:
|
|
399
|
+
fallback = "bounded structural coverage"
|
|
400
|
+
candidates: list[str] = []
|
|
401
|
+
for relationship in relationships:
|
|
402
|
+
if relationship["source"] in selected or relationship["test"] in selected:
|
|
403
|
+
candidates.extend((relationship["source"], relationship["test"]))
|
|
404
|
+
matched_components = {_component_for(path, repository) for path in selected}
|
|
405
|
+
if selected:
|
|
406
|
+
parents = {str(Path(path).parent) for path in selected}
|
|
407
|
+
candidates.extend(path for path in paths if str(Path(path).parent) in parents)
|
|
408
|
+
candidates.extend(
|
|
409
|
+
path for path in paths if _component_for(path, repository) in matched_components
|
|
410
|
+
)
|
|
411
|
+
structural = sorted(
|
|
412
|
+
(
|
|
413
|
+
path for path in paths
|
|
414
|
+
if kinds[path] in ({"source", "test", "manifest"} if requires_tests else {"source", "manifest"})
|
|
415
|
+
),
|
|
416
|
+
key=lambda path: (
|
|
417
|
+
0 if requires_tests and kinds[path] == "test" else 1 if kinds[path] == "source" else 2,
|
|
418
|
+
-scores[path],
|
|
419
|
+
path,
|
|
420
|
+
),
|
|
421
|
+
)
|
|
422
|
+
candidates.extend(structural)
|
|
423
|
+
if any(term in task.casefold() for term in ("architecture", "setup", "build", "command")):
|
|
424
|
+
candidates.extend(path for path in paths if kinds[path] == "documentation")
|
|
425
|
+
for path in candidates:
|
|
426
|
+
if path in selected or path in fallback_paths:
|
|
427
|
+
continue
|
|
428
|
+
fallback_paths.append(path)
|
|
429
|
+
if len(fallback_paths) >= configured.max_fallback_files:
|
|
430
|
+
break
|
|
431
|
+
selected.extend(fallback_paths)
|
|
432
|
+
selected = selected[: configured.max_files]
|
|
433
|
+
|
|
434
|
+
snippets: dict[str, str] = {}
|
|
435
|
+
remaining = configured.max_chars
|
|
436
|
+
for path in selected:
|
|
437
|
+
if remaining <= 0:
|
|
438
|
+
break
|
|
439
|
+
try:
|
|
440
|
+
content = workspace.read_file(path, max_chars=min(configured.max_per_file_chars, remaining))
|
|
441
|
+
except (OSError, UnicodeError, SafetyError):
|
|
442
|
+
continue
|
|
443
|
+
snippets[path] = content
|
|
444
|
+
remaining -= len(content)
|
|
445
|
+
|
|
446
|
+
selected = list(snippets)
|
|
447
|
+
return {
|
|
448
|
+
"terms": terms,
|
|
449
|
+
"ranked_paths": selected,
|
|
450
|
+
"matches": matches[:100],
|
|
451
|
+
"snippets": snippets,
|
|
452
|
+
"relationships": [
|
|
453
|
+
relationship
|
|
454
|
+
for relationship in relationships
|
|
455
|
+
if relationship["source"] in selected or relationship["test"] in selected
|
|
456
|
+
],
|
|
457
|
+
"diagnostics": {
|
|
458
|
+
"repository_files": len(inventory),
|
|
459
|
+
"inventory_truncated": len(workspace.list_files(limit=configured.inventory_max_files)) >= configured.inventory_max_files,
|
|
460
|
+
"exact_lexical_matches": exact_lexical_matches,
|
|
461
|
+
"normalized_lexical_matches": normalized_lexical_matches,
|
|
462
|
+
"fallback": fallback,
|
|
463
|
+
"fallback_paths": fallback_paths,
|
|
464
|
+
"selected": selected,
|
|
465
|
+
"relationship_count": len(relationships),
|
|
466
|
+
"budgets": asdict(configured),
|
|
467
|
+
},
|
|
468
|
+
"repo_map": [
|
|
469
|
+
{
|
|
470
|
+
"path": component.path,
|
|
471
|
+
"languages": component.languages,
|
|
472
|
+
"frameworks": component.frameworks,
|
|
473
|
+
"manifests": component.manifests,
|
|
474
|
+
"tests": component.test_locations,
|
|
475
|
+
}
|
|
476
|
+
for component in repository.components
|
|
477
|
+
],
|
|
478
|
+
}
|
devagent/safety.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import fnmatch
|
|
4
|
+
import os
|
|
5
|
+
import shlex
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Sequence
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SafetyError(RuntimeError):
|
|
11
|
+
"""Raised when a requested local operation crosses a safety boundary."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
SKIP_DIRECTORIES = frozenset(
|
|
15
|
+
{
|
|
16
|
+
".git", ".devagent", ".venv", "venv", "node_modules", "dist", "build",
|
|
17
|
+
"target", "__pycache__", "vendor", "coverage", "htmlcov", ".tox", ".nox",
|
|
18
|
+
".pytest_cache", ".mypy_cache", ".ruff_cache", ".next", "site-packages",
|
|
19
|
+
}
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
_SECRET_PATTERNS = (
|
|
23
|
+
".env",
|
|
24
|
+
".env.*",
|
|
25
|
+
"*.pem",
|
|
26
|
+
"*.key",
|
|
27
|
+
"id_rsa",
|
|
28
|
+
"id_ed25519",
|
|
29
|
+
"credentials*",
|
|
30
|
+
"secrets*",
|
|
31
|
+
"*private_key*",
|
|
32
|
+
"*apikey*",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def is_secret_path(path: Path) -> bool:
|
|
37
|
+
lowered_parts = tuple(part.lower() for part in path.parts)
|
|
38
|
+
if ".ssh" in lowered_parts or ".aws" in lowered_parts:
|
|
39
|
+
return True
|
|
40
|
+
name = path.name.lower()
|
|
41
|
+
return any(fnmatch.fnmatch(name, pattern) for pattern in _SECRET_PATTERNS)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class PathPolicy:
|
|
45
|
+
def __init__(self, root: Path | str) -> None:
|
|
46
|
+
self.root = Path(root).expanduser().resolve()
|
|
47
|
+
if not self.root.is_dir():
|
|
48
|
+
raise SafetyError(f"Workspace is not a directory: {self.root}")
|
|
49
|
+
|
|
50
|
+
def resolve(self, relative: str, *, allow_missing: bool = True) -> Path:
|
|
51
|
+
if not relative or "\x00" in relative:
|
|
52
|
+
raise SafetyError("Path must be a non-empty text path")
|
|
53
|
+
candidate = (self.root / relative).resolve(strict=False)
|
|
54
|
+
try:
|
|
55
|
+
candidate.relative_to(self.root)
|
|
56
|
+
except ValueError as exc:
|
|
57
|
+
raise SafetyError(f"Path escapes workspace: {relative}") from exc
|
|
58
|
+
if is_secret_path(candidate.relative_to(self.root)):
|
|
59
|
+
raise SafetyError(f"Access to sensitive file is blocked: {relative}")
|
|
60
|
+
if not allow_missing and not candidate.exists():
|
|
61
|
+
raise SafetyError(f"Path does not exist: {relative}")
|
|
62
|
+
return candidate
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class CommandPolicy:
|
|
66
|
+
"""Token-aware policy for non-shell subprocess execution."""
|
|
67
|
+
|
|
68
|
+
_BLOCKED_PROGRAMS = {
|
|
69
|
+
"sudo",
|
|
70
|
+
"su",
|
|
71
|
+
"rm",
|
|
72
|
+
"rmdir",
|
|
73
|
+
"mkfs",
|
|
74
|
+
"shutdown",
|
|
75
|
+
"reboot",
|
|
76
|
+
"poweroff",
|
|
77
|
+
"curl",
|
|
78
|
+
"wget",
|
|
79
|
+
"scp",
|
|
80
|
+
"ssh",
|
|
81
|
+
"sftp",
|
|
82
|
+
"rsync",
|
|
83
|
+
"nc",
|
|
84
|
+
"netcat",
|
|
85
|
+
"busybox",
|
|
86
|
+
"env",
|
|
87
|
+
}
|
|
88
|
+
_READ_ONLY_GIT = {"diff", "status", "rev-parse", "log", "show", "ls-files", "grep"}
|
|
89
|
+
_SHELL_PROGRAMS = {"sh", "bash", "zsh", "fish", "cmd", "powershell", "pwsh"}
|
|
90
|
+
_INSTALLERS = {("pip", "install"), ("pip3", "install"), ("npm", "install"), ("npm", "i"), ("pnpm", "add"), ("yarn", "add")}
|
|
91
|
+
|
|
92
|
+
@classmethod
|
|
93
|
+
def parse(cls, command: str | Sequence[str]) -> tuple[str, ...]:
|
|
94
|
+
if isinstance(command, str):
|
|
95
|
+
try:
|
|
96
|
+
tokens = tuple(shlex.split(command, posix=os.name != "nt"))
|
|
97
|
+
except ValueError as exc:
|
|
98
|
+
raise SafetyError(f"Invalid command quoting: {exc}") from exc
|
|
99
|
+
else:
|
|
100
|
+
tokens = tuple(str(item) for item in command)
|
|
101
|
+
if not tokens or any(not token or "\x00" in token for token in tokens):
|
|
102
|
+
raise SafetyError("Command must contain non-empty arguments")
|
|
103
|
+
return tokens
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def validate(cls, command: str | Sequence[str]) -> tuple[str, ...]:
|
|
107
|
+
tokens = cls.parse(command)
|
|
108
|
+
executable = Path(tokens[0]).name.lower()
|
|
109
|
+
lowered = tuple(token.lower() for token in tokens)
|
|
110
|
+
if executable in cls._BLOCKED_PROGRAMS or executable in cls._SHELL_PROGRAMS:
|
|
111
|
+
raise SafetyError(f"Blocked command program: {executable}")
|
|
112
|
+
if executable == "git" and (len(lowered) < 2 or lowered[1] not in cls._READ_ONLY_GIT):
|
|
113
|
+
operation = lowered[1] if len(lowered) > 1 else "(missing)"
|
|
114
|
+
raise SafetyError(f"Only explicit read-only Git operations are allowed; blocked: git {operation}")
|
|
115
|
+
if executable in {"python", "python3", "node", "ruby", "perl"} and any(
|
|
116
|
+
token in {"-c", "-e", "--eval"} for token in lowered[1:]
|
|
117
|
+
):
|
|
118
|
+
raise SafetyError("Inline interpreter execution is blocked")
|
|
119
|
+
if executable in {"python", "python3"} and len(lowered) > 2 and lowered[1:3] == ("-m", "pip"):
|
|
120
|
+
raise SafetyError("Package installation and pip execution are blocked during a run")
|
|
121
|
+
if len(lowered) > 1 and (executable, lowered[1]) in cls._INSTALLERS:
|
|
122
|
+
raise SafetyError(f"Package installation is blocked during an engineering run: {' '.join(tokens[:2])}")
|
|
123
|
+
for token in tokens:
|
|
124
|
+
lowered_token = token.lower()
|
|
125
|
+
if is_secret_path(Path(lowered_token)):
|
|
126
|
+
raise SafetyError("Command references a sensitive path")
|
|
127
|
+
if lowered_token.startswith(("http://", "https://", "ftp://")):
|
|
128
|
+
raise SafetyError("Network URLs are blocked in engineering commands")
|
|
129
|
+
if any(operator in token for operator in ("\n", "\r", "`", "$(", ">${", "<${")):
|
|
130
|
+
raise SafetyError("Shell syntax is not allowed in structured commands")
|
|
131
|
+
return tokens
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
from devagent.models import AgentState
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class InvalidTransition(RuntimeError):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
_TRANSITIONS: dict[AgentState, set[AgentState]] = {
|
|
13
|
+
AgentState.PREFLIGHT: {AgentState.DISCOVER, AgentState.BLOCKED},
|
|
14
|
+
AgentState.DISCOVER: {AgentState.UNDERSTAND, AgentState.BLOCKED},
|
|
15
|
+
AgentState.UNDERSTAND: {AgentState.TASK_SPEC, AgentState.GATHER_CONTEXT, AgentState.BLOCKED},
|
|
16
|
+
AgentState.TASK_SPEC: {AgentState.BASELINE, AgentState.BLOCKED},
|
|
17
|
+
AgentState.BASELINE: {AgentState.PLAN, AgentState.BLOCKED},
|
|
18
|
+
AgentState.PLAN: {AgentState.GATHER_CONTEXT, AgentState.BLOCKED},
|
|
19
|
+
AgentState.GATHER_CONTEXT: {AgentState.UNDERSTAND, AgentState.REPRODUCE, AgentState.PLAN, AgentState.BLOCKED},
|
|
20
|
+
AgentState.REPRODUCE: {AgentState.IMPLEMENT, AgentState.GATHER_CONTEXT, AgentState.BLOCKED},
|
|
21
|
+
AgentState.IMPLEMENT: {AgentState.VERIFY_TARGETED, AgentState.BLOCKED},
|
|
22
|
+
AgentState.VERIFY_TARGETED: {AgentState.DIAGNOSE, AgentState.VERIFY_BROAD, AgentState.BLOCKED},
|
|
23
|
+
AgentState.DIAGNOSE: {AgentState.PLAN, AgentState.IMPLEMENT, AgentState.BLOCKED},
|
|
24
|
+
AgentState.VERIFY_BROAD: {AgentState.REVIEW, AgentState.DIAGNOSE, AgentState.PARTIALLY_VERIFIED},
|
|
25
|
+
AgentState.REVIEW: {AgentState.IMPLEMENT, AgentState.QUALITY_CHECK, AgentState.BLOCKED},
|
|
26
|
+
AgentState.QUALITY_CHECK: {AgentState.IMPLEMENT, AgentState.FINAL_VERIFY, AgentState.BLOCKED},
|
|
27
|
+
AgentState.FINAL_VERIFY: {AgentState.LEARN, AgentState.DIAGNOSE, AgentState.PARTIALLY_VERIFIED},
|
|
28
|
+
AgentState.LEARN: {AgentState.REPORT},
|
|
29
|
+
AgentState.REPORT: {AgentState.SUCCESS, AgentState.PARTIALLY_VERIFIED, AgentState.BLOCKED},
|
|
30
|
+
AgentState.SUCCESS: set(),
|
|
31
|
+
AgentState.PARTIALLY_VERIFIED: set(),
|
|
32
|
+
AgentState.BLOCKED: set(),
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class Lifecycle:
|
|
38
|
+
state: AgentState = AgentState.PREFLIGHT
|
|
39
|
+
history: list[AgentState] = field(default_factory=lambda: [AgentState.PREFLIGHT])
|
|
40
|
+
|
|
41
|
+
def transition(self, target: AgentState) -> None:
|
|
42
|
+
if target not in _TRANSITIONS[self.state]:
|
|
43
|
+
raise InvalidTransition(f"Invalid DevAgent transition: {self.state.value} -> {target.value}")
|
|
44
|
+
self.state = target
|
|
45
|
+
self.history.append(target)
|