fuzzprep 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. fuzzprep/__init__.py +3 -0
  2. fuzzprep/__main__.py +4 -0
  3. fuzzprep/agents/harness_builder/SKILL.md +156 -0
  4. fuzzprep/agents/library_builder/SKILL.md +147 -0
  5. fuzzprep/agents/scripts/check_build.sh +87 -0
  6. fuzzprep/agents/scripts/check_build_in_container.sh +73 -0
  7. fuzzprep/agents/scripts/check_dockerfile_from_scratch.sh +33 -0
  8. fuzzprep/cli.py +1109 -0
  9. fuzzprep/core/__init__.py +0 -0
  10. fuzzprep/core/agent_stream.py +304 -0
  11. fuzzprep/core/files.py +35 -0
  12. fuzzprep/core/paths.py +25 -0
  13. fuzzprep/core/reporting.py +218 -0
  14. fuzzprep/core/repos.py +217 -0
  15. fuzzprep/core/resources.py +41 -0
  16. fuzzprep/core/subprocesses.py +197 -0
  17. fuzzprep/feature_extractor/__init__.py +0 -0
  18. fuzzprep/feature_extractor/benchmark_yaml.py +92 -0
  19. fuzzprep/feature_extractor/extraction.py +184 -0
  20. fuzzprep/feature_extractor/models.py +96 -0
  21. fuzzprep/feature_extractor/native/.clang-format +1 -0
  22. fuzzprep/feature_extractor/native/CMakeLists.txt +90 -0
  23. fuzzprep/feature_extractor/native/include/feature_extractor.hpp +136 -0
  24. fuzzprep/feature_extractor/native/src/extraction_action.cpp +294 -0
  25. fuzzprep/feature_extractor/native/src/json_writer.cpp +154 -0
  26. fuzzprep/feature_extractor/native/src/macro_callbacks.cpp +75 -0
  27. fuzzprep/feature_extractor/native/src/main.cpp +130 -0
  28. fuzzprep/feature_extractor/native_build.py +99 -0
  29. fuzzprep/library_builder/__init__.py +0 -0
  30. fuzzprep/library_builder/agents.py +472 -0
  31. fuzzprep/library_builder/analysis.py +145 -0
  32. fuzzprep/library_builder/build_parameters.py +158 -0
  33. fuzzprep/library_builder/dependency_resolution.py +139 -0
  34. fuzzprep/library_builder/environments/__init__.py +0 -0
  35. fuzzprep/library_builder/environments/base.py +89 -0
  36. fuzzprep/library_builder/environments/gate.py +96 -0
  37. fuzzprep/library_builder/environments/local.py +205 -0
  38. fuzzprep/library_builder/environments/oss_fuzz.py +485 -0
  39. fuzzprep/library_builder/environments/verification.py +125 -0
  40. fuzzprep/library_builder/exploration.py +217 -0
  41. fuzzprep/library_builder/generation.py +325 -0
  42. fuzzprep/library_builder/harness_explorer.py +257 -0
  43. fuzzprep/library_builder/models.py +169 -0
  44. fuzzprep/library_builder/package_names.json +33 -0
  45. fuzzprep/library_builder/package_names.py +40 -0
  46. fuzzprep/library_builder/scripts.py +389 -0
  47. fuzzprep/library_builder/stats.py +102 -0
  48. fuzzprep/library_builder/symbol_patterns.json +65 -0
  49. fuzzprep/library_builder/timeouts.py +14 -0
  50. fuzzprep/library_builder/workspace.py +250 -0
  51. fuzzprep-0.1.0.dist-info/METADATA +255 -0
  52. fuzzprep-0.1.0.dist-info/RECORD +56 -0
  53. fuzzprep-0.1.0.dist-info/WHEEL +4 -0
  54. fuzzprep-0.1.0.dist-info/entry_points.txt +3 -0
  55. fuzzprep-0.1.0.dist-info/licenses/LICENSE +202 -0
  56. fuzzprep-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +52 -0
@@ -0,0 +1,205 @@
1
+ from __future__ import annotations
2
+
3
+ import dataclasses
4
+ import tempfile
5
+ from pathlib import Path
6
+ from typing import TYPE_CHECKING
7
+
8
+ from fuzzprep.core.subprocesses import run_command
9
+ from fuzzprep.library_builder.environments.base import Environment, EnvironmentUnavailableError
10
+ from fuzzprep.library_builder.timeouts import DEFAULT_BUILD_TIMEOUT_SECONDS
11
+
12
+ if TYPE_CHECKING:
13
+ from fuzzprep.library_builder.build_parameters import BuildParameters
14
+ from fuzzprep.library_builder.models import (
15
+ AnalysisResult,
16
+ BuildExplorationResult,
17
+ HarnessExplorationResult,
18
+ Language,
19
+ )
20
+
21
+ # A cold clang link on a loaded machine takes longer than the 10s the Docker daemon probe
22
+ # allows, and a probe that times out reads to the user as a missing toolchain.
23
+ _TOOLCHAIN_PROBE_TIMEOUT_SECONDS = 60
24
+
25
+ # The smallest translation unit that still forces the libFuzzer runtime archive to be linked:
26
+ # the driver supplies main(), so an unresolved LLVMFuzzerTestOneInput is the only other symbol.
27
+ _PROBE_SOURCE = (
28
+ "#include <stdint.h>\n"
29
+ "#include <stddef.h>\n"
30
+ "int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; }\n"
31
+ )
32
+
33
+
34
+ def _requests_libfuzzer(harness_cflags: str) -> bool:
35
+ """Whether these flags ask the compiler to link libFuzzer's runtime archive.
36
+
37
+ `-fsanitize=fuzzer-no-link` deliberately does not link it, so a plain substring test would
38
+ probe a link the run never performs and demand a package it never needs.
39
+ """
40
+ for flag in harness_cflags.split():
41
+ if not flag.startswith("-fsanitize="):
42
+ continue
43
+ if "fuzzer" in flag.removeprefix("-fsanitize=").split(","):
44
+ return True
45
+ return False
46
+
47
+
48
+ def libfuzzer_link_failure(parameters: BuildParameters) -> str | None:
49
+ """Return why this toolchain cannot link a libFuzzer harness, or None if it can.
50
+
51
+ Kept apart from check_availability so a caller that only wants the answer -- a test deciding
52
+ whether the host can run it -- gets it without catching a startup failure. Probes exactly
53
+ what the run will do: the configured compiler, with the configured harness flags.
54
+ """
55
+ with tempfile.TemporaryDirectory() as probe_dir:
56
+ source = Path(probe_dir) / "probe.c"
57
+ source.write_text(_PROBE_SOURCE)
58
+ command = [
59
+ parameters.cc,
60
+ *parameters.harness_cflags.split(),
61
+ str(source),
62
+ "-o",
63
+ str(Path(probe_dir) / "probe"),
64
+ ]
65
+ try:
66
+ result = run_command(command, Path(probe_dir), _TOOLCHAIN_PROBE_TIMEOUT_SECONDS)
67
+ except OSError as exc:
68
+ # run_command converts a timeout into a result but lets this through, and a --cc
69
+ # that is not on PATH is the plainest form of an unusable toolchain.
70
+ return str(exc)
71
+ if result.exit_code == 0:
72
+ return None
73
+ return result.stderr.strip() or result.stdout.strip() or f"exit code {result.exit_code}"
74
+
75
+
76
+ def _unavailable(parameters: BuildParameters, detail: str) -> EnvironmentUnavailableError:
77
+ """Build the startup failure for a host that cannot link the configured fuzzing engine."""
78
+ return EnvironmentUnavailableError(
79
+ f"'{parameters.cc}' cannot link a libFuzzer harness with "
80
+ f"'{parameters.harness_cflags}'. The fuzzing runtime is packaged separately: on "
81
+ f"Debian/Ubuntu install libclang-rt-dev, elsewhere the compiler-rt package for your "
82
+ f"clang. Pass --harness-cflags to use a different fuzzing engine. "
83
+ f"Compiler output: {detail}",
84
+ Environment.LOCAL,
85
+ )
86
+
87
+
88
+ class LocalExecutor:
89
+ """Runs each pipeline stage as a host subprocess.
90
+
91
+ Each stage's pass/fail comes from its own probe. The shared gate (check_build.sh, the same
92
+ script a repair agent is told to run) runs once, after the harness probe succeeds: it runs
93
+ build.sh and asserts the artifacts, so the published install/ and out/ come from a build
94
+ the gate itself accepted. Whether it first deletes install/ and build/ depends on the lane
95
+ -- see gate.apply_to_harness_result.
96
+ """
97
+
98
+ def __init__(
99
+ self, *, base_image: str | None = None, bypass_scratch_validation: bool = False
100
+ ) -> None:
101
+ self._project_name: str | None = None
102
+ # A local run never builds the generated Dockerfile, but the output ships it, so the
103
+ # choice still has to be honoured here.
104
+ self._base_image = base_image
105
+ self._bypass_scratch_validation = bypass_scratch_validation
106
+
107
+ def check_availability(self, parameters: BuildParameters | None = None) -> None:
108
+ """Fail at startup if the host cannot link a libFuzzer harness.
109
+
110
+ The host itself is always reachable, but unlike the OSS-Fuzz base image it does not
111
+ come with the fuzzing runtime: clang links `-fsanitize=fuzzer` against compiler-rt's
112
+ archive, which ships in a separate package. Without this the run gets that far only
113
+ after building the library, and reports a link error instead of a missing package.
114
+
115
+ Only the default engine flag is probed. A caller who passed its own --harness-cflags
116
+ chose a toolchain contract this check cannot predict, so it stays out of the way. One
117
+ probe covers C and C++ alike, since both link the same archive.
118
+ """
119
+ from fuzzprep.library_builder.build_parameters import BuildParameters
120
+
121
+ parameters = parameters or BuildParameters.defaults()
122
+ if not _requests_libfuzzer(parameters.harness_cflags):
123
+ return
124
+ failure = libfuzzer_link_failure(parameters)
125
+ if failure is not None:
126
+ raise _unavailable(parameters, failure)
127
+
128
+ def run_library_build(
129
+ self,
130
+ analysis: AnalysisResult,
131
+ workdir: Path,
132
+ *,
133
+ timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
134
+ parameters: BuildParameters | None = None,
135
+ ) -> BuildExplorationResult:
136
+ from fuzzprep.library_builder import workspace
137
+ from fuzzprep.library_builder.build_parameters import (
138
+ BuildParameters,
139
+ compile_commands_capture_environment,
140
+ )
141
+ from fuzzprep.library_builder.environments import verification
142
+ from fuzzprep.library_builder.exploration import explore
143
+
144
+ workdir = workdir.resolve()
145
+ parameters = parameters or BuildParameters.defaults()
146
+ self._project_name = analysis.project_name
147
+ # Before the build attempt, not after: the gate a repair agent is handed needs the
148
+ # harness scaffold to exist whether or not a build was ever attempted.
149
+ workspace.materialize(workdir, analysis, parameters=parameters, base_image=self._base_image)
150
+
151
+ # Capture here and not only in the gate: CMake writes compile_commands.json during
152
+ # configure, and the gate skips the library build whenever this build already ran cold.
153
+ with parameters.library_environment(), compile_commands_capture_environment():
154
+ result = explore(
155
+ analysis,
156
+ workdir,
157
+ timeout=timeout,
158
+ environment=Environment.LOCAL,
159
+ parameters=parameters,
160
+ )
161
+ if result.command:
162
+ return result
163
+ # No build was attempted (unidentified build system), so there is no command to
164
+ # report. Point at the gate, which is what an agent's fix has to satisfy.
165
+ return dataclasses.replace(
166
+ result,
167
+ command=verification.verification_command(
168
+ workdir, environment=Environment.LOCAL, project_name=analysis.project_name
169
+ ),
170
+ )
171
+
172
+ def run_harness_compile( # noqa: PLR0913 -- paths and build configuration are independent inputs
173
+ self,
174
+ install_dir: Path,
175
+ workdir: Path,
176
+ language: Language,
177
+ *,
178
+ extra_include_paths: list[str] | None = None,
179
+ extra_library_paths: list[str] | None = None,
180
+ parameters: BuildParameters | None = None,
181
+ library_llm_used: bool = False,
182
+ ) -> HarnessExplorationResult:
183
+ from fuzzprep.library_builder.build_parameters import BuildParameters
184
+ from fuzzprep.library_builder.environments import gate
185
+ from fuzzprep.library_builder.harness_explorer import explore_harness_compilation
186
+
187
+ workdir = workdir.resolve()
188
+ parameters = parameters or BuildParameters.defaults()
189
+ with parameters.harness_environment():
190
+ harness_result = explore_harness_compilation(
191
+ install_dir,
192
+ workdir,
193
+ language,
194
+ extra_include_paths=extra_include_paths,
195
+ extra_library_paths=extra_library_paths,
196
+ environment=Environment.LOCAL,
197
+ )
198
+ return gate.apply_to_harness_result(
199
+ harness_result,
200
+ workdir,
201
+ environment=Environment.LOCAL,
202
+ project_name=self._project_name or workdir.name,
203
+ library_llm_used=library_llm_used,
204
+ bypass_scratch_validation=self._bypass_scratch_validation,
205
+ )
@@ -0,0 +1,485 @@
1
+ from __future__ import annotations
2
+
3
+ import dataclasses
4
+ import json
5
+ import logging
6
+ import os
7
+ import re
8
+ import shlex
9
+ from pathlib import Path
10
+ from typing import TYPE_CHECKING
11
+
12
+ from fuzzprep.core.subprocesses import Runner, RunResult, run_command, run_command_streaming
13
+ from fuzzprep.library_builder.environments import verification
14
+ from fuzzprep.library_builder.environments.base import Environment, EnvironmentUnavailableError
15
+ from fuzzprep.library_builder.timeouts import DEFAULT_BUILD_TIMEOUT_SECONDS
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ if TYPE_CHECKING:
20
+ from fuzzprep.library_builder.build_parameters import BuildParameters
21
+ from fuzzprep.library_builder.models import (
22
+ AnalysisResult,
23
+ BuildExplorationResult,
24
+ HarnessExplorationResult,
25
+ Language,
26
+ )
27
+
28
+ _AVAILABILITY_TIMEOUT_SECONDS = 10
29
+ _IMAGE_BUILD_TIMEOUT_SECONDS = 600
30
+
31
+ # Docker pull/network-failure phrases. An unreachable environment is not a build failure,
32
+ # so it never routes to agent fallback.
33
+ _UNAVAILABLE_PATTERN = re.compile(
34
+ "|".join((r"Error response from daemon", r"no such host", r"i/o timeout")),
35
+ re.IGNORECASE,
36
+ )
37
+
38
+
39
+ class _ImageBuildError(Exception):
40
+ """The run-scoped image failed to build for a reason other than Docker/network
41
+ unavailability."""
42
+
43
+ def __init__(self, command: list[str], result: RunResult) -> None:
44
+ super().__init__(result.stderr or result.stdout)
45
+ self.command = command
46
+ self.result = result
47
+
48
+
49
+ def _is_environment_unavailable(combined_output: str) -> bool:
50
+ return bool(_UNAVAILABLE_PATTERN.search(combined_output))
51
+
52
+
53
+ def _is_within(path: Path, base: Path) -> bool:
54
+ try:
55
+ path.resolve().relative_to(base.resolve())
56
+ except ValueError:
57
+ return False
58
+ return True
59
+
60
+
61
+ _CONTAINER_SRC_DIR = "/src"
62
+
63
+
64
+ def _ensure_source_symlink(workdir: Path, analysis: AnalysisResult) -> None:
65
+ """Symlink workdir/src to the real source when it lives elsewhere.
66
+
67
+ is_standard_source_layout then accepts the layout, so build_library.sh gets the portable
68
+ $SCRIPT_DIR/src path instead of a host-only absolute path no container can resolve.
69
+ """
70
+ link = workdir / "src"
71
+ source = analysis.source_path.resolve()
72
+ if link.resolve() == source:
73
+ return
74
+ if link.is_symlink() or link.exists():
75
+ link.unlink()
76
+ link.symlink_to(source)
77
+
78
+
79
+ def _docker_run_factory(
80
+ image_tag: str, extra_mounts: list[Path], environment_variables: dict[str, str] | None = None
81
+ ) -> Runner:
82
+ """Build a Runner that executes command inside image_tag via `docker run --entrypoint bash`.
83
+
84
+ cwd is bind-mounted at /src, not at its own host path, because OSS-Fuzz tooling (the
85
+ base image's `compile`, $LIB_FUZZING_ENGINE setup) is hardwired to /src. Mounting
86
+ elsewhere would leave that tooling operating on the stale snapshot baked into the image
87
+ instead of the live workspace harness-link discovery rewrites between attempts.
88
+
89
+ extra_mounts covers a source tree outside workdir, which keeps mounting at its own host
90
+ path. Used for internal probing only; the pass/fail gate goes through verification.py.
91
+ """
92
+
93
+ def run(command: list[str], cwd: Path, timeout: int) -> RunResult:
94
+ mounts = ["-v", f"{cwd.resolve()}:{_CONTAINER_SRC_DIR}"]
95
+ for mount in dict.fromkeys(str(p.resolve()) for p in extra_mounts):
96
+ mounts += ["-v", f"{mount}:{mount}"]
97
+ docker_command = ["docker", "run", "--rm", "--entrypoint", "bash"]
98
+ for name, value in (environment_variables or {}).items():
99
+ docker_command += ["-e", f"{name}={value}"]
100
+ docker_command += mounts
101
+ docker_command += ["-w", _CONTAINER_SRC_DIR, image_tag, "-c", shlex.join(command)]
102
+ logger.debug("Running docker command from %s:\n%s", str(cwd), shlex.join(docker_command))
103
+ result = run_command_streaming(docker_command, cwd, timeout)
104
+ _restore_host_ownership(image_tag, mounts, cwd)
105
+ return result
106
+
107
+ return run
108
+
109
+
110
+ _CHOWN_TIMEOUT_SECONDS = 60
111
+
112
+
113
+ def _restore_host_ownership(image_tag: str, mounts: list[str], cwd: Path) -> None:
114
+ """Give the host user back what the container just wrote into the bind mount.
115
+
116
+ The OSS-Fuzz image runs as root, so build/, install/, and out/ come back owned by uid 0
117
+ and the next attempt's rmtree of install/ fails. -h chowns <workspace>/src itself rather
118
+ than following it, since a non-standard-layout run leaves it a symlink into the user's
119
+ own tree. Best-effort: on Docker Desktop there is nothing to undo.
120
+ """
121
+ command = [
122
+ *["docker", "run", "--rm", *mounts, "--entrypoint", "chown", image_tag],
123
+ *["-Rh", f"{os.getuid()}:{os.getgid()}", _CONTAINER_SRC_DIR],
124
+ ]
125
+ result = run_command(command, cwd, _CHOWN_TIMEOUT_SECONDS)
126
+ if result.exit_code != 0:
127
+ logger.debug("Could not restore host ownership of %s: %s", cwd, result.output.strip())
128
+
129
+
130
+ # What the base image already sets correctly for a fuzzing build. Only values that differ
131
+ # travel into the container: forwarding an empty CFLAGS would wipe the image's sanitizer
132
+ # flags.
133
+ _UNCHOSEN_COMPILER_SETTINGS = {"CC": "clang", "CXX": "clang++", "CFLAGS": "", "CXXFLAGS": ""}
134
+ _UNCHOSEN_HARNESS_FLAGS = "-fsanitize=fuzzer"
135
+
136
+
137
+ def _chosen_settings(values: dict[str, str], *, unchosen: dict[str, str]) -> dict[str, str]:
138
+ return {name: value for name, value in values.items() if value != unchosen.get(name, "")}
139
+
140
+
141
+ def _container_build_environment(parameters: BuildParameters) -> dict[str, str]:
142
+ """The library-build settings to forward into the container.
143
+
144
+ CMAKE_EXPORT_COMPILE_COMMANDS is named here rather than left to the host process, because
145
+ `docker run` forwards nothing: compile_commands_capture_environment() sets it on the host
146
+ and CMake inside the container never sees it. Unconditional, unlike the compiler settings
147
+ -- it is capture configuration, not a build choice a user can make.
148
+ """
149
+ return {
150
+ "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
151
+ **_chosen_settings(
152
+ {
153
+ "CC": parameters.cc,
154
+ "CXX": parameters.cxx,
155
+ "CFLAGS": parameters.library_cflags,
156
+ "CXXFLAGS": parameters.library_cxxflags,
157
+ },
158
+ unchosen=_UNCHOSEN_COMPILER_SETTINGS,
159
+ ),
160
+ }
161
+
162
+
163
+ def _container_harness_environment(parameters: BuildParameters) -> dict[str, str]:
164
+ """The harness-compile settings to forward into the container.
165
+
166
+ The default harness flags are dropped rather than forwarded: replacing CFLAGS with just
167
+ `-fsanitize=fuzzer` would throw away the sanitizer configuration `compile` adds. A
168
+ genuinely chosen value still forwards, since `compile` appends SANITIZER_FLAGS on top of
169
+ whatever CFLAGS it inherits rather than replacing them.
170
+ """
171
+ return _chosen_settings(
172
+ {
173
+ "CC": parameters.cc,
174
+ "CXX": parameters.cxx,
175
+ "CFLAGS": parameters.harness_cflags,
176
+ "CXXFLAGS": parameters.harness_cxxflags,
177
+ },
178
+ unchosen={
179
+ **_UNCHOSEN_COMPILER_SETTINGS,
180
+ "CFLAGS": _UNCHOSEN_HARNESS_FLAGS,
181
+ "CXXFLAGS": _UNCHOSEN_HARNESS_FLAGS,
182
+ },
183
+ )
184
+
185
+
186
+ _TOKEN_PATH_PATTERN = re.compile(r"^(?P<flag>--?[A-Za-z][A-Za-z-]*=?)?(?P<path>/src(?:/.*)?)$")
187
+
188
+
189
+ def _rewrite_path_field(value: str, host_prefix: str) -> str:
190
+ if value == _CONTAINER_SRC_DIR or value.startswith(_CONTAINER_SRC_DIR + "/"):
191
+ return host_prefix + value[len(_CONTAINER_SRC_DIR) :]
192
+ return value
193
+
194
+
195
+ def _rewrite_argument_token(token: str, host_prefix: str) -> str:
196
+ """Rewrite a single compiler-argument token whose path is /src or /src/... glued
197
+ directly after a short flag (-I, -L, -isystem, --sysroot=, ...)."""
198
+ match = _TOKEN_PATH_PATTERN.match(token)
199
+ if match is None:
200
+ return token
201
+ flag = match.group("flag") or ""
202
+ path = match.group("path")
203
+ return flag + host_prefix + path[len(_CONTAINER_SRC_DIR) :]
204
+
205
+
206
+ def _rewrite_directory_field(entry: dict, host_prefix: str) -> bool:
207
+ directory = entry.get("directory")
208
+ if not isinstance(directory, str):
209
+ return False
210
+ rewritten = _rewrite_path_field(directory, host_prefix)
211
+ if rewritten == directory:
212
+ return False
213
+ entry["directory"] = rewritten
214
+ return True
215
+
216
+
217
+ def _rewrite_file_field(entry: dict, host_prefix: str) -> bool:
218
+ file_path = entry.get("file")
219
+ if not isinstance(file_path, str) or not file_path.startswith("/"):
220
+ return False
221
+ rewritten = _rewrite_path_field(file_path, host_prefix)
222
+ if rewritten == file_path:
223
+ return False
224
+ entry["file"] = rewritten
225
+ return True
226
+
227
+
228
+ def _rewrite_arguments_field(entry: dict, host_prefix: str) -> bool:
229
+ arguments = entry.get("arguments")
230
+ if not isinstance(arguments, list) or not all(isinstance(a, str) for a in arguments):
231
+ return False
232
+ new_arguments = [_rewrite_argument_token(a, host_prefix) for a in arguments]
233
+ if new_arguments == arguments:
234
+ return False
235
+ entry["arguments"] = new_arguments
236
+ return True
237
+
238
+
239
+ def _rewrite_command_field(entry: dict, host_prefix: str) -> bool:
240
+ command = entry.get("command")
241
+ if not isinstance(command, str):
242
+ return False
243
+ tokens = shlex.split(command)
244
+ new_tokens = [_rewrite_argument_token(t, host_prefix) for t in tokens]
245
+ if new_tokens == tokens:
246
+ return False
247
+ entry["command"] = shlex.join(new_tokens)
248
+ return True
249
+
250
+
251
+ def _rewrite_compile_commands_entry(entry: dict, host_prefix: str) -> bool:
252
+ """Rewrite one compile_commands.json entry's directory/file/command/arguments
253
+ fields in place. Returns whether anything changed."""
254
+ field_rewriters = (
255
+ _rewrite_directory_field,
256
+ _rewrite_file_field,
257
+ _rewrite_arguments_field,
258
+ _rewrite_command_field,
259
+ )
260
+ # Collected into a list first: any() would short-circuit and skip the later fields once
261
+ # one rewrite succeeded.
262
+ results = [rewrite(entry, host_prefix) for rewrite in field_rewriters]
263
+ return any(results)
264
+
265
+
266
+ def rewrite_compile_commands_paths(compile_commands_path: Path, host_workdir: Path) -> None:
267
+ """Rewrite the /src paths a docker-mounted build baked into compile_commands.json back
268
+ to their host-side workdir path.
269
+
270
+ Called from environments/gate.py, right after the container build that wrote those paths.
271
+
272
+ The native feature extractor runs as a plain host subprocess and fatal-errors trying to
273
+ chdir into a /src path that doesn't exist there. host_workdir <-> /src is a fixed
274
+ prefix, so the substitution is exact.
275
+ """
276
+ host_prefix = str(host_workdir.resolve())
277
+ entries = json.loads(compile_commands_path.read_text())
278
+
279
+ changed = False
280
+ for entry in entries:
281
+ changed = _rewrite_compile_commands_entry(entry, host_prefix) or changed
282
+
283
+ if changed:
284
+ compile_commands_path.write_text(json.dumps(entries, indent=2))
285
+
286
+
287
+ class OssFuzzExecutor:
288
+ """Runs each pipeline stage against the real OSS-Fuzz project layout in the workspace.
289
+
290
+ Stateful per instance: run_library_build builds a run-scoped image from the workspace
291
+ Dockerfile, used for the bind-mounted exploration build and for harness-link discovery's
292
+ retry loop; run_harness_compile reuses it.
293
+
294
+ Each stage's pass/fail comes from its own probe. The shared gate (check_build.sh, run in
295
+ the container by check_build_in_container.sh) runs once, after the harness probe
296
+ succeeds, with the workspace mounted at /src and its out/ mounted at the image's own
297
+ $OUT=/out, so its artifacts — the harness binaries included — land on the host. Those
298
+ mounts are also why a run finishes with run_from_scratch_docker_verification: a mounted
299
+ gate can pass while the Dockerfile's own clone or apt layers are broken.
300
+ """
301
+
302
+ def __init__(
303
+ self, *, base_image: str | None = None, bypass_scratch_validation: bool = False
304
+ ) -> None:
305
+ self._image_tag: str | None = None
306
+ self._project_name: str | None = None
307
+ self._base_image = base_image
308
+ self._bypass_scratch_validation = bypass_scratch_validation
309
+
310
+ def check_availability(self, parameters: BuildParameters | None = None) -> None:
311
+ # The base image provides the harness toolchain, so the run's compiler settings say
312
+ # nothing about whether this environment is usable. Only the daemon does.
313
+ del parameters
314
+ result = run_command(["docker", "info"], Path.cwd(), _AVAILABILITY_TIMEOUT_SECONDS)
315
+ if result.exit_code != 0:
316
+ detail = (
317
+ result.stderr.strip() or result.stdout.strip() or f"exit code {result.exit_code}"
318
+ )
319
+ raise EnvironmentUnavailableError(
320
+ f"Docker daemon not reachable: {detail}", Environment.OSS_FUZZ
321
+ )
322
+
323
+ def run_library_build(
324
+ self,
325
+ analysis: AnalysisResult,
326
+ workdir: Path,
327
+ *,
328
+ timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
329
+ parameters: BuildParameters | None = None,
330
+ ) -> BuildExplorationResult:
331
+ from fuzzprep.library_builder import workspace as workspace_layout
332
+ from fuzzprep.library_builder.build_parameters import BuildParameters
333
+ from fuzzprep.library_builder.exploration import explore, write_build_library_script
334
+ from fuzzprep.library_builder.models import BuildExplorationResult
335
+
336
+ workdir = workdir.resolve()
337
+ parameters = parameters or BuildParameters.defaults()
338
+ self._project_name = analysis.project_name
339
+ workspace_layout.materialize(
340
+ workdir, analysis, parameters=parameters, base_image=self._base_image
341
+ )
342
+ _ensure_source_symlink(workdir, analysis)
343
+ # Before _ensure_image: the Dockerfile's COPY of build_library.sh needs it on disk.
344
+ write_build_library_script(analysis, workdir, parameters=parameters)
345
+
346
+ try:
347
+ self._ensure_image(workdir, analysis)
348
+ except _ImageBuildError as exc:
349
+ logger.debug("Failed to ensure image: %s", exc.result.stderr)
350
+ return BuildExplorationResult(
351
+ build_system=analysis.build_system,
352
+ succeeded=False,
353
+ command=exc.command,
354
+ stdout=exc.result.stdout,
355
+ stderr=exc.result.stderr,
356
+ exit_code=exc.result.exit_code,
357
+ duration_seconds=exc.result.duration_seconds,
358
+ environment=Environment.OSS_FUZZ,
359
+ )
360
+ if self._image_tag is None:
361
+ raise RuntimeError("_ensure_image did not set an image tag")
362
+
363
+ extra_mounts = (
364
+ [] if _is_within(analysis.source_path, workdir) else [analysis.source_path.resolve()]
365
+ )
366
+ # `docker run` does not forward the host process environment BuildParameters sets,
367
+ # so the compiler settings are passed explicitly or they are silently ignored here.
368
+ run = _docker_run_factory(
369
+ self._image_tag, extra_mounts, _container_build_environment(parameters)
370
+ )
371
+ exploration_result = explore(
372
+ analysis,
373
+ workdir,
374
+ timeout=timeout,
375
+ environment=Environment.OSS_FUZZ,
376
+ run=run,
377
+ parameters=parameters,
378
+ )
379
+ if exploration_result.command:
380
+ return exploration_result
381
+ # No build was attempted (unidentified build system), so there is no command to
382
+ # report. Point at the gate, which is what an agent's fix has to satisfy.
383
+ return dataclasses.replace(
384
+ exploration_result,
385
+ command=verification.verification_command(
386
+ workdir,
387
+ environment=Environment.OSS_FUZZ,
388
+ project_name=analysis.project_name,
389
+ ),
390
+ )
391
+
392
+ def run_harness_compile( # noqa: PLR0913 -- paths and build configuration are independent inputs
393
+ self,
394
+ install_dir: Path,
395
+ workdir: Path,
396
+ language: Language,
397
+ *,
398
+ extra_include_paths: list[str] | None = None,
399
+ extra_library_paths: list[str] | None = None,
400
+ parameters: BuildParameters | None = None,
401
+ library_llm_used: bool = False,
402
+ ) -> HarnessExplorationResult:
403
+ from fuzzprep.library_builder.build_parameters import BuildParameters
404
+ from fuzzprep.library_builder.environments import gate
405
+ from fuzzprep.library_builder.harness_explorer import explore_harness_compilation
406
+
407
+ if self._image_tag is None or self._project_name is None:
408
+ raise RuntimeError(
409
+ "OssFuzzExecutor.run_harness_compile requires a prior successful "
410
+ "run_library_build on this instance to establish the run-scoped image"
411
+ )
412
+ workdir = workdir.resolve()
413
+ parameters = parameters or BuildParameters.defaults()
414
+ # Discovery's retry loop execs directly against the already-built image, so it does
415
+ # not pay for a `docker build` per attempt.
416
+ run = _docker_run_factory(self._image_tag, [], _container_harness_environment(parameters))
417
+ harness_result = explore_harness_compilation(
418
+ install_dir,
419
+ workdir,
420
+ language,
421
+ extra_include_paths=extra_include_paths,
422
+ extra_library_paths=extra_library_paths,
423
+ environment=Environment.OSS_FUZZ,
424
+ run=run,
425
+ )
426
+ gated = gate.apply_to_harness_result(
427
+ harness_result,
428
+ workdir,
429
+ environment=Environment.OSS_FUZZ,
430
+ project_name=self._project_name,
431
+ library_llm_used=library_llm_used,
432
+ bypass_scratch_validation=self._bypass_scratch_validation,
433
+ )
434
+ if gated.llm_used is False and _is_environment_unavailable(gated.output):
435
+ raise EnvironmentUnavailableError(
436
+ f"Docker became unavailable during verification: {gated.stdout.strip()}",
437
+ Environment.OSS_FUZZ,
438
+ )
439
+ return gated
440
+
441
+ def _ensure_image(self, workdir: Path, analysis: AnalysisResult) -> None:
442
+ """Build a run-scoped image from the workspace's real Dockerfile, for internal
443
+ probing. The gate builds its own image from the same Dockerfile, to the same tag.
444
+
445
+ Always invokes `docker build`, relying on Docker's layer cache when the Dockerfile
446
+ is unchanged. Caching on the discovered package list instead would miss Dockerfile
447
+ edits an agent made directly, leaving probes to run against a stale image.
448
+ """
449
+ tag = f"fuzzprep-dev/{analysis.project_name}:latest"
450
+ command = ["docker", "build", "-t", tag, "."]
451
+ result = run_command(command, workdir, _IMAGE_BUILD_TIMEOUT_SECONDS)
452
+
453
+ if result.exit_code != 0:
454
+ if _is_environment_unavailable(result.output):
455
+ raise EnvironmentUnavailableError(
456
+ f"Failed to build the oss-fuzz image: {result.stderr.strip()}",
457
+ Environment.OSS_FUZZ,
458
+ )
459
+ raise _ImageBuildError(command, result)
460
+
461
+ self._require_compile(tag)
462
+ self._image_tag = tag
463
+
464
+ def _require_compile(self, image_tag: str) -> None:
465
+ """Reject a base image that has no `compile`, before anything tries to build in it.
466
+
467
+ Every stage enters the build through `compile`, and the workspace Dockerfile is
468
+ written in terms of $SRC, so --base-image selects among OSS-Fuzz base images rather
469
+ than accepting any image. Reported as unavailable rather than as a build failure: no
470
+ edit a repair agent makes to build.sh puts `compile` into the image.
471
+ """
472
+ from fuzzprep.library_builder.workspace import DEFAULT_BASE_IMAGE
473
+
474
+ command = ["docker", "run", "--rm", "--entrypoint", "bash", image_tag]
475
+ command += ["-c", "command -v compile"]
476
+ if run_command(command, Path.cwd(), _AVAILABILITY_TIMEOUT_SECONDS).exit_code == 0:
477
+ return
478
+ raise EnvironmentUnavailableError(
479
+ f"The base image behind {image_tag} has no `compile` on PATH, so it is not an "
480
+ "OSS-Fuzz base image. --base-image selects among OSS-Fuzz base images (for example "
481
+ f"{DEFAULT_BASE_IMAGE}); it cannot take an arbitrary image, because every "
482
+ "verification stage enters the build through `compile` and the generated Dockerfile "
483
+ "is written in terms of $SRC.",
484
+ Environment.OSS_FUZZ,
485
+ )