fuzzprep 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fuzzprep/__init__.py +3 -0
- fuzzprep/__main__.py +4 -0
- fuzzprep/agents/harness_builder/SKILL.md +156 -0
- fuzzprep/agents/library_builder/SKILL.md +147 -0
- fuzzprep/agents/scripts/check_build.sh +87 -0
- fuzzprep/agents/scripts/check_build_in_container.sh +73 -0
- fuzzprep/agents/scripts/check_dockerfile_from_scratch.sh +33 -0
- fuzzprep/cli.py +1109 -0
- fuzzprep/core/__init__.py +0 -0
- fuzzprep/core/agent_stream.py +304 -0
- fuzzprep/core/files.py +35 -0
- fuzzprep/core/paths.py +25 -0
- fuzzprep/core/reporting.py +218 -0
- fuzzprep/core/repos.py +217 -0
- fuzzprep/core/resources.py +41 -0
- fuzzprep/core/subprocesses.py +197 -0
- fuzzprep/feature_extractor/__init__.py +0 -0
- fuzzprep/feature_extractor/benchmark_yaml.py +92 -0
- fuzzprep/feature_extractor/extraction.py +184 -0
- fuzzprep/feature_extractor/models.py +96 -0
- fuzzprep/feature_extractor/native/.clang-format +1 -0
- fuzzprep/feature_extractor/native/CMakeLists.txt +90 -0
- fuzzprep/feature_extractor/native/include/feature_extractor.hpp +136 -0
- fuzzprep/feature_extractor/native/src/extraction_action.cpp +294 -0
- fuzzprep/feature_extractor/native/src/json_writer.cpp +154 -0
- fuzzprep/feature_extractor/native/src/macro_callbacks.cpp +75 -0
- fuzzprep/feature_extractor/native/src/main.cpp +130 -0
- fuzzprep/feature_extractor/native_build.py +99 -0
- fuzzprep/library_builder/__init__.py +0 -0
- fuzzprep/library_builder/agents.py +472 -0
- fuzzprep/library_builder/analysis.py +145 -0
- fuzzprep/library_builder/build_parameters.py +158 -0
- fuzzprep/library_builder/dependency_resolution.py +139 -0
- fuzzprep/library_builder/environments/__init__.py +0 -0
- fuzzprep/library_builder/environments/base.py +89 -0
- fuzzprep/library_builder/environments/gate.py +96 -0
- fuzzprep/library_builder/environments/local.py +205 -0
- fuzzprep/library_builder/environments/oss_fuzz.py +485 -0
- fuzzprep/library_builder/environments/verification.py +125 -0
- fuzzprep/library_builder/exploration.py +217 -0
- fuzzprep/library_builder/generation.py +325 -0
- fuzzprep/library_builder/harness_explorer.py +257 -0
- fuzzprep/library_builder/models.py +169 -0
- fuzzprep/library_builder/package_names.json +33 -0
- fuzzprep/library_builder/package_names.py +40 -0
- fuzzprep/library_builder/scripts.py +389 -0
- fuzzprep/library_builder/stats.py +102 -0
- fuzzprep/library_builder/symbol_patterns.json +65 -0
- fuzzprep/library_builder/timeouts.py +14 -0
- fuzzprep/library_builder/workspace.py +250 -0
- fuzzprep-0.1.0.dist-info/METADATA +255 -0
- fuzzprep-0.1.0.dist-info/RECORD +56 -0
- fuzzprep-0.1.0.dist-info/WHEEL +4 -0
- fuzzprep-0.1.0.dist-info/entry_points.txt +3 -0
- fuzzprep-0.1.0.dist-info/licenses/LICENSE +202 -0
- fuzzprep-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +52 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import tempfile
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
from fuzzprep.core.subprocesses import run_command
|
|
9
|
+
from fuzzprep.library_builder.environments.base import Environment, EnvironmentUnavailableError
|
|
10
|
+
from fuzzprep.library_builder.timeouts import DEFAULT_BUILD_TIMEOUT_SECONDS
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from fuzzprep.library_builder.build_parameters import BuildParameters
|
|
14
|
+
from fuzzprep.library_builder.models import (
|
|
15
|
+
AnalysisResult,
|
|
16
|
+
BuildExplorationResult,
|
|
17
|
+
HarnessExplorationResult,
|
|
18
|
+
Language,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
# A cold clang link on a loaded machine takes longer than the 10s the Docker daemon probe
|
|
22
|
+
# allows, and a probe that times out reads to the user as a missing toolchain.
|
|
23
|
+
_TOOLCHAIN_PROBE_TIMEOUT_SECONDS = 60
|
|
24
|
+
|
|
25
|
+
# The smallest translation unit that still forces the libFuzzer runtime archive to be linked:
|
|
26
|
+
# the driver supplies main(), so an unresolved LLVMFuzzerTestOneInput is the only other symbol.
|
|
27
|
+
_PROBE_SOURCE = (
|
|
28
|
+
"#include <stdint.h>\n"
|
|
29
|
+
"#include <stddef.h>\n"
|
|
30
|
+
"int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; }\n"
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _requests_libfuzzer(harness_cflags: str) -> bool:
|
|
35
|
+
"""Whether these flags ask the compiler to link libFuzzer's runtime archive.
|
|
36
|
+
|
|
37
|
+
`-fsanitize=fuzzer-no-link` deliberately does not link it, so a plain substring test would
|
|
38
|
+
probe a link the run never performs and demand a package it never needs.
|
|
39
|
+
"""
|
|
40
|
+
for flag in harness_cflags.split():
|
|
41
|
+
if not flag.startswith("-fsanitize="):
|
|
42
|
+
continue
|
|
43
|
+
if "fuzzer" in flag.removeprefix("-fsanitize=").split(","):
|
|
44
|
+
return True
|
|
45
|
+
return False
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def libfuzzer_link_failure(parameters: BuildParameters) -> str | None:
|
|
49
|
+
"""Return why this toolchain cannot link a libFuzzer harness, or None if it can.
|
|
50
|
+
|
|
51
|
+
Kept apart from check_availability so a caller that only wants the answer -- a test deciding
|
|
52
|
+
whether the host can run it -- gets it without catching a startup failure. Probes exactly
|
|
53
|
+
what the run will do: the configured compiler, with the configured harness flags.
|
|
54
|
+
"""
|
|
55
|
+
with tempfile.TemporaryDirectory() as probe_dir:
|
|
56
|
+
source = Path(probe_dir) / "probe.c"
|
|
57
|
+
source.write_text(_PROBE_SOURCE)
|
|
58
|
+
command = [
|
|
59
|
+
parameters.cc,
|
|
60
|
+
*parameters.harness_cflags.split(),
|
|
61
|
+
str(source),
|
|
62
|
+
"-o",
|
|
63
|
+
str(Path(probe_dir) / "probe"),
|
|
64
|
+
]
|
|
65
|
+
try:
|
|
66
|
+
result = run_command(command, Path(probe_dir), _TOOLCHAIN_PROBE_TIMEOUT_SECONDS)
|
|
67
|
+
except OSError as exc:
|
|
68
|
+
# run_command converts a timeout into a result but lets this through, and a --cc
|
|
69
|
+
# that is not on PATH is the plainest form of an unusable toolchain.
|
|
70
|
+
return str(exc)
|
|
71
|
+
if result.exit_code == 0:
|
|
72
|
+
return None
|
|
73
|
+
return result.stderr.strip() or result.stdout.strip() or f"exit code {result.exit_code}"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _unavailable(parameters: BuildParameters, detail: str) -> EnvironmentUnavailableError:
|
|
77
|
+
"""Build the startup failure for a host that cannot link the configured fuzzing engine."""
|
|
78
|
+
return EnvironmentUnavailableError(
|
|
79
|
+
f"'{parameters.cc}' cannot link a libFuzzer harness with "
|
|
80
|
+
f"'{parameters.harness_cflags}'. The fuzzing runtime is packaged separately: on "
|
|
81
|
+
f"Debian/Ubuntu install libclang-rt-dev, elsewhere the compiler-rt package for your "
|
|
82
|
+
f"clang. Pass --harness-cflags to use a different fuzzing engine. "
|
|
83
|
+
f"Compiler output: {detail}",
|
|
84
|
+
Environment.LOCAL,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class LocalExecutor:
|
|
89
|
+
"""Runs each pipeline stage as a host subprocess.
|
|
90
|
+
|
|
91
|
+
Each stage's pass/fail comes from its own probe. The shared gate (check_build.sh, the same
|
|
92
|
+
script a repair agent is told to run) runs once, after the harness probe succeeds: it runs
|
|
93
|
+
build.sh and asserts the artifacts, so the published install/ and out/ come from a build
|
|
94
|
+
the gate itself accepted. Whether it first deletes install/ and build/ depends on the lane
|
|
95
|
+
-- see gate.apply_to_harness_result.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(
|
|
99
|
+
self, *, base_image: str | None = None, bypass_scratch_validation: bool = False
|
|
100
|
+
) -> None:
|
|
101
|
+
self._project_name: str | None = None
|
|
102
|
+
# A local run never builds the generated Dockerfile, but the output ships it, so the
|
|
103
|
+
# choice still has to be honoured here.
|
|
104
|
+
self._base_image = base_image
|
|
105
|
+
self._bypass_scratch_validation = bypass_scratch_validation
|
|
106
|
+
|
|
107
|
+
def check_availability(self, parameters: BuildParameters | None = None) -> None:
|
|
108
|
+
"""Fail at startup if the host cannot link a libFuzzer harness.
|
|
109
|
+
|
|
110
|
+
The host itself is always reachable, but unlike the OSS-Fuzz base image it does not
|
|
111
|
+
come with the fuzzing runtime: clang links `-fsanitize=fuzzer` against compiler-rt's
|
|
112
|
+
archive, which ships in a separate package. Without this the run gets that far only
|
|
113
|
+
after building the library, and reports a link error instead of a missing package.
|
|
114
|
+
|
|
115
|
+
Only the default engine flag is probed. A caller who passed its own --harness-cflags
|
|
116
|
+
chose a toolchain contract this check cannot predict, so it stays out of the way. One
|
|
117
|
+
probe covers C and C++ alike, since both link the same archive.
|
|
118
|
+
"""
|
|
119
|
+
from fuzzprep.library_builder.build_parameters import BuildParameters
|
|
120
|
+
|
|
121
|
+
parameters = parameters or BuildParameters.defaults()
|
|
122
|
+
if not _requests_libfuzzer(parameters.harness_cflags):
|
|
123
|
+
return
|
|
124
|
+
failure = libfuzzer_link_failure(parameters)
|
|
125
|
+
if failure is not None:
|
|
126
|
+
raise _unavailable(parameters, failure)
|
|
127
|
+
|
|
128
|
+
def run_library_build(
|
|
129
|
+
self,
|
|
130
|
+
analysis: AnalysisResult,
|
|
131
|
+
workdir: Path,
|
|
132
|
+
*,
|
|
133
|
+
timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
|
|
134
|
+
parameters: BuildParameters | None = None,
|
|
135
|
+
) -> BuildExplorationResult:
|
|
136
|
+
from fuzzprep.library_builder import workspace
|
|
137
|
+
from fuzzprep.library_builder.build_parameters import (
|
|
138
|
+
BuildParameters,
|
|
139
|
+
compile_commands_capture_environment,
|
|
140
|
+
)
|
|
141
|
+
from fuzzprep.library_builder.environments import verification
|
|
142
|
+
from fuzzprep.library_builder.exploration import explore
|
|
143
|
+
|
|
144
|
+
workdir = workdir.resolve()
|
|
145
|
+
parameters = parameters or BuildParameters.defaults()
|
|
146
|
+
self._project_name = analysis.project_name
|
|
147
|
+
# Before the build attempt, not after: the gate a repair agent is handed needs the
|
|
148
|
+
# harness scaffold to exist whether or not a build was ever attempted.
|
|
149
|
+
workspace.materialize(workdir, analysis, parameters=parameters, base_image=self._base_image)
|
|
150
|
+
|
|
151
|
+
# Capture here and not only in the gate: CMake writes compile_commands.json during
|
|
152
|
+
# configure, and the gate skips the library build whenever this build already ran cold.
|
|
153
|
+
with parameters.library_environment(), compile_commands_capture_environment():
|
|
154
|
+
result = explore(
|
|
155
|
+
analysis,
|
|
156
|
+
workdir,
|
|
157
|
+
timeout=timeout,
|
|
158
|
+
environment=Environment.LOCAL,
|
|
159
|
+
parameters=parameters,
|
|
160
|
+
)
|
|
161
|
+
if result.command:
|
|
162
|
+
return result
|
|
163
|
+
# No build was attempted (unidentified build system), so there is no command to
|
|
164
|
+
# report. Point at the gate, which is what an agent's fix has to satisfy.
|
|
165
|
+
return dataclasses.replace(
|
|
166
|
+
result,
|
|
167
|
+
command=verification.verification_command(
|
|
168
|
+
workdir, environment=Environment.LOCAL, project_name=analysis.project_name
|
|
169
|
+
),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
def run_harness_compile( # noqa: PLR0913 -- paths and build configuration are independent inputs
|
|
173
|
+
self,
|
|
174
|
+
install_dir: Path,
|
|
175
|
+
workdir: Path,
|
|
176
|
+
language: Language,
|
|
177
|
+
*,
|
|
178
|
+
extra_include_paths: list[str] | None = None,
|
|
179
|
+
extra_library_paths: list[str] | None = None,
|
|
180
|
+
parameters: BuildParameters | None = None,
|
|
181
|
+
library_llm_used: bool = False,
|
|
182
|
+
) -> HarnessExplorationResult:
|
|
183
|
+
from fuzzprep.library_builder.build_parameters import BuildParameters
|
|
184
|
+
from fuzzprep.library_builder.environments import gate
|
|
185
|
+
from fuzzprep.library_builder.harness_explorer import explore_harness_compilation
|
|
186
|
+
|
|
187
|
+
workdir = workdir.resolve()
|
|
188
|
+
parameters = parameters or BuildParameters.defaults()
|
|
189
|
+
with parameters.harness_environment():
|
|
190
|
+
harness_result = explore_harness_compilation(
|
|
191
|
+
install_dir,
|
|
192
|
+
workdir,
|
|
193
|
+
language,
|
|
194
|
+
extra_include_paths=extra_include_paths,
|
|
195
|
+
extra_library_paths=extra_library_paths,
|
|
196
|
+
environment=Environment.LOCAL,
|
|
197
|
+
)
|
|
198
|
+
return gate.apply_to_harness_result(
|
|
199
|
+
harness_result,
|
|
200
|
+
workdir,
|
|
201
|
+
environment=Environment.LOCAL,
|
|
202
|
+
project_name=self._project_name or workdir.name,
|
|
203
|
+
library_llm_used=library_llm_used,
|
|
204
|
+
bypass_scratch_validation=self._bypass_scratch_validation,
|
|
205
|
+
)
|
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import shlex
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
from fuzzprep.core.subprocesses import Runner, RunResult, run_command, run_command_streaming
|
|
13
|
+
from fuzzprep.library_builder.environments import verification
|
|
14
|
+
from fuzzprep.library_builder.environments.base import Environment, EnvironmentUnavailableError
|
|
15
|
+
from fuzzprep.library_builder.timeouts import DEFAULT_BUILD_TIMEOUT_SECONDS
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from fuzzprep.library_builder.build_parameters import BuildParameters
|
|
21
|
+
from fuzzprep.library_builder.models import (
|
|
22
|
+
AnalysisResult,
|
|
23
|
+
BuildExplorationResult,
|
|
24
|
+
HarnessExplorationResult,
|
|
25
|
+
Language,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
_AVAILABILITY_TIMEOUT_SECONDS = 10
|
|
29
|
+
_IMAGE_BUILD_TIMEOUT_SECONDS = 600
|
|
30
|
+
|
|
31
|
+
# Docker pull/network-failure phrases. An unreachable environment is not a build failure,
|
|
32
|
+
# so it never routes to agent fallback.
|
|
33
|
+
_UNAVAILABLE_PATTERN = re.compile(
|
|
34
|
+
"|".join((r"Error response from daemon", r"no such host", r"i/o timeout")),
|
|
35
|
+
re.IGNORECASE,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _ImageBuildError(Exception):
|
|
40
|
+
"""The run-scoped image failed to build for a reason other than Docker/network
|
|
41
|
+
unavailability."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, command: list[str], result: RunResult) -> None:
|
|
44
|
+
super().__init__(result.stderr or result.stdout)
|
|
45
|
+
self.command = command
|
|
46
|
+
self.result = result
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _is_environment_unavailable(combined_output: str) -> bool:
|
|
50
|
+
return bool(_UNAVAILABLE_PATTERN.search(combined_output))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _is_within(path: Path, base: Path) -> bool:
|
|
54
|
+
try:
|
|
55
|
+
path.resolve().relative_to(base.resolve())
|
|
56
|
+
except ValueError:
|
|
57
|
+
return False
|
|
58
|
+
return True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
_CONTAINER_SRC_DIR = "/src"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _ensure_source_symlink(workdir: Path, analysis: AnalysisResult) -> None:
|
|
65
|
+
"""Symlink workdir/src to the real source when it lives elsewhere.
|
|
66
|
+
|
|
67
|
+
is_standard_source_layout then accepts the layout, so build_library.sh gets the portable
|
|
68
|
+
$SCRIPT_DIR/src path instead of a host-only absolute path no container can resolve.
|
|
69
|
+
"""
|
|
70
|
+
link = workdir / "src"
|
|
71
|
+
source = analysis.source_path.resolve()
|
|
72
|
+
if link.resolve() == source:
|
|
73
|
+
return
|
|
74
|
+
if link.is_symlink() or link.exists():
|
|
75
|
+
link.unlink()
|
|
76
|
+
link.symlink_to(source)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _docker_run_factory(
|
|
80
|
+
image_tag: str, extra_mounts: list[Path], environment_variables: dict[str, str] | None = None
|
|
81
|
+
) -> Runner:
|
|
82
|
+
"""Build a Runner that executes command inside image_tag via `docker run --entrypoint bash`.
|
|
83
|
+
|
|
84
|
+
cwd is bind-mounted at /src, not at its own host path, because OSS-Fuzz tooling (the
|
|
85
|
+
base image's `compile`, $LIB_FUZZING_ENGINE setup) is hardwired to /src. Mounting
|
|
86
|
+
elsewhere would leave that tooling operating on the stale snapshot baked into the image
|
|
87
|
+
instead of the live workspace harness-link discovery rewrites between attempts.
|
|
88
|
+
|
|
89
|
+
extra_mounts covers a source tree outside workdir, which keeps mounting at its own host
|
|
90
|
+
path. Used for internal probing only; the pass/fail gate goes through verification.py.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def run(command: list[str], cwd: Path, timeout: int) -> RunResult:
|
|
94
|
+
mounts = ["-v", f"{cwd.resolve()}:{_CONTAINER_SRC_DIR}"]
|
|
95
|
+
for mount in dict.fromkeys(str(p.resolve()) for p in extra_mounts):
|
|
96
|
+
mounts += ["-v", f"{mount}:{mount}"]
|
|
97
|
+
docker_command = ["docker", "run", "--rm", "--entrypoint", "bash"]
|
|
98
|
+
for name, value in (environment_variables or {}).items():
|
|
99
|
+
docker_command += ["-e", f"{name}={value}"]
|
|
100
|
+
docker_command += mounts
|
|
101
|
+
docker_command += ["-w", _CONTAINER_SRC_DIR, image_tag, "-c", shlex.join(command)]
|
|
102
|
+
logger.debug("Running docker command from %s:\n%s", str(cwd), shlex.join(docker_command))
|
|
103
|
+
result = run_command_streaming(docker_command, cwd, timeout)
|
|
104
|
+
_restore_host_ownership(image_tag, mounts, cwd)
|
|
105
|
+
return result
|
|
106
|
+
|
|
107
|
+
return run
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
_CHOWN_TIMEOUT_SECONDS = 60
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _restore_host_ownership(image_tag: str, mounts: list[str], cwd: Path) -> None:
|
|
114
|
+
"""Give the host user back what the container just wrote into the bind mount.
|
|
115
|
+
|
|
116
|
+
The OSS-Fuzz image runs as root, so build/, install/, and out/ come back owned by uid 0
|
|
117
|
+
and the next attempt's rmtree of install/ fails. -h chowns <workspace>/src itself rather
|
|
118
|
+
than following it, since a non-standard-layout run leaves it a symlink into the user's
|
|
119
|
+
own tree. Best-effort: on Docker Desktop there is nothing to undo.
|
|
120
|
+
"""
|
|
121
|
+
command = [
|
|
122
|
+
*["docker", "run", "--rm", *mounts, "--entrypoint", "chown", image_tag],
|
|
123
|
+
*["-Rh", f"{os.getuid()}:{os.getgid()}", _CONTAINER_SRC_DIR],
|
|
124
|
+
]
|
|
125
|
+
result = run_command(command, cwd, _CHOWN_TIMEOUT_SECONDS)
|
|
126
|
+
if result.exit_code != 0:
|
|
127
|
+
logger.debug("Could not restore host ownership of %s: %s", cwd, result.output.strip())
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# What the base image already sets correctly for a fuzzing build. Only values that differ
|
|
131
|
+
# travel into the container: forwarding an empty CFLAGS would wipe the image's sanitizer
|
|
132
|
+
# flags.
|
|
133
|
+
_UNCHOSEN_COMPILER_SETTINGS = {"CC": "clang", "CXX": "clang++", "CFLAGS": "", "CXXFLAGS": ""}
|
|
134
|
+
_UNCHOSEN_HARNESS_FLAGS = "-fsanitize=fuzzer"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _chosen_settings(values: dict[str, str], *, unchosen: dict[str, str]) -> dict[str, str]:
|
|
138
|
+
return {name: value for name, value in values.items() if value != unchosen.get(name, "")}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _container_build_environment(parameters: BuildParameters) -> dict[str, str]:
|
|
142
|
+
"""The library-build settings to forward into the container.
|
|
143
|
+
|
|
144
|
+
CMAKE_EXPORT_COMPILE_COMMANDS is named here rather than left to the host process, because
|
|
145
|
+
`docker run` forwards nothing: compile_commands_capture_environment() sets it on the host
|
|
146
|
+
and CMake inside the container never sees it. Unconditional, unlike the compiler settings
|
|
147
|
+
-- it is capture configuration, not a build choice a user can make.
|
|
148
|
+
"""
|
|
149
|
+
return {
|
|
150
|
+
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
|
|
151
|
+
**_chosen_settings(
|
|
152
|
+
{
|
|
153
|
+
"CC": parameters.cc,
|
|
154
|
+
"CXX": parameters.cxx,
|
|
155
|
+
"CFLAGS": parameters.library_cflags,
|
|
156
|
+
"CXXFLAGS": parameters.library_cxxflags,
|
|
157
|
+
},
|
|
158
|
+
unchosen=_UNCHOSEN_COMPILER_SETTINGS,
|
|
159
|
+
),
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _container_harness_environment(parameters: BuildParameters) -> dict[str, str]:
|
|
164
|
+
"""The harness-compile settings to forward into the container.
|
|
165
|
+
|
|
166
|
+
The default harness flags are dropped rather than forwarded: replacing CFLAGS with just
|
|
167
|
+
`-fsanitize=fuzzer` would throw away the sanitizer configuration `compile` adds. A
|
|
168
|
+
genuinely chosen value still forwards, since `compile` appends SANITIZER_FLAGS on top of
|
|
169
|
+
whatever CFLAGS it inherits rather than replacing them.
|
|
170
|
+
"""
|
|
171
|
+
return _chosen_settings(
|
|
172
|
+
{
|
|
173
|
+
"CC": parameters.cc,
|
|
174
|
+
"CXX": parameters.cxx,
|
|
175
|
+
"CFLAGS": parameters.harness_cflags,
|
|
176
|
+
"CXXFLAGS": parameters.harness_cxxflags,
|
|
177
|
+
},
|
|
178
|
+
unchosen={
|
|
179
|
+
**_UNCHOSEN_COMPILER_SETTINGS,
|
|
180
|
+
"CFLAGS": _UNCHOSEN_HARNESS_FLAGS,
|
|
181
|
+
"CXXFLAGS": _UNCHOSEN_HARNESS_FLAGS,
|
|
182
|
+
},
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
_TOKEN_PATH_PATTERN = re.compile(r"^(?P<flag>--?[A-Za-z][A-Za-z-]*=?)?(?P<path>/src(?:/.*)?)$")
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _rewrite_path_field(value: str, host_prefix: str) -> str:
|
|
190
|
+
if value == _CONTAINER_SRC_DIR or value.startswith(_CONTAINER_SRC_DIR + "/"):
|
|
191
|
+
return host_prefix + value[len(_CONTAINER_SRC_DIR) :]
|
|
192
|
+
return value
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _rewrite_argument_token(token: str, host_prefix: str) -> str:
|
|
196
|
+
"""Rewrite a single compiler-argument token whose path is /src or /src/... glued
|
|
197
|
+
directly after a short flag (-I, -L, -isystem, --sysroot=, ...)."""
|
|
198
|
+
match = _TOKEN_PATH_PATTERN.match(token)
|
|
199
|
+
if match is None:
|
|
200
|
+
return token
|
|
201
|
+
flag = match.group("flag") or ""
|
|
202
|
+
path = match.group("path")
|
|
203
|
+
return flag + host_prefix + path[len(_CONTAINER_SRC_DIR) :]
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _rewrite_directory_field(entry: dict, host_prefix: str) -> bool:
|
|
207
|
+
directory = entry.get("directory")
|
|
208
|
+
if not isinstance(directory, str):
|
|
209
|
+
return False
|
|
210
|
+
rewritten = _rewrite_path_field(directory, host_prefix)
|
|
211
|
+
if rewritten == directory:
|
|
212
|
+
return False
|
|
213
|
+
entry["directory"] = rewritten
|
|
214
|
+
return True
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _rewrite_file_field(entry: dict, host_prefix: str) -> bool:
|
|
218
|
+
file_path = entry.get("file")
|
|
219
|
+
if not isinstance(file_path, str) or not file_path.startswith("/"):
|
|
220
|
+
return False
|
|
221
|
+
rewritten = _rewrite_path_field(file_path, host_prefix)
|
|
222
|
+
if rewritten == file_path:
|
|
223
|
+
return False
|
|
224
|
+
entry["file"] = rewritten
|
|
225
|
+
return True
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _rewrite_arguments_field(entry: dict, host_prefix: str) -> bool:
|
|
229
|
+
arguments = entry.get("arguments")
|
|
230
|
+
if not isinstance(arguments, list) or not all(isinstance(a, str) for a in arguments):
|
|
231
|
+
return False
|
|
232
|
+
new_arguments = [_rewrite_argument_token(a, host_prefix) for a in arguments]
|
|
233
|
+
if new_arguments == arguments:
|
|
234
|
+
return False
|
|
235
|
+
entry["arguments"] = new_arguments
|
|
236
|
+
return True
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _rewrite_command_field(entry: dict, host_prefix: str) -> bool:
|
|
240
|
+
command = entry.get("command")
|
|
241
|
+
if not isinstance(command, str):
|
|
242
|
+
return False
|
|
243
|
+
tokens = shlex.split(command)
|
|
244
|
+
new_tokens = [_rewrite_argument_token(t, host_prefix) for t in tokens]
|
|
245
|
+
if new_tokens == tokens:
|
|
246
|
+
return False
|
|
247
|
+
entry["command"] = shlex.join(new_tokens)
|
|
248
|
+
return True
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _rewrite_compile_commands_entry(entry: dict, host_prefix: str) -> bool:
|
|
252
|
+
"""Rewrite one compile_commands.json entry's directory/file/command/arguments
|
|
253
|
+
fields in place. Returns whether anything changed."""
|
|
254
|
+
field_rewriters = (
|
|
255
|
+
_rewrite_directory_field,
|
|
256
|
+
_rewrite_file_field,
|
|
257
|
+
_rewrite_arguments_field,
|
|
258
|
+
_rewrite_command_field,
|
|
259
|
+
)
|
|
260
|
+
# Collected into a list first: any() would short-circuit and skip the later fields once
|
|
261
|
+
# one rewrite succeeded.
|
|
262
|
+
results = [rewrite(entry, host_prefix) for rewrite in field_rewriters]
|
|
263
|
+
return any(results)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def rewrite_compile_commands_paths(compile_commands_path: Path, host_workdir: Path) -> None:
|
|
267
|
+
"""Rewrite the /src paths a docker-mounted build baked into compile_commands.json back
|
|
268
|
+
to their host-side workdir path.
|
|
269
|
+
|
|
270
|
+
Called from environments/gate.py, right after the container build that wrote those paths.
|
|
271
|
+
|
|
272
|
+
The native feature extractor runs as a plain host subprocess and fatal-errors trying to
|
|
273
|
+
chdir into a /src path that doesn't exist there. host_workdir <-> /src is a fixed
|
|
274
|
+
prefix, so the substitution is exact.
|
|
275
|
+
"""
|
|
276
|
+
host_prefix = str(host_workdir.resolve())
|
|
277
|
+
entries = json.loads(compile_commands_path.read_text())
|
|
278
|
+
|
|
279
|
+
changed = False
|
|
280
|
+
for entry in entries:
|
|
281
|
+
changed = _rewrite_compile_commands_entry(entry, host_prefix) or changed
|
|
282
|
+
|
|
283
|
+
if changed:
|
|
284
|
+
compile_commands_path.write_text(json.dumps(entries, indent=2))
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class OssFuzzExecutor:
|
|
288
|
+
"""Runs each pipeline stage against the real OSS-Fuzz project layout in the workspace.
|
|
289
|
+
|
|
290
|
+
Stateful per instance: run_library_build builds a run-scoped image from the workspace
|
|
291
|
+
Dockerfile, used for the bind-mounted exploration build and for harness-link discovery's
|
|
292
|
+
retry loop; run_harness_compile reuses it.
|
|
293
|
+
|
|
294
|
+
Each stage's pass/fail comes from its own probe. The shared gate (check_build.sh, run in
|
|
295
|
+
the container by check_build_in_container.sh) runs once, after the harness probe
|
|
296
|
+
succeeds, with the workspace mounted at /src and its out/ mounted at the image's own
|
|
297
|
+
$OUT=/out, so its artifacts — the harness binaries included — land on the host. Those
|
|
298
|
+
mounts are also why a run finishes with run_from_scratch_docker_verification: a mounted
|
|
299
|
+
gate can pass while the Dockerfile's own clone or apt layers are broken.
|
|
300
|
+
"""
|
|
301
|
+
|
|
302
|
+
def __init__(
|
|
303
|
+
self, *, base_image: str | None = None, bypass_scratch_validation: bool = False
|
|
304
|
+
) -> None:
|
|
305
|
+
self._image_tag: str | None = None
|
|
306
|
+
self._project_name: str | None = None
|
|
307
|
+
self._base_image = base_image
|
|
308
|
+
self._bypass_scratch_validation = bypass_scratch_validation
|
|
309
|
+
|
|
310
|
+
def check_availability(self, parameters: BuildParameters | None = None) -> None:
|
|
311
|
+
# The base image provides the harness toolchain, so the run's compiler settings say
|
|
312
|
+
# nothing about whether this environment is usable. Only the daemon does.
|
|
313
|
+
del parameters
|
|
314
|
+
result = run_command(["docker", "info"], Path.cwd(), _AVAILABILITY_TIMEOUT_SECONDS)
|
|
315
|
+
if result.exit_code != 0:
|
|
316
|
+
detail = (
|
|
317
|
+
result.stderr.strip() or result.stdout.strip() or f"exit code {result.exit_code}"
|
|
318
|
+
)
|
|
319
|
+
raise EnvironmentUnavailableError(
|
|
320
|
+
f"Docker daemon not reachable: {detail}", Environment.OSS_FUZZ
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
def run_library_build(
|
|
324
|
+
self,
|
|
325
|
+
analysis: AnalysisResult,
|
|
326
|
+
workdir: Path,
|
|
327
|
+
*,
|
|
328
|
+
timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
|
|
329
|
+
parameters: BuildParameters | None = None,
|
|
330
|
+
) -> BuildExplorationResult:
|
|
331
|
+
from fuzzprep.library_builder import workspace as workspace_layout
|
|
332
|
+
from fuzzprep.library_builder.build_parameters import BuildParameters
|
|
333
|
+
from fuzzprep.library_builder.exploration import explore, write_build_library_script
|
|
334
|
+
from fuzzprep.library_builder.models import BuildExplorationResult
|
|
335
|
+
|
|
336
|
+
workdir = workdir.resolve()
|
|
337
|
+
parameters = parameters or BuildParameters.defaults()
|
|
338
|
+
self._project_name = analysis.project_name
|
|
339
|
+
workspace_layout.materialize(
|
|
340
|
+
workdir, analysis, parameters=parameters, base_image=self._base_image
|
|
341
|
+
)
|
|
342
|
+
_ensure_source_symlink(workdir, analysis)
|
|
343
|
+
# Before _ensure_image: the Dockerfile's COPY of build_library.sh needs it on disk.
|
|
344
|
+
write_build_library_script(analysis, workdir, parameters=parameters)
|
|
345
|
+
|
|
346
|
+
try:
|
|
347
|
+
self._ensure_image(workdir, analysis)
|
|
348
|
+
except _ImageBuildError as exc:
|
|
349
|
+
logger.debug("Failed to ensure image: %s", exc.result.stderr)
|
|
350
|
+
return BuildExplorationResult(
|
|
351
|
+
build_system=analysis.build_system,
|
|
352
|
+
succeeded=False,
|
|
353
|
+
command=exc.command,
|
|
354
|
+
stdout=exc.result.stdout,
|
|
355
|
+
stderr=exc.result.stderr,
|
|
356
|
+
exit_code=exc.result.exit_code,
|
|
357
|
+
duration_seconds=exc.result.duration_seconds,
|
|
358
|
+
environment=Environment.OSS_FUZZ,
|
|
359
|
+
)
|
|
360
|
+
if self._image_tag is None:
|
|
361
|
+
raise RuntimeError("_ensure_image did not set an image tag")
|
|
362
|
+
|
|
363
|
+
extra_mounts = (
|
|
364
|
+
[] if _is_within(analysis.source_path, workdir) else [analysis.source_path.resolve()]
|
|
365
|
+
)
|
|
366
|
+
# `docker run` does not forward the host process environment BuildParameters sets,
|
|
367
|
+
# so the compiler settings are passed explicitly or they are silently ignored here.
|
|
368
|
+
run = _docker_run_factory(
|
|
369
|
+
self._image_tag, extra_mounts, _container_build_environment(parameters)
|
|
370
|
+
)
|
|
371
|
+
exploration_result = explore(
|
|
372
|
+
analysis,
|
|
373
|
+
workdir,
|
|
374
|
+
timeout=timeout,
|
|
375
|
+
environment=Environment.OSS_FUZZ,
|
|
376
|
+
run=run,
|
|
377
|
+
parameters=parameters,
|
|
378
|
+
)
|
|
379
|
+
if exploration_result.command:
|
|
380
|
+
return exploration_result
|
|
381
|
+
# No build was attempted (unidentified build system), so there is no command to
|
|
382
|
+
# report. Point at the gate, which is what an agent's fix has to satisfy.
|
|
383
|
+
return dataclasses.replace(
|
|
384
|
+
exploration_result,
|
|
385
|
+
command=verification.verification_command(
|
|
386
|
+
workdir,
|
|
387
|
+
environment=Environment.OSS_FUZZ,
|
|
388
|
+
project_name=analysis.project_name,
|
|
389
|
+
),
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
def run_harness_compile( # noqa: PLR0913 -- paths and build configuration are independent inputs
|
|
393
|
+
self,
|
|
394
|
+
install_dir: Path,
|
|
395
|
+
workdir: Path,
|
|
396
|
+
language: Language,
|
|
397
|
+
*,
|
|
398
|
+
extra_include_paths: list[str] | None = None,
|
|
399
|
+
extra_library_paths: list[str] | None = None,
|
|
400
|
+
parameters: BuildParameters | None = None,
|
|
401
|
+
library_llm_used: bool = False,
|
|
402
|
+
) -> HarnessExplorationResult:
|
|
403
|
+
from fuzzprep.library_builder.build_parameters import BuildParameters
|
|
404
|
+
from fuzzprep.library_builder.environments import gate
|
|
405
|
+
from fuzzprep.library_builder.harness_explorer import explore_harness_compilation
|
|
406
|
+
|
|
407
|
+
if self._image_tag is None or self._project_name is None:
|
|
408
|
+
raise RuntimeError(
|
|
409
|
+
"OssFuzzExecutor.run_harness_compile requires a prior successful "
|
|
410
|
+
"run_library_build on this instance to establish the run-scoped image"
|
|
411
|
+
)
|
|
412
|
+
workdir = workdir.resolve()
|
|
413
|
+
parameters = parameters or BuildParameters.defaults()
|
|
414
|
+
# Discovery's retry loop execs directly against the already-built image, so it does
|
|
415
|
+
# not pay for a `docker build` per attempt.
|
|
416
|
+
run = _docker_run_factory(self._image_tag, [], _container_harness_environment(parameters))
|
|
417
|
+
harness_result = explore_harness_compilation(
|
|
418
|
+
install_dir,
|
|
419
|
+
workdir,
|
|
420
|
+
language,
|
|
421
|
+
extra_include_paths=extra_include_paths,
|
|
422
|
+
extra_library_paths=extra_library_paths,
|
|
423
|
+
environment=Environment.OSS_FUZZ,
|
|
424
|
+
run=run,
|
|
425
|
+
)
|
|
426
|
+
gated = gate.apply_to_harness_result(
|
|
427
|
+
harness_result,
|
|
428
|
+
workdir,
|
|
429
|
+
environment=Environment.OSS_FUZZ,
|
|
430
|
+
project_name=self._project_name,
|
|
431
|
+
library_llm_used=library_llm_used,
|
|
432
|
+
bypass_scratch_validation=self._bypass_scratch_validation,
|
|
433
|
+
)
|
|
434
|
+
if gated.llm_used is False and _is_environment_unavailable(gated.output):
|
|
435
|
+
raise EnvironmentUnavailableError(
|
|
436
|
+
f"Docker became unavailable during verification: {gated.stdout.strip()}",
|
|
437
|
+
Environment.OSS_FUZZ,
|
|
438
|
+
)
|
|
439
|
+
return gated
|
|
440
|
+
|
|
441
|
+
def _ensure_image(self, workdir: Path, analysis: AnalysisResult) -> None:
|
|
442
|
+
"""Build a run-scoped image from the workspace's real Dockerfile, for internal
|
|
443
|
+
probing. The gate builds its own image from the same Dockerfile, to the same tag.
|
|
444
|
+
|
|
445
|
+
Always invokes `docker build`, relying on Docker's layer cache when the Dockerfile
|
|
446
|
+
is unchanged. Caching on the discovered package list instead would miss Dockerfile
|
|
447
|
+
edits an agent made directly, leaving probes to run against a stale image.
|
|
448
|
+
"""
|
|
449
|
+
tag = f"fuzzprep-dev/{analysis.project_name}:latest"
|
|
450
|
+
command = ["docker", "build", "-t", tag, "."]
|
|
451
|
+
result = run_command(command, workdir, _IMAGE_BUILD_TIMEOUT_SECONDS)
|
|
452
|
+
|
|
453
|
+
if result.exit_code != 0:
|
|
454
|
+
if _is_environment_unavailable(result.output):
|
|
455
|
+
raise EnvironmentUnavailableError(
|
|
456
|
+
f"Failed to build the oss-fuzz image: {result.stderr.strip()}",
|
|
457
|
+
Environment.OSS_FUZZ,
|
|
458
|
+
)
|
|
459
|
+
raise _ImageBuildError(command, result)
|
|
460
|
+
|
|
461
|
+
self._require_compile(tag)
|
|
462
|
+
self._image_tag = tag
|
|
463
|
+
|
|
464
|
+
def _require_compile(self, image_tag: str) -> None:
|
|
465
|
+
"""Reject a base image that has no `compile`, before anything tries to build in it.
|
|
466
|
+
|
|
467
|
+
Every stage enters the build through `compile`, and the workspace Dockerfile is
|
|
468
|
+
written in terms of $SRC, so --base-image selects among OSS-Fuzz base images rather
|
|
469
|
+
than accepting any image. Reported as unavailable rather than as a build failure: no
|
|
470
|
+
edit a repair agent makes to build.sh puts `compile` into the image.
|
|
471
|
+
"""
|
|
472
|
+
from fuzzprep.library_builder.workspace import DEFAULT_BASE_IMAGE
|
|
473
|
+
|
|
474
|
+
command = ["docker", "run", "--rm", "--entrypoint", "bash", image_tag]
|
|
475
|
+
command += ["-c", "command -v compile"]
|
|
476
|
+
if run_command(command, Path.cwd(), _AVAILABILITY_TIMEOUT_SECONDS).exit_code == 0:
|
|
477
|
+
return
|
|
478
|
+
raise EnvironmentUnavailableError(
|
|
479
|
+
f"The base image behind {image_tag} has no `compile` on PATH, so it is not an "
|
|
480
|
+
"OSS-Fuzz base image. --base-image selects among OSS-Fuzz base images (for example "
|
|
481
|
+
f"{DEFAULT_BASE_IMAGE}); it cannot take an arbitrary image, because every "
|
|
482
|
+
"verification stage enters the build through `compile` and the generated Dockerfile "
|
|
483
|
+
"is written in terms of $SRC.",
|
|
484
|
+
Environment.OSS_FUZZ,
|
|
485
|
+
)
|