fuzzprep 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fuzzprep/__init__.py +3 -0
- fuzzprep/__main__.py +4 -0
- fuzzprep/agents/harness_builder/SKILL.md +156 -0
- fuzzprep/agents/library_builder/SKILL.md +147 -0
- fuzzprep/agents/scripts/check_build.sh +87 -0
- fuzzprep/agents/scripts/check_build_in_container.sh +73 -0
- fuzzprep/agents/scripts/check_dockerfile_from_scratch.sh +33 -0
- fuzzprep/cli.py +1109 -0
- fuzzprep/core/__init__.py +0 -0
- fuzzprep/core/agent_stream.py +304 -0
- fuzzprep/core/files.py +35 -0
- fuzzprep/core/paths.py +25 -0
- fuzzprep/core/reporting.py +218 -0
- fuzzprep/core/repos.py +217 -0
- fuzzprep/core/resources.py +41 -0
- fuzzprep/core/subprocesses.py +197 -0
- fuzzprep/feature_extractor/__init__.py +0 -0
- fuzzprep/feature_extractor/benchmark_yaml.py +92 -0
- fuzzprep/feature_extractor/extraction.py +184 -0
- fuzzprep/feature_extractor/models.py +96 -0
- fuzzprep/feature_extractor/native/.clang-format +1 -0
- fuzzprep/feature_extractor/native/CMakeLists.txt +90 -0
- fuzzprep/feature_extractor/native/include/feature_extractor.hpp +136 -0
- fuzzprep/feature_extractor/native/src/extraction_action.cpp +294 -0
- fuzzprep/feature_extractor/native/src/json_writer.cpp +154 -0
- fuzzprep/feature_extractor/native/src/macro_callbacks.cpp +75 -0
- fuzzprep/feature_extractor/native/src/main.cpp +130 -0
- fuzzprep/feature_extractor/native_build.py +99 -0
- fuzzprep/library_builder/__init__.py +0 -0
- fuzzprep/library_builder/agents.py +472 -0
- fuzzprep/library_builder/analysis.py +145 -0
- fuzzprep/library_builder/build_parameters.py +158 -0
- fuzzprep/library_builder/dependency_resolution.py +139 -0
- fuzzprep/library_builder/environments/__init__.py +0 -0
- fuzzprep/library_builder/environments/base.py +89 -0
- fuzzprep/library_builder/environments/gate.py +96 -0
- fuzzprep/library_builder/environments/local.py +205 -0
- fuzzprep/library_builder/environments/oss_fuzz.py +485 -0
- fuzzprep/library_builder/environments/verification.py +125 -0
- fuzzprep/library_builder/exploration.py +217 -0
- fuzzprep/library_builder/generation.py +325 -0
- fuzzprep/library_builder/harness_explorer.py +257 -0
- fuzzprep/library_builder/models.py +169 -0
- fuzzprep/library_builder/package_names.json +33 -0
- fuzzprep/library_builder/package_names.py +40 -0
- fuzzprep/library_builder/scripts.py +389 -0
- fuzzprep/library_builder/stats.py +102 -0
- fuzzprep/library_builder/symbol_patterns.json +65 -0
- fuzzprep/library_builder/timeouts.py +14 -0
- fuzzprep/library_builder/workspace.py +250 -0
- fuzzprep-0.1.0.dist-info/METADATA +255 -0
- fuzzprep-0.1.0.dist-info/RECORD +56 -0
- fuzzprep-0.1.0.dist-info/WHEEL +4 -0
- fuzzprep-0.1.0.dist-info/entry_points.txt +3 -0
- fuzzprep-0.1.0.dist-info/licenses/LICENSE +202 -0
- fuzzprep-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +52 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""The build gate: one script that decides whether a build passed, and where to run it.
|
|
2
|
+
|
|
3
|
+
`agents/scripts/check_build.sh` is the only definition of "the build passed". This module
|
|
4
|
+
decides only where it runs — as a host subprocess, or inside the workspace's own OSS-Fuzz
|
|
5
|
+
image with the workspace mounted. The pipeline and every repair agent go through here, so an
|
|
6
|
+
agent verifies its fix against the same assertions the pipeline applies.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from fuzzprep.core.resources import agent_script
|
|
15
|
+
from fuzzprep.core.subprocesses import MergedOutput, run_command_streaming
|
|
16
|
+
from fuzzprep.library_builder.environments.base import Environment
|
|
17
|
+
from fuzzprep.library_builder.timeouts import DEFAULT_BUILD_TIMEOUT_SECONDS
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class VerificationResult(MergedOutput):
|
|
22
|
+
"""The atomic pass/fail outcome of the shared check_build.sh script."""
|
|
23
|
+
|
|
24
|
+
passed: bool
|
|
25
|
+
command: list[str]
|
|
26
|
+
stdout: str
|
|
27
|
+
stderr: str
|
|
28
|
+
duration_seconds: float
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def gate_keeps_artifacts(*, library_llm_used: bool, bypass_scratch_validation: bool) -> bool:
|
|
32
|
+
"""Whether the gate may reuse the workspace's install/ instead of rebuilding the library.
|
|
33
|
+
|
|
34
|
+
True only when a valid install/ is already there and nothing has changed the build since:
|
|
35
|
+
the deterministic library build produced it, or the caller traded the from-scratch
|
|
36
|
+
guarantee for speed with --bypass-scratch-validation.
|
|
37
|
+
|
|
38
|
+
One definition, because two callers must reach the same answer -- the pipeline's own gate,
|
|
39
|
+
and the command a repair agent is told to run. When they disagreed, the agent paid a full
|
|
40
|
+
cold rebuild of the library the pipeline had deliberately decided to keep.
|
|
41
|
+
"""
|
|
42
|
+
return bypass_scratch_validation or not library_llm_used
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def verification_command(
|
|
46
|
+
workspace: Path, *, environment: Environment, project_name: str, keep_artifacts: bool = False
|
|
47
|
+
) -> list[str]:
|
|
48
|
+
"""The gate invocation for workspace, without running it.
|
|
49
|
+
|
|
50
|
+
Shared by run_verification and by callers that already know the answer — a probe that
|
|
51
|
+
failed — and only need a command to report as "reproduce with".
|
|
52
|
+
|
|
53
|
+
keep_artifacts drops the gate's deletion of install/ and build/, so the library build is
|
|
54
|
+
skipped rather than repeated. Only for a caller that already built the library from
|
|
55
|
+
nothing; see the comment on that deletion in check_build.sh.
|
|
56
|
+
|
|
57
|
+
The path is resolved because every gate script cds to the workspace it is handed while
|
|
58
|
+
_run also sets cwd to it, so a relative path would be applied twice. It also makes the
|
|
59
|
+
reported command runnable from any directory.
|
|
60
|
+
"""
|
|
61
|
+
workspace = workspace.resolve()
|
|
62
|
+
options = ["--keep-artifacts"] if keep_artifacts else []
|
|
63
|
+
if environment is Environment.OSS_FUZZ:
|
|
64
|
+
return [
|
|
65
|
+
"bash",
|
|
66
|
+
str(agent_script("check_build_in_container.sh")),
|
|
67
|
+
str(workspace),
|
|
68
|
+
project_name,
|
|
69
|
+
*options,
|
|
70
|
+
]
|
|
71
|
+
return ["bash", str(agent_script("check_build.sh")), str(workspace), *options]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def run_verification(
|
|
75
|
+
workspace: Path,
|
|
76
|
+
*,
|
|
77
|
+
environment: Environment,
|
|
78
|
+
project_name: str,
|
|
79
|
+
keep_artifacts: bool = False,
|
|
80
|
+
timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
|
|
81
|
+
) -> VerificationResult:
|
|
82
|
+
"""Run the gate for workspace in the given environment."""
|
|
83
|
+
command = verification_command(
|
|
84
|
+
workspace,
|
|
85
|
+
environment=environment,
|
|
86
|
+
project_name=project_name,
|
|
87
|
+
keep_artifacts=keep_artifacts,
|
|
88
|
+
)
|
|
89
|
+
return _run(command, cwd=workspace.resolve(), timeout=timeout)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def run_from_scratch_docker_verification(
|
|
93
|
+
project_dir: Path,
|
|
94
|
+
*,
|
|
95
|
+
project_name: str,
|
|
96
|
+
timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
|
|
97
|
+
) -> VerificationResult:
|
|
98
|
+
"""Build project_dir's Dockerfile with nothing mounted and run OSS-Fuzz's `compile`.
|
|
99
|
+
|
|
100
|
+
The gate mounts the workspace, which is what puts the artifacts on the host — and also
|
|
101
|
+
what lets a broken clone or apt layer pass unnoticed, since the mount supplies what the
|
|
102
|
+
image failed to. This check keeps the mounted gate honest, so it runs once per successful
|
|
103
|
+
oss-fuzz run, immediately before generation.
|
|
104
|
+
|
|
105
|
+
project_dir is resolved for the same reason verification_command resolves it.
|
|
106
|
+
"""
|
|
107
|
+
project_dir = project_dir.resolve()
|
|
108
|
+
command = [
|
|
109
|
+
"bash",
|
|
110
|
+
str(agent_script("check_dockerfile_from_scratch.sh")),
|
|
111
|
+
str(project_dir),
|
|
112
|
+
project_name,
|
|
113
|
+
]
|
|
114
|
+
return _run(command, cwd=project_dir, timeout=timeout)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _run(command: list[str], *, cwd: Path, timeout: int) -> VerificationResult:
|
|
118
|
+
result = run_command_streaming(command, cwd, timeout)
|
|
119
|
+
return VerificationResult(
|
|
120
|
+
passed=result.exit_code == 0,
|
|
121
|
+
command=command,
|
|
122
|
+
stdout=result.stdout,
|
|
123
|
+
stderr=result.stderr,
|
|
124
|
+
duration_seconds=result.duration_seconds,
|
|
125
|
+
)
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import shutil
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from fuzzprep.core.files import write_executable
|
|
9
|
+
from fuzzprep.core.subprocesses import Runner, run_command_streaming
|
|
10
|
+
from fuzzprep.library_builder.build_parameters import BuildParameters
|
|
11
|
+
from fuzzprep.library_builder.environments.base import Environment
|
|
12
|
+
from fuzzprep.library_builder.models import (
|
|
13
|
+
AgentReport,
|
|
14
|
+
AnalysisResult,
|
|
15
|
+
BuildExplorationResult,
|
|
16
|
+
BuildPaths,
|
|
17
|
+
BuildSystem,
|
|
18
|
+
)
|
|
19
|
+
from fuzzprep.library_builder.scripts import build_library_script
|
|
20
|
+
from fuzzprep.library_builder.timeouts import DEFAULT_BUILD_TIMEOUT_SECONDS
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
_BEAR_NOT_FOUND_ERROR = (
|
|
25
|
+
"bear not found on PATH; install bear to capture compile_commands.json for "
|
|
26
|
+
"Make/Autotools builds"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
_MAKE_LIKE_SYSTEMS = (BuildSystem.MAKEFILE, BuildSystem.AUTOTOOLS)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def is_standard_source_layout(analysis: AnalysisResult, workdir: Path) -> bool:
|
|
33
|
+
"""True when the source was cloned to workdir/src, the layout generated output expects.
|
|
34
|
+
|
|
35
|
+
build_library.sh's paths can then be $SCRIPT_DIR-relative, so the same script works
|
|
36
|
+
unmodified from the workspace and from a generated output directory.
|
|
37
|
+
"""
|
|
38
|
+
return analysis.source_path.resolve() == (workdir / "src").resolve()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def write_build_library_script(
|
|
42
|
+
analysis: AnalysisResult, workdir: Path, *, parameters: BuildParameters | None = None
|
|
43
|
+
) -> tuple[Path, bool]:
|
|
44
|
+
"""Write build_library.sh into workdir/build_library.sh.
|
|
45
|
+
|
|
46
|
+
With the standard workdir/src layout the paths are $SCRIPT_DIR-relative, so the script
|
|
47
|
+
can be published verbatim; otherwise they fall back to absolute. Returns (script_path,
|
|
48
|
+
standard_layout) so callers know whether BuildExplorationResult.script_path can be set.
|
|
49
|
+
|
|
50
|
+
Separate from explore() because OssFuzzExecutor needs this file on disk before building
|
|
51
|
+
the workspace image, which COPYs it.
|
|
52
|
+
"""
|
|
53
|
+
workdir = workdir.resolve()
|
|
54
|
+
standard_layout = is_standard_source_layout(analysis, workdir)
|
|
55
|
+
source_dir = "$SCRIPT_DIR/src" if standard_layout else str(analysis.source_path.resolve())
|
|
56
|
+
|
|
57
|
+
parameters = parameters or BuildParameters.defaults()
|
|
58
|
+
script = build_library_script(
|
|
59
|
+
analysis.build_system,
|
|
60
|
+
BuildPaths(
|
|
61
|
+
source_dir=source_dir,
|
|
62
|
+
build_dir="$BUILD_PREFIX/build",
|
|
63
|
+
install_dir="$BUILD_PREFIX/install",
|
|
64
|
+
),
|
|
65
|
+
autotools_setup=analysis.autotools_setup,
|
|
66
|
+
configure_args=parameters.library_configure_args,
|
|
67
|
+
cc=parameters.cc,
|
|
68
|
+
cxx=parameters.cxx,
|
|
69
|
+
cflags=parameters.library_cflags,
|
|
70
|
+
cxxflags=parameters.library_cxxflags,
|
|
71
|
+
)
|
|
72
|
+
script_path = write_executable(workdir / "build_library.sh", script)
|
|
73
|
+
return script_path, standard_layout
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def explore( # noqa: PLR0913 -- 4 keyword-only inputs, each independently meaningful
|
|
77
|
+
analysis: AnalysisResult,
|
|
78
|
+
workdir: Path,
|
|
79
|
+
*,
|
|
80
|
+
timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
|
|
81
|
+
environment: Environment = Environment.LOCAL,
|
|
82
|
+
run: Runner | None = None,
|
|
83
|
+
parameters: BuildParameters | None = None,
|
|
84
|
+
) -> BuildExplorationResult:
|
|
85
|
+
"""Write a build_library.sh into workdir and run it in the given environment.
|
|
86
|
+
|
|
87
|
+
environment decides whether the build command is wrapped with `bear`, and is recorded on
|
|
88
|
+
the returned result; the script text itself is environment-independent. run defaults to
|
|
89
|
+
streaming as a host subprocess; a caller building inside a container passes a run
|
|
90
|
+
primitive that wraps the command in `docker run`.
|
|
91
|
+
"""
|
|
92
|
+
workdir = workdir.resolve()
|
|
93
|
+
build_dir = workdir / "build"
|
|
94
|
+
install_dir = workdir / "install"
|
|
95
|
+
|
|
96
|
+
if build_dir.exists():
|
|
97
|
+
shutil.rmtree(build_dir)
|
|
98
|
+
build_dir.mkdir(parents=True)
|
|
99
|
+
|
|
100
|
+
if install_dir.exists():
|
|
101
|
+
shutil.rmtree(install_dir)
|
|
102
|
+
install_dir.mkdir(parents=True)
|
|
103
|
+
|
|
104
|
+
script_path, standard_layout = write_build_library_script(
|
|
105
|
+
analysis, workdir, parameters=parameters
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
if analysis.build_system == BuildSystem.UNKNOWN:
|
|
109
|
+
return BuildExplorationResult(
|
|
110
|
+
build_system=analysis.build_system,
|
|
111
|
+
succeeded=False,
|
|
112
|
+
command=[],
|
|
113
|
+
stdout="",
|
|
114
|
+
stderr="",
|
|
115
|
+
exit_code=-1,
|
|
116
|
+
duration_seconds=0.0,
|
|
117
|
+
environment=environment,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
command = _build_command(analysis.build_system, environment, script_path)
|
|
121
|
+
|
|
122
|
+
runner = run if run is not None else run_command_streaming
|
|
123
|
+
result = runner(command, workdir, timeout)
|
|
124
|
+
|
|
125
|
+
succeeded = result.exit_code == 0
|
|
126
|
+
stderr = result.stderr
|
|
127
|
+
if succeeded:
|
|
128
|
+
validation_errors = validate_install_artifacts(install_dir)
|
|
129
|
+
if validation_errors:
|
|
130
|
+
succeeded = False
|
|
131
|
+
stderr += "\n" + "\n".join(validation_errors)
|
|
132
|
+
|
|
133
|
+
return BuildExplorationResult(
|
|
134
|
+
build_system=analysis.build_system,
|
|
135
|
+
succeeded=succeeded,
|
|
136
|
+
command=command,
|
|
137
|
+
stdout=result.stdout,
|
|
138
|
+
stderr=stderr,
|
|
139
|
+
exit_code=result.exit_code,
|
|
140
|
+
duration_seconds=result.duration_seconds,
|
|
141
|
+
install_dir=install_dir,
|
|
142
|
+
script_path=script_path if standard_layout else None,
|
|
143
|
+
environment=environment,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _build_command(
|
|
148
|
+
build_system: BuildSystem, environment: Environment, script_path: Path
|
|
149
|
+
) -> list[str]:
|
|
150
|
+
"""Choose the canonical build invocation, wrapping Make/Autotools with `bear --`.
|
|
151
|
+
|
|
152
|
+
The wrap is unconditional in the oss-fuzz environment, where bear is always present. On the
|
|
153
|
+
local host it is best-effort: a missing bear must not fail the build, and it is the gate's
|
|
154
|
+
own bear wrap that decides whether a capture ships (see workspace.find_compile_commands).
|
|
155
|
+
"""
|
|
156
|
+
plain = ["bash", str(script_path.name)]
|
|
157
|
+
if build_system not in _MAKE_LIKE_SYSTEMS:
|
|
158
|
+
return plain
|
|
159
|
+
if environment is Environment.OSS_FUZZ or shutil.which("bear") is not None:
|
|
160
|
+
return ["bear", "--", *plain]
|
|
161
|
+
return plain
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def compile_commands_absent_reason(build_system: BuildSystem) -> str:
|
|
165
|
+
"""Why the gate's build left no compile_commands.json behind.
|
|
166
|
+
|
|
167
|
+
CMake and Meson emit one themselves, so for those a missing file means the build did not
|
|
168
|
+
reach the point of writing it. Make and Autotools have no equivalent and need bear, which
|
|
169
|
+
is only best-effort on a host.
|
|
170
|
+
"""
|
|
171
|
+
if build_system in _MAKE_LIKE_SYSTEMS and shutil.which("bear") is None:
|
|
172
|
+
return _BEAR_NOT_FOUND_ERROR
|
|
173
|
+
return "the build produced none"
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _string_list(value: object) -> list[str]:
|
|
177
|
+
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
178
|
+
return []
|
|
179
|
+
return [str(item) for item in value]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def read_agent_report(workdir: Path) -> AgentReport | None:
|
|
183
|
+
"""Read and delete workdir/agent_report.json, tolerantly parsing its contents.
|
|
184
|
+
|
|
185
|
+
Returns None if the file is absent, is not valid JSON, or is not a JSON object. Deletes
|
|
186
|
+
the file whenever it existed, so a later invocation cannot pick up a stale report.
|
|
187
|
+
"""
|
|
188
|
+
report_path = workdir / "agent_report.json"
|
|
189
|
+
if not report_path.exists():
|
|
190
|
+
return None
|
|
191
|
+
try:
|
|
192
|
+
data = json.loads(report_path.read_text())
|
|
193
|
+
if not isinstance(data, dict):
|
|
194
|
+
return None
|
|
195
|
+
summary = data.get("summary")
|
|
196
|
+
return AgentReport(
|
|
197
|
+
summary=summary if isinstance(summary, str) else None,
|
|
198
|
+
missing_libs=_string_list(data.get("missing_libs")),
|
|
199
|
+
missing_apt_packages=_string_list(data.get("missing_apt_packages")),
|
|
200
|
+
extra_include_paths=_string_list(data.get("extra_include_paths")),
|
|
201
|
+
extra_library_paths=_string_list(data.get("extra_library_paths")),
|
|
202
|
+
)
|
|
203
|
+
except (json.JSONDecodeError, OSError):
|
|
204
|
+
return None
|
|
205
|
+
finally:
|
|
206
|
+
report_path.unlink(missing_ok=True)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def validate_install_artifacts(install_dir: Path) -> list[str]:
|
|
210
|
+
errors = []
|
|
211
|
+
lib_dir = install_dir / "lib"
|
|
212
|
+
if not lib_dir.exists() or not any(lib_dir.glob("*.a")):
|
|
213
|
+
errors.append(f"no static libraries (*.a) found in {lib_dir}")
|
|
214
|
+
include_dir = install_dir / "include"
|
|
215
|
+
if not include_dir.exists() or not any(include_dir.iterdir()):
|
|
216
|
+
errors.append(f"no headers found in {include_dir}")
|
|
217
|
+
return errors
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Publishing the verified workspace as the output directory.
|
|
2
|
+
|
|
3
|
+
There is one output shape, whichever environment verified it: the workspace copied verbatim,
|
|
4
|
+
so the shipped scripts are the ones that passed, plus what only makes sense outside the
|
|
5
|
+
workspace — `setup.sh`, the built `install/` tree, the build's `compile_commands.json`, and a
|
|
6
|
+
README naming the environment this run actually exercised.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import shutil
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from fuzzprep.core.files import copy_executable, write_executable
|
|
17
|
+
from fuzzprep.library_builder import workspace
|
|
18
|
+
from fuzzprep.library_builder.environments.base import Environment
|
|
19
|
+
from fuzzprep.library_builder.models import (
|
|
20
|
+
AnalysisResult,
|
|
21
|
+
BuildExplorationResult,
|
|
22
|
+
GenerationResult,
|
|
23
|
+
HarnessExplorationResult,
|
|
24
|
+
)
|
|
25
|
+
from fuzzprep.library_builder.scripts import HARNESS_SOURCE_DIR
|
|
26
|
+
|
|
27
|
+
# Copied from the workspace verbatim: the validated scripts, plus the OSS-Fuzz project files
|
|
28
|
+
# around them.
|
|
29
|
+
_COPIED_SCRIPTS = ("build.sh", "build_library.sh", "compile_harness.sh", "compile_harnesses.sh")
|
|
30
|
+
_COPIED_FILES = ("project.yaml",)
|
|
31
|
+
|
|
32
|
+
# Build products and run metadata: worth shipping, but never part of the docker build context,
|
|
33
|
+
# since the container rebuilds from its own fresh clone.
|
|
34
|
+
_DOCKERIGNORE = ("install/\n", "compile_commands.json\n", "stats.json\n", "logs/\n", "out/\n")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class MissingInstallTreeError(Exception):
|
|
38
|
+
"""Generation was asked to publish a build whose install tree it cannot find."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class GenerationInputs:
|
|
43
|
+
"""Everything generation needs beyond the workspace itself.
|
|
44
|
+
|
|
45
|
+
Grouped because they all answer one question — what this run actually did — and each
|
|
46
|
+
appears in the README as well as in a generated file.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
analysis: AnalysisResult
|
|
50
|
+
build: BuildExplorationResult
|
|
51
|
+
harness: HarnessExplorationResult
|
|
52
|
+
system_packages: list[str]
|
|
53
|
+
environment: Environment
|
|
54
|
+
agent_backend: str | None
|
|
55
|
+
scratch_validation_bypassed: bool = False
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def generate(workspace_dir: Path, output_path: Path, inputs: GenerationInputs) -> GenerationResult:
|
|
59
|
+
"""Copy the verified workspace to output_path and add the host-side resources.
|
|
60
|
+
|
|
61
|
+
The output is itself an OSS-Fuzz project and a standalone host build tree at once:
|
|
62
|
+
`docker build .` works, and so does `setup.sh` followed by `build_library.sh`.
|
|
63
|
+
"""
|
|
64
|
+
output_path.mkdir(parents=True)
|
|
65
|
+
files: list[Path] = [
|
|
66
|
+
*_copy_project_files(workspace_dir, output_path, inputs.system_packages),
|
|
67
|
+
*_copy_harness_source(workspace_dir, output_path),
|
|
68
|
+
_write_setup_sh(output_path, inputs.analysis, inputs.system_packages),
|
|
69
|
+
_write_dockerignore(output_path),
|
|
70
|
+
]
|
|
71
|
+
_copy_install_tree(inputs.build, output_path)
|
|
72
|
+
compile_commands = _copy_compile_commands(workspace_dir, output_path)
|
|
73
|
+
if compile_commands is not None:
|
|
74
|
+
files.append(compile_commands)
|
|
75
|
+
files.append(
|
|
76
|
+
write_readme(output_path, inputs, build_tree=workspace_dir if compile_commands else None)
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
return GenerationResult(
|
|
80
|
+
project_name=inputs.analysis.project_name,
|
|
81
|
+
output_path=output_path,
|
|
82
|
+
files=files,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _copy_project_files(
|
|
87
|
+
workspace_dir: Path, output_path: Path, system_packages: list[str]
|
|
88
|
+
) -> list[Path]:
|
|
89
|
+
"""Copy the scripts and project files the workspace validated, verbatim.
|
|
90
|
+
|
|
91
|
+
The Dockerfile is the exception: the shipped copy must not depend on `bear`, which the
|
|
92
|
+
workspace image carries only so compile_commands.json can be captured.
|
|
93
|
+
"""
|
|
94
|
+
copied = [copy_executable(workspace_dir / name, output_path / name) for name in _COPIED_SCRIPTS]
|
|
95
|
+
for name in _COPIED_FILES:
|
|
96
|
+
shutil.copy2(workspace_dir / name, output_path / name)
|
|
97
|
+
copied.append(output_path / name)
|
|
98
|
+
|
|
99
|
+
dockerfile = output_path / "Dockerfile"
|
|
100
|
+
dockerfile.write_text(
|
|
101
|
+
workspace.strip_bear_dependency((workspace_dir / "Dockerfile").read_text())
|
|
102
|
+
)
|
|
103
|
+
workspace.inject_apt_packages(output_path, system_packages)
|
|
104
|
+
copied.append(dockerfile)
|
|
105
|
+
return copied
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _copy_harness_source(workspace_dir: Path, output_path: Path) -> list[Path]:
|
|
109
|
+
"""Copy harness_source/* verbatim, including whichever default_fuzzer extension discovery
|
|
110
|
+
settled on."""
|
|
111
|
+
source_dir = workspace_dir / HARNESS_SOURCE_DIR
|
|
112
|
+
destination_dir = output_path / HARNESS_SOURCE_DIR
|
|
113
|
+
destination_dir.mkdir()
|
|
114
|
+
copied: list[Path] = []
|
|
115
|
+
for entry in sorted(source_dir.iterdir()):
|
|
116
|
+
destination = destination_dir / entry.name
|
|
117
|
+
shutil.copy2(entry, destination)
|
|
118
|
+
copied.append(destination)
|
|
119
|
+
return copied
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _copy_install_tree(build: BuildExplorationResult, output_path: Path) -> None:
|
|
123
|
+
"""Publish the built library — the artifact a user links their first harness against.
|
|
124
|
+
|
|
125
|
+
Shipped for every environment: compile_harness.sh links against install/, so an output
|
|
126
|
+
directory without it hands the user a script that cannot run. Raises rather than skipping,
|
|
127
|
+
because generation only runs after a verified build: a missing tree here means the result
|
|
128
|
+
lost track of one that exists, and skipping it published an unusable project as a success.
|
|
129
|
+
"""
|
|
130
|
+
install_dir = build.install_dir
|
|
131
|
+
if install_dir is None:
|
|
132
|
+
msg = (
|
|
133
|
+
"the verified build result records no install tree, so the generated project has "
|
|
134
|
+
"no library for compile_harness.sh to link against"
|
|
135
|
+
)
|
|
136
|
+
raise MissingInstallTreeError(msg)
|
|
137
|
+
if not install_dir.is_dir():
|
|
138
|
+
msg = f"the verified build's install tree is missing at {install_dir}"
|
|
139
|
+
raise MissingInstallTreeError(msg)
|
|
140
|
+
shutil.copytree(install_dir, output_path / "install", symlinks=True)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _copy_compile_commands(workspace_dir: Path, output_path: Path) -> Path | None:
|
|
144
|
+
"""Copy compile_commands.json into the output directory, keeping its workspace paths.
|
|
145
|
+
|
|
146
|
+
A compilation database only means anything next to the build tree it describes, and the
|
|
147
|
+
output directory ships `install/` alone -- no `src/`, no `build/`. So the paths are left
|
|
148
|
+
pointing at `.fuzzprep/<project>/`, where the tree that produced them still stands;
|
|
149
|
+
rewriting them to the output directory named files that were never copied there, and
|
|
150
|
+
tooling chdirs into each entry's `directory` before it reads anything.
|
|
151
|
+
"""
|
|
152
|
+
captured = workspace.find_compile_commands(workspace_dir)
|
|
153
|
+
if captured is None:
|
|
154
|
+
return None
|
|
155
|
+
destination = output_path / "compile_commands.json"
|
|
156
|
+
destination.write_text(usable_compile_commands(captured.read_text()))
|
|
157
|
+
return destination
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _write_dockerignore(output_path: Path) -> Path:
|
|
161
|
+
"""Keep the build products out of the docker build context.
|
|
162
|
+
|
|
163
|
+
The image rebuilds the library from its own clone, so sending install/ — hundreds of MB
|
|
164
|
+
for a large library — would only slow every build down.
|
|
165
|
+
"""
|
|
166
|
+
path = output_path / ".dockerignore"
|
|
167
|
+
path.write_text("".join(_DOCKERIGNORE))
|
|
168
|
+
return path
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _write_setup_sh(
|
|
172
|
+
output_path: Path, analysis: AnalysisResult, system_packages: list[str]
|
|
173
|
+
) -> Path:
|
|
174
|
+
"""Write setup.sh — clone the library and install its build dependencies.
|
|
175
|
+
|
|
176
|
+
The host counterpart of the Dockerfile's clone and apt layers, so it uses the same
|
|
177
|
+
`apt-get update && apt-get install` form. `sudo` is resolved when the script runs, since
|
|
178
|
+
the generating host's uid says nothing about the consuming host's.
|
|
179
|
+
"""
|
|
180
|
+
lines = [
|
|
181
|
+
"#!/bin/bash\n",
|
|
182
|
+
"set -euo pipefail\n",
|
|
183
|
+
"\n",
|
|
184
|
+
'SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n',
|
|
185
|
+
"\n",
|
|
186
|
+
f'git clone --recursive {analysis.clone_url} "$SCRIPT_DIR/src"\n',
|
|
187
|
+
]
|
|
188
|
+
if analysis.repo_ref is not None:
|
|
189
|
+
lines.append(f'git -C "$SCRIPT_DIR/src" checkout {analysis.repo_ref}\n')
|
|
190
|
+
lines.append("\n")
|
|
191
|
+
if system_packages:
|
|
192
|
+
lines += [
|
|
193
|
+
'if [ "$(id -u)" -eq 0 ]; then\n',
|
|
194
|
+
' SUDO=""\n',
|
|
195
|
+
"else\n",
|
|
196
|
+
' SUDO="sudo"\n',
|
|
197
|
+
"fi\n",
|
|
198
|
+
"$SUDO apt-get update\n",
|
|
199
|
+
f"$SUDO apt-get install -y --no-install-recommends {' '.join(system_packages)}\n",
|
|
200
|
+
]
|
|
201
|
+
else:
|
|
202
|
+
lines.append("# No build dependencies were discovered for this library.\n")
|
|
203
|
+
return write_executable(output_path / "setup.sh", "".join(lines))
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def usable_compile_commands(text: str) -> str:
|
|
207
|
+
"""Drop the entries a clang tool cannot replay, keeping the rest verbatim.
|
|
208
|
+
|
|
209
|
+
`bear` records every compiler exec the build made, not just the ones that compiled library
|
|
210
|
+
sources. Three kinds of entry come back unusable, and each is fatal rather than noisy:
|
|
211
|
+
a `directory` CMake has since deleted (its `CMakeScratch/TryCompile-*` dirs) aborts the
|
|
212
|
+
whole run, because a clang tool chdirs there before it parses; a raw `-cc1` invocation is
|
|
213
|
+
rejected argument by argument when the driver is asked to replay it; and CMake's own
|
|
214
|
+
`CMakeFiles/` probe sources are not part of the library at all.
|
|
215
|
+
"""
|
|
216
|
+
entries = json.loads(text)
|
|
217
|
+
return json.dumps([entry for entry in entries if _entry_is_usable(entry)], indent=2)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _entry_is_usable(entry: dict) -> bool:
|
|
221
|
+
directory, file = entry.get("directory"), entry.get("file")
|
|
222
|
+
if not isinstance(directory, str) or not isinstance(file, str):
|
|
223
|
+
return False
|
|
224
|
+
if not Path(directory).is_dir() or not Path(file).is_file():
|
|
225
|
+
return False
|
|
226
|
+
if "CMakeFiles" in Path(file).parts:
|
|
227
|
+
return False
|
|
228
|
+
arguments = entry.get("arguments")
|
|
229
|
+
if isinstance(arguments, list) and "-cc1" in arguments:
|
|
230
|
+
return False
|
|
231
|
+
command = entry.get("command")
|
|
232
|
+
return not (isinstance(command, str) and "-cc1" in command.split())
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _compile_commands_section(build_tree: Path | None) -> str:
|
|
236
|
+
"""The README's account of where the shipped compilation database points.
|
|
237
|
+
|
|
238
|
+
A database is only readable next to the tree it describes, and this directory is not that
|
|
239
|
+
tree, so the section names the one it is — and says nothing at all when no database shipped.
|
|
240
|
+
"""
|
|
241
|
+
if build_tree is None:
|
|
242
|
+
return ""
|
|
243
|
+
return f"""
|
|
244
|
+
## compile_commands.json
|
|
245
|
+
|
|
246
|
+
`compile_commands.json` describes the build this run made, so its paths point into that
|
|
247
|
+
build tree at `{build_tree}` — not into this directory, which ships `install/` alone.
|
|
248
|
+
It reads correctly for as long as that tree stands; re-run FuzzPrep to refresh it.
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def write_readme(output_path: Path, inputs: GenerationInputs, *, build_tree: Path | None) -> Path:
|
|
253
|
+
"""Write README.md — what to run, and what this run actually proved.
|
|
254
|
+
|
|
255
|
+
The directory provisions both environments but only one was exercised, so the README says
|
|
256
|
+
which. Otherwise it implies the host scripts and the Dockerfile were both verified.
|
|
257
|
+
|
|
258
|
+
build_tree is the workspace the shipped compile_commands.json describes, or None when the
|
|
259
|
+
build captured none; naming it is what stops a reader from taking those paths for this
|
|
260
|
+
directory's own.
|
|
261
|
+
"""
|
|
262
|
+
analysis = inputs.analysis
|
|
263
|
+
verified, unverified = (
|
|
264
|
+
("oss-fuzz (in the container)", "The host scripts (setup.sh, build_library.sh) are")
|
|
265
|
+
if inputs.environment is Environment.OSS_FUZZ
|
|
266
|
+
else (
|
|
267
|
+
"local (on the host)",
|
|
268
|
+
"The container path (Dockerfile, build.sh under `compile`) is",
|
|
269
|
+
)
|
|
270
|
+
)
|
|
271
|
+
harness_stub = f"{HARNESS_SOURCE_DIR}/default_fuzzer.*"
|
|
272
|
+
scratch_note = (
|
|
273
|
+
"\n- **Not proven from scratch.** This run used `--bypass-scratch-validation`, so the "
|
|
274
|
+
"library was never rebuilt into an empty tree to confirm the result. The artifacts "
|
|
275
|
+
"here are the ones the run produced, not ones shown to reproduce."
|
|
276
|
+
if inputs.scratch_validation_bypassed
|
|
277
|
+
else ""
|
|
278
|
+
)
|
|
279
|
+
text = f"""# {analysis.project_name} — fuzzing build
|
|
280
|
+
|
|
281
|
+
Prepared by FuzzPrep. This directory is both an OSS-Fuzz project and a standalone
|
|
282
|
+
host build tree.
|
|
283
|
+
|
|
284
|
+
## What this run verified
|
|
285
|
+
|
|
286
|
+
- **Verified: {verified}.** {unverified} generated but was not executed by this run.
|
|
287
|
+
- Build system: {analysis.build_system.value}
|
|
288
|
+
- Repository: {analysis.clone_url} (ref: {analysis.repo_ref or "default branch"})
|
|
289
|
+
- Repair agent: {inputs.agent_backend or "none"}
|
|
290
|
+
- Verified with: `{" ".join(inputs.harness.command or inputs.build.command) or "n/a"}`{scratch_note}
|
|
291
|
+
|
|
292
|
+
## Run it on the host
|
|
293
|
+
|
|
294
|
+
```bash
|
|
295
|
+
./setup.sh # clone the library and install its build dependencies
|
|
296
|
+
./build_library.sh # build and install it into install/
|
|
297
|
+
./compile_harnesses.sh # compile every harness in {HARNESS_SOURCE_DIR}/ into out/
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
`install/` already holds the library this run built, so `compile_harnesses.sh` works
|
|
301
|
+
without re-running the first two steps.
|
|
302
|
+
|
|
303
|
+
## Run it as an OSS-Fuzz project
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
docker build -t {analysis.project_name}:fuzz .
|
|
307
|
+
docker run --rm --entrypoint bash {analysis.project_name}:fuzz -c compile
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
Or copy this directory into `oss-fuzz/projects/{analysis.project_name}` and use
|
|
311
|
+
`infra/helper.py build_fuzzers {analysis.project_name}`.
|
|
312
|
+
|
|
313
|
+
## Write a real harness
|
|
314
|
+
|
|
315
|
+
`{harness_stub}` is a stub: it returns 0 without calling the library. Drop your own
|
|
316
|
+
harness source into `{HARNESS_SOURCE_DIR}/` — `compile_harnesses.sh` compiles every
|
|
317
|
+
`.c`/`.cc`/`.cpp`/`.cxx` file in there and names each binary after its source.
|
|
318
|
+
|
|
319
|
+
Note that the link line was proven against the stub, so it confirms the static archives
|
|
320
|
+
and their system dependencies link — not that the installed headers in `install/include`
|
|
321
|
+
are usable.
|
|
322
|
+
{_compile_commands_section(build_tree)}"""
|
|
323
|
+
path = output_path / "README.md"
|
|
324
|
+
path.write_text(text)
|
|
325
|
+
return path
|