fuzzprep 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fuzzprep/__init__.py +3 -0
- fuzzprep/__main__.py +4 -0
- fuzzprep/agents/harness_builder/SKILL.md +156 -0
- fuzzprep/agents/library_builder/SKILL.md +147 -0
- fuzzprep/agents/scripts/check_build.sh +87 -0
- fuzzprep/agents/scripts/check_build_in_container.sh +73 -0
- fuzzprep/agents/scripts/check_dockerfile_from_scratch.sh +33 -0
- fuzzprep/cli.py +1109 -0
- fuzzprep/core/__init__.py +0 -0
- fuzzprep/core/agent_stream.py +304 -0
- fuzzprep/core/files.py +35 -0
- fuzzprep/core/paths.py +25 -0
- fuzzprep/core/reporting.py +218 -0
- fuzzprep/core/repos.py +217 -0
- fuzzprep/core/resources.py +41 -0
- fuzzprep/core/subprocesses.py +197 -0
- fuzzprep/feature_extractor/__init__.py +0 -0
- fuzzprep/feature_extractor/benchmark_yaml.py +92 -0
- fuzzprep/feature_extractor/extraction.py +184 -0
- fuzzprep/feature_extractor/models.py +96 -0
- fuzzprep/feature_extractor/native/.clang-format +1 -0
- fuzzprep/feature_extractor/native/CMakeLists.txt +90 -0
- fuzzprep/feature_extractor/native/include/feature_extractor.hpp +136 -0
- fuzzprep/feature_extractor/native/src/extraction_action.cpp +294 -0
- fuzzprep/feature_extractor/native/src/json_writer.cpp +154 -0
- fuzzprep/feature_extractor/native/src/macro_callbacks.cpp +75 -0
- fuzzprep/feature_extractor/native/src/main.cpp +130 -0
- fuzzprep/feature_extractor/native_build.py +99 -0
- fuzzprep/library_builder/__init__.py +0 -0
- fuzzprep/library_builder/agents.py +472 -0
- fuzzprep/library_builder/analysis.py +145 -0
- fuzzprep/library_builder/build_parameters.py +158 -0
- fuzzprep/library_builder/dependency_resolution.py +139 -0
- fuzzprep/library_builder/environments/__init__.py +0 -0
- fuzzprep/library_builder/environments/base.py +89 -0
- fuzzprep/library_builder/environments/gate.py +96 -0
- fuzzprep/library_builder/environments/local.py +205 -0
- fuzzprep/library_builder/environments/oss_fuzz.py +485 -0
- fuzzprep/library_builder/environments/verification.py +125 -0
- fuzzprep/library_builder/exploration.py +217 -0
- fuzzprep/library_builder/generation.py +325 -0
- fuzzprep/library_builder/harness_explorer.py +257 -0
- fuzzprep/library_builder/models.py +169 -0
- fuzzprep/library_builder/package_names.json +33 -0
- fuzzprep/library_builder/package_names.py +40 -0
- fuzzprep/library_builder/scripts.py +389 -0
- fuzzprep/library_builder/stats.py +102 -0
- fuzzprep/library_builder/symbol_patterns.json +65 -0
- fuzzprep/library_builder/timeouts.py +14 -0
- fuzzprep/library_builder/workspace.py +250 -0
- fuzzprep-0.1.0.dist-info/METADATA +255 -0
- fuzzprep-0.1.0.dist-info/RECORD +56 -0
- fuzzprep-0.1.0.dist-info/WHEEL +4 -0
- fuzzprep-0.1.0.dist-info/entry_points.txt +3 -0
- fuzzprep-0.1.0.dist-info/licenses/LICENSE +202 -0
- fuzzprep-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +52 -0
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
"""Generators for the shell scripts FuzzPrep validates and then ships.
|
|
2
|
+
|
|
3
|
+
Every script here is environment-independent: the same text runs as a host subprocess, inside
|
|
4
|
+
the OSS-Fuzz base-builder container, and from the generated output directory. Each variable
|
|
5
|
+
the two environments disagree about is read with a fallback (`${CC:-clang}`,
|
|
6
|
+
`${OUT:-$SCRIPT_DIR/out}`, `${LIB_FUZZING_ENGINE:--fsanitize=fuzzer}`), so an environment that
|
|
7
|
+
defines it wins and one that does not still gets a working value.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from fuzzprep.library_builder.models import (
|
|
16
|
+
AutotoolsSetup,
|
|
17
|
+
BuildPaths,
|
|
18
|
+
BuildSystem,
|
|
19
|
+
Language,
|
|
20
|
+
LinkConfiguration,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# The one name for the harness source directory, in the workspace and in generated output.
|
|
24
|
+
HARNESS_SOURCE_DIR = "harness_source"
|
|
25
|
+
|
|
26
|
+
# libFuzzer's flag, which is what OSS-Fuzz's `compile` exports as LIB_FUZZING_ENGINE. Used as
|
|
27
|
+
# a fallback, so the environment's value wins whenever a script runs under `compile`.
|
|
28
|
+
_DEFAULT_FUZZING_ENGINE = "-fsanitize=fuzzer"
|
|
29
|
+
|
|
30
|
+
# Capped rather than -j$(nproc): builds run in containers and CI with memory limits far below
|
|
31
|
+
# what a large host's core count implies.
|
|
32
|
+
_MAX_BUILD_JOBS = 4
|
|
33
|
+
|
|
34
|
+
_JOB_COUNT = (
|
|
35
|
+
f'JOBS="$(nproc)"\nif [ "$JOBS" -gt {_MAX_BUILD_JOBS} ]; then JOBS={_MAX_BUILD_JOBS}; fi\n'
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
# Autotools variants that bootstrap by running a script in the source tree, mapped to that
|
|
39
|
+
# script's name. Both generate configure; only the filename differs.
|
|
40
|
+
_AUTOTOOLS_BOOTSTRAP_SCRIPTS: dict[AutotoolsSetup | None, str] = {
|
|
41
|
+
AutotoolsSetup.AUTOGEN: "autogen.sh",
|
|
42
|
+
AutotoolsSetup.BOOTSTRAP: "bootstrap",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
_CONFIGURE_ARGS_VARIABLE = "CONFIGURE_ARGS"
|
|
46
|
+
_CONFIGURE_ARGS_EXPANSION = f' "${{{_CONFIGURE_ARGS_VARIABLE}[@]}}"'
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def build_library_script( # noqa: PLR0913 -- each argument is a distinct input to the text
|
|
50
|
+
build_system: BuildSystem,
|
|
51
|
+
paths: BuildPaths,
|
|
52
|
+
*,
|
|
53
|
+
autotools_setup: AutotoolsSetup | None = None,
|
|
54
|
+
configure_args: tuple[str, ...] = (),
|
|
55
|
+
cc: str = "clang",
|
|
56
|
+
cxx: str = "clang++",
|
|
57
|
+
cflags: str = "",
|
|
58
|
+
cxxflags: str = "",
|
|
59
|
+
) -> str:
|
|
60
|
+
"""Generate a build_library.sh script with parameterized paths.
|
|
61
|
+
|
|
62
|
+
The compiler settings are baked in as fallbacks rather than read from the environment at
|
|
63
|
+
run time, because the script must reproduce the validated build from a bare shell — which
|
|
64
|
+
is how both the gate and a user following the generated README run it. An environment that
|
|
65
|
+
does define them, such as the base image's sanitizer configuration, still wins.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
build_system: detected build system.
|
|
69
|
+
paths: source/build/install path strings for the generated script.
|
|
70
|
+
autotools_setup: autotools bootstrap variant (only used when build_system is AUTOTOOLS).
|
|
71
|
+
configure_args: build-system-level configure options, baked into the script text
|
|
72
|
+
because cmake and meson have no environment equivalent.
|
|
73
|
+
cc: C compiler for the library build.
|
|
74
|
+
cxx: C++ compiler for the library build.
|
|
75
|
+
cflags: C flags for the library build.
|
|
76
|
+
cxxflags: C++ flags for the library build.
|
|
77
|
+
"""
|
|
78
|
+
header = (
|
|
79
|
+
"#!/bin/bash\n"
|
|
80
|
+
"set -euo pipefail\n"
|
|
81
|
+
'SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n'
|
|
82
|
+
'BUILD_PREFIX="${BUILD_PREFIX:-$SCRIPT_DIR}"\n'
|
|
83
|
+
"\n"
|
|
84
|
+
f'CC="${{CC:-{_double_quoted_shell_value(cc)}}}"\n'
|
|
85
|
+
f'CXX="${{CXX:-{_double_quoted_shell_value(cxx)}}}"\n'
|
|
86
|
+
f'CFLAGS="${{CFLAGS:-{_double_quoted_shell_value(cflags)}}}"\n'
|
|
87
|
+
f'CXXFLAGS="${{CXXFLAGS:-{_double_quoted_shell_value(cxxflags)}}}"\n'
|
|
88
|
+
)
|
|
89
|
+
header += _JOB_COUNT
|
|
90
|
+
header += _configure_args_block(configure_args)
|
|
91
|
+
header += _skip_if_already_built(paths.install_dir)
|
|
92
|
+
body = _build_body(
|
|
93
|
+
build_system,
|
|
94
|
+
paths.source_dir,
|
|
95
|
+
paths.build_dir,
|
|
96
|
+
paths.install_dir,
|
|
97
|
+
autotools_setup,
|
|
98
|
+
use_configure_args=bool(configure_args),
|
|
99
|
+
)
|
|
100
|
+
return header + body
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _configure_args_block(configure_args: tuple[str, ...]) -> str:
|
|
104
|
+
"""Bake configure options into a bash array, overridable from the environment.
|
|
105
|
+
|
|
106
|
+
An array rather than a string, so an option containing spaces stays one option. An
|
|
107
|
+
EXTRA_CONFIGURE_ARGS in the environment replaces the baked list wholesale.
|
|
108
|
+
"""
|
|
109
|
+
if not configure_args:
|
|
110
|
+
return ""
|
|
111
|
+
baked = " ".join(_single_quoted_shell_value(arg) for arg in configure_args)
|
|
112
|
+
return (
|
|
113
|
+
f"\n{_CONFIGURE_ARGS_VARIABLE}=({baked})\n"
|
|
114
|
+
'if [ -n "${EXTRA_CONFIGURE_ARGS:-}" ]; then\n'
|
|
115
|
+
f' read -r -a {_CONFIGURE_ARGS_VARIABLE} <<<"$EXTRA_CONFIGURE_ARGS"\n'
|
|
116
|
+
"fi\n"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _skip_if_already_built(install_dir: str) -> str:
|
|
121
|
+
"""Exit early when install_dir already has real artifacts (*.a + non-empty include/).
|
|
122
|
+
|
|
123
|
+
A repeat invocation that does not first clear install_dir — a harness-discovery retry
|
|
124
|
+
loop, say, unlike explore() — would otherwise redo the full compile for nothing.
|
|
125
|
+
`rm -rf install_dir` forces a real rebuild.
|
|
126
|
+
"""
|
|
127
|
+
return (
|
|
128
|
+
"\n"
|
|
129
|
+
f'if compgen -G "{install_dir}/lib/*.a" > /dev/null '
|
|
130
|
+
f'&& [ -d "{install_dir}/include" ] '
|
|
131
|
+
f'&& [ -n "$(ls -A "{install_dir}/include" 2>/dev/null)" ]; then\n'
|
|
132
|
+
f' echo "Artifacts already present in {install_dir}; skipping build '
|
|
133
|
+
f'(rm -rf {install_dir} to force a rebuild)."\n'
|
|
134
|
+
" exit 0\n"
|
|
135
|
+
"fi\n"
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _build_body( # noqa: PLR0913 -- one branch per build system; each needs every path
|
|
140
|
+
build_system: BuildSystem,
|
|
141
|
+
source_dir: str,
|
|
142
|
+
build_dir: str,
|
|
143
|
+
install_dir: str,
|
|
144
|
+
autotools_setup: AutotoolsSetup | None = None,
|
|
145
|
+
*,
|
|
146
|
+
use_configure_args: bool = False,
|
|
147
|
+
) -> str:
|
|
148
|
+
"""Generate the build-system-specific half of build_library.sh.
|
|
149
|
+
|
|
150
|
+
Every path is wrapped in double quotes rather than interpolated bare. The three paths are
|
|
151
|
+
usually `$SCRIPT_DIR`/`$BUILD_PREFIX` expansions, and $SCRIPT_DIR resolves to wherever the
|
|
152
|
+
user unpacked the output directory: unquoted, a directory with a space in its name word-
|
|
153
|
+
splits into two arguments and the shipped script fails. Double quotes, not single, because
|
|
154
|
+
the expansion still has to happen.
|
|
155
|
+
"""
|
|
156
|
+
extra = _CONFIGURE_ARGS_EXPANSION if use_configure_args else ""
|
|
157
|
+
if build_system == BuildSystem.CMAKE:
|
|
158
|
+
return (
|
|
159
|
+
"\n"
|
|
160
|
+
"# build system: cmake\n"
|
|
161
|
+
"\n"
|
|
162
|
+
f'cmake -B "{build_dir}" -S "{source_dir}" \\\n'
|
|
163
|
+
' -DCMAKE_C_COMPILER="$CC" \\\n'
|
|
164
|
+
' -DCMAKE_CXX_COMPILER="$CXX" \\\n'
|
|
165
|
+
' -DCMAKE_C_FLAGS="$CFLAGS" \\\n'
|
|
166
|
+
' -DCMAKE_CXX_FLAGS="$CXXFLAGS" \\\n'
|
|
167
|
+
f' "-DCMAKE_INSTALL_PREFIX={install_dir}" \\\n'
|
|
168
|
+
" -DBUILD_SHARED_LIBS=OFF \\\n"
|
|
169
|
+
f" -DBUILD_TESTING=OFF{extra}\n"
|
|
170
|
+
f'cmake --build "{build_dir}" -- -j"$JOBS"\n'
|
|
171
|
+
f'cmake --install "{build_dir}"\n'
|
|
172
|
+
)
|
|
173
|
+
if build_system == BuildSystem.MESON:
|
|
174
|
+
return (
|
|
175
|
+
"\n"
|
|
176
|
+
"# build system: meson\n"
|
|
177
|
+
"\n"
|
|
178
|
+
'CC="$CC" CXX="$CXX" CFLAGS="$CFLAGS" CXXFLAGS="$CXXFLAGS" \\\n'
|
|
179
|
+
f' meson setup "{build_dir}" "{source_dir}" \\\n'
|
|
180
|
+
f' "--prefix={install_dir}" --default-library=static{extra}\n'
|
|
181
|
+
f'ninja -C "{build_dir}" -j"$JOBS"\n'
|
|
182
|
+
f'ninja -C "{build_dir}" install\n'
|
|
183
|
+
)
|
|
184
|
+
if build_system == BuildSystem.AUTOTOOLS:
|
|
185
|
+
bootstrap_script = _AUTOTOOLS_BOOTSTRAP_SCRIPTS.get(autotools_setup)
|
|
186
|
+
if bootstrap_script is not None:
|
|
187
|
+
# The script sometimes runs configure itself; distclean resets that state.
|
|
188
|
+
setup_step = f'(cd "{source_dir}" && ./{bootstrap_script} && make distclean || true)\n'
|
|
189
|
+
elif autotools_setup == AutotoolsSetup.AUTORECONF:
|
|
190
|
+
setup_step = f'(cd "{source_dir}" && autoreconf -fiv)\n'
|
|
191
|
+
else:
|
|
192
|
+
setup_step = ""
|
|
193
|
+
return (
|
|
194
|
+
"\n"
|
|
195
|
+
"# build system: autotools\n"
|
|
196
|
+
"\n" + setup_step + f'mkdir -p "{build_dir}"\n'
|
|
197
|
+
"(\n"
|
|
198
|
+
f' cd "{build_dir}"\n'
|
|
199
|
+
' CC="$CC" CXX="$CXX" CFLAGS="$CFLAGS" CXXFLAGS="$CXXFLAGS" \\\n'
|
|
200
|
+
f' "{source_dir}/configure" "--prefix={install_dir}" '
|
|
201
|
+
f"--enable-static --disable-shared{extra}\n"
|
|
202
|
+
' make -j"$JOBS"\n'
|
|
203
|
+
" make install\n"
|
|
204
|
+
")\n"
|
|
205
|
+
)
|
|
206
|
+
if build_system == BuildSystem.MAKEFILE:
|
|
207
|
+
return (
|
|
208
|
+
"\n"
|
|
209
|
+
"# build system: makefile\n"
|
|
210
|
+
"\n"
|
|
211
|
+
f'make -C "{source_dir}" -j"$JOBS" \\\n'
|
|
212
|
+
' CC="$CC" CXX="$CXX" CFLAGS="$CFLAGS" CXXFLAGS="$CXXFLAGS" \\\n'
|
|
213
|
+
f' "PREFIX={install_dir}"{extra}\n'
|
|
214
|
+
f'make -C "{source_dir}" install "PREFIX={install_dir}"{extra}\n'
|
|
215
|
+
)
|
|
216
|
+
return "\n# build system: unknown\n"
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
DEFAULT_FUZZER_C = (
|
|
220
|
+
"#include <stddef.h>\n"
|
|
221
|
+
"#include <stdint.h>\n"
|
|
222
|
+
"\n"
|
|
223
|
+
"int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n"
|
|
224
|
+
" // TODO: Add fuzzing logic using the target library.\n"
|
|
225
|
+
" return 0;\n"
|
|
226
|
+
"}\n"
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
DEFAULT_FUZZER_CC = (
|
|
230
|
+
"#include <stddef.h>\n"
|
|
231
|
+
"#include <stdint.h>\n"
|
|
232
|
+
"\n"
|
|
233
|
+
'extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n'
|
|
234
|
+
" // TODO: Add fuzzing logic using the target library.\n"
|
|
235
|
+
" return 0;\n"
|
|
236
|
+
"}\n"
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def write_default_fuzzer(harness_dir: Path, language: Language) -> Path:
|
|
241
|
+
"""Write a default LLVMFuzzer stub to harness_dir for the given language."""
|
|
242
|
+
ext = "c" if language == Language.C else "cc"
|
|
243
|
+
source = DEFAULT_FUZZER_C if language == Language.C else DEFAULT_FUZZER_CC
|
|
244
|
+
path = harness_dir / f"default_fuzzer.{ext}"
|
|
245
|
+
path.write_text(source)
|
|
246
|
+
return path
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def build_harness_script(
|
|
250
|
+
link: LinkConfiguration,
|
|
251
|
+
*,
|
|
252
|
+
whole_archive: bool = False,
|
|
253
|
+
harness_cflags: str | None = None,
|
|
254
|
+
harness_cxxflags: str | None = None,
|
|
255
|
+
) -> str:
|
|
256
|
+
"""Generate a script that compiles one harness source into one binary.
|
|
257
|
+
|
|
258
|
+
Args:
|
|
259
|
+
link: the archives and flags the harness must link against.
|
|
260
|
+
whole_archive: link with --whole-archive, forcing every library symbol in so every
|
|
261
|
+
undefined transitive dependency surfaces. Used by the discovery probe, not by
|
|
262
|
+
the shipped script.
|
|
263
|
+
harness_cflags: C flags baked in as the default, for a run of the shipped script with
|
|
264
|
+
no CFLAGS in the environment. Defaults to libFuzzer's flag.
|
|
265
|
+
harness_cxxflags: the same for C++.
|
|
266
|
+
"""
|
|
267
|
+
lib_lines = "".join(f' "$INSTALL_DIR/lib/{path.name}"\n' for path in link.static_libs)
|
|
268
|
+
extra = " ".join(link.transitive_link_flags)
|
|
269
|
+
extra_line = f'EXTRA_LINK_FLAGS="{extra}"\n' if extra else "EXTRA_LINK_FLAGS=\n"
|
|
270
|
+
|
|
271
|
+
extra_lib_paths = " ".join(f"-L{path}" for path in link.extra_library_paths)
|
|
272
|
+
extra_lib_paths_line = (
|
|
273
|
+
f'EXTRA_LIB_PATHS="{extra_lib_paths}"\n' if extra_lib_paths else "EXTRA_LIB_PATHS=\n"
|
|
274
|
+
)
|
|
275
|
+
extra_include_flags = "".join(f' "-I{path}"' for path in link.extra_include_paths)
|
|
276
|
+
|
|
277
|
+
if whole_archive:
|
|
278
|
+
static_libs_str = '-Wl,--whole-archive "${STATIC_LIBS[@]-}" -Wl,--no-whole-archive'
|
|
279
|
+
else:
|
|
280
|
+
static_libs_str = '"${STATIC_LIBS[@]-}"'
|
|
281
|
+
|
|
282
|
+
cc = os.environ.get("CC", "clang")
|
|
283
|
+
cxx = os.environ.get("CXX", "clang++")
|
|
284
|
+
cflags = harness_cflags or _DEFAULT_FUZZING_ENGINE
|
|
285
|
+
cxxflags = harness_cxxflags or _DEFAULT_FUZZING_ENGINE
|
|
286
|
+
preamble = (
|
|
287
|
+
"#!/bin/bash\n"
|
|
288
|
+
"set -euo pipefail\n"
|
|
289
|
+
"\n"
|
|
290
|
+
'SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n'
|
|
291
|
+
'BUILD_PREFIX="${BUILD_PREFIX:-$SCRIPT_DIR}"\n'
|
|
292
|
+
"\n"
|
|
293
|
+
f'CC="${{CC:-{_double_quoted_shell_value(cc)}}}"\n'
|
|
294
|
+
f'CXX="${{CXX:-{_double_quoted_shell_value(cxx)}}}"\n'
|
|
295
|
+
f'CFLAGS="${{CFLAGS:-{_double_quoted_shell_value(cflags)}}}"\n'
|
|
296
|
+
f'CXXFLAGS="${{CXXFLAGS:-{_double_quoted_shell_value(cxxflags)}}}"\n'
|
|
297
|
+
f'LIB_FUZZING_ENGINE="${{LIB_FUZZING_ENGINE:-{_DEFAULT_FUZZING_ENGINE}}}"\n'
|
|
298
|
+
"\n"
|
|
299
|
+
'INSTALL_DIR="$BUILD_PREFIX/install"\n'
|
|
300
|
+
'HARNESS_SOURCE="${1:?usage: compile_harness.sh SOURCE OUTPUT}"\n'
|
|
301
|
+
'OUTPUT_BINARY="${2:?usage: compile_harness.sh SOURCE OUTPUT}"\n'
|
|
302
|
+
'mkdir -p "$(dirname "$OUTPUT_BINARY")"\n'
|
|
303
|
+
"\n"
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
return (
|
|
307
|
+
preamble
|
|
308
|
+
+ "STATIC_LIBS=(\n"
|
|
309
|
+
+ lib_lines
|
|
310
|
+
+ ")\n"
|
|
311
|
+
+ "\n"
|
|
312
|
+
+ extra_line
|
|
313
|
+
+ extra_lib_paths_line
|
|
314
|
+
+ "\n"
|
|
315
|
+
+ 'case "$HARNESS_SOURCE" in\n'
|
|
316
|
+
+ " *.c)\n"
|
|
317
|
+
+ (
|
|
318
|
+
f' "$CC" $CFLAGS "-I$INSTALL_DIR/include"{extra_include_flags} '
|
|
319
|
+
'"$HARNESS_SOURCE" \\\n'
|
|
320
|
+
)
|
|
321
|
+
+ (
|
|
322
|
+
f" {static_libs_str} $EXTRA_LIB_PATHS $EXTRA_LINK_FLAGS"
|
|
323
|
+
' "$LIB_FUZZING_ENGINE" -o "$OUTPUT_BINARY"\n'
|
|
324
|
+
)
|
|
325
|
+
+ " ;;\n"
|
|
326
|
+
+ " *.cc|*.cpp|*.cxx)\n"
|
|
327
|
+
+ (
|
|
328
|
+
f' "$CXX" $CXXFLAGS "-I$INSTALL_DIR/include"{extra_include_flags} '
|
|
329
|
+
'"$HARNESS_SOURCE" \\\n'
|
|
330
|
+
)
|
|
331
|
+
+ (
|
|
332
|
+
f" {static_libs_str} $EXTRA_LIB_PATHS $EXTRA_LINK_FLAGS"
|
|
333
|
+
' "$LIB_FUZZING_ENGINE" -o "$OUTPUT_BINARY"\n'
|
|
334
|
+
)
|
|
335
|
+
+ " ;;\n"
|
|
336
|
+
+ " *)\n"
|
|
337
|
+
+ ' echo "Unsupported harness extension: $HARNESS_SOURCE" >&2\n'
|
|
338
|
+
+ " exit 2\n"
|
|
339
|
+
+ " ;;\n"
|
|
340
|
+
+ "esac\n"
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _double_quoted_shell_value(value: str) -> str:
|
|
345
|
+
"""Escape a configured default that is embedded inside a shell double-quoted value."""
|
|
346
|
+
return value.replace("\\", "\\\\").replace('"', '\\"').replace("$", "\\$").replace("`", "\\`")
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _single_quoted_shell_value(value: str) -> str:
|
|
350
|
+
"""Wrap a configure option in single quotes so spaces and $ survive verbatim."""
|
|
351
|
+
return "'" + value.replace("'", "'\\''") + "'"
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def build_harnesses_script() -> str:
|
|
355
|
+
"""Generate a deterministic batch wrapper around ``compile_harness.sh``.
|
|
356
|
+
|
|
357
|
+
Takes optional source and output directories and nothing else: every compiler and linker
|
|
358
|
+
decision stays in the single-harness script. $OUT is honoured when the environment defines
|
|
359
|
+
it and falls back to $SCRIPT_DIR/out, so one script serves both environments.
|
|
360
|
+
"""
|
|
361
|
+
return (
|
|
362
|
+
"#!/bin/bash\n"
|
|
363
|
+
"set -euo pipefail\n"
|
|
364
|
+
"\n"
|
|
365
|
+
'SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n'
|
|
366
|
+
f'HARNESS_DIR="${{1:-$SCRIPT_DIR/{HARNESS_SOURCE_DIR}}}"\n'
|
|
367
|
+
'OUT_DIR="${2:-${OUT:-$SCRIPT_DIR/out}}"\n'
|
|
368
|
+
'mkdir -p "$OUT_DIR"\n'
|
|
369
|
+
"shopt -s nullglob\n"
|
|
370
|
+
"harnesses=(\n"
|
|
371
|
+
' "$HARNESS_DIR"/*.c "$HARNESS_DIR"/*.cc\n'
|
|
372
|
+
' "$HARNESS_DIR"/*.cpp "$HARNESS_DIR"/*.cxx\n'
|
|
373
|
+
")\n"
|
|
374
|
+
"outputs=()\n"
|
|
375
|
+
'for harness in "${harnesses[@]}"; do\n'
|
|
376
|
+
' name="$(basename "${harness%.*}")"\n'
|
|
377
|
+
' for output in "${outputs[@]}"; do\n'
|
|
378
|
+
' if [ "$output" = "$name" ]; then\n'
|
|
379
|
+
' echo "Duplicate harness output name: $name" >&2\n'
|
|
380
|
+
" exit 2\n"
|
|
381
|
+
" fi\n"
|
|
382
|
+
" done\n"
|
|
383
|
+
' outputs+=("$name")\n'
|
|
384
|
+
"done\n"
|
|
385
|
+
'for harness in "${harnesses[@]}"; do\n'
|
|
386
|
+
' name="$(basename "${harness%.*}")"\n'
|
|
387
|
+
' "$SCRIPT_DIR/compile_harness.sh" "$harness" "$OUT_DIR/$name"\n'
|
|
388
|
+
"done\n"
|
|
389
|
+
)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from enum import Enum
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from fuzzprep.library_builder.environments.base import Environment
|
|
9
|
+
from fuzzprep.library_builder.models import BuildExplorationResult, HarnessExplorationResult
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class RunStatus(Enum):
|
|
13
|
+
SUCCESS = "success"
|
|
14
|
+
FAILED_LIBRARY_BUILD = "failed_library_build"
|
|
15
|
+
FAILED_HARNESS_BUILD = "failed_harness_build"
|
|
16
|
+
FAILED_DOCKERFILE_VERIFICATION = "failed_dockerfile_verification"
|
|
17
|
+
FAILED_OUTPUT_GENERATION = "failed_output_generation"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class AgentPhaseStats:
|
|
22
|
+
"""One phase's agent accounting, or the record that no agent ran.
|
|
23
|
+
|
|
24
|
+
Every numeric field is None when no agent was invoked, so stats.json carries JSON nulls
|
|
25
|
+
rather than an "N/A" string a consumer would have to type-check around. `invoked` is what
|
|
26
|
+
says which case it is.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
invoked: bool
|
|
30
|
+
duration_seconds: float | None = None
|
|
31
|
+
cost_usd: float | None = None
|
|
32
|
+
input_tokens: int | None = None
|
|
33
|
+
output_tokens: int | None = None
|
|
34
|
+
summary: str | None = None
|
|
35
|
+
# Why the agent stopped short, when it did. None means it ran to a verdict on the build --
|
|
36
|
+
# so a failed run with a stop_reason set is not evidence the build is unfixable.
|
|
37
|
+
stop_reason: str | None = None
|
|
38
|
+
|
|
39
|
+
def to_dict(self) -> dict[str, object]:
|
|
40
|
+
return {
|
|
41
|
+
"invoked": self.invoked,
|
|
42
|
+
"duration_seconds": self.duration_seconds,
|
|
43
|
+
"cost_usd": self.cost_usd,
|
|
44
|
+
"input_tokens": self.input_tokens,
|
|
45
|
+
"output_tokens": self.output_tokens,
|
|
46
|
+
"summary": self.summary,
|
|
47
|
+
"stop_reason": self.stop_reason,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def not_invoked_agent_stats() -> AgentPhaseStats:
|
|
52
|
+
return AgentPhaseStats(invoked=False)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def agent_phase_stats(result: BuildExplorationResult | HarnessExplorationResult) -> AgentPhaseStats:
|
|
56
|
+
"""The agent accounting carried by either stage's result.
|
|
57
|
+
|
|
58
|
+
One function for both, since both report it through the same AgentOutcome fields.
|
|
59
|
+
"""
|
|
60
|
+
if not result.llm_used:
|
|
61
|
+
return not_invoked_agent_stats()
|
|
62
|
+
return AgentPhaseStats(
|
|
63
|
+
invoked=True,
|
|
64
|
+
duration_seconds=result.duration_seconds,
|
|
65
|
+
cost_usd=result.cost_usd,
|
|
66
|
+
input_tokens=result.input_tokens,
|
|
67
|
+
output_tokens=result.output_tokens,
|
|
68
|
+
summary=result.agent_summary,
|
|
69
|
+
stop_reason=result.agent_stop_reason.value if result.agent_stop_reason else None,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class RunStats:
|
|
75
|
+
total_duration_seconds: float
|
|
76
|
+
library_build_agent: AgentPhaseStats
|
|
77
|
+
harness_build_agent: AgentPhaseStats
|
|
78
|
+
status: RunStatus
|
|
79
|
+
environment: Environment = Environment.LOCAL
|
|
80
|
+
compile_commands_path: str | None = None
|
|
81
|
+
# The literal command the gate was invoked with, so a person can reproduce the result.
|
|
82
|
+
verification_command: str | None = None
|
|
83
|
+
build_parameters: dict[str, str | list[str]] | None = None
|
|
84
|
+
# True when --bypass-scratch-validation ran: no from-scratch rebuild proved this result.
|
|
85
|
+
scratch_validation_bypassed: bool = False
|
|
86
|
+
|
|
87
|
+
def to_dict(self) -> dict[str, object]:
|
|
88
|
+
return {
|
|
89
|
+
"total_duration_seconds": self.total_duration_seconds,
|
|
90
|
+
"library_build_agent": self.library_build_agent.to_dict(),
|
|
91
|
+
"harness_build_agent": self.harness_build_agent.to_dict(),
|
|
92
|
+
"status": self.status.value,
|
|
93
|
+
"environment": self.environment.value,
|
|
94
|
+
"compile_commands_path": self.compile_commands_path,
|
|
95
|
+
"verification_command": self.verification_command,
|
|
96
|
+
"build_parameters": self.build_parameters,
|
|
97
|
+
"scratch_validation_bypassed": self.scratch_validation_bypassed,
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def write_run_stats(path: Path, stats: RunStats) -> None:
|
|
102
|
+
path.write_text(json.dumps(stats.to_dict(), indent=2))
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"-lm": [
|
|
3
|
+
"^(sin|cos|tan|asin|acos|atan2?)(h?)([fl]?)$",
|
|
4
|
+
"^(pow|exp2?|expm1|log|log2|log10|log1p)([fl]?)$",
|
|
5
|
+
"^(sqrt|cbrt|hypot)([fl]?)$",
|
|
6
|
+
"^(fabs|floor|ceil|round|trunc|rint|nearbyint)([fl]?)$",
|
|
7
|
+
"^(fma|fmod|remainder|fdim|fmax|fmin)([fl]?)$",
|
|
8
|
+
"^(erf|erfc|lgamma|tgamma)([fl]?)$",
|
|
9
|
+
"^(ldexp|frexp|modf|scalbn|ilogb|logb)([fl]?)$"
|
|
10
|
+
],
|
|
11
|
+
"-lz": [
|
|
12
|
+
"^(inflate|deflate)",
|
|
13
|
+
"^compress2?$",
|
|
14
|
+
"^uncompress2?$",
|
|
15
|
+
"^gz",
|
|
16
|
+
"^adler32",
|
|
17
|
+
"^crc32",
|
|
18
|
+
"^zlib"
|
|
19
|
+
],
|
|
20
|
+
"-lpthread": ["^pthread_"],
|
|
21
|
+
"-lssl": ["^SSL_", "^TLS_"],
|
|
22
|
+
"-lcrypto": [
|
|
23
|
+
"^BIO_", "^X509_", "^EVP_", "^OPENSSL_", "^ERR_",
|
|
24
|
+
"^AES_", "^RAND_",
|
|
25
|
+
"^d2i_", "^i2d_", "^PEM_",
|
|
26
|
+
"^CRYPTO_", "^EC_", "^RSA_", "^DSA_", "^DH_"
|
|
27
|
+
],
|
|
28
|
+
"-lbz2": ["^BZ2_"],
|
|
29
|
+
"-llzma": ["^lzma_"],
|
|
30
|
+
"-ldl": ["^dl(open|sym|close|error)$"],
|
|
31
|
+
"-lrt": [
|
|
32
|
+
"^clock_(gettime|getres|settime)$",
|
|
33
|
+
"^shm_(open|unlink)$",
|
|
34
|
+
"^aio_(read|write|error|return|suspend|cancel|fsync)$",
|
|
35
|
+
"^timer_(create|delete|gettime|settime|getoverrun)$",
|
|
36
|
+
"^mq_"
|
|
37
|
+
],
|
|
38
|
+
"-lresolv": [
|
|
39
|
+
"^res_(init|query|search|querydomain|send|mkquery|nmkquery)$",
|
|
40
|
+
"^ns_initparse$",
|
|
41
|
+
"^dn_expand$"
|
|
42
|
+
],
|
|
43
|
+
"-lzstd": ["^ZSTD_"],
|
|
44
|
+
"-llz4": ["^LZ4_", "^LZ4F_"],
|
|
45
|
+
"-liconv": ["^iconv"],
|
|
46
|
+
"-lxml2": ["^xml", "^XML_"],
|
|
47
|
+
"-lsqlite3": ["^sqlite3_"],
|
|
48
|
+
"-lpcre2-8": ["^pcre2_"],
|
|
49
|
+
"-lffi": ["^ffi_"],
|
|
50
|
+
"-lcurl": ["^curl_"],
|
|
51
|
+
"-levent": ["^event_base_", "^event_new$", "^event_add$", "^event_del$", "^evbuffer_", "^bufferevent_", "^evhttp_", "^evdns_"],
|
|
52
|
+
"-luv": ["^uv_"],
|
|
53
|
+
"-lpng": ["^png_"],
|
|
54
|
+
"-ljpeg": ["^jpeg_"],
|
|
55
|
+
"-lwebp": ["^WebP", "^WebPDecode", "^WebPEncode"],
|
|
56
|
+
"-lfreetype": ["^FT_"],
|
|
57
|
+
"-lsodium": ["^sodium_", "^crypto_sign", "^crypto_box", "^crypto_secretbox", "^crypto_hash", "^crypto_auth", "^randombytes_"],
|
|
58
|
+
"-lgcrypt": ["^gcry_"],
|
|
59
|
+
"-lssh2": ["^libssh2_"],
|
|
60
|
+
"-lseccomp": ["^seccomp_"],
|
|
61
|
+
"-lgmp": ["^mpz_", "^mpq_", "^mpf_", "^mpn_", "^__gmp"],
|
|
62
|
+
"-larchive": ["^archive_"],
|
|
63
|
+
"-lmagic": ["^magic_"],
|
|
64
|
+
"-lgit2": ["^git_"]
|
|
65
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""The wall-clock ceilings the library builder enforces, in one place.
|
|
2
|
+
|
|
3
|
+
The two are far apart because they bound different things:
|
|
4
|
+
|
|
5
|
+
* `DEFAULT_BUILD_TIMEOUT_SECONDS` bounds a full library build (configure, compile, install) in
|
|
6
|
+
either environment. Real libraries take minutes.
|
|
7
|
+
* `HARNESS_PROBE_TIMEOUT_SECONDS` bounds one harness compile-and-link attempt — one stub
|
|
8
|
+
translation unit plus a link, so an overrun means a hung linker, not a slow build.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
DEFAULT_BUILD_TIMEOUT_SECONDS = 900
|
|
14
|
+
HARNESS_PROBE_TIMEOUT_SECONDS = 60
|