fuzzprep 0.1.0__py3-none-any.whl

fuzzprep/library_builder/harness_explorer.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from pathlib import Path
+
+from fuzzprep.core.files import write_executable
+from fuzzprep.core.subprocesses import Runner, run_command
+from fuzzprep.library_builder.environments.base import Environment
+from fuzzprep.library_builder.models import (
+    HarnessExplorationResult,
+    Language,
+    LinkConfiguration,
+)
+from fuzzprep.library_builder.scripts import (
+    HARNESS_SOURCE_DIR,
+    build_harness_script,
+    build_harnesses_script,
+    write_default_fuzzer,
+)
+from fuzzprep.library_builder.timeouts import HARNESS_PROBE_TIMEOUT_SECONDS
+
+logger = logging.getLogger(__name__)
+
+_MAX_ATTEMPTS = 5
+
+# Always linked rather than discovered: glibc's pthread_* entry points are weak symbols, so
+# omitting -lpthread silently links no-op stubs instead of erroring, and _resolve_flags never
+# sees an undefined symbol to react to.
+_DEFAULT_LINK_FLAGS = ["-lpthread"]
+
+_PATTERNS_FILE = Path(__file__).parent / "symbol_patterns.json"
+
+# Each flag maps to a compiled regex that ORs all of its patterns together.
+# Patterns use anchors (^/$) to avoid false substring matches.
+_LIB_PATTERNS: dict[str, re.Pattern[str]] = {
+    flag: re.compile("|".join(patterns))
+    for flag, patterns in json.loads(_PATTERNS_FILE.read_text()).items()
+}
+
+# C++ ABI symbols that appear when a C library pulls in a C++ static archive.
+# Matches operator new/delete, exception-handling (__cxa_), and personality routine.
+_CXX_ABI_RE = re.compile(r"operator (?:new|delete)\b|__cxa_|__gxx_personality")
+
+
+def explore_harness_compilation(  # noqa: PLR0913 -- public API; every param is a distinct required input
+    install_dir: Path,
+    workdir: Path,
+    language: Language,
+    *,
+    extra_include_paths: list[str] | None = None,
+    extra_library_paths: list[str] | None = None,
+    environment: Environment = Environment.LOCAL,
+    run: Runner | None = None,
+) -> HarnessExplorationResult:
+    """Test harness compilation against install artifacts to discover transitive deps.
+
+    Links with --whole-archive to force every library symbol in, so every undefined
+    transitive dependency surfaces. Retries up to _MAX_ATTEMPTS times, accumulating the -l
+    flags resolved from each attempt's linker errors on top of _DEFAULT_LINK_FLAGS. Always
+    returns a result; callers read .succeeded.
+
+    extra_include_paths/extra_library_paths are fixed inputs (typically from a prior agent's
+    AgentReport) threaded unchanged into the returned result.
+
+    environment.harness_probe_command decides how each attempt enters the build. run defaults
+    to a host subprocess; a caller probing inside a container passes a run primitive that
+    wraps the command in `docker run`.
+    """
+    extra_include_paths = extra_include_paths or []
+    extra_library_paths = extra_library_paths or []
+    lib_dir = install_dir / "lib"
+    include_dir = install_dir / "include"
+
+    static_libs = sorted(lib_dir.glob("*.a"))
+    logger.debug(
+        "Discovered static libraries [%s] from dir %s",
+        " ".join(str(p) for p in static_libs),
+        lib_dir,
+    )
+    if not static_libs:
+        return HarnessExplorationResult(
+            succeeded=False,
+            command=[],
+            static_libs=[],
+            include_dir=include_dir,
+            transitive_link_flags=[],
+            stdout="",
+            stderr="no *.a files found in install/lib",
+            exit_code=-1,
+            extra_include_paths=extra_include_paths,
+            extra_library_paths=extra_library_paths,
+            environment=environment,
+        )
+
+    use_cpp = language == Language.CPP
+    harness_src_dir = workdir / HARNESS_SOURCE_DIR
+    harness_src_dir.mkdir(exist_ok=True)
+    # Idempotent: workspace materialization already wrote this stub. Discovery upgrades this
+    # same file's extension in place on a C++ finding rather than probing with a throwaway
+    # copy, so the discovered language cannot desync from what generation ships.
+    harness_path = write_default_fuzzer(harness_src_dir, language)
+
+    script_path = workdir / "compile_harness.sh"
+    batch_script_path = workdir / "compile_harnesses.sh"
+    runner = run if run is not None else run_command
+    extra_flags: list[str] = list(_DEFAULT_LINK_FLAGS)
+    seen_flags: set[str] = set(_DEFAULT_LINK_FLAGS)
+    last_stdout = ""
+    last_stderr = ""
+    last_exit = -1
+    diagnostic = ""
+
+    for _ in range(_MAX_ATTEMPTS):
+        write_executable(
+            script_path,
+            build_harness_script(
+                LinkConfiguration(
+                    static_libs=static_libs,
+                    transitive_link_flags=extra_flags,
+                    extra_library_paths=extra_library_paths,
+                    extra_include_paths=extra_include_paths,
+                ),
+                whole_archive=True,
+                harness_cflags=os.environ.get("CFLAGS"),
+                harness_cxxflags=os.environ.get("CXXFLAGS"),
+            ),
+        )
+        write_executable(batch_script_path, build_harnesses_script())
+        command = environment.harness_probe_command
+        result = runner(command, workdir, HARNESS_PROBE_TIMEOUT_SECONDS)
+        last_stdout = result.stdout
+        last_stderr = result.stderr
+        last_exit = result.exit_code
+        diagnostic = result.output
+
+        if result.exit_code == 0:
+            return HarnessExplorationResult(
+                succeeded=True,
+                command=command,
+                static_libs=static_libs,
+                include_dir=include_dir,
+                transitive_link_flags=extra_flags,
+                stdout=last_stdout,
+                stderr=last_stderr,
+                exit_code=last_exit,
+                script_path=script_path,
+                extra_include_paths=extra_include_paths,
+                extra_library_paths=extra_library_paths,
+                environment=environment,
+            )
+
+        upgraded_to_cxx = False
+        if not use_cpp and _requires_cxx(diagnostic):
+            logger.debug("Upgrading harness build to use CXX instead of CC")
+            harness_path.unlink(missing_ok=True)
+            harness_path = write_default_fuzzer(harness_src_dir, Language.CPP)
+            use_cpp = True
+            upgraded_to_cxx = True
+
+        new_flags = _resolve_flags(diagnostic, seen_flags)
+        if not new_flags and not upgraded_to_cxx:
+            break
+        extra_flags.extend(new_flags)
+        seen_flags.update(new_flags)
+
+    return HarnessExplorationResult(
+        succeeded=False,
+        command=[],
+        static_libs=static_libs,
+        include_dir=include_dir,
+        transitive_link_flags=extra_flags,
+        stdout=last_stdout,
+        stderr=last_stderr,
+        exit_code=last_exit,
+        missing_system_libs=extract_missing_system_libs(diagnostic),
+        extra_include_paths=extra_include_paths,
+        extra_library_paths=extra_library_paths,
+        environment=environment,
+    )
+
+
+def _resolve_flags(stderr: str, already_seen: set[str]) -> list[str]:
+    symbols = _extract_undefined_symbols(stderr)
+    resolved: dict[str, str] = {}
+    for sym in symbols:
+        flag = _symbol_to_flag(sym)
+        if flag and flag not in already_seen and flag not in resolved:
+            resolved[flag] = flag
+    return list(resolved.values())
+
+
+def _extract_undefined_symbols(stderr: str) -> list[str]:
+    symbols: list[str] = []
+    # Linux ld: undefined reference to `symbol'
+    for m in re.finditer(r"undefined reference to `([^']+)'", stderr):
+        symbols.append(m.group(1))
+    # macOS ld (compact): Undefined symbol: _symbol
+    for m in re.finditer(r'[Uu]ndefined symbol[: ]+[_"]?([A-Za-z_][A-Za-z0-9_]*)', stderr):
+        symbols.append(m.group(1))
+    # macOS ld (verbose): "  "_Symbol", referenced from:"
+    for m in re.finditer(r'"_([A-Za-z_][A-Za-z0-9_]*)",\s+referenced from:', stderr):
+        symbols.append(m.group(1))
+    return symbols
+
+
+def _requires_cxx(stderr: str) -> bool:
+    if _CXX_ABI_RE.search(stderr):
+        return True
+    return any(
+        symbol.startswith("_Z") or "::" in symbol for symbol in _extract_undefined_symbols(stderr)
+    )
+
+
+def extract_missing_system_libs(stderr: str) -> list[str]:
+    libs: list[str] = []
+    # macOS ld: library 'zstd' not found
+    for m in re.finditer(r"ld: library '([^']+)' not found", stderr):
+        libs.append(m.group(1))
+    # Linux ld: cannot find -lzstd
+    for m in re.finditer(r"ld: cannot find -l([^\s:]+)", stderr):
+        libs.append(m.group(1))
+    return list(dict.fromkeys(libs))  # deduplicate, preserve order
+
+
+def lib_names_from_link_flags(flags: list[str]) -> list[str]:
+    """Strip the "-l" prefix from transitive_link_flags entries.
+
+    Every entry is "-l<name>", since every key in symbol_patterns.json is. The bare names
+    are what package_names.translate() takes.
+    """
+    return [flag.removeprefix("-l") for flag in flags]
+
+
+def _symbol_to_flag(symbol: str) -> str | None:
+    for flag, pattern in _LIB_PATTERNS.items():
+        if pattern.search(symbol):
+            return flag
+    return None
+
+
+def validate_harness_artifacts(workdir: Path) -> list[str]:
+    """Check that a harness binary really exists on the host, in every environment.
+
+    An agent's exit code is not evidence that it linked anything, so what it claims to have
+    produced is checked directly. One host path serves both environments because the gate puts
+    the binaries there: a local build writes <workdir>/out itself, and
+    check_build_in_container.sh bind-mounts that same directory over the base image's
+    $OUT=/out, which is outside the /src mount and would otherwise be discarded with the
+    container.
+    """
+    out_dir = workdir / "out"
+    if not out_dir.exists() or not any(out_dir.iterdir()):
+        return [f"no compiled harness binary found in {out_dir}"]
+    return []

fuzzprep/library_builder/models.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum, StrEnum
+from pathlib import Path
+
+from fuzzprep.core.subprocesses import MergedOutput
+from fuzzprep.library_builder.environments.base import Environment
+
+
+class BuildSystem(Enum):
+    CMAKE = "cmake"
+    MESON = "meson"
+    AUTOTOOLS = "autotools"
+    MAKEFILE = "makefile"
+    UNKNOWN = "unknown"
+
+
+class AutotoolsSetup(Enum):
+    CONFIGURE = "configure"  # configure script already present
+    AUTOGEN = "autogen"  # autogen.sh present, must run before configure
+    BOOTSTRAP = "bootstrap"  # bootstrap present (gnulib convention), same role as autogen.sh
+    AUTORECONF = "autoreconf"  # only configure.ac / configure.in, need autoreconf -fiv
+
+
+class Language(Enum):
+    C = "c"
+    CPP = "c++"
+
+
+@dataclass(frozen=True)
+class AnalysisResult:
+    """What static analysis concluded about a repository — immutable pipeline input.
+
+    Frozen because every later phase reads it and none owns it. Anything a later phase
+    discovers travels onward as an explicit argument, not as a mutation of this record.
+    """
+
+    project_name: str
+    source_path: Path
+    build_system: BuildSystem
+    language: Language
+    clone_url: str
+    repo_ref: str | None
+    warnings: list[str] = field(default_factory=list)
+    autotools_setup: AutotoolsSetup | None = None
+
+
+@dataclass
+class AgentReport:
+    """Parsed contents of an agent invocation's agent_report.json, or "nothing reported"."""
+
+    summary: str | None = None
+    missing_libs: list[str] = field(default_factory=list)
+    missing_apt_packages: list[str] = field(default_factory=list)
+    extra_include_paths: list[str] = field(default_factory=list)
+    extra_library_paths: list[str] = field(default_factory=list)
+
+
+class AgentStopReason(StrEnum):
+    """Why a repair agent stopped without fixing the build.
+
+    None of these is a verdict on the build: one needs a person, one needs time to pass, and
+    one needs only a retry. Reported on the result, so "the build did not pass" stays one
+    control path -- but a consumer counting repair failures must exclude them, because in all
+    three cases the agent never finished judging the build.
+    """
+
+    ACTION_REQUIRED = "action_required"
+    BUDGET_LIMITED = "budget_limited"
+    SERVICE_UNAVAILABLE = "service_unavailable"
+
+
+@dataclass(kw_only=True)
+class AgentOutcome:
+    """What a repair-agent attempt contributes to a stage's result.
+
+    Both stage results carry this same block, so it is declared once here and inherited.
+    Keyword-only, so the two results can still declare their own required fields.
+    """
+
+    llm_used: bool = False
+    # The validated script to publish, set once a stage (or an agent's repair) proved it
+    # works and its paths are portable enough to copy verbatim.
+    script_path: Path | None = None
+    agent_stop_reason: AgentStopReason | None = None
+    # Non-empty only when FuzzPrep rejected a repair the agent reported as done, because
+    # the artifacts it claims to have produced are not there. That is a different thing to
+    # tell the user than an agent that failed and said so.
+    validation_errors: list[str] = field(default_factory=list)
+    cost_usd: float | None = None
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    transcript_path: Path | None = None
+    agent_summary: str | None = None
+    missing_apt_packages: list[str] = field(default_factory=list)
+    extra_include_paths: list[str] = field(default_factory=list)
+    extra_library_paths: list[str] = field(default_factory=list)
+    environment: Environment = Environment.LOCAL
+
+
+@dataclass
+class LinkConfiguration:
+    """Everything a harness link line needs: the library's own archives, plus what they
+    transitively pull in and where to find it.
+
+    Separate from HarnessExplorationResult so the retry loop, which rewrites
+    compile_harness.sh with one more resolved flag per attempt, proposes a link line rather
+    than appearing to report an outcome.
+    """
+
+    static_libs: list[Path] = field(default_factory=list)
+    transitive_link_flags: list[str] = field(default_factory=list)
+    extra_library_paths: list[str] = field(default_factory=list)
+    extra_include_paths: list[str] = field(default_factory=list)
+
+
+@dataclass
+class BuildExplorationResult(MergedOutput, AgentOutcome):
+    build_system: BuildSystem
+    succeeded: bool
+    command: list[str]
+    stdout: str
+    stderr: str
+    exit_code: int
+    duration_seconds: float
+    # The validated install tree. Set even when the source lives outside the workspace and
+    # the build script therefore cannot be copied verbatim.
+    install_dir: Path | None = None
+
+
+@dataclass
+class HarnessExplorationResult(MergedOutput, AgentOutcome):
+    succeeded: bool
+    command: list[str]
+    static_libs: list[Path]
+    include_dir: Path
+    transitive_link_flags: list[str]
+    stdout: str
+    stderr: str
+    exit_code: int
+    missing_system_libs: list[str] = field(default_factory=list)
+    duration_seconds: float = 0.0
+    # Whether the gate that produced this result reused the workspace's install/ rather than
+    # rebuilding the library from nothing. Stamped by environments/gate.py, so a repair agent
+    # is told to run the same gate the pipeline ran instead of recomputing the decision and
+    # paying for a cold rebuild the pipeline had decided to skip. See
+    # verification.gate_keeps_artifacts.
+    gate_keeps_artifacts: bool = False
+
+
+@dataclass
+class BuildPaths:
+    source_dir: str
+    build_dir: str
+    install_dir: str
+
+
+@dataclass
+class HarnessPaths:
+    install_dir: Path
+    workdir: Path
+
+
+@dataclass
+class GenerationResult:
+    project_name: str
+    output_path: Path
+    files: list[Path]

fuzzprep/library_builder/package_names.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "system_libs": ["m", "pthread", "dl", "rt", "resolv", "c", "gcc_s", "stdc++"],
+  "apt_packages": {
+    "z":        "zlib1g-dev",
+    "zstd":     "libzstd-dev",
+    "lz4":      "liblz4-dev",
+    "lzma":     "liblzma-dev",
+    "bz2":      "libbz2-dev",
+    "ssl":      "libssl-dev",
+    "crypto":   "libssl-dev",
+    "iconv":    "libc6-dev",
+    "xml2":     "libxml2-dev",
+    "sqlite3":  "libsqlite3-dev",
+    "pcre2-8":  "libpcre2-dev",
+    "ffi":      "libffi-dev",
+    "curl":     "libcurl4-openssl-dev",
+    "event":    "libevent-dev",
+    "uv":       "libuv1-dev",
+    "png":      "libpng-dev",
+    "jpeg":     "libjpeg-dev",
+    "webp":     "libwebp-dev",
+    "freetype": "libfreetype6-dev",
+    "sodium":   "libsodium-dev",
+    "gcrypt":   "libgcrypt20-dev",
+    "ssh2":     "libssh2-1-dev",
+    "seccomp":  "libseccomp-dev",
+    "gmp":      "libgmp-dev",
+    "archive":  "libarchive-dev",
+    "magic":    "libmagic-dev",
+    "git2":     "libgit2-dev"
+  }
+}

fuzzprep/library_builder/package_names.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+_DATA_FILE = Path(__file__).parent / "package_names.json"
+_DATA = json.loads(_DATA_FILE.read_text())
+_SYSTEM_LIBS: frozenset[str] = frozenset(_DATA["system_libs"])
+_APT_PACKAGES: dict[str, str] = _DATA["apt_packages"]
+
+
+@dataclass
+class PackageTranslation:
+    apt_packages: list[str]
+    unknown_libs: list[str]
+
+
+def translate(lib_names: list[str]) -> PackageTranslation:
+    """Translate raw lib names (e.g. ["zstd"]) to installable apt package names.
+
+    System libs (m, pthread, dl, rt, resolv, c, gcc_s, stdc++) are silently dropped.
+    Libs with no mapping go into unknown_libs. Deduplication preserves order.
+    """
+    apt: list[str] = []
+    unknown: list[str] = []
+
+    for lib in lib_names:
+        if lib in _SYSTEM_LIBS:
+            continue
+        package = _APT_PACKAGES.get(lib)
+        if package is None:
+            unknown.append(lib)
+            continue
+        apt.append(package)
+
+    return PackageTranslation(
+        apt_packages=list(dict.fromkeys(apt)),
+        unknown_libs=list(dict.fromkeys(unknown)),
+    )