fuzzprep 0.1.0__py3-none-any.whl

fuzzprep/library_builder/analysis.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from fuzzprep.core.repos import RepoSource
+from fuzzprep.library_builder.models import (
+    AnalysisResult,
+    AutotoolsSetup,
+    BuildSystem,
+    Language,
+)
+
+logger = logging.getLogger(__name__)
+_C_HEADER_EXTENSIONS: frozenset[str] = frozenset({".h", ".hpp", ".hxx", ".hh"})
+_C_SOURCE_EXTENSIONS: frozenset[str] = frozenset({".c"})
+# `.C` is C++ by convention, and Path.suffix keeps the case the file was stored with.
+_CPP_SOURCE_EXTENSIONS: frozenset[str] = frozenset({".cc", ".cpp", ".cxx", ".c++", ".C"})
+_VCS_DIRS: frozenset[str] = frozenset({".git", ".hg", ".svn"})
+_READ_CHUNK_BYTES = 1 << 20
+
+# Bare Makefile is the last-resort signal: a project shipping meson.build and a convenience
+# top-level Makefile is a meson project.
+_BUILD_SYSTEM_CHECKS: list[tuple[BuildSystem, list[str]]] = [
+    (BuildSystem.CMAKE, ["CMakeLists.txt"]),
+    (BuildSystem.AUTOTOOLS, ["configure.ac", "configure.in", "configure"]),
+    (BuildSystem.MESON, ["meson.build"]),
+    (BuildSystem.MAKEFILE, ["Makefile", "makefile"]),
+]
+
+
+class UnsupportedRepositoryError(Exception):
+    """Repository has no recognizable C/C++ signals."""
+
+
+def analyze(source: RepoSource) -> AnalysisResult:
+    """Run deterministic static analysis on a repository directory."""
+    build_system, build_files = _detect_build_system(source.source_path)
+    has_headers, c_lines, cpp_lines = _scan_sources(source.source_path)
+    warnings: list[str] = []
+    language = _select_language(c_lines, cpp_lines, warnings)
+
+    if not build_files and not has_headers:
+        raise UnsupportedRepositoryError(
+            f"No C/C++ signals found in {source.source_path}: "
+            "no recognized build system files and no C/C++ headers."
+        )
+
+    if build_system == BuildSystem.UNKNOWN:
+        warnings.append(f"No recognized build system found in {source.source_path}.")
+
+    if not has_headers:
+        warnings.append(f"No C/C++ header files found in {source.source_path}.")
+
+    autotools_setup = (
+        _detect_autotools_setup(source.source_path)
+        if build_system == BuildSystem.AUTOTOOLS
+        else None
+    )
+
+    return AnalysisResult(
+        project_name=source.project_name,
+        source_path=source.source_path,
+        build_system=build_system,
+        language=language,
+        clone_url=source.clone_url,
+        repo_ref=source.repo_ref,
+        warnings=warnings,
+        autotools_setup=autotools_setup,
+    )
+
+
+def _detect_autotools_setup(root: Path) -> AutotoolsSetup:
+    """Detect how to bootstrap autotools for this repository.
+
+    Priority: an existing configure > autogen.sh > bootstrap > autoreconf from configure.ac.
+    Only reached once the tree is known to be autotools, so a bare `bootstrap` here is the
+    gnulib-style wrapper rather than some other project's setup script.
+    """
+    if (root / "configure").exists():
+        return AutotoolsSetup.CONFIGURE
+    if (root / "autogen.sh").exists():
+        return AutotoolsSetup.AUTOGEN
+    if (root / "bootstrap").exists():
+        return AutotoolsSetup.BOOTSTRAP
+    return AutotoolsSetup.AUTORECONF
+
+
+def _detect_build_system(root: Path) -> tuple[BuildSystem, list[Path]]:
+    """Detect the primary build system in priority order."""
+    for build_system, filenames in _BUILD_SYSTEM_CHECKS:
+        found = [root / name for name in filenames if (root / name).exists()]
+        if found:
+            return build_system, found
+    return BuildSystem.UNKNOWN, []
+
+
+def _scan_sources(root: Path) -> tuple[bool, int, int]:
+    """Walk root once for header presence and C/C++ source line counts.
+
+    Returns whether a C/C++ header exists, the C line count, and the C++ line count. Files under
+    a VCS directory do not count. Headers stay out of the line counts because `.h` belongs to
+    both languages.
+    """
+    has_headers = False
+    c_lines = 0
+    cpp_lines = 0
+    for path in root.rglob("*"):
+        if any(part in _VCS_DIRS for part in path.relative_to(root).parts) or not path.is_file():
+            continue
+        suffix = path.suffix
+        if suffix in _C_HEADER_EXTENSIONS:
+            has_headers = True
+        elif suffix in _C_SOURCE_EXTENSIONS:
+            c_lines += _count_lines(path)
+        elif suffix in _CPP_SOURCE_EXTENSIONS:
+            cpp_lines += _count_lines(path)
+    return has_headers, c_lines, cpp_lines
+
+
+def _count_lines(path: Path) -> int:
+    """Count the lines in a file, or 0 when it cannot be read."""
+    try:
+        with path.open("rb") as handle:
+            return sum(
+                chunk.count(b"\n") for chunk in iter(lambda: handle.read(_READ_CHUNK_BYTES), b"")
+            )
+    except OSError as error:
+        logger.debug("Skipping %s while counting lines: %s", path, error)
+        return 0
+
+
+def _select_language(c_lines: int, cpp_lines: int, warnings: list[str]) -> Language:
+    """Pick the dominant language from the C and C++ source line counts.
+
+    Falls back to C++ with a warning when the tree holds neither: C++ is the safe guess, since a
+    C library compiled by a C++ driver links while the reverse does not.
+    """
+    logger.debug("C LoC: %d, CPP LoC %d", c_lines, cpp_lines)
+    if c_lines == 0 and cpp_lines == 0:
+        warnings.append(
+            "Could not determine the source language (no C or C++ sources); assuming C++."
+        )
+        return Language.CPP
+    return Language.CPP if cpp_lines > c_lines else Language.C

fuzzprep/library_builder/build_parameters.py

@@ -0,0 +1,158 @@
+"""Compiler settings used while preparing and emitting a local fuzzing build."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+_DEFAULT_HARNESS_FLAGS = "-fsanitize=fuzzer"
+
+
+@dataclass(frozen=True)
+class BuildParameters:
+    """Separate compiler settings for library and final-harness compilation."""
+
+    cc: str
+    cxx: str
+    library_cflags: str
+    library_cxxflags: str
+    harness_cflags: str
+    harness_cxxflags: str
+    # Build-system-level configure options: cmake cache variables, meson options, autotools
+    # --enable-* switches, make variables. Baked into the generated script rather than passed
+    # through the environment, since cmake and meson have no environment equivalent.
+    library_configure_args: tuple[str, ...] = ()
+
+    @classmethod
+    def from_args(cls, args: object) -> BuildParameters:
+        """Resolve CLI arguments against the caller's existing compiler environment."""
+        return cls(
+            cc=_argument_or_environment(args, "cc", "CC", "clang"),
+            cxx=_argument_or_environment(args, "cxx", "CXX", "clang++"),
+            library_cflags=_argument_or_environment(args, "library_cflags", "CFLAGS", ""),
+            library_cxxflags=_argument_or_environment(args, "library_cxxflags", "CXXFLAGS", ""),
+            harness_cflags=_argument_or_default(args, "harness_cflags", _DEFAULT_HARNESS_FLAGS),
+            harness_cxxflags=_argument_or_default(args, "harness_cxxflags", _DEFAULT_HARNESS_FLAGS),
+            library_configure_args=_repeated_argument(args, "library_configure_args"),
+        )
+
+    @classmethod
+    def defaults(cls) -> BuildParameters:
+        """The settings a run with no CLI arguments would use.
+
+        For a caller that builds outside a `generate` invocation and still needs the compiler
+        resolution rules the CLI applies.
+        """
+        return cls.from_args(_NO_ARGUMENTS)
+
+    @contextmanager
+    def library_environment(self) -> Iterator[None]:
+        """Expose library settings to build commands and repair agents."""
+        with _temporary_environment(
+            {
+                "CC": self.cc,
+                "CXX": self.cxx,
+                "CFLAGS": self.library_cflags,
+                "CXXFLAGS": self.library_cxxflags,
+            }
+        ):
+            yield
+
+    @contextmanager
+    def harness_environment(self) -> Iterator[None]:
+        """Expose final-harness settings while validating and emitting its compiler."""
+        with _temporary_environment(
+            {
+                "CC": self.cc,
+                "CXX": self.cxx,
+                "CFLAGS": self.harness_cflags,
+                "CXXFLAGS": self.harness_cxxflags,
+            }
+        ):
+            yield
+
+    def to_dict(self) -> dict[str, str | list[str]]:
+        """Return the effective settings suitable for published run metadata."""
+        return {
+            "cc": self.cc,
+            "cxx": self.cxx,
+            "library_cflags": self.library_cflags,
+            "library_cxxflags": self.library_cxxflags,
+            "harness_cflags": self.harness_cflags,
+            "harness_cxxflags": self.harness_cxxflags,
+            "library_configure_args": list(self.library_configure_args),
+        }
+
+
+# A sentinel with no attributes, so every getattr in from_args misses and each field falls
+# back to the environment-or-default rules.
+_NO_ARGUMENTS = object()
+
+
+def _argument_or_environment(args: object, argument: str, environment: str, default: str) -> str:
+    value = getattr(args, argument, None)
+    return value if isinstance(value, str) else os.environ.get(environment, default)
+
+
+def _argument_or_default(args: object, argument: str, default: str) -> str:
+    value = getattr(args, argument, None)
+    return value if isinstance(value, str) else default
+
+
+def _repeated_argument(args: object, argument: str) -> tuple[str, ...]:
+    value = getattr(args, argument, None)
+    if not isinstance(value, list):
+        return ()
+    return tuple(str(item) for item in value)
+
+
+_COMPILER_ENVIRONMENT_NAMES = ("CC", "CXX", "CFLAGS", "CXXFLAGS")
+
+
+@contextmanager
+def neutral_compiler_environment() -> Iterator[None]:
+    """Unset CC/CXX/CFLAGS/CXXFLAGS for the duration.
+
+    For anything that runs both the library build and the harness compile in one invocation:
+    the build gate, and a repair agent, which reaches the gate through check_build.sh and so
+    passes on whatever its own shell exports. Each generated script bakes in its own settings,
+    so with nothing exported both get the right ones. Leaving one stage's environment in place
+    applies its flags to both, and the harness flags in particular (`-fsanitize=fuzzer`
+    supplies its own main) make cmake's compiler check and autotools' configure link test fail.
+    """
+    original = {name: os.environ.pop(name, None) for name in _COMPILER_ENVIRONMENT_NAMES}
+    try:
+        yield
+    finally:
+        for name, previous in original.items():
+            if previous is not None:
+                os.environ[name] = previous
+
+
+@contextmanager
+def compile_commands_capture_environment() -> Iterator[None]:
+    """Ask CMake to write build/compile_commands.json for the duration.
+
+    CMake reads this as the default for the cache entry, so the flag never has to appear in the
+    generated build_library.sh -- capture stays something the harness applies, and the shipped
+    script is unaffected. Meson's Ninja backend writes the file unconditionally; Make and
+    Autotools have no equivalent and rely on the gate's `bear` wrap instead.
+    """
+    with _temporary_environment({"CMAKE_EXPORT_COMPILE_COMMANDS": "ON"}):
+        yield
+
+
+@contextmanager
+def _temporary_environment(values: dict[str, str]) -> Iterator[None]:
+    original = {name: os.environ.get(name) for name in values}
+    os.environ.update(values)
+    try:
+        yield
+    finally:
+        for name, previous in original.items():
+            if previous is None:
+                os.environ.pop(name, None)
+            else:
+                os.environ[name] = previous

fuzzprep/library_builder/dependency_resolution.py

@@ -0,0 +1,139 @@
+"""Single dependency-resolution/merge point for library_builder.
+
+Every dependency, whatever discovered it, becomes a `LibraryDependency` tagged with a
+`DependencySource` and reaches the persisted state through one `merge()`.
+"""
+
+from __future__ import annotations
+
+import itertools
+import json
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+
+
+class DependencySource(StrEnum):
+    LINKER = "linker"
+    LIBRARY_AGENT = "library_agent"
+    HARNESS_AGENT = "harness_agent"
+
+
+@dataclass(frozen=True)
+class LibraryDependency:
+    """One dependency, possibly only partially resolved. Only `source` is required."""
+
+    source: DependencySource
+    name: str | None = None
+    apt_package: str | None = None
+
+
+@dataclass
+class DependencyState:
+    """Persisted dependency-resolution state — the on-disk shape of state.json."""
+
+    version: int = 1
+    apt_packages: list[str] = field(default_factory=list)
+    sources: dict[str, list[str]] = field(default_factory=dict)
+
+
+def load_state(path: Path) -> DependencyState:
+    """Load state.json for a project; return an empty DependencyState if absent/malformed."""
+    if not path.exists():
+        return DependencyState()
+    try:
+        data = json.loads(path.read_text())
+    except (json.JSONDecodeError, OSError):
+        return DependencyState()
+
+    state = DependencyState()
+    if isinstance(data.get("apt_packages"), list):
+        state.apt_packages = [str(p) for p in data["apt_packages"]]
+    if isinstance(data.get("sources"), dict):
+        state.sources = {
+            k: [str(p) for p in v] for k, v in data["sources"].items() if isinstance(v, list)
+        }
+    return state
+
+
+def save_state(path: Path, state: DependencyState) -> None:
+    """Write state.json for a project."""
+    path.write_text(
+        json.dumps(
+            {
+                "version": state.version,
+                "apt_packages": state.apt_packages,
+                "sources": state.sources,
+            },
+            indent=2,
+        )
+    )
+
+
+def merge(state: DependencyState, dependencies: list[LibraryDependency]) -> None:
+    """Union dependencies into state in-place, deduplicating while preserving order.
+
+    The only function permitted to mutate DependencyState.apt_packages/sources, and
+    idempotent: merging the same list twice leaves the same state as merging it once.
+    """
+    new_apt: list[str] = []
+    new_apt_by_source: dict[str, list[str]] = {}
+
+    for dep in dependencies:
+        if dep.apt_package is not None:
+            new_apt.append(dep.apt_package)
+            new_apt_by_source.setdefault(dep.source.value, []).append(dep.apt_package)
+
+    state.apt_packages = list(dict.fromkeys(state.apt_packages + new_apt))
+    for source_tag, apt_names in new_apt_by_source.items():
+        existing = state.sources.get(source_tag, [])
+        state.sources[source_tag] = list(dict.fromkeys(existing + apt_names))
+
+
+def from_deterministic_probe(
+    missing_system_libs: list[str], transitive_link_flags: list[str]
+) -> list[LibraryDependency]:
+    """Translate the deterministic harness-link probe's outputs into LibraryDependency entries.
+
+    Unions the libs the linker reported missing with the bare names in transitive_link_flags,
+    which it resolved silently because the exploration host already had them. Each name is
+    translated on its own, so a system library — which translate() drops without recording it
+    as unknown — is excluded rather than misclassified.
+    """
+    from fuzzprep.library_builder.harness_explorer import lib_names_from_link_flags
+    from fuzzprep.library_builder.package_names import translate
+
+    names_from_flags = lib_names_from_link_flags(transitive_link_flags)
+    names = list(dict.fromkeys(missing_system_libs + names_from_flags))
+
+    dependencies: list[LibraryDependency] = []
+    for name in names:
+        translation = translate([name])
+        if not (translation.apt_packages or translation.unknown_libs):
+            continue  # a system library, which translate() drops silently
+        dependencies.append(
+            LibraryDependency(
+                source=DependencySource.LINKER,
+                name=name,
+                apt_package=translation.apt_packages[0] if translation.apt_packages else None,
+            )
+        )
+    return dependencies
+
+
+def from_agent_report(
+    missing_libs: list[str],
+    missing_apt_packages: list[str],
+    *,
+    source: DependencySource,
+) -> list[LibraryDependency]:
+    """Wrap an agent's self-reported dependency lists into LibraryDependency entries.
+
+    Zips the two lists positionally: index i's name and apt package are only *assumed* to
+    describe the same dependency. Nothing guarantees that when an agent reports more than one
+    dependency in a run — a known limitation.
+    """
+    return [
+        LibraryDependency(source=source, name=name, apt_package=apt)
+        for name, apt in itertools.zip_longest(missing_libs, missing_apt_packages)
+    ]

fuzzprep/library_builder/environments/base.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from enum import Enum
+from pathlib import Path
+from typing import TYPE_CHECKING, Protocol
+
+from fuzzprep.library_builder.timeouts import DEFAULT_BUILD_TIMEOUT_SECONDS
+
+if TYPE_CHECKING:
+    from fuzzprep.library_builder.build_parameters import BuildParameters
+    from fuzzprep.library_builder.models import (
+        AnalysisResult,
+        BuildExplorationResult,
+        HarnessExplorationResult,
+        Language,
+    )
+
+
+class Environment(Enum):
+    LOCAL = "local"
+    OSS_FUZZ = "oss-fuzz"
+
+    @property
+    def harness_probe_command(self) -> list[str]:
+        """How to recompile the harnesses in this environment.
+
+        The generated scripts are environment-independent; entering them is not. OSS-Fuzz goes
+        through the base image's `compile`, which assembles what the image only half-provides:
+        it resolves SANITIZER_FLAGS into CFLAGS/CXXFLAGS, and exports
+        LIB_FUZZING_ENGINE=-fsanitize=fuzzer in place of the deprecated archive path the
+        image's ENV names. Running compile_harnesses.sh directly there fails on that missing
+        archive, or — if only the engine flag is patched — links an uninstrumented harness.
+
+        `compile` runs build.sh, so it re-enters the library build too. build_library.sh's
+        skip-if-already-built guard keeps that cheap enough to repeat per discovery attempt,
+        and the real link line is what makes discovery faithful: sanitizer flags change which
+        symbols come out undefined.
+
+        Hence --base-image accepts OSS-Fuzz-style images only, and OssFuzzExecutor checks for
+        `compile` before probing.
+        """
+        if self is Environment.OSS_FUZZ:
+            return ["bash", "-c", "compile"]
+        return ["bash", "compile_harnesses.sh"]
+
+
+class EnvironmentUnavailableError(Exception):
+    """The selected environment itself is not usable (e.g. Docker daemon unreachable).
+
+    Not a build failure: callers must not route it to agent fallback, since no edit to a
+    build script fixes an unavailable environment.
+    """
+
+    def __init__(self, message: str, environment: Environment) -> None:
+        super().__init__(message)
+        self.environment = environment
+
+
+class EnvironmentExecutor(Protocol):
+    """Runs the two generate pipeline stages in a specific target environment."""
+
+    def run_library_build(
+        self,
+        analysis: AnalysisResult,
+        workdir: Path,
+        *,
+        timeout: int = DEFAULT_BUILD_TIMEOUT_SECONDS,
+        parameters: BuildParameters | None = None,
+    ) -> BuildExplorationResult: ...
+
+    def run_harness_compile(  # noqa: PLR0913 -- paths and build configuration are independent inputs
+        self,
+        install_dir: Path,
+        workdir: Path,
+        language: Language,
+        *,
+        extra_include_paths: list[str] | None = None,
+        extra_library_paths: list[str] | None = None,
+        parameters: BuildParameters | None = None,
+        library_llm_used: bool = False,
+    ) -> HarnessExplorationResult: ...
+
+    def check_availability(self, parameters: BuildParameters | None = None) -> None:
+        """Raise EnvironmentUnavailableError if this environment cannot be used.
+
+        `parameters` is the run's resolved compiler settings, since what an environment needs
+        can depend on them: the local host must provide the harness toolchain itself.
+        """
+        ...

fuzzprep/library_builder/environments/gate.py

@@ -0,0 +1,96 @@
+"""Applying the shared build gate to a stage's probe result.
+
+Both executors reach the same conclusion the same way, so the decision lives here once:
+run the gate when the probe found something worth gating, and otherwise report the command
+that would reproduce the failure without paying to rerun it.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from pathlib import Path
+
+from fuzzprep.library_builder import workspace
+from fuzzprep.library_builder.build_parameters import (
+    compile_commands_capture_environment,
+    neutral_compiler_environment,
+)
+from fuzzprep.library_builder.environments import verification
+from fuzzprep.library_builder.environments.base import Environment
+from fuzzprep.library_builder.models import HarnessExplorationResult
+
+
+def apply_to_harness_result(  # noqa: PLR0913 -- 4 keyword-only inputs, each independently meaningful
+    harness_result: HarnessExplorationResult,
+    workdir: Path,
+    *,
+    environment: Environment,
+    project_name: str,
+    library_llm_used: bool = False,
+    bypass_scratch_validation: bool = False,
+) -> HarnessExplorationResult:
+    """Gate a harness probe's outcome with agents/scripts/check_build.sh.
+
+    Skipped when the probe already failed or had nothing to link against: the gate would only
+    reconfirm the same failure at the cost of a full rebuild. Its command is still reported, so
+    the diagnostic says how to reproduce it.
+
+    The gate rebuilds the library from nothing only when something changed since the last
+    cold build, which is exactly when a repair agent produced this one. On the deterministic
+    lane explore() already deleted build/ and install/ before building, so a rebuild here
+    would repeat that build to reach the same state -- the run's single largest avoidable
+    cost. bypass_scratch_validation drops the rebuild on the agent lane too, trading the
+    from-scratch guarantee for speed at the caller's explicit request.
+    """
+    keep_artifacts = verification.gate_keeps_artifacts(
+        library_llm_used=library_llm_used,
+        bypass_scratch_validation=bypass_scratch_validation,
+    )
+    command = verification.verification_command(
+        workdir,
+        environment=environment,
+        project_name=project_name,
+        keep_artifacts=keep_artifacts,
+    )
+    if not harness_result.static_libs or not harness_result.succeeded:
+        return dataclasses.replace(
+            harness_result, command=command, gate_keeps_artifacts=keep_artifacts
+        )
+
+    # The gate builds the library and compiles harnesses in one invocation, so no stage's
+    # compiler environment may be in effect: each generated script bakes in its own.
+    with neutral_compiler_environment(), compile_commands_capture_environment():
+        result = verification.run_verification(
+            workdir,
+            environment=environment,
+            project_name=project_name,
+            keep_artifacts=keep_artifacts,
+        )
+    _localize_compile_commands(workdir, environment=environment)
+    return dataclasses.replace(
+        harness_result,
+        succeeded=result.passed,
+        command=result.command,
+        stdout=result.stdout,
+        stderr=result.stderr,
+        exit_code=0 if result.passed else 1,
+        duration_seconds=result.duration_seconds,
+        gate_keeps_artifacts=keep_artifacts,
+    )
+
+
+def _localize_compile_commands(workdir: Path, *, environment: Environment) -> None:
+    """Rewrite a container-built capture's /src paths back to the host workspace.
+
+    Done here because here is where the container build happens, so every lane -- deterministic
+    or agent-repaired -- gets the same treatment. Keyed to the library-build result instead, it
+    was skipped whenever a repair agent produced the build, publishing container paths.
+    """
+    if environment is not Environment.OSS_FUZZ:
+        return
+    # Imported here: oss_fuzz imports this module, so a module-level import would be circular.
+    from fuzzprep.library_builder.environments.oss_fuzz import rewrite_compile_commands_paths
+
+    captured = workspace.find_compile_commands(workdir)
+    if captured is not None:
+        rewrite_compile_commands_paths(captured, workdir)