fuzzprep 0.1.0__py3-none-any.whl

fuzzprep/core/repos.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import logging
+import re
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from fuzzprep.core.paths import project_state_file
+
+logger = logging.getLogger(__name__)
+
+_PROJECT_NAME_PATTERN = re.compile(r"[a-z0-9][a-z0-9_.+-]*")
+
+
+class RepositoryNotFoundError(Exception):
+    """Local path does not exist or is not a directory."""
+
+
+class NoCloneableOriginError(Exception):
+    """Local repository has no cloneable git remote origin."""
+
+
+class CloneFailedError(Exception):
+    """A git operation against the remote repository failed, carrying git's own stderr."""
+
+
+class LocalRepoRefError(Exception):
+    """--repo-ref was combined with a local path, where it cannot be honoured."""
+
+
+class InvalidProjectNameError(Exception):
+    """The project name is not a single safe path component, or names a workspace outside
+    the state directory."""
+
+
+@dataclass
+class RepoSource:
+    source_path: Path
+    clone_url: str
+    project_name: str
+    repo_ref: str | None = None
+
+
+def name_from_url(url: str) -> str:
+    """Infer a project name from a repository URL basename."""
+    name = url.rstrip("/").split("/")[-1]
+    if name.endswith(".git"):
+        name = name[:-4]
+    return name or "project"
+
+
+def _validated_project_name(name: str) -> str:
+    """Return name if it is a single path component safe to join onto the state directory.
+
+    The name becomes a directory under the state directory, and that directory is emptied on
+    every re-run. `Path("state") / "/tmp/x"` is `/tmp/x` and `Path("state") / ".."` is the state
+    directory's parent, so an unchecked name aims the reset at an arbitrary directory. It also
+    becomes a Docker image name component, which is why the character set is the one Docker
+    accepts rather than merely everything a filesystem tolerates.
+
+    Raises InvalidProjectNameError if the name is empty, `.`, `..`, contains a path separator,
+    or holds a character outside [a-z0-9_.+-].
+    """
+    if not _PROJECT_NAME_PATTERN.fullmatch(name):
+        raise InvalidProjectNameError(
+            f"Invalid project name {name!r}: it becomes a directory under the state directory "
+            "and a Docker image name, so it must be a single component starting with a letter "
+            "or digit and holding only [a-z0-9_.+-] — no path separators, `.`, or `..`. "
+            "Pass --project-name NAME to choose one."
+        )
+    return name
+
+
+def clean_project_dir(project_dir: Path, keep: set[Path], *, state_dir: Path) -> None:
+    """Empty project_dir, preserving the paths in keep and any child that contains one.
+
+    A kept path is not always a direct child, so containment is matched as well as equality:
+    deleting the child that holds a nested kept path would take it along too.
+
+    Raises InvalidProjectNameError if project_dir is not a direct child of state_dir. Name
+    validation already rejects the names that would break that, so this is the backstop for
+    the next caller that builds a project directory some other way — including through a
+    symlink planted where the workspace goes.
+    """
+    if project_dir.resolve().parent != state_dir.resolve():
+        raise InvalidProjectNameError(
+            f"Refusing to reset {project_dir}: it is not a direct child of the state directory "
+            f"{state_dir}, and resetting it would delete files FuzzPrep does not own."
+        )
+    keep_resolved = {p.resolve() for p in keep}
+    for child in project_dir.iterdir():
+        resolved = child.resolve()
+        if any(kept == resolved or kept.is_relative_to(resolved) for kept in keep_resolved):
+            continue
+        if child.is_dir() and not child.is_symlink():
+            shutil.rmtree(child)
+        else:
+            child.unlink()
+
+
+def _run_git(command: list[str], *, cwd: Path | None = None) -> None:
+    """Run a git command, raising CloneFailedError with git's own stderr on failure.
+
+    `check=True` would surface an unreachable host or a bad ref as a CalledProcessError
+    traceback; callers need a typed failure to report as an ingestion diagnostic.
+    """
+    result = subprocess.run(command, cwd=cwd, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise CloneFailedError(
+            f"git {' '.join(command[1:])} failed (exit {result.returncode}):\n"
+            f"{result.stderr.strip()}"
+        )
+
+
+def ingest_url(
+    url: str,
+    *,
+    project_name: str | None = None,
+    repo_ref: str | None = None,
+    state_dir: Path,
+) -> RepoSource:
+    """Clone a remote repository into state_dir and return a RepoSource.
+
+    The repo_ref checkout and the submodule update run on the fresh-clone and already-cloned
+    paths alike: `git clean -fdx` wipes untracked submodule content and a checkout can move
+    submodule pointers, so without the update a re-run builds against empty or stale trees.
+
+    state.json survives the workspace wipe, so the dependencies learned across prior runs are
+    not discarded on every re-run.
+
+    Raises InvalidProjectNameError if the project name — given or inferred from the URL — is not
+    a single safe path component.
+    Raises CloneFailedError if a git operation fails.
+    """
+    given_or_inferred = project_name if project_name is not None else name_from_url(url)
+    name = _validated_project_name(given_or_inferred.lower())
+    project_dir = state_dir / name
+    dest = project_dir / "src"
+    repo_source = RepoSource(source_path=dest, clone_url=url, project_name=name, repo_ref=repo_ref)
+    state_file = project_state_file(state_dir, name)
+
+    if not dest.exists():
+        _run_git(["git", "clone", "--recursive", url, str(dest)])
+    else:
+        clean_project_dir(project_dir, keep={dest, state_file}, state_dir=state_dir)
+        _run_git(["git", "reset", "--hard"], cwd=dest)
+        _run_git(["git", "clean", "-fdx"], cwd=dest)
+    if repo_ref:
+        _run_git(["git", "checkout", repo_ref], cwd=dest)
+    _run_git(["git", "submodule", "update", "--init", "--recursive"], cwd=dest)
+    return repo_source
+
+
+def ingest_local(
+    path: Path,
+    *,
+    project_name: str | None = None,
+    repo_ref: str | None = None,
+    state_dir: Path,
+) -> RepoSource:
+    """Validate a local repository path and return a RepoSource.
+
+    Resets the project workspace as ingest_url does, so a previous run's Dockerfile, scripts,
+    and agent report cannot be mistaken for this run's. The user's source directory is never
+    touched, including when it sits inside the workspace being reset — a deliberate layout,
+    since staging a copy at <state_dir>/<project>/src is what earns $SCRIPT_DIR/src-relative
+    scripts in the output. Passing it to `keep` is what makes that safe.
+
+    Raises RepositoryNotFoundError if the path does not exist or is not a directory.
+    Raises InvalidProjectNameError if the project name is not a single safe path component.
+    Raises NoCloneableOriginError if the repository has no cloneable git remote origin.
+    Raises LocalRepoRefError if repo_ref is set: honouring it would check out a ref in a
+    working tree the user owns, and ignoring it would ship a setup.sh and Dockerfile pinning a
+    ref that was never built.
+    """
+    if not path.exists() or not path.is_dir():
+        raise RepositoryNotFoundError(f"Local path does not exist or is not a directory: {path}")
+    if repo_ref is not None:
+        raise LocalRepoRefError(
+            f"--repo-ref {repo_ref!r} cannot be applied to the local path {path}: "
+            "FuzzPrep will not check out a ref in a working tree you own, and the "
+            "generated output would otherwise claim a ref it never built. "
+            f"Check out {repo_ref!r} yourself first, or pass the repository URL instead."
+        )
+    given_or_inferred = project_name if project_name is not None else path.name
+    name = _validated_project_name(given_or_inferred.lower())
+    origin = _get_git_origin(path)
+    if origin is None:
+        raise NoCloneableOriginError(
+            f"Local repository has no cloneable git origin: {path}. "
+            "Generated Dockerfiles require a cloneable URL. "
+            "Add a remote with: git remote add origin <url>"
+        )
+    project_directory = state_dir / name
+    if project_directory.is_dir():
+        clean_project_dir(
+            project_directory,
+            keep={project_state_file(state_dir, name), path.resolve()},
+            state_dir=state_dir,
+        )
+    return RepoSource(source_path=path, clone_url=origin, project_name=name, repo_ref=repo_ref)
+
+
+def _get_git_origin(path: Path) -> str | None:
+    """Return the git remote origin URL, or None if unavailable."""
+    result = subprocess.run(
+        ["git", "remote", "get-url", "origin"],
+        cwd=path,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode == 0:
+        url = result.stdout.strip()
+        return url if url else None
+    return None

fuzzprep/core/resources.py

@@ -0,0 +1,41 @@
+"""Locating the data files FuzzPrep ships: agent skills and the build gate scripts.
+
+These live under `src/fuzzprep/agents/` and resolve through `importlib.resources`, so a
+git checkout and an installed wheel find them the same way. A missing file is an error, not a
+degraded mode: without the gate scripts no build can be verified, and without a skill file a
+repair agent gets a one-sentence prompt instead of its instructions.
+"""
+
+from __future__ import annotations
+
+from importlib.resources import as_file, files
+from pathlib import Path
+
+_AGENTS_PACKAGE = "fuzzprep.agents"
+
+
+def agent_script(name: str) -> Path:
+    """Return the filesystem path of agents/scripts/<name>.
+
+    A path rather than a stream: these are shell scripts passed as subprocess arguments, and
+    one of them is bind-mounted into a container.
+    """
+    return _resource_path(f"scripts/{name}")
+
+
+def skill_instructions(agent_name: str) -> str:
+    """Return the text of agents/<agent_name>/SKILL.md — a repair agent's instructions."""
+    return _resource_path(f"{agent_name}/SKILL.md").read_text()
+
+
+def _resource_path(relative_path: str) -> Path:
+    resource = files(_AGENTS_PACKAGE).joinpath(relative_path)
+    # as_file materializes the resource if the distribution is zipped. FuzzPrep installs
+    # unzipped, so the path stays valid after the context exits.
+    with as_file(resource) as path:
+        if not path.is_file():
+            raise FileNotFoundError(
+                f"FuzzPrep resource {relative_path!r} is missing from the "
+                f"{_AGENTS_PACKAGE} package — the installation is incomplete."
+            )
+        return path

fuzzprep/core/subprocesses.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import contextlib
+import contextvars
+import os
+import signal
+import subprocess
+import threading
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import IO
+
+
+class MergedOutput:
+    """Adds `output` to a result type that carries a command's two output streams.
+
+    Whether `stderr` holds anything depends on the runner: `run_command_streaming` merges it
+    into stdout and leaves it empty, while `run_command` keeps the two apart. A caller
+    scanning for diagnostics wants both and should not have to know which runner it got.
+    """
+
+    stdout: str
+    stderr: str
+
+    @property
+    def output(self) -> str:
+        return self.stdout + self.stderr
+
+
+@dataclass
+class RunResult(MergedOutput):
+    stdout: str
+    stderr: str
+    exit_code: int
+    duration_seconds: float
+
+
+# (command, cwd, timeout) -> RunResult — the one shape host-subprocess and docker-wrapped
+# execution share, so a caller can swap how a command runs without touching its own logic.
+Runner = Callable[[list[str], Path, int], RunResult]
+
+_quiet: contextvars.ContextVar[bool] = contextvars.ContextVar("fuzzprep_quiet", default=False)
+_log_path: contextvars.ContextVar[Path | None] = contextvars.ContextVar(
+    "fuzzprep_log_path", default=None
+)
+
+
+@contextlib.contextmanager
+def streaming_context(*, quiet: bool = False, log_path: Path | None = None) -> Iterator[None]:
+    """Scope `run_command_streaming`'s live-printing and log-file destination for every call
+    made while this context is active.
+
+    Set once at a phase boundary in cli.py and read implicitly wherever
+    `run_command_streaming` is invoked, so the modules in between need no quiet/log_path
+    parameter of their own.
+    """
+    quiet_token = _quiet.set(quiet)
+    log_path_token = _log_path.set(log_path)
+    try:
+        yield
+    finally:
+        _quiet.reset(quiet_token)
+        _log_path.reset(log_path_token)
+
+
+def _write_log(log_path: Path | None, text: str) -> None:
+    if log_path is None:
+        return
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    log_path.write_text(text)
+
+
+# How long to wait for the reader thread to notice the pipe closed after a kill. The thread
+# is a daemon, so an unresponsive grandchild holding stdout costs this much delay and is then
+# abandoned rather than deadlocking the run.
+_DRAIN_GRACE_SECONDS = 5
+
+
+def _drain(stream: IO[str], lines: list[str], *, quiet: bool) -> None:
+    """Read stream to EOF, collecting each line and optionally echoing it live."""
+    for line in stream:
+        if not quiet:
+            print(line, end="", flush=True)
+        lines.append(line)
+
+
+def _terminate_process_group(proc: subprocess.Popen[str]) -> None:
+    """Kill the child and anything it spawned.
+
+    A build runs `bash build_library.sh`, which spawns make, which spawns compilers. Killing
+    only the shell leaves that tree running and holding the stdout pipe open, so a timed-out
+    run never returns. `run_command_streaming` starts the child in its own session so the
+    whole tree can be signalled by process group here.
+    """
+    try:
+        os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+    except (ProcessLookupError, PermissionError):
+        proc.kill()
+
+
+def run_command_streaming(command: list[str], cwd: Path, timeout: int) -> RunResult:
+    """Run command, printing each line in real-time while capturing combined output.
+
+    `timeout` bounds the whole call, not just the wait after the child closes stdout, so a
+    command that hangs while producing no output is killed at the deadline and reported as
+    `exit_code=-1`, matching `run_command`. Output from before the deadline is still returned.
+
+    The active `streaming_context` decides whether the live per-line printing happens. The
+    full output always goes to that context's log_path, on success, failure, or timeout.
+    """
+    quiet = _quiet.get()
+    log_path = _log_path.get()
+    start = time.monotonic()
+    lines: list[str] = []
+    proc = subprocess.Popen(
+        command,
+        cwd=cwd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        start_new_session=True,
+    )
+    assert proc.stdout is not None
+    reader = threading.Thread(
+        target=_drain, args=(proc.stdout, lines), kwargs={"quiet": quiet}, daemon=True
+    )
+    reader.start()
+    try:
+        exit_code = _await_exit(proc, reader, timeout=timeout, start=start)
+    except KeyboardInterrupt:
+        # start_new_session detached the child, so the terminal's Ctrl-C never reached it.
+        # Pass it on rather than orphaning a running build.
+        _terminate_process_group(proc)
+        raise
+    _write_log(log_path, "".join(lines))
+    return RunResult(
+        stdout="".join(lines),
+        stderr="",
+        exit_code=exit_code,
+        duration_seconds=time.monotonic() - start,
+    )
+
+
+def _await_exit(
+    proc: subprocess.Popen[str], reader: threading.Thread, *, timeout: int, start: float
+) -> int:
+    """Wait for proc to finish within the deadline, killing it if it doesn't.
+
+    Returns its exit code, or -1 on timeout.
+    """
+    reader.join(timeout)
+    if not reader.is_alive():
+        remaining = max(timeout - (time.monotonic() - start), 0.0)
+        with contextlib.suppress(subprocess.TimeoutExpired):
+            return proc.wait(timeout=remaining)
+    _terminate_process_group(proc)
+    reader.join(_DRAIN_GRACE_SECONDS)
+    proc.wait()
+    return -1
+
+
+def run_command(command: list[str], cwd: Path, timeout: int) -> RunResult:
+    start = time.monotonic()
+    try:
+        completed = subprocess.run(
+            command,
+            cwd=cwd,
+            timeout=timeout,
+            capture_output=True,
+            text=True,
+        )
+        duration = time.monotonic() - start
+        return RunResult(
+            stdout=completed.stdout,
+            stderr=completed.stderr,
+            exit_code=completed.returncode,
+            duration_seconds=duration,
+        )
+    except subprocess.TimeoutExpired as exc:
+        duration = time.monotonic() - start
+        return RunResult(
+            stdout=_decode_output(exc.stdout),
+            stderr=_decode_output(exc.stderr),
+            exit_code=-1,
+            duration_seconds=duration,
+        )
+
+
+def _decode_output(output: bytes | str | None) -> str:
+    if output is None:
+        return ""
+    if isinstance(output, bytes):
+        return output.decode(errors="replace")
+    return output

fuzzprep/feature_extractor/benchmark_yaml.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from fuzzprep.feature_extractor.extraction import (
+    MissingFeatureArtifactError,
+    load_feature_artifact,
+)
+from fuzzprep.feature_extractor.models import (
+    BenchmarkFunction,
+    BenchmarkYaml,
+    FunctionSignature,
+)
+from fuzzprep.library_builder.models import Language
+
+_DEFAULT_TARGET_NAME = "default_fuzzer"
+_INCLUDE_HEADERS = []
+_EXT_BY_LANGUAGE = {Language.C: "c", Language.CPP: "cc"}
+
+
+def select_public(f: FunctionSignature) -> bool:
+    return f.is_public_api
+
+
+def select_by_header(f: FunctionSignature) -> bool:
+    return Path(f.header_path).name in _INCLUDE_HEADERS
+
+
+def generate_benchmark(
+    output_dir: Path,
+    headers: list[str],
+    target_name: str | None = None,
+    target_path: str | None = None,
+) -> BenchmarkYaml:
+    """Convert <output_dir>/features.json into an oss-fuzz-gen-compatible YAML benchmark.
+
+    Keeps only public-API functions, or only those declared in `headers` when it is non-empty,
+    and overwrites <output_dir>/<project_name>.yaml.
+    """
+    check_include_func = select_public
+    if len(headers) > 0:
+        global _INCLUDE_HEADERS
+        _INCLUDE_HEADERS = headers
+        check_include_func = select_by_header
+    features_json = output_dir / "features.json"
+    if not features_json.is_file():
+        raise MissingFeatureArtifactError(
+            f"{features_json} not found. Run 'fuzzprep extract-features' first."
+        )
+    artifact = load_feature_artifact(features_json)
+
+    resolved_target_name = target_name or _DEFAULT_TARGET_NAME
+    ext = _EXT_BY_LANGUAGE.get(artifact.language, "c")
+    resolved_target_path = target_path or f"/src/harness_source/{resolved_target_name}.{ext}"
+
+    benchmark = BenchmarkYaml(
+        project=artifact.project_name,
+        language=artifact.language.value,
+        target_name=resolved_target_name,
+        target_path=resolved_target_path,
+        functions=[
+            BenchmarkFunction(
+                name=f.name, signature=f.signature, return_type=f.return_type, params=f.params
+            )
+            for f in artifact.functions
+            if check_include_func(f)
+        ],
+    )
+
+    output_path = output_dir / f"{artifact.project_name}.yaml"
+    output_path.write_text(yaml.safe_dump(_to_yaml_dict(benchmark), sort_keys=False))
+    return benchmark
+
+
+def _to_yaml_dict(benchmark: BenchmarkYaml) -> dict:
+    return {
+        "project": benchmark.project,
+        "language": benchmark.language,
+        "target_name": benchmark.target_name,
+        "target_path": benchmark.target_path,
+        "functions": [
+            {
+                "name": f.name,
+                "signature": f.signature,
+                "return_type": f.return_type,
+                "params": [{"name": p.name, "type": p.type} for p in f.params],
+            }
+            for f in benchmark.functions
+        ],
+    }