fuzzprep 0.1.0__py3-none-any.whl

fuzzprep/feature_extractor/extraction.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from fuzzprep.feature_extractor.models import (
+    SCHEMA_VERSION,
+    EnumDefinition,
+    Enumerator,
+    FeatureArtifactSet,
+    Field,
+    FunctionSignature,
+    MacroDefinition,
+    Param,
+    StructUnionDefinition,
+    Typedef,
+)
+from fuzzprep.library_builder.models import Language
+
+_REQUIRED_FIELDS = (
+    "schema_version",
+    "project_name",
+    "language",
+    "functions",
+    "typedefs",
+    "macros",
+    "enums",
+    "records",
+    "warnings",
+)
+
+_LANGUAGE_BY_VALUE = {lang.value: lang for lang in Language}
+
+
+class FeatureArtifactError(Exception):
+    """The feature artifact JSON is missing, malformed, or from an incompatible schema."""
+
+
+class MissingCompileCommandsError(Exception):
+    """The output directory has no compile_commands.json to extract features from."""
+
+
+class MissingFeatureArtifactError(Exception):
+    """The output directory has no features.json; extract-features must run first."""
+
+
+def extract_features(output_dir: Path) -> FeatureArtifactSet:
+    """Extract a library's declarations into features.json, returning the parsed result.
+
+    Reads <output_dir>/compile_commands.json, builds and invokes the native tool, and
+    overwrites <output_dir>/features.json with the result.
+    """
+    from fuzzprep.core.subprocesses import run_command_streaming
+    from fuzzprep.feature_extractor.native_build import build_native_tool
+
+    compile_commands = output_dir / "compile_commands.json"
+    if not compile_commands.is_file():
+        raise MissingCompileCommandsError(
+            f"{compile_commands} not found. Generate one with CMake "
+            "(-DCMAKE_EXPORT_COMPILE_COMMANDS=ON) or by wrapping a non-CMake build "
+            "with 'bear -- <build command>'."
+        )
+
+    binary = build_native_tool()
+    features_json = output_dir / "features.json"
+    project_name = output_dir.resolve().name
+    cmd = [str(binary.resolve()), str(output_dir.resolve()), str(features_json), project_name]
+    result = run_command_streaming(
+        cmd,
+        Path.cwd(),
+        timeout=600,
+    )
+    if result.exit_code != 0:
+        raise FeatureArtifactError(
+            f"Native feature_extractor tool failed\n"
+            "Run below command to reproduce:\n"
+            f"cd {Path.cwd()} && "
+            f"{' '.join(cmd)}"
+        )
+    features_json.write_text(json.dumps(json.loads(features_json.read_text()), indent=2) + "\n")
+    return load_feature_artifact(features_json)
+
+
+def load_feature_artifact(path: Path) -> FeatureArtifactSet:
+    """Parse and validate a features.json file into a typed FeatureArtifactSet.
+
+    Raises FeatureArtifactError with an actionable message on a missing file,
+    malformed JSON, missing/mistyped fields, or a schema_version mismatch.
+    """
+    if not path.exists():
+        raise FeatureArtifactError(f"{path} does not exist. Run 'fuzzprep extract-features' first.")
+    try:
+        data = json.loads(path.read_text())
+    except json.JSONDecodeError as exc:
+        raise FeatureArtifactError(f"{path} is not valid JSON: {exc}") from exc
+
+    if not isinstance(data, dict):
+        raise FeatureArtifactError(f"{path} does not contain a JSON object.")
+
+    project_name, language = _validate_header_fields(path, data)
+
+    try:
+        return FeatureArtifactSet(
+            schema_version=data["schema_version"],
+            project_name=project_name,
+            language=language,
+            functions=[_parse_function(f) for f in data["functions"]],
+            typedefs=[_parse_typedef(t) for t in data["typedefs"]],
+            macros=[_parse_macro(m) for m in data["macros"]],
+            enums=[_parse_enum(e) for e in data["enums"]],
+            records=[_parse_record(r) for r in data["records"]],
+            warnings=[str(w) for w in data["warnings"]],
+        )
+    except (KeyError, TypeError) as exc:
+        raise FeatureArtifactError(f"{path} has a malformed entry: {exc}") from exc
+
+
+def _validate_header_fields(path: Path, data: dict) -> tuple[str, Language]:
+    """Validate schema_version/project_name/language, returning the latter two."""
+    missing = [key for key in _REQUIRED_FIELDS if key not in data]
+    if missing:
+        raise FeatureArtifactError(f"{path} is missing required field(s): {', '.join(missing)}")
+
+    if data["schema_version"] != SCHEMA_VERSION:
+        raise FeatureArtifactError(
+            f"{path} has schema_version {data['schema_version']!r}, "
+            f"expected {SCHEMA_VERSION}. Re-run 'fuzzprep extract-features'."
+        )
+
+    project_name = data["project_name"]
+    if not isinstance(project_name, str) or not project_name:
+        raise FeatureArtifactError(f"{path} has an empty or non-string project_name.")
+
+    language = _LANGUAGE_BY_VALUE.get(data["language"])
+    if language is None:
+        raise FeatureArtifactError(f"{path} has an unrecognized language: {data['language']!r}")
+
+    return project_name, language
+
+
+def _parse_function(data: dict) -> FunctionSignature:
+    return FunctionSignature(
+        name=data["name"],
+        return_type=data["return_type"],
+        params=[Param(name=p["name"], type=p["type"]) for p in data["params"]],
+        signature=data["signature"],
+        is_public_api=data["is_public_api"],
+        header_path=data["header_path"],
+    )
+
+
+def _parse_typedef(data: dict) -> Typedef:
+    return Typedef(
+        name=data["name"],
+        underlying_type=data["underlying_type"],
+        header_path=data["header_path"],
+    )
+
+
+def _parse_macro(data: dict) -> MacroDefinition:
+    return MacroDefinition(
+        name=data["name"],
+        is_function_like=data["is_function_like"],
+        params=[str(p) for p in data["params"]],
+        value=data["value"],
+        header_path=data["header_path"],
+    )
+
+
+def _parse_enum(data: dict) -> EnumDefinition:
+    return EnumDefinition(
+        name=data["name"],
+        enumerators=[Enumerator(name=e["name"], value=e["value"]) for e in data["enumerators"]],
+        header_path=data["header_path"],
+    )
+
+
+def _parse_record(data: dict) -> StructUnionDefinition:
+    return StructUnionDefinition(
+        name=data["name"],
+        kind=data["kind"],
+        fields=[Field(name=f["name"], type=f["type"]) for f in data["fields"]],
+        header_path=data["header_path"],
+    )

fuzzprep/feature_extractor/models.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from fuzzprep.library_builder.models import Language
+
+SCHEMA_VERSION = 1
+
+
+@dataclass
+class Param:
+    name: str
+    type: str
+
+
+@dataclass
+class FunctionSignature:
+    name: str
+    return_type: str
+    params: list[Param]
+    signature: str
+    is_public_api: bool
+    header_path: str
+
+
+@dataclass
+class Typedef:
+    name: str
+    underlying_type: str
+    header_path: str
+
+
+@dataclass
+class MacroDefinition:
+    name: str
+    is_function_like: bool
+    params: list[str]
+    value: str
+    header_path: str
+
+
+@dataclass
+class Enumerator:
+    name: str
+    value: int
+
+
+@dataclass
+class EnumDefinition:
+    name: str | None
+    enumerators: list[Enumerator]
+    header_path: str
+
+
+@dataclass
+class Field:
+    name: str
+    type: str
+
+
+@dataclass
+class StructUnionDefinition:
+    name: str | None
+    kind: str  # "struct" | "union"
+    fields: list[Field]
+    header_path: str
+
+
+@dataclass
+class FeatureArtifactSet:
+    schema_version: int
+    project_name: str
+    language: Language
+    functions: list[FunctionSignature] = field(default_factory=list)
+    typedefs: list[Typedef] = field(default_factory=list)
+    macros: list[MacroDefinition] = field(default_factory=list)
+    enums: list[EnumDefinition] = field(default_factory=list)
+    records: list[StructUnionDefinition] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+
+@dataclass
+class BenchmarkFunction:
+    name: str
+    signature: str
+    return_type: str
+    params: list[Param]
+
+
+@dataclass
+class BenchmarkYaml:
+    project: str
+    language: str
+    target_name: str
+    target_path: str
+    functions: list[BenchmarkFunction] = field(default_factory=list)

fuzzprep/feature_extractor/native/.clang-format

@@ -0,0 +1 @@
+BasedOnStyle: LLVM

fuzzprep/feature_extractor/native/CMakeLists.txt

@@ -0,0 +1,90 @@
+cmake_minimum_required(VERSION 3.20)
+project(feature_extractor CXX C)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
+
+find_package(Clang REQUIRED CONFIG)
+find_package(LLVM REQUIRED CONFIG)
+
+message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
+message(STATUS "Using ClangConfig.cmake in: ${Clang_DIR}")
+
+list(GET LLVM_LIBRARY_DIRS 0 _llvm_library_dir)
+set(CLANG_RESOURCE_DIR "${_llvm_library_dir}/clang/${LLVM_VERSION_MAJOR}")
+message(STATUS "Using clang resource dir: ${CLANG_RESOURCE_DIR}")
+
+# -resource-dir only covers clang's own builtin headers (stddef.h, stdarg.h, ...) — on
+# macOS, platform libc headers (inttypes.h, stdio.h, ...) live under the Xcode SDK and
+# are found only via -isysroot, which main.cpp adds explicitly rather than relying on
+# ClangTool's ad hoc driver invocation to auto-detect it.
+set(CLANG_SYSROOT "")
+if(APPLE)
+  execute_process(
+    COMMAND xcrun --show-sdk-path
+    OUTPUT_VARIABLE CLANG_SYSROOT
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET
+  )
+  message(STATUS "Using macOS SDK sysroot: ${CLANG_SYSROOT}")
+endif()
+
+# cJSON is linked statically into feature_extractor, so its MIT notice has to travel with any
+# redistributed binary. THIRD_PARTY_NOTICES.md at the repository root carries that notice —
+# update it when this pin moves or another dependency is added here.
+include(FetchContent)
+set(ENABLE_CJSON_TEST OFF CACHE BOOL "" FORCE)
+set(ENABLE_CJSON_UTILS OFF CACHE BOOL "" FORCE)
+set(CJSON_OVERRIDE_BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
+set(CJSON_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
+FetchContent_Declare(
+  cjson
+  GIT_REPOSITORY https://github.com/DaveGamble/cJSON.git
+  GIT_TAG c859b25da02955fef659d658b8f324b5cde87be3 # v1.7.19
+  GIT_SHALLOW TRUE
+)
+FetchContent_MakeAvailable(cjson)
+
+add_executable(feature_extractor
+  src/main.cpp
+  src/extraction_action.cpp
+  src/macro_callbacks.cpp
+  src/json_writer.cpp
+)
+
+target_include_directories(feature_extractor PRIVATE include)
+target_include_directories(feature_extractor SYSTEM PRIVATE
+  ${LLVM_INCLUDE_DIRS}
+  ${CLANG_INCLUDE_DIRS}
+  ${cjson_SOURCE_DIR}
+)
+target_compile_definitions(feature_extractor PRIVATE ${LLVM_DEFINITIONS})
+target_compile_definitions(feature_extractor PRIVATE
+  CLANG_RESOURCE_DIR="${CLANG_RESOURCE_DIR}"
+  CLANG_SYSROOT="${CLANG_SYSROOT}"
+)
+
+if(NOT LLVM_ENABLE_RTTI)
+  target_compile_options(feature_extractor PRIVATE -fno-rtti)
+endif()
+
+target_compile_options(feature_extractor PRIVATE
+  -Wall -Wextra -Werror -Wpedantic -Wshadow
+)
+
+target_link_libraries(feature_extractor PRIVATE
+  clangTooling
+  clangASTMatchers
+  clangBasic
+  clangFrontend
+  clangAST
+  clangLex
+  cjson
+)
+
+install(TARGETS feature_extractor RUNTIME DESTINATION bin)

fuzzprep/feature_extractor/native/include/feature_extractor.hpp

@@ -0,0 +1,136 @@
+#pragma once
+
+#include <clang/Basic/LangOptions.h>
+#include <clang/Basic/SourceManager.h>
+#include <clang/Frontend/FrontendAction.h>
+#include <clang/Lex/PPCallbacks.h>
+#include <clang/Tooling/Tooling.h>
+#include <llvm/ADT/StringRef.h>
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+namespace feature_extractor {
+
+struct Param {
+  std::string name;
+  std::string type;
+};
+
+struct FunctionInfo {
+  std::string name;
+  std::string return_type;
+  std::vector<Param> params;
+  std::string signature;
+  bool is_public_api = false;
+  std::string header_path;
+};
+
+struct TypedefInfo {
+  std::string name;
+  std::string underlying_type;
+  std::string header_path;
+};
+
+struct MacroInfo {
+  std::string name;
+  bool is_function_like = false;
+  std::vector<std::string> params;
+  std::string value;
+  std::string header_path;
+};
+
+struct Enumerator {
+  std::string name;
+  long long value = 0;
+};
+
+struct EnumInfo {
+  std::optional<std::string> name;
+  std::vector<Enumerator> enumerators;
+  std::string header_path;
+};
+
+struct Field {
+  std::string name;
+  std::string type;
+};
+
+struct RecordInfo {
+  std::optional<std::string> name;
+  std::string kind; // "struct" | "union"
+  std::vector<Field> fields;
+  std::string header_path;
+};
+
+struct FeatureArtifact {
+  int schema_version = 1;
+  std::string project_name;
+  std::string language; // "c" | "c++" — must match fuzzprep.library_builder.models.Language
+  std::vector<FunctionInfo> functions;
+  std::vector<TypedefInfo> typedefs;
+  std::vector<MacroInfo> macros;
+  std::vector<EnumInfo> enums;
+  std::vector<RecordInfo> records;
+  std::vector<std::string> warnings;
+};
+
+// The project source root: the common ancestor of every translation unit's file
+// path, which for an out-of-tree build is not where compile_commands.json lives.
+// header_path values are relative to it, and a declaration inside it counts as
+// library-owned rather than third-party or system.
+struct ProjectContext {
+  std::string project_root;
+};
+
+// Not thread-safe. ClangTool runs one translation unit at a time, so one
+// collector can be shared across every TU to accumulate and deduplicate
+// declarations that appear in more than one (via a shared header, say).
+class FeatureCollector {
+public:
+  void addFunction(FunctionInfo info);
+  void addTypedef(TypedefInfo info);
+  void addMacro(MacroInfo info);
+  void addEnum(EnumInfo info);
+  void addRecord(RecordInfo info);
+  void addWarning(std::string warning);
+
+  const FeatureArtifact &artifact() const { return artifact_; }
+
+private:
+  FeatureArtifact artifact_;
+  std::unordered_set<std::string> seen_functions_;
+  std::unordered_set<std::string> seen_typedefs_;
+  std::unordered_set<std::string> seen_macros_;
+  std::unordered_set<std::string> seen_enums_;
+  std::unordered_set<std::string> seen_records_;
+};
+
+// True when abs_path is inside ctx.project_root, i.e. library-owned rather than a
+// system or third-party header.
+bool isWithinProject(const ProjectContext &ctx, llvm::StringRef abs_path);
+
+// abs_path relative to ctx.project_root when it is within the project, and
+// unchanged otherwise.
+std::string relativeHeaderPath(const ProjectContext &ctx,
+                               llvm::StringRef abs_path);
+
+// Builds the ASTFrontendAction that extracts functions, typedefs, enums, and
+// records from one translation unit, and registers the macro-extracting
+// PPCallbacks alongside it. Both write into collector.
+std::unique_ptr<clang::tooling::FrontendActionFactory>
+newExtractionActionFactory(FeatureCollector &collector, ProjectContext ctx);
+
+// Registers a PPCallbacks that extracts macro definitions into collector.
+std::unique_ptr<clang::PPCallbacks>
+newMacroCollectorCallbacks(FeatureCollector &collector, ProjectContext ctx,
+                           const clang::SourceManager &sm,
+                           const clang::LangOptions &lang_opts);
+
+// Serializes artifact to the JSON shape features.json readers expect.
+std::string writeJson(const FeatureArtifact &artifact);
+
+} // namespace feature_extractor