fuzzprep 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. fuzzprep/__init__.py +3 -0
  2. fuzzprep/__main__.py +4 -0
  3. fuzzprep/agents/harness_builder/SKILL.md +156 -0
  4. fuzzprep/agents/library_builder/SKILL.md +147 -0
  5. fuzzprep/agents/scripts/check_build.sh +87 -0
  6. fuzzprep/agents/scripts/check_build_in_container.sh +73 -0
  7. fuzzprep/agents/scripts/check_dockerfile_from_scratch.sh +33 -0
  8. fuzzprep/cli.py +1109 -0
  9. fuzzprep/core/__init__.py +0 -0
  10. fuzzprep/core/agent_stream.py +304 -0
  11. fuzzprep/core/files.py +35 -0
  12. fuzzprep/core/paths.py +25 -0
  13. fuzzprep/core/reporting.py +218 -0
  14. fuzzprep/core/repos.py +217 -0
  15. fuzzprep/core/resources.py +41 -0
  16. fuzzprep/core/subprocesses.py +197 -0
  17. fuzzprep/feature_extractor/__init__.py +0 -0
  18. fuzzprep/feature_extractor/benchmark_yaml.py +92 -0
  19. fuzzprep/feature_extractor/extraction.py +184 -0
  20. fuzzprep/feature_extractor/models.py +96 -0
  21. fuzzprep/feature_extractor/native/.clang-format +1 -0
  22. fuzzprep/feature_extractor/native/CMakeLists.txt +90 -0
  23. fuzzprep/feature_extractor/native/include/feature_extractor.hpp +136 -0
  24. fuzzprep/feature_extractor/native/src/extraction_action.cpp +294 -0
  25. fuzzprep/feature_extractor/native/src/json_writer.cpp +154 -0
  26. fuzzprep/feature_extractor/native/src/macro_callbacks.cpp +75 -0
  27. fuzzprep/feature_extractor/native/src/main.cpp +130 -0
  28. fuzzprep/feature_extractor/native_build.py +99 -0
  29. fuzzprep/library_builder/__init__.py +0 -0
  30. fuzzprep/library_builder/agents.py +472 -0
  31. fuzzprep/library_builder/analysis.py +145 -0
  32. fuzzprep/library_builder/build_parameters.py +158 -0
  33. fuzzprep/library_builder/dependency_resolution.py +139 -0
  34. fuzzprep/library_builder/environments/__init__.py +0 -0
  35. fuzzprep/library_builder/environments/base.py +89 -0
  36. fuzzprep/library_builder/environments/gate.py +96 -0
  37. fuzzprep/library_builder/environments/local.py +205 -0
  38. fuzzprep/library_builder/environments/oss_fuzz.py +485 -0
  39. fuzzprep/library_builder/environments/verification.py +125 -0
  40. fuzzprep/library_builder/exploration.py +217 -0
  41. fuzzprep/library_builder/generation.py +325 -0
  42. fuzzprep/library_builder/harness_explorer.py +257 -0
  43. fuzzprep/library_builder/models.py +169 -0
  44. fuzzprep/library_builder/package_names.json +33 -0
  45. fuzzprep/library_builder/package_names.py +40 -0
  46. fuzzprep/library_builder/scripts.py +389 -0
  47. fuzzprep/library_builder/stats.py +102 -0
  48. fuzzprep/library_builder/symbol_patterns.json +65 -0
  49. fuzzprep/library_builder/timeouts.py +14 -0
  50. fuzzprep/library_builder/workspace.py +250 -0
  51. fuzzprep-0.1.0.dist-info/METADATA +255 -0
  52. fuzzprep-0.1.0.dist-info/RECORD +56 -0
  53. fuzzprep-0.1.0.dist-info/WHEEL +4 -0
  54. fuzzprep-0.1.0.dist-info/entry_points.txt +3 -0
  55. fuzzprep-0.1.0.dist-info/licenses/LICENSE +202 -0
  56. fuzzprep-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +52 -0
@@ -0,0 +1,294 @@
1
+ #include "feature_extractor.hpp"
2
+
3
+ #include <clang/AST/ASTConsumer.h>
4
+ #include <clang/AST/Decl.h>
5
+ #include <clang/AST/DeclCXX.h>
6
+ #include <clang/AST/RecursiveASTVisitor.h>
7
+ #include <clang/Frontend/CompilerInstance.h>
8
+ #include <llvm/ADT/SmallString.h>
9
+ #include <llvm/Support/Path.h>
10
+
11
+ namespace feature_extractor {
12
+
13
+ bool isWithinProject(const ProjectContext &ctx, llvm::StringRef abs_path) {
14
+ if (ctx.project_root.empty() || abs_path.empty()) {
15
+ return false;
16
+ }
17
+ llvm::SmallString<256> root(ctx.project_root);
18
+ llvm::sys::path::remove_dots(root, /*remove_dot_dot=*/true);
19
+ llvm::SmallString<256> path(abs_path);
20
+ llvm::sys::path::remove_dots(path, /*remove_dot_dot=*/true);
21
+ return path.str().starts_with(root.str());
22
+ }
23
+
24
+ std::string relativeHeaderPath(const ProjectContext &ctx,
25
+ llvm::StringRef abs_path) {
26
+ if (!isWithinProject(ctx, abs_path)) {
27
+ return std::string(abs_path);
28
+ }
29
+ llvm::StringRef rel = abs_path.drop_front(ctx.project_root.size());
30
+ while (!rel.empty() && rel.front() == '/') {
31
+ rel = rel.drop_front(1);
32
+ }
33
+ return std::string(rel);
34
+ }
35
+
36
+ void FeatureCollector::addFunction(FunctionInfo info) {
37
+ if (!seen_functions_.insert(info.signature).second) {
38
+ return;
39
+ }
40
+ artifact_.functions.push_back(std::move(info));
41
+ }
42
+
43
+ void FeatureCollector::addTypedef(TypedefInfo info) {
44
+ std::string key = info.name + "|" + info.header_path;
45
+ if (!seen_typedefs_.insert(key).second) {
46
+ return;
47
+ }
48
+ artifact_.typedefs.push_back(std::move(info));
49
+ }
50
+
51
+ void FeatureCollector::addMacro(MacroInfo info) {
52
+ std::string key = info.name + "|" + info.value;
53
+ if (!seen_macros_.insert(key).second) {
54
+ return;
55
+ }
56
+ artifact_.macros.push_back(std::move(info));
57
+ }
58
+
59
+ void FeatureCollector::addEnum(EnumInfo info) {
60
+ std::string key = info.name.value_or("") + "|" + info.header_path;
61
+ if (!seen_enums_.insert(key).second) {
62
+ return;
63
+ }
64
+ artifact_.enums.push_back(std::move(info));
65
+ }
66
+
67
+ void FeatureCollector::addRecord(RecordInfo info) {
68
+ std::string key =
69
+ info.kind + "|" + info.name.value_or("") + "|" + info.header_path;
70
+ if (!seen_records_.insert(key).second) {
71
+ return;
72
+ }
73
+ artifact_.records.push_back(std::move(info));
74
+ }
75
+
76
+ void FeatureCollector::addWarning(std::string warning) {
77
+ artifact_.warnings.push_back(std::move(warning));
78
+ }
79
+
80
+ namespace {
81
+
82
+ bool hasSourceExtension(llvm::StringRef path) {
83
+ return path.ends_with(".c") || path.ends_with(".cc") ||
84
+ path.ends_with(".cpp") || path.ends_with(".cxx") ||
85
+ path.ends_with(".m") || path.ends_with(".mm");
86
+ }
87
+
88
+ // Public API means external linkage plus a declaration inside a library-owned
89
+ // header — not a system header, and not a .c/.cpp translation unit.
90
+ bool isPublicApiLocation(const clang::NamedDecl &decl,
91
+ const ProjectContext &ctx,
92
+ const clang::SourceManager &sm) {
93
+ if (decl.getLinkageInternal() != clang::Linkage::External) {
94
+ return false;
95
+ }
96
+ clang::SourceLocation loc = sm.getSpellingLoc(decl.getLocation());
97
+ if (loc.isInvalid() || sm.isInSystemHeader(loc)) {
98
+ return false;
99
+ }
100
+ llvm::StringRef filename = sm.getFilename(loc);
101
+ if (filename.empty() || !isWithinProject(ctx, filename)) {
102
+ return false;
103
+ }
104
+ return !hasSourceExtension(filename);
105
+ }
106
+
107
+ std::string buildSignature(const FunctionInfo &info) {
108
+ std::string sig = info.return_type + " " + info.name + "(";
109
+ for (size_t i = 0; i < info.params.size(); ++i) {
110
+ if (i > 0) {
111
+ sig += ", ";
112
+ }
113
+ sig += info.params[i].type;
114
+ }
115
+ sig += ")";
116
+ return sig;
117
+ }
118
+
119
+ class ExtractionVisitor : public clang::RecursiveASTVisitor<ExtractionVisitor> {
120
+ public:
121
+ ExtractionVisitor(FeatureCollector &collector, const ProjectContext &ctx,
122
+ const clang::SourceManager &sm)
123
+ : collector_(collector), ctx_(ctx), sm_(sm) {}
124
+
125
+ bool VisitFunctionDecl(clang::FunctionDecl *decl) {
126
+ if (decl->isImplicit() || llvm::isa<clang::CXXMethodDecl>(decl)) {
127
+ return true;
128
+ }
129
+ clang::SourceLocation loc = sm_.getSpellingLoc(decl->getLocation());
130
+ if (loc.isInvalid()) {
131
+ return true;
132
+ }
133
+ llvm::StringRef filename = sm_.getFilename(loc);
134
+ if (filename.empty() || !isWithinProject(ctx_, filename)) {
135
+ return true;
136
+ }
137
+
138
+ FunctionInfo info;
139
+ info.name = decl->getNameAsString();
140
+ if (info.name.empty()) {
141
+ return true;
142
+ }
143
+ info.return_type = decl->getReturnType().getAsString();
144
+ for (const clang::ParmVarDecl *param : decl->parameters()) {
145
+ info.params.push_back(
146
+ Param{param->getNameAsString(), param->getType().getAsString()});
147
+ }
148
+ info.signature = buildSignature(info);
149
+ info.is_public_api = isPublicApiLocation(*decl, ctx_, sm_);
150
+ info.header_path = relativeHeaderPath(ctx_, filename);
151
+ collector_.addFunction(std::move(info));
152
+ return true;
153
+ }
154
+
155
+ bool VisitTypedefNameDecl(clang::TypedefNameDecl *decl) {
156
+ if (decl->isImplicit()) {
157
+ return true;
158
+ }
159
+ clang::SourceLocation loc = sm_.getSpellingLoc(decl->getLocation());
160
+ if (loc.isInvalid()) {
161
+ return true;
162
+ }
163
+ llvm::StringRef filename = sm_.getFilename(loc);
164
+ if (filename.empty() || !isWithinProject(ctx_, filename)) {
165
+ return true;
166
+ }
167
+ TypedefInfo info;
168
+ info.name = decl->getNameAsString();
169
+ info.underlying_type = decl->getUnderlyingType().getAsString();
170
+ info.header_path = relativeHeaderPath(ctx_, filename);
171
+ collector_.addTypedef(std::move(info));
172
+ return true;
173
+ }
174
+
175
+ bool VisitEnumDecl(clang::EnumDecl *decl) {
176
+ if (!decl->isThisDeclarationADefinition()) {
177
+ return true;
178
+ }
179
+ clang::SourceLocation loc = sm_.getSpellingLoc(decl->getLocation());
180
+ if (loc.isInvalid()) {
181
+ return true;
182
+ }
183
+ llvm::StringRef filename = sm_.getFilename(loc);
184
+ if (filename.empty() || !isWithinProject(ctx_, filename)) {
185
+ return true;
186
+ }
187
+ EnumInfo info;
188
+ if (decl->getIdentifier() != nullptr) {
189
+ info.name = decl->getNameAsString();
190
+ }
191
+ for (const clang::EnumConstantDecl *ecd : decl->enumerators()) {
192
+ info.enumerators.push_back(
193
+ Enumerator{ecd->getNameAsString(), ecd->getInitVal().getSExtValue()});
194
+ }
195
+ info.header_path = relativeHeaderPath(ctx_, filename);
196
+ collector_.addEnum(std::move(info));
197
+ return true;
198
+ }
199
+
200
+ bool VisitRecordDecl(clang::RecordDecl *decl) {
201
+ if (!decl->isThisDeclarationADefinition() ||
202
+ (!decl->isStruct() && !decl->isUnion())) {
203
+ return true;
204
+ }
205
+ clang::SourceLocation loc = sm_.getSpellingLoc(decl->getLocation());
206
+ if (loc.isInvalid()) {
207
+ return true;
208
+ }
209
+ llvm::StringRef filename = sm_.getFilename(loc);
210
+ if (filename.empty() || !isWithinProject(ctx_, filename)) {
211
+ return true;
212
+ }
213
+ RecordInfo info;
214
+ if (decl->getIdentifier() != nullptr) {
215
+ info.name = decl->getNameAsString();
216
+ }
217
+ info.kind = decl->isUnion() ? "union" : "struct";
218
+ for (const clang::FieldDecl *field : decl->fields()) {
219
+ info.fields.push_back(
220
+ Field{field->getNameAsString(), field->getType().getAsString()});
221
+ }
222
+ info.header_path = relativeHeaderPath(ctx_, filename);
223
+ collector_.addRecord(std::move(info));
224
+ return true;
225
+ }
226
+
227
+ private:
228
+ FeatureCollector &collector_;
229
+ const ProjectContext &ctx_;
230
+ const clang::SourceManager &sm_;
231
+ };
232
+
233
+ class ExtractionASTConsumer : public clang::ASTConsumer {
234
+ public:
235
+ ExtractionASTConsumer(FeatureCollector &collector, ProjectContext ctx)
236
+ : collector_(collector), ctx_(std::move(ctx)) {}
237
+
238
+ void HandleTranslationUnit(clang::ASTContext &context) override {
239
+ ExtractionVisitor visitor(collector_, ctx_, context.getSourceManager());
240
+ visitor.TraverseDecl(context.getTranslationUnitDecl());
241
+ }
242
+
243
+ private:
244
+ FeatureCollector &collector_;
245
+ ProjectContext ctx_;
246
+ };
247
+
248
+ class ExtractionFrontendAction : public clang::ASTFrontendAction {
249
+ public:
250
+ ExtractionFrontendAction(FeatureCollector &collector, ProjectContext ctx)
251
+ : collector_(collector), ctx_(std::move(ctx)) {}
252
+
253
+ bool BeginSourceFileAction(clang::CompilerInstance &ci) override {
254
+ ci.getPreprocessor().addPPCallbacks(newMacroCollectorCallbacks(
255
+ collector_, ctx_, ci.getSourceManager(), ci.getLangOpts()));
256
+ return true;
257
+ }
258
+
259
+ std::unique_ptr<clang::ASTConsumer>
260
+ CreateASTConsumer(clang::CompilerInstance & /*ci*/,
261
+ llvm::StringRef /*file*/) override {
262
+ return std::make_unique<ExtractionASTConsumer>(collector_, ctx_);
263
+ }
264
+
265
+ private:
266
+ FeatureCollector &collector_;
267
+ ProjectContext ctx_;
268
+ };
269
+
270
+ class ExtractionFrontendActionFactory
271
+ : public clang::tooling::FrontendActionFactory {
272
+ public:
273
+ ExtractionFrontendActionFactory(FeatureCollector &collector,
274
+ ProjectContext ctx)
275
+ : collector_(collector), ctx_(std::move(ctx)) {}
276
+
277
+ std::unique_ptr<clang::FrontendAction> create() override {
278
+ return std::make_unique<ExtractionFrontendAction>(collector_, ctx_);
279
+ }
280
+
281
+ private:
282
+ FeatureCollector &collector_;
283
+ ProjectContext ctx_;
284
+ };
285
+
286
+ } // namespace
287
+
288
+ std::unique_ptr<clang::tooling::FrontendActionFactory>
289
+ newExtractionActionFactory(FeatureCollector &collector, ProjectContext ctx) {
290
+ return std::make_unique<ExtractionFrontendActionFactory>(collector,
291
+ std::move(ctx));
292
+ }
293
+
294
+ } // namespace feature_extractor
@@ -0,0 +1,154 @@
1
+ #include "feature_extractor.hpp"
2
+
3
+ #include <cJSON.h>
4
+
5
+ namespace feature_extractor {
6
+
7
+ namespace {
8
+
9
+ cJSON *createParamArray(const std::vector<Param> &params) {
10
+ cJSON *array = cJSON_CreateArray();
11
+ for (const auto &param : params) {
12
+ cJSON *item = cJSON_CreateObject();
13
+ cJSON_AddStringToObject(item, "name", param.name.c_str());
14
+ cJSON_AddStringToObject(item, "type", param.type.c_str());
15
+ cJSON_AddItemToArray(array, item);
16
+ }
17
+ return array;
18
+ }
19
+
20
+ cJSON *createStringArray(const std::vector<std::string> &values) {
21
+ cJSON *array = cJSON_CreateArray();
22
+ for (const auto &value : values) {
23
+ cJSON_AddItemToArray(array, cJSON_CreateString(value.c_str()));
24
+ }
25
+ return array;
26
+ }
27
+
28
+ void addOptionalString(cJSON *object, const char *key,
29
+ const std::optional<std::string> &value) {
30
+ if (value.has_value()) {
31
+ cJSON_AddStringToObject(object, key, value->c_str());
32
+ } else {
33
+ cJSON_AddNullToObject(object, key);
34
+ }
35
+ }
36
+
37
+ cJSON *createFunction(const FunctionInfo &fn) {
38
+ cJSON *object = cJSON_CreateObject();
39
+ cJSON_AddStringToObject(object, "name", fn.name.c_str());
40
+ cJSON_AddStringToObject(object, "return_type", fn.return_type.c_str());
41
+ cJSON_AddItemToObject(object, "params", createParamArray(fn.params));
42
+ cJSON_AddStringToObject(object, "signature", fn.signature.c_str());
43
+ cJSON_AddBoolToObject(object, "is_public_api", fn.is_public_api);
44
+ cJSON_AddStringToObject(object, "header_path", fn.header_path.c_str());
45
+ return object;
46
+ }
47
+
48
+ cJSON *createTypedef(const TypedefInfo &td) {
49
+ cJSON *object = cJSON_CreateObject();
50
+ cJSON_AddStringToObject(object, "name", td.name.c_str());
51
+ cJSON_AddStringToObject(object, "underlying_type", td.underlying_type.c_str());
52
+ cJSON_AddStringToObject(object, "header_path", td.header_path.c_str());
53
+ return object;
54
+ }
55
+
56
+ cJSON *createMacro(const MacroInfo &macro) {
57
+ cJSON *object = cJSON_CreateObject();
58
+ cJSON_AddStringToObject(object, "name", macro.name.c_str());
59
+ cJSON_AddBoolToObject(object, "is_function_like", macro.is_function_like);
60
+ cJSON_AddItemToObject(object, "params", createStringArray(macro.params));
61
+ cJSON_AddStringToObject(object, "value", macro.value.c_str());
62
+ cJSON_AddStringToObject(object, "header_path", macro.header_path.c_str());
63
+ return object;
64
+ }
65
+
66
+ cJSON *createEnumerator(const Enumerator &e) {
67
+ cJSON *object = cJSON_CreateObject();
68
+ cJSON_AddStringToObject(object, "name", e.name.c_str());
69
+ // cJSON stores numbers as double, so a value beyond +-2^53 loses precision.
70
+ // Real-world enum constants stay well inside that range.
71
+ cJSON_AddNumberToObject(object, "value", static_cast<double>(e.value));
72
+ return object;
73
+ }
74
+
75
+ cJSON *createEnum(const EnumInfo &e) {
76
+ cJSON *object = cJSON_CreateObject();
77
+ addOptionalString(object, "name", e.name);
78
+ cJSON *enumerators = cJSON_CreateArray();
79
+ for (const auto &enumerator : e.enumerators) {
80
+ cJSON_AddItemToArray(enumerators, createEnumerator(enumerator));
81
+ }
82
+ cJSON_AddItemToObject(object, "enumerators", enumerators);
83
+ cJSON_AddStringToObject(object, "header_path", e.header_path.c_str());
84
+ return object;
85
+ }
86
+
87
+ cJSON *createField(const Field &field) {
88
+ cJSON *object = cJSON_CreateObject();
89
+ cJSON_AddStringToObject(object, "name", field.name.c_str());
90
+ cJSON_AddStringToObject(object, "type", field.type.c_str());
91
+ return object;
92
+ }
93
+
94
+ cJSON *createRecord(const RecordInfo &record) {
95
+ cJSON *object = cJSON_CreateObject();
96
+ addOptionalString(object, "name", record.name);
97
+ cJSON_AddStringToObject(object, "kind", record.kind.c_str());
98
+ cJSON *fields = cJSON_CreateArray();
99
+ for (const auto &field : record.fields) {
100
+ cJSON_AddItemToArray(fields, createField(field));
101
+ }
102
+ cJSON_AddItemToObject(object, "fields", fields);
103
+ cJSON_AddStringToObject(object, "header_path", record.header_path.c_str());
104
+ return object;
105
+ }
106
+
107
+ } // namespace
108
+
109
+ std::string writeJson(const FeatureArtifact &artifact) {
110
+ cJSON *root = cJSON_CreateObject();
111
+ cJSON_AddNumberToObject(root, "schema_version", artifact.schema_version);
112
+ cJSON_AddStringToObject(root, "project_name", artifact.project_name.c_str());
113
+ cJSON_AddStringToObject(root, "language", artifact.language.c_str());
114
+
115
+ cJSON *functions = cJSON_CreateArray();
116
+ for (const auto &fn : artifact.functions) {
117
+ cJSON_AddItemToArray(functions, createFunction(fn));
118
+ }
119
+ cJSON_AddItemToObject(root, "functions", functions);
120
+
121
+ cJSON *typedefs = cJSON_CreateArray();
122
+ for (const auto &td : artifact.typedefs) {
123
+ cJSON_AddItemToArray(typedefs, createTypedef(td));
124
+ }
125
+ cJSON_AddItemToObject(root, "typedefs", typedefs);
126
+
127
+ cJSON *macros = cJSON_CreateArray();
128
+ for (const auto &macro : artifact.macros) {
129
+ cJSON_AddItemToArray(macros, createMacro(macro));
130
+ }
131
+ cJSON_AddItemToObject(root, "macros", macros);
132
+
133
+ cJSON *enums = cJSON_CreateArray();
134
+ for (const auto &e : artifact.enums) {
135
+ cJSON_AddItemToArray(enums, createEnum(e));
136
+ }
137
+ cJSON_AddItemToObject(root, "enums", enums);
138
+
139
+ cJSON *records = cJSON_CreateArray();
140
+ for (const auto &record : artifact.records) {
141
+ cJSON_AddItemToArray(records, createRecord(record));
142
+ }
143
+ cJSON_AddItemToObject(root, "records", records);
144
+
145
+ cJSON_AddItemToObject(root, "warnings", createStringArray(artifact.warnings));
146
+
147
+ char *rendered = cJSON_PrintUnformatted(root);
148
+ std::string result(rendered);
149
+ cJSON_free(rendered);
150
+ cJSON_Delete(root);
151
+ return result;
152
+ }
153
+
154
+ } // namespace feature_extractor
@@ -0,0 +1,75 @@
1
+ #include "feature_extractor.hpp"
2
+
3
+ #include <clang/Lex/Lexer.h>
4
+ #include <clang/Lex/MacroInfo.h>
5
+ #include <clang/Lex/Token.h>
6
+
7
+ namespace feature_extractor {
8
+
9
+ namespace {
10
+
11
+ class MacroCollectorCallbacks : public clang::PPCallbacks {
12
+ public:
13
+ MacroCollectorCallbacks(FeatureCollector &collector, ProjectContext ctx,
14
+ const clang::SourceManager &sm,
15
+ const clang::LangOptions &lang_opts)
16
+ : collector_(collector), ctx_(std::move(ctx)), sm_(sm),
17
+ lang_opts_(lang_opts) {}
18
+
19
+ void MacroDefined(const clang::Token &name_tok,
20
+ const clang::MacroDirective *md) override {
21
+ const clang::MacroInfo *mi = md->getMacroInfo();
22
+ if (mi == nullptr || name_tok.getIdentifierInfo() == nullptr) {
23
+ return;
24
+ }
25
+ clang::SourceLocation loc = sm_.getSpellingLoc(mi->getDefinitionLoc());
26
+ if (loc.isInvalid() || sm_.isInSystemHeader(loc)) {
27
+ return;
28
+ }
29
+ llvm::StringRef filename = sm_.getFilename(loc);
30
+ if (filename.empty() || !isWithinProject(ctx_, filename)) {
31
+ return;
32
+ }
33
+
34
+ MacroInfo info;
35
+ info.name = name_tok.getIdentifierInfo()->getName().str();
36
+ info.is_function_like = mi->isFunctionLike();
37
+ if (info.is_function_like) {
38
+ for (const clang::IdentifierInfo *param : mi->params()) {
39
+ info.params.push_back(param->getName().str());
40
+ }
41
+ }
42
+ info.value = macroValueText(*mi);
43
+ info.header_path = relativeHeaderPath(ctx_, filename);
44
+ collector_.addMacro(std::move(info));
45
+ }
46
+
47
+ private:
48
+ std::string macroValueText(const clang::MacroInfo &mi) const {
49
+ std::string value;
50
+ for (const clang::Token &tok : mi.tokens()) {
51
+ if (!value.empty()) {
52
+ value += " ";
53
+ }
54
+ value += clang::Lexer::getSpelling(tok, sm_, lang_opts_);
55
+ }
56
+ return value;
57
+ }
58
+
59
+ FeatureCollector &collector_;
60
+ ProjectContext ctx_;
61
+ const clang::SourceManager &sm_;
62
+ const clang::LangOptions &lang_opts_;
63
+ };
64
+
65
+ } // namespace
66
+
67
+ std::unique_ptr<clang::PPCallbacks>
68
+ newMacroCollectorCallbacks(FeatureCollector &collector, ProjectContext ctx,
69
+ const clang::SourceManager &sm,
70
+ const clang::LangOptions &lang_opts) {
71
+ return std::make_unique<MacroCollectorCallbacks>(collector, std::move(ctx),
72
+ sm, lang_opts);
73
+ }
74
+
75
+ } // namespace feature_extractor
@@ -0,0 +1,130 @@
1
+ #include "feature_extractor.hpp"
2
+
3
+ #include <clang/Tooling/CompilationDatabase.h>
4
+ #include <clang/Tooling/Tooling.h>
5
+ #include <llvm/Support/FileSystem.h>
6
+ #include <llvm/Support/Path.h>
7
+ #include <llvm/Support/raw_ostream.h>
8
+
9
+ #include <fstream>
10
+ #include <string>
11
+ #include <vector>
12
+
13
+ namespace {
14
+
15
+ bool looksLikeCxx(llvm::StringRef path) {
16
+ return path.ends_with(".cc") || path.ends_with(".cpp") ||
17
+ path.ends_with(".cxx") || path.ends_with(".hpp") ||
18
+ path.ends_with(".hh") || path.ends_with(".hxx") ||
19
+ path.ends_with(".mm");
20
+ }
21
+
22
+ std::string detectLanguage(const std::vector<std::string> &files) {
23
+ // Must match models.Language's values ("c" | "c++"), which extraction.py parses.
24
+ for (const std::string &file : files) {
25
+ if (looksLikeCxx(file)) {
26
+ return "c++";
27
+ }
28
+ }
29
+ return "c";
30
+ }
31
+
32
+ // The compile-commands directory (argv[1]) is often a separate build directory,
33
+ // not the source checkout, so it cannot serve as the project root. Recover the
34
+ // real root as the common ancestor of every translation unit's absolute path.
35
+ std::string computeProjectRoot(const std::vector<std::string> &files) {
36
+ std::vector<llvm::StringRef> common;
37
+ bool first = true;
38
+ for (const std::string &file : files) {
39
+ llvm::SmallString<256> dir(file);
40
+ llvm::sys::path::remove_filename(dir);
41
+ llvm::sys::path::remove_dots(dir, /*remove_dot_dot=*/true);
42
+ std::vector<llvm::StringRef> components(llvm::sys::path::begin(dir),
43
+ llvm::sys::path::end(dir));
44
+ if (first) {
45
+ common = std::move(components);
46
+ first = false;
47
+ continue;
48
+ }
49
+ size_t limit = std::min(common.size(), components.size());
50
+ size_t matched = 0;
51
+ while (matched < limit && common[matched] == components[matched]) {
52
+ ++matched;
53
+ }
54
+ common.resize(matched);
55
+ }
56
+ llvm::SmallString<256> root;
57
+ for (llvm::StringRef component : common) {
58
+ llvm::sys::path::append(root, component);
59
+ }
60
+ return std::string(root);
61
+ }
62
+
63
+ } // namespace
64
+
65
+ int main(int argc, char **argv) {
66
+ if (argc != 4) {
67
+ llvm::errs()
68
+ << "usage: feature_extractor <compile-commands-dir> <output-json-path> "
69
+ "<project-name>\n";
70
+ return 1;
71
+ }
72
+
73
+ llvm::SmallString<256> compile_commands_dir(argv[1]);
74
+ llvm::sys::fs::make_absolute(compile_commands_dir);
75
+ llvm::sys::path::remove_dots(compile_commands_dir, /*remove_dot_dot=*/true);
76
+ const std::string output_json_path = argv[2];
77
+ const std::string project_name = argv[3];
78
+
79
+ std::string error_message;
80
+ std::unique_ptr<clang::tooling::CompilationDatabase> db =
81
+ clang::tooling::CompilationDatabase::autoDetectFromDirectory(
82
+ compile_commands_dir, error_message);
83
+ if (db == nullptr) {
84
+ llvm::errs() << "error loading compile_commands.json from "
85
+ << compile_commands_dir << ": " << error_message << "\n";
86
+ return 1;
87
+ }
88
+
89
+ std::vector<std::string> files = db->getAllFiles();
90
+ feature_extractor::FeatureCollector collector;
91
+ feature_extractor::ProjectContext ctx{computeProjectRoot(files)};
92
+
93
+ clang::tooling::ClangTool tool(*db, files);
94
+ tool.appendArgumentsAdjuster(clang::tooling::getInsertArgumentAdjuster(
95
+ "-resource-dir=" CLANG_RESOURCE_DIR,
96
+ clang::tooling::ArgumentInsertPosition::BEGIN));
97
+ // -resource-dir covers only clang's builtin headers (stddef.h, stdarg.h, ...).
98
+ // Platform libc headers still need an explicit -isysroot on macOS, since
99
+ // ClangTool's ad hoc driver invocation does not auto-detect the Xcode SDK the way
100
+ // a plain `clang` does. CLANG_SYSROOT is empty elsewhere, where the default
101
+ // system include paths apply.
102
+ const std::string clang_sysroot = CLANG_SYSROOT;
103
+ if (!clang_sysroot.empty()) {
104
+ tool.appendArgumentsAdjuster(clang::tooling::getInsertArgumentAdjuster(
105
+ {"-isysroot", clang_sysroot},
106
+ clang::tooling::ArgumentInsertPosition::BEGIN));
107
+ }
108
+ std::unique_ptr<clang::tooling::FrontendActionFactory> factory =
109
+ feature_extractor::newExtractionActionFactory(collector, ctx);
110
+ int run_result = tool.run(factory.get());
111
+
112
+ feature_extractor::FeatureArtifact artifact = collector.artifact();
113
+ artifact.project_name = project_name;
114
+ artifact.language = detectLanguage(files);
115
+ if (run_result != 0) {
116
+ artifact.warnings.push_back(
117
+ "one or more translation units in compile_commands.json failed to "
118
+ "compile "
119
+ "cleanly; extraction may be incomplete for those files");
120
+ }
121
+
122
+ std::ofstream out(output_json_path);
123
+ if (!out) {
124
+ llvm::errs() << "error: could not open " << output_json_path
125
+ << " for writing\n";
126
+ return 1;
127
+ }
128
+ out << feature_extractor::writeJson(artifact);
129
+ return 0;
130
+ }