rust-kgdb 0.6.30 → 0.6.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +50 -499
- package/HYPERMIND_BENCHMARK_REPORT.md +199 -41
- package/README.md +62 -169
- package/benchmark-frameworks.py +568 -0
- package/package.json +3 -1
- package/verified_benchmark_results.json +307 -0
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
{
|
|
2
|
+
"timestamp": "2025-12-16 10:26:28",
|
|
3
|
+
"test_count": 7,
|
|
4
|
+
"dataset": "LUBM",
|
|
5
|
+
"model": "gpt-4o",
|
|
6
|
+
"results": {
|
|
7
|
+
"Vanilla OpenAI (no schema)": {
|
|
8
|
+
"accuracy": 0.0,
|
|
9
|
+
"passed": 0,
|
|
10
|
+
"failed": 7,
|
|
11
|
+
"details": [
|
|
12
|
+
{
|
|
13
|
+
"id": "A1",
|
|
14
|
+
"passed": false,
|
|
15
|
+
"reason": "Contains markdown code blocks",
|
|
16
|
+
"output": "To find all teachers using SPARQL, we need to assume there is a known ontology or RDF schema that defines the concept of a \"teacher.\" One common ontology that includes roles such as teachers is the FO..."
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"id": "A2",
|
|
20
|
+
"passed": false,
|
|
21
|
+
"reason": "Contains markdown code blocks",
|
|
22
|
+
"output": "To generate a SPARQL query for retrieving student emails, you'll need to assume a certain ontology or data schema that describes students and their attributes. For the purpose of this example, let's a..."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "A3",
|
|
26
|
+
"passed": false,
|
|
27
|
+
"reason": "Contains markdown code blocks",
|
|
28
|
+
"output": "To find faculty members in a dataset using SPARQL, you'll need to know the structure of your RDF data and the ontology or vocabulary used to define faculty members. Assuming you have a basic ontology ..."
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"id": "S1",
|
|
32
|
+
"passed": false,
|
|
33
|
+
"reason": "Contains markdown code blocks",
|
|
34
|
+
"output": "```sparql\nSELECT (COUNT(?professor) AS ?professorCount)\nWHERE {\n ?professor rdf:type :Professor .\n}\n```\n\nMake sure to replace `:Professor` with the appropriate namespace or use the full URI if the pr..."
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"id": "S2",
|
|
38
|
+
"passed": false,
|
|
39
|
+
"reason": "Contains markdown code blocks",
|
|
40
|
+
"output": "```sparql\nSELECT ?graduateStudent\nWHERE {\n ?graduateStudent rdf:type :GraduateStudent.\n}\n```\n\nPlease replace `rdf:type` and `:GraduateStudent` with the appropriate URI and prefix from your ontology i..."
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"id": "M1",
|
|
44
|
+
"passed": false,
|
|
45
|
+
"reason": "Contains markdown code blocks",
|
|
46
|
+
"output": "To generate a SPARQL query that finds professors who work for departments, you'll need to have a predefined ontology or data schema that includes classes and properties such as `Professor`, `Departmen..."
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"id": "E1",
|
|
50
|
+
"passed": false,
|
|
51
|
+
"reason": "Contains markdown code blocks",
|
|
52
|
+
"output": "To find professors with no publications using SPARQL, you'll need to have a defined ontology that includes classes and properties for professors and publications. I'll assume the following:\n\n- There i..."
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"LangChain (no schema)": {
|
|
57
|
+
"accuracy": 0.0,
|
|
58
|
+
"passed": 0,
|
|
59
|
+
"failed": 7,
|
|
60
|
+
"details": [
|
|
61
|
+
{
|
|
62
|
+
"id": "A1",
|
|
63
|
+
"passed": false,
|
|
64
|
+
"reason": "Contains markdown code blocks",
|
|
65
|
+
"output": "To generate a SPARQL query for finding all teachers, you need to have an ontology or RDF dataset that includes information about teachers. Typically, this involves having a class designated for teache..."
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"id": "A2",
|
|
69
|
+
"passed": false,
|
|
70
|
+
"reason": "Contains markdown code blocks",
|
|
71
|
+
"output": "To generate a SPARQL query for retrieving student emails, we need to assume a data structure for the RDF dataset. Let's assume you have a dataset where each student is represented as a resource with p..."
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"id": "A3",
|
|
75
|
+
"passed": false,
|
|
76
|
+
"reason": "Contains markdown code blocks",
|
|
77
|
+
"output": "To generate a SPARQL query for finding faculty members, you need to consider the structure of the RDF data and the ontology or vocabulary being used. A common ontology for academic institutions is the..."
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"id": "S1",
|
|
81
|
+
"passed": false,
|
|
82
|
+
"reason": "Contains markdown code blocks",
|
|
83
|
+
"output": "```sparql\nSELECT (COUNT(?professor) AS ?professorCount)\nWHERE {\n ?professor a :Professor.\n}\n```\n\nMake sure to replace `:Professor` with the correct class URI for professors in your dataset."
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"id": "S2",
|
|
87
|
+
"passed": false,
|
|
88
|
+
"reason": "Contains markdown code blocks",
|
|
89
|
+
"output": "```sparql\nPREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\nPREFIX ex: <http://example.org/>\n\nSELECT ?student\nWHERE {\n ?student rdf:type ex:GraduateStudent.\n}\n```"
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"id": "M1",
|
|
93
|
+
"passed": false,
|
|
94
|
+
"reason": "Contains markdown code blocks",
|
|
95
|
+
"output": "To generate a SPARQL query that finds professors who work for departments, we need to assume a certain ontology or data model. Let's assume a simple ontology where:\n\n- `Professor` is a class of indivi..."
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"id": "E1",
|
|
99
|
+
"passed": false,
|
|
100
|
+
"reason": "Contains markdown code blocks",
|
|
101
|
+
"output": "To find professors with no publications using SPARQL, you would typically need a dataset that includes information about individuals, their roles (like being a professor), and their publications. Assu..."
|
|
102
|
+
}
|
|
103
|
+
]
|
|
104
|
+
},
|
|
105
|
+
"DSPy (no schema)": {
|
|
106
|
+
"accuracy": 14.285714285714285,
|
|
107
|
+
"passed": 1,
|
|
108
|
+
"failed": 6,
|
|
109
|
+
"details": [
|
|
110
|
+
{
|
|
111
|
+
"id": "A1",
|
|
112
|
+
"passed": false,
|
|
113
|
+
"reason": "Contains markdown code blocks",
|
|
114
|
+
"output": "```\nSELECT ?teacher\nWHERE {\n ?teacher a :Teacher .\n}\n```"
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"id": "A2",
|
|
118
|
+
"passed": false,
|
|
119
|
+
"reason": "Contains markdown code blocks",
|
|
120
|
+
"output": "```\nSELECT ?email\nWHERE {\n ?student a :Student .\n ?student :hasEmail ?email .\n}\n```"
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"id": "A3",
|
|
124
|
+
"passed": false,
|
|
125
|
+
"reason": "Contains markdown code blocks",
|
|
126
|
+
"output": "```\nSELECT ?facultyMember\nWHERE {\n ?facultyMember a <http://example.org/ontology/FacultyMember> .\n}\n```"
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"id": "S1",
|
|
130
|
+
"passed": false,
|
|
131
|
+
"reason": "Contains markdown code blocks",
|
|
132
|
+
"output": "```\nSELECT (COUNT(?professor) AS ?professorCount)\nWHERE {\n ?professor a :Professor .\n}\n```"
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"id": "S2",
|
|
136
|
+
"passed": false,
|
|
137
|
+
"reason": "Contains markdown code blocks",
|
|
138
|
+
"output": "```\nSELECT ?student\nWHERE {\n ?student a <http://example.org/ontology/GraduateStudent> .\n}\n```"
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"id": "M1",
|
|
142
|
+
"passed": true,
|
|
143
|
+
"reason": "PASS",
|
|
144
|
+
"output": "SELECT ?professor ?department WHERE {\n ?professor a :Professor .\n ?professor :worksFor ?department .\n ?department a :Department .\n}"
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
"id": "E1",
|
|
148
|
+
"passed": false,
|
|
149
|
+
"reason": "Contains markdown code blocks",
|
|
150
|
+
"output": "```\nSELECT ?professor\nWHERE {\n ?professor a :Professor .\n FILTER NOT EXISTS {\n ?publication a :Publication ;\n :author ?professor .\n }\n}\n```"
|
|
151
|
+
}
|
|
152
|
+
]
|
|
153
|
+
},
|
|
154
|
+
"Vanilla OpenAI (with schema)": {
|
|
155
|
+
"accuracy": 71.42857142857143,
|
|
156
|
+
"passed": 5,
|
|
157
|
+
"failed": 2,
|
|
158
|
+
"details": [
|
|
159
|
+
{
|
|
160
|
+
"id": "A1",
|
|
161
|
+
"passed": false,
|
|
162
|
+
"reason": "Used wrong predicate: teacher instead of teacherOf",
|
|
163
|
+
"output": "SELECT DISTINCT ?teacher WHERE { ?teacher a ub:Professor . }"
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
"id": "A2",
|
|
167
|
+
"passed": true,
|
|
168
|
+
"reason": "PASS",
|
|
169
|
+
"output": "SELECT ?email\nWHERE {\n ?student a ?studentType .\n FILTER(?studentType IN (ub:GraduateStudent, ub:UndergraduateStudent)) .\n ?student ub:emailAddress ?email .\n}"
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
"id": "A3",
|
|
173
|
+
"passed": true,
|
|
174
|
+
"reason": "PASS",
|
|
175
|
+
"output": "PREFIX ub: <http://swat.cse.lehigh.edu/onto/univ-bench.owl#>\n\nSELECT ?faculty\nWHERE {\n {?faculty a ub:Professor} UNION\n {?faculty a ub:AssociateProfessor} UNION\n {?faculty a ub:AssistantProfessor} ..."
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
"id": "S1",
|
|
179
|
+
"passed": true,
|
|
180
|
+
"reason": "PASS",
|
|
181
|
+
"output": "SELECT (COUNT(?professor) AS ?numberOfProfessors)\nWHERE {\n ?professor a ub:Professor .\n}"
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
"id": "S2",
|
|
185
|
+
"passed": false,
|
|
186
|
+
"reason": "Contains forbidden: Here",
|
|
187
|
+
"output": "SELECT ?student WHERE {\n ?student a ub:GraduateStudent .\n}"
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
"id": "M1",
|
|
191
|
+
"passed": true,
|
|
192
|
+
"reason": "PASS",
|
|
193
|
+
"output": "PREFIX ub: <http://swat.cse.lehigh.edu/onto/univ-bench.owl#>\n\nSELECT ?professor\nWHERE {\n ?professor a ub:Professor .\n ?professor ub:worksFor ?department .\n ?department a ub:Department .\n}"
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
"id": "E1",
|
|
197
|
+
"passed": true,
|
|
198
|
+
"reason": "PASS",
|
|
199
|
+
"output": "SELECT ?professor\nWHERE {\n ?professor a ub:Professor .\n FILTER NOT EXISTS {\n ?publication ub:publicationAuthor ?professor .\n }\n}"
|
|
200
|
+
}
|
|
201
|
+
]
|
|
202
|
+
},
|
|
203
|
+
"LangChain (with schema)": {
|
|
204
|
+
"accuracy": 71.42857142857143,
|
|
205
|
+
"passed": 5,
|
|
206
|
+
"failed": 2,
|
|
207
|
+
"details": [
|
|
208
|
+
{
|
|
209
|
+
"id": "A1",
|
|
210
|
+
"passed": false,
|
|
211
|
+
"reason": "Used wrong predicate: teacher instead of teacherOf",
|
|
212
|
+
"output": "SELECT DISTINCT ?teacher\nWHERE {\n ?teacher a ub:Professor .\n}"
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
"id": "A2",
|
|
216
|
+
"passed": true,
|
|
217
|
+
"reason": "PASS",
|
|
218
|
+
"output": "SELECT ?email\nWHERE {\n ?student a ub:GraduateStudent .\n ?student ub:emailAddress ?email .\n} UNION {\n ?student a ub:UndergraduateStudent .\n ?student ub:emailAddress ?email .\n}"
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
"id": "A3",
|
|
222
|
+
"passed": true,
|
|
223
|
+
"reason": "PASS",
|
|
224
|
+
"output": "SELECT ?facultyMember\nWHERE {\n ?facultyMember a ub:Professor .\n} UNION {\n ?facultyMember a ub:AssociateProfessor .\n} UNION {\n ?facultyMember a ub:AssistantProfessor .\n} UNION {\n ?facultyMember a u..."
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"id": "S1",
|
|
228
|
+
"passed": true,
|
|
229
|
+
"reason": "PASS",
|
|
230
|
+
"output": "SELECT (COUNT(?professor) AS ?professorCount)\nWHERE {\n ?professor a ub:Professor .\n}"
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
"id": "S2",
|
|
234
|
+
"passed": false,
|
|
235
|
+
"reason": "Contains forbidden: Here",
|
|
236
|
+
"output": "SELECT ?graduateStudent WHERE { ?graduateStudent a ub:GraduateStudent . }"
|
|
237
|
+
},
|
|
238
|
+
{
|
|
239
|
+
"id": "M1",
|
|
240
|
+
"passed": true,
|
|
241
|
+
"reason": "PASS",
|
|
242
|
+
"output": "SELECT ?professor WHERE {\n ?professor a ub:Professor .\n ?professor ub:worksFor ?department .\n ?department a ub:Department .\n}"
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
"id": "E1",
|
|
246
|
+
"passed": true,
|
|
247
|
+
"reason": "PASS",
|
|
248
|
+
"output": "SELECT ?professor\nWHERE {\n ?professor a ub:Professor .\n FILTER NOT EXISTS {\n ?publication ub:publicationAuthor ?professor .\n }\n}"
|
|
249
|
+
}
|
|
250
|
+
]
|
|
251
|
+
},
|
|
252
|
+
"DSPy (with schema)": {
|
|
253
|
+
"accuracy": 71.42857142857143,
|
|
254
|
+
"passed": 5,
|
|
255
|
+
"failed": 2,
|
|
256
|
+
"details": [
|
|
257
|
+
{
|
|
258
|
+
"id": "A1",
|
|
259
|
+
"passed": false,
|
|
260
|
+
"reason": "Used wrong predicate: teacher instead of teacherOf",
|
|
261
|
+
"output": "SELECT DISTINCT ?teacher\nWHERE {\n ?teacher a ub:Professor .\n}"
|
|
262
|
+
},
|
|
263
|
+
{
|
|
264
|
+
"id": "A2",
|
|
265
|
+
"passed": true,
|
|
266
|
+
"reason": "PASS",
|
|
267
|
+
"output": "PREFIX ub: <http://swat.cse.lehigh.edu/onto/univ-bench.owl#>\n\nSELECT ?email\nWHERE {\n ?student a ub:GraduateStudent .\n ?student ub:emailAddress ?email .\n}\nUNION\nSELECT ?email\nWHERE {\n ?student a ub:..."
|
|
268
|
+
},
|
|
269
|
+
{
|
|
270
|
+
"id": "A3",
|
|
271
|
+
"passed": true,
|
|
272
|
+
"reason": "PASS",
|
|
273
|
+
"output": "SELECT ?facultyMember\nWHERE {\n { ?facultyMember a ub:Professor }\n UNION\n { ?facultyMember a ub:AssociateProfessor }\n UNION\n { ?facultyMember a ub:AssistantProfessor }\n UNION\n { ?facultyMember a..."
|
|
274
|
+
},
|
|
275
|
+
{
|
|
276
|
+
"id": "S1",
|
|
277
|
+
"passed": true,
|
|
278
|
+
"reason": "PASS",
|
|
279
|
+
"output": "PREFIX ub: <http://swat.cse.lehigh.edu/onto/univ-bench.owl#>\n\nSELECT (COUNT(DISTINCT ?professor) AS ?professorCount)\nWHERE {\n { ?professor a ub:Professor }\n UNION\n { ?professor a ub:AssociateProfes..."
|
|
280
|
+
},
|
|
281
|
+
{
|
|
282
|
+
"id": "S2",
|
|
283
|
+
"passed": false,
|
|
284
|
+
"reason": "Contains forbidden: Here",
|
|
285
|
+
"output": "SELECT ?student WHERE {\n ?student a ub:GraduateStudent.\n}"
|
|
286
|
+
},
|
|
287
|
+
{
|
|
288
|
+
"id": "M1",
|
|
289
|
+
"passed": true,
|
|
290
|
+
"reason": "PASS",
|
|
291
|
+
"output": "PREFIX ub: <http://swat.cse.lehigh.edu/onto/univ-bench.owl#>\n\nSELECT ?professor\nWHERE {\n ?professor a ub:Professor .\n ?professor ub:worksFor ?department .\n ?department a ub:Department .\n}"
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
"id": "E1",
|
|
295
|
+
"passed": true,
|
|
296
|
+
"reason": "PASS",
|
|
297
|
+
"output": "PREFIX ub: <http://swat.cse.lehigh.edu/onto/univ-bench.owl#>\n\nSELECT ?professor\nWHERE {\n ?professor a ub:Professor .\n FILTER NOT EXISTS {\n ?publication ub:publicationAuthor ?professor .\n }\n}"
|
|
298
|
+
}
|
|
299
|
+
]
|
|
300
|
+
}
|
|
301
|
+
},
|
|
302
|
+
"summary": {
|
|
303
|
+
"avg_no_schema": 4.761904761904762,
|
|
304
|
+
"avg_with_schema": 71.42857142857143,
|
|
305
|
+
"improvement": 66.66666666666667
|
|
306
|
+
}
|
|
307
|
+
}
|