nucliadb-sdk 0.0.0.post2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nucliadb_sdk/__init__.py +24 -0
- nucliadb_sdk/client.py +420 -0
- nucliadb_sdk/entities.py +33 -0
- nucliadb_sdk/file.py +27 -0
- nucliadb_sdk/knowledgebox.py +368 -0
- nucliadb_sdk/labels.py +43 -0
- nucliadb_sdk/py.typed +0 -0
- nucliadb_sdk/resource.py +369 -0
- nucliadb_sdk/search.py +104 -0
- nucliadb_sdk/tests/__init__.py +18 -0
- nucliadb_sdk/tests/conftest.py +22 -0
- nucliadb_sdk/tests/fixtures.py +93 -0
- nucliadb_sdk/tests/test_create.py +29 -0
- nucliadb_sdk/tests/test_dict_resource.py +51 -0
- nucliadb_sdk/tests/test_search_resource.py +346 -0
- nucliadb_sdk/tests/test_set_resource.py +126 -0
- nucliadb_sdk/utils.py +133 -0
- nucliadb_sdk/vectors.py +51 -0
- nucliadb_sdk-0.0.0.post2.dist-info/METADATA +25 -0
- nucliadb_sdk-0.0.0.post2.dist-info/RECORD +23 -0
- nucliadb_sdk-0.0.0.post2.dist-info/WHEEL +5 -0
- nucliadb_sdk-0.0.0.post2.dist-info/top_level.txt +1 -0
- nucliadb_sdk-0.0.0.post2.dist-info/zip-safe +1 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
2
|
+
#
|
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
5
|
+
#
|
|
6
|
+
# AGPL:
|
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
10
|
+
# License, or (at your option) any later version.
|
|
11
|
+
#
|
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
# GNU Affero General Public License for more details.
|
|
16
|
+
#
|
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
|
+
|
|
20
|
+
from typing import Any, Dict
|
|
21
|
+
|
|
22
|
+
import pytest
|
|
23
|
+
|
|
24
|
+
from nucliadb_sdk.knowledgebox import KnowledgeBox
|
|
25
|
+
from nucliadb_sdk.labels import Label
|
|
26
|
+
from nucliadb_sdk.vectors import Vector
|
|
27
|
+
|
|
28
|
+
DATA: Dict[str, Any] = {
|
|
29
|
+
"text": [
|
|
30
|
+
"Sunday afternoon walking through Venice in the sun with @user ️ ️ ️ @ Abbot Kinney, Venice",
|
|
31
|
+
"Time for some BBQ and whiskey libations. Chomp, belch, chomp! (@ Lucille's Smokehouse Bar-B-Que)",
|
|
32
|
+
"Love love love all these people ️ ️ ️ #friends #bff #celebrate #blessed #sundayfunday @ San…",
|
|
33
|
+
'️ ️ ️ ️ @ Toys"R"Us',
|
|
34
|
+
"Man these are the funniest kids ever!! That face! #HappyBirthdayBubb @ FLIPnOUT Xtreme",
|
|
35
|
+
"#sandiego @ San Diego, California",
|
|
36
|
+
"My little ️ ️ ️ ️ ️ #ObsessedWithMyDog @ Cafe Solstice Capitol Hill",
|
|
37
|
+
"More #tinyepic things #tinyepicwestern, this one is crazy @user I may be one of your…",
|
|
38
|
+
"Last night ️ @ Omnia Night Club At Caesars Palace",
|
|
39
|
+
"friendship at its finest. ....#pixar #toystory #buzz #woody #friends #friendship #bff…",
|
|
40
|
+
"I L VE working for a cause! Yesterday's balloon decor for SNN 11th Annual Back 2 School Health…",
|
|
41
|
+
"Dem shoes tho! Lol! ...I'm getting some! Get ready copleypriceymca…",
|
|
42
|
+
"I love weekends with you, calebrancourt @ Cannery Row At Monterey Bay, CA",
|
|
43
|
+
"Yesss! August is full of love and birthdays ️ @ Los Angeles,…",
|
|
44
|
+
"I took a pic with Eminem! *still not over that performance vibe from #parrisproject!…",
|
|
45
|
+
"I L VE working for a cause! Yesterday's balloon decor for SNN 11th Annual Back 2 School Health…",
|
|
46
|
+
"Sundays are all about the cute babies and dogs! #Ballard #sundaymarket #littelestmodel…",
|
|
47
|
+
"it was too hot .. trying to find shade behind @user at the roadraceengineering booth…",
|
|
48
|
+
"Sometimes you just have to be a kid Skyspace, Los Angeles. #CRStylesTravel #FunDay #LALife…",
|
|
49
|
+
"had an amazing night, with some pretty amazing people @ Pandora Summer",
|
|
50
|
+
"Thank you @user for an incredible night last night @ Shrine Auditorium & Expo Hall",
|
|
51
|
+
"thanks for the amazing night @ Pandora Summer",
|
|
52
|
+
"@user draw the line ~",
|
|
53
|
+
"This is why I hate when Taylor visits #Bunny #NotCaturday @ Our House",
|
|
54
|
+
"thank you to my boys for performing so well last night, I love you all very much @ Pandora…",
|
|
55
|
+
"2nd theater and 3rd movie of the day a new record #badmoms #suicidesquad #sausageparty (at @user",
|
|
56
|
+
"In honor of #happygayunclesday ️ #happygunclesday #guncle #guncles @ Chula Vista, California",
|
|
57
|
+
"I think it's nap time for us @ Los Angeles, California",
|
|
58
|
+
"I L VE working for a cause! Yesterday's balloon decor for SNN 11th Annual Back 2 School Health…",
|
|
59
|
+
"Chasing #mclarenp1 in Malibu today.________________________________________ : credit: @user",
|
|
60
|
+
"Sunday afternoon at home! Drinking good wine and enjoying this incredible view to the sound of…",
|
|
61
|
+
"#Lunch @user best #Ribs in #SanDiego (at @user in San Diego, CA)",
|
|
62
|
+
"Here's to the start of a great adventure. Niners today, Alaska tomorrow. #grams @ Levi's Stadium",
|
|
63
|
+
"Really horrible seats for a really awful Sunday afternoon! Life is really bad #soundersfc.…",
|
|
64
|
+
"Enjoy your last days of summer! Clearly, mauricethewhale is enjoying his! #ladykiller…",
|
|
65
|
+
"Surprised everyone by actually surviving a weekend in the wild (however still came close to…",
|
|
66
|
+
"In the zone | : @user #colorsworldwide #RnBOnly @ The Regent",
|
|
67
|
+
"“Me & Catherine Sutherland!!!” #PowerMorphicon2016 #PowerRangersZeo #PowerRangersTurbo…",
|
|
68
|
+
"Taped up and dripped out. Know what I'm talking bout #paint #painting #diy #homeimprovement…",
|
|
69
|
+
"princesscruises @ Port of Seattle, Pier 91",
|
|
70
|
+
"Stay classy #SanDiego (@ San Diego International Airport - @user in San Diego, CA)",
|
|
71
|
+
"I ️ the people I work with. Enjoying our Sunday watching the first…",
|
|
72
|
+
"Another big happy 18th birthday to my partner in crime ️ I love u and all the crazy times we've…",
|
|
73
|
+
"Day is DONE! Thank u INSTA-LIKERS! Rollicking day at FRINGE! C U THURSDAY~~~and then some .…",
|
|
74
|
+
"SnakeShred recording guitars for the new SnakeSkin record #guitar #recording #api #apipegged…",
|
|
75
|
+
"My sisters, my best friends #hamessisters #andlewisandclark…",
|
|
76
|
+
"my absolute favorite place. @ La Jolla Cove Seal Beach",
|
|
77
|
+
"To live is an awfully big adventure #Seattle #spaceneedle @ Space Needle",
|
|
78
|
+
"Finally met Snow White ! She was my favorite when i was little :) #SnowWhite #Disneyland @user",
|
|
79
|
+
"@user loved your coverage at TI as always, now need to get you on some of those Sounders matches...",
|
|
80
|
+
"Saturdayzzz w/ Lionel Richie @ Harvey's Outdoor Ampitheater",
|
|
81
|
+
"All the beautiful ladies that helped make last night #APositiveExperience @ Upscale Salon",
|
|
82
|
+
"Sunday funday indeed #sundayfunday #igers #igdaily #instalike #enjoylife #asian #LA…",
|
|
83
|
+
"Lights, Camera, Action @ Hollywood Walk of Fame",
|
|
84
|
+
"Found your Unicorn @user it's a pillow/stuffed toy and every pride weekend it…",
|
|
85
|
+
"Harry Potter ️ @ TCL Chinese Theatres",
|
|
86
|
+
"My key was still here...thank the Lawd #littlehoneyvee #doodles #doodleoftheday #illustration…",
|
|
87
|
+
"️ ️ ️ @ Los Angeles, California",
|
|
88
|
+
"Absolutely in love with this ethereal glow on holytolidoitseric. Who would rock this look?…",
|
|
89
|
+
"Getting up that ice wall one ax at a time : pechoi11 @ Mount Shasta, California",
|
|
90
|
+
"Sephora is my favorite place @ SEPHORA",
|
|
91
|
+
"Tutorial time w/ @user @user @ Stern Grove Festival",
|
|
92
|
+
"I love my people! @ Paso Robles, California",
|
|
93
|
+
"Rad Fashion at the midway SF on a Sunday ️ ️ ️ @ The Midway SF",
|
|
94
|
+
"Just like the movie... Radiator Springs ️ #cars #inlove…",
|
|
95
|
+
"Cheers to Bruce on his birthday. ️#westhollywood…",
|
|
96
|
+
"My bears! #cityfest ️ ️ ️ @ Babycakes San Diego",
|
|
97
|
+
"gonna miss you guys so much. thank you for an amazing experience ️#ballyhoocarrie…",
|
|
98
|
+
"MATTRESS HELPER - new again SAGGING #BED #FIRMER #fix #guaranteed #sleepless #needsleep…",
|
|
99
|
+
"Just that sound alone makes the world stop ️ @ Sausalito, California",
|
|
100
|
+
"#ExclusiveBaseballPicks are KILLING the #Bookie like always. #WeeklyRecap 8/8-8/14. 96-3…",
|
|
101
|
+
"My lady love beautiful friend #doubleLtrouble #leilavie #beachour @ Stinson Beach, California",
|
|
102
|
+
"My date for tonight @ Hotel Las Rocas",
|
|
103
|
+
"This weekend love was truly in the air but out of all the celeb weddings I have to say…",
|
|
104
|
+
"@user hi from Tijuana B.C. See you soon",
|
|
105
|
+
"️ #gramma @ Living Way Christian Fellowship",
|
|
106
|
+
"Acquired this stunner today at my new favorite spot Sharks Tooth #crassula.…",
|
|
107
|
+
"Birthday cheers with my favorite people!!#26again #justafewofmygirls #justafewofmygirls…",
|
|
108
|
+
"today was my first day of work at Victoria Secret SO happy and SO…",
|
|
109
|
+
"Taking bae to all my favorite #LA spots. #gettycenter @ Getty Center",
|
|
110
|
+
"So much fun this weekend celebrating the bride to be. #MalliesLastRally @ Las Vegas, Nevada",
|
|
111
|
+
"Love this picture of me and baby Eve at Disneyland earlier today …",
|
|
112
|
+
"The one photo I've taken with my Mother that I like for once. -08/13/16, Sat. @ El Dorado…",
|
|
113
|
+
"Love my bed @ Downey, California",
|
|
114
|
+
"So cute!! #KnottsBerryFarm #Charlie #Snoopy #Woodstock @ Knott's Berry Farm",
|
|
115
|
+
"I ️ Faure Requiem @user oboyddd @ Walt Disney Concert Hall",
|
|
116
|
+
"️ @ Columbia River",
|
|
117
|
+
"No I don't have any carrots!! #seattlepoloclub #horse #horses @ Seattle Polo & Equestrian Club",
|
|
118
|
+
"My favourite place in California #YosemiteNationalPark #California @ Yosemite Wilderness",
|
|
119
|
+
"#sundayfunday #live at allure... @user @user @user",
|
|
120
|
+
"My babies ️ ️ ️ #cat #dog #catsofinstagram #dogsofinstagram #lovemyfurbabies @ Port Hueneme,…",
|
|
121
|
+
"Photo taken Thursday at SDCC2016 by MannyLLanura Photography. #widowmaker #overwatch…",
|
|
122
|
+
"Hello there, my king Jareth #bowie #goblinking #labyrinth @ Los Angeles, California",
|
|
123
|
+
"These Teens @ ARIA Resort & Casino",
|
|
124
|
+
"Sunday views #hike #waterfall #waterfallhike #latourellfalls #hikeoregon #hikepnw #whyihike…",
|
|
125
|
+
'"Beauty begins the moment you decide to be yourself" -Coco Chanel love this!Skin is the largest…',
|
|
126
|
+
"Caught by the lens of Yc Wong. photo by 8888tiger8888 suit by virginblak #ljifff…",
|
|
127
|
+
"Weldone champ #babrain #alikhamis #rio #rio2016 #olympics @ Los…",
|
|
128
|
+
"Took a lil trip to Tahoe #iputhehoeintahoe @ Incline Village At…",
|
|
129
|
+
"About to hang out with a bunch of #Armenians and #Russians...trying to fit in Sporting my…",
|
|
130
|
+
],
|
|
131
|
+
"label": [
|
|
132
|
+
12,
|
|
133
|
+
19,
|
|
134
|
+
0,
|
|
135
|
+
0,
|
|
136
|
+
2,
|
|
137
|
+
11,
|
|
138
|
+
0,
|
|
139
|
+
19,
|
|
140
|
+
0,
|
|
141
|
+
7,
|
|
142
|
+
1,
|
|
143
|
+
2,
|
|
144
|
+
9,
|
|
145
|
+
0,
|
|
146
|
+
1,
|
|
147
|
+
1,
|
|
148
|
+
1,
|
|
149
|
+
4,
|
|
150
|
+
19,
|
|
151
|
+
8,
|
|
152
|
+
7,
|
|
153
|
+
8,
|
|
154
|
+
5,
|
|
155
|
+
2,
|
|
156
|
+
8,
|
|
157
|
+
2,
|
|
158
|
+
0,
|
|
159
|
+
1,
|
|
160
|
+
1,
|
|
161
|
+
10,
|
|
162
|
+
8,
|
|
163
|
+
14,
|
|
164
|
+
1,
|
|
165
|
+
16,
|
|
166
|
+
2,
|
|
167
|
+
14,
|
|
168
|
+
18,
|
|
169
|
+
6,
|
|
170
|
+
6,
|
|
171
|
+
8,
|
|
172
|
+
14,
|
|
173
|
+
0,
|
|
174
|
+
0,
|
|
175
|
+
9,
|
|
176
|
+
19,
|
|
177
|
+
13,
|
|
178
|
+
3,
|
|
179
|
+
8,
|
|
180
|
+
0,
|
|
181
|
+
16,
|
|
182
|
+
7,
|
|
183
|
+
7,
|
|
184
|
+
6,
|
|
185
|
+
6,
|
|
186
|
+
2,
|
|
187
|
+
0,
|
|
188
|
+
0,
|
|
189
|
+
0,
|
|
190
|
+
1,
|
|
191
|
+
18,
|
|
192
|
+
7,
|
|
193
|
+
2,
|
|
194
|
+
8,
|
|
195
|
+
0,
|
|
196
|
+
0,
|
|
197
|
+
0,
|
|
198
|
+
0,
|
|
199
|
+
0,
|
|
200
|
+
7,
|
|
201
|
+
0,
|
|
202
|
+
19,
|
|
203
|
+
1,
|
|
204
|
+
1,
|
|
205
|
+
0,
|
|
206
|
+
14,
|
|
207
|
+
0,
|
|
208
|
+
1,
|
|
209
|
+
14,
|
|
210
|
+
1,
|
|
211
|
+
8,
|
|
212
|
+
3,
|
|
213
|
+
1,
|
|
214
|
+
5,
|
|
215
|
+
1,
|
|
216
|
+
1,
|
|
217
|
+
0,
|
|
218
|
+
12,
|
|
219
|
+
2,
|
|
220
|
+
1,
|
|
221
|
+
6,
|
|
222
|
+
0,
|
|
223
|
+
10,
|
|
224
|
+
0,
|
|
225
|
+
14,
|
|
226
|
+
12,
|
|
227
|
+
13,
|
|
228
|
+
18,
|
|
229
|
+
7,
|
|
230
|
+
8,
|
|
231
|
+
19,
|
|
232
|
+
],
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def test_search_resource(knowledgebox: KnowledgeBox):
|
|
237
|
+
# Lets create a bunch of resources
|
|
238
|
+
|
|
239
|
+
text: str
|
|
240
|
+
for index, text in enumerate(DATA["text"]):
|
|
241
|
+
if index == 50:
|
|
242
|
+
break
|
|
243
|
+
label = DATA["label"][index]
|
|
244
|
+
knowledgebox.upload(
|
|
245
|
+
text=text,
|
|
246
|
+
labels=[f"emoji/{label}"],
|
|
247
|
+
vectors={"all-MiniLM-L6-v2": [1.0, 2.0, 3.0, 2.0]},
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
assert len(knowledgebox) == 50
|
|
251
|
+
labels = knowledgebox.get_uploaded_labels()
|
|
252
|
+
|
|
253
|
+
assert labels["emoji"].count == 50
|
|
254
|
+
assert labels["emoji"].labels["0"] == 9
|
|
255
|
+
|
|
256
|
+
resources = knowledgebox.search(text="love")
|
|
257
|
+
assert resources.fulltext.total == 5
|
|
258
|
+
assert len(resources.resources) == 5
|
|
259
|
+
|
|
260
|
+
resources = knowledgebox.search(filter=[Label(labelset="emoji", label="0")])
|
|
261
|
+
|
|
262
|
+
assert resources.fulltext.total == 9
|
|
263
|
+
|
|
264
|
+
resources = knowledgebox.search(filter=["emoji/0"])
|
|
265
|
+
|
|
266
|
+
assert resources.fulltext.total == 9
|
|
267
|
+
|
|
268
|
+
vector_q = [1.0, 2.0, 3.0, 2.0]
|
|
269
|
+
resources = knowledgebox.search(
|
|
270
|
+
vector=vector_q, vectorset="all-MiniLM-L6-v2", min_score=0.70
|
|
271
|
+
)
|
|
272
|
+
assert len(resources.sentences.results) == 11
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _test_search_resource_simple_label(knowledgebox: KnowledgeBox):
|
|
276
|
+
# Lets create a bunch of resources
|
|
277
|
+
|
|
278
|
+
text: str
|
|
279
|
+
for index, text in enumerate(DATA["text"]):
|
|
280
|
+
if index == 50:
|
|
281
|
+
break
|
|
282
|
+
label = DATA["label"][index]
|
|
283
|
+
knowledgebox.upload(
|
|
284
|
+
text=text,
|
|
285
|
+
labels=[str(label)],
|
|
286
|
+
vectors={"all-MiniLM-L6-v2": [1.0, 2.0, 3.0, 2.0]},
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
assert len(knowledgebox) == 50
|
|
290
|
+
labels = knowledgebox.get_uploaded_labels()
|
|
291
|
+
|
|
292
|
+
assert labels["default"].count == 50
|
|
293
|
+
assert labels["default"].labels["0"] == 9
|
|
294
|
+
|
|
295
|
+
resources = knowledgebox.search(text="love")
|
|
296
|
+
assert resources.fulltext.total == 5
|
|
297
|
+
assert len(resources.resources) == 5
|
|
298
|
+
|
|
299
|
+
resources = knowledgebox.search(filter=["12"])
|
|
300
|
+
|
|
301
|
+
vector_q = [1.0, 2.0, 3.0, 2.0]
|
|
302
|
+
resources = knowledgebox.search(
|
|
303
|
+
vector=vector_q,
|
|
304
|
+
vectorset="all-MiniLM-L6-v2",
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
@pytest.mark.asyncio
|
|
309
|
+
async def _test_search_resource_async(knowledgebox: KnowledgeBox):
|
|
310
|
+
knowledgebox.new_vectorset("all-MiniLM-L6-v2", 4)
|
|
311
|
+
|
|
312
|
+
text: str
|
|
313
|
+
for index, text in enumerate(DATA["text"]):
|
|
314
|
+
if index == 50:
|
|
315
|
+
break
|
|
316
|
+
label = DATA["label"][index]
|
|
317
|
+
await knowledgebox.async_upload(
|
|
318
|
+
text=text,
|
|
319
|
+
labels=[Label(label=str(label), labelset="emoji")],
|
|
320
|
+
vectors=[
|
|
321
|
+
Vector(
|
|
322
|
+
value=[1.0, 2.0, 3.0, 2.0],
|
|
323
|
+
vectorset="all-MiniLM-L6-v2",
|
|
324
|
+
)
|
|
325
|
+
],
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
assert len(knowledgebox) == 50
|
|
329
|
+
labels = await knowledgebox.async_get_uploaded_labels()
|
|
330
|
+
|
|
331
|
+
assert labels["emoji"].count == 50
|
|
332
|
+
assert labels["emoji"].labels["0"] == 9
|
|
333
|
+
|
|
334
|
+
resources = await knowledgebox.async_search(text="love")
|
|
335
|
+
assert resources.fulltext.total == 5
|
|
336
|
+
assert len(resources.resources) == 5
|
|
337
|
+
|
|
338
|
+
resources = await knowledgebox.async_search(
|
|
339
|
+
filter=[Label(labelset="emoji", label="12")]
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
vector_q = [1.0, 2.0, 3.0, 2.0]
|
|
343
|
+
resources = await knowledgebox.async_search(
|
|
344
|
+
vector=vector_q,
|
|
345
|
+
vectorset="all-MiniLM-L6-v2",
|
|
346
|
+
)
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
2
|
+
#
|
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
5
|
+
#
|
|
6
|
+
# AGPL:
|
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
10
|
+
# License, or (at your option) any later version.
|
|
11
|
+
#
|
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
# GNU Affero General Public License for more details.
|
|
16
|
+
#
|
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
|
+
|
|
20
|
+
import pytest
|
|
21
|
+
|
|
22
|
+
from nucliadb_sdk.entities import Entity
|
|
23
|
+
from nucliadb_sdk.file import File
|
|
24
|
+
from nucliadb_sdk.knowledgebox import KnowledgeBox
|
|
25
|
+
from nucliadb_sdk.vectors import Vector
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_create_resource(knowledgebox: KnowledgeBox):
|
|
29
|
+
assert knowledgebox.get("mykey1") is None
|
|
30
|
+
|
|
31
|
+
resource_id = knowledgebox.upload(
|
|
32
|
+
key="mykey1",
|
|
33
|
+
binary=File(data=b"asd", filename="data"),
|
|
34
|
+
text="I'm Ramon",
|
|
35
|
+
labels=["labelset/positive"],
|
|
36
|
+
entities=[Entity(type="NAME", value="Ramon", positions=[(5, 9)])],
|
|
37
|
+
vectors=[Vector(value=[1.0, 0.2], vectorset="base")],
|
|
38
|
+
)
|
|
39
|
+
resource2 = knowledgebox[resource_id]
|
|
40
|
+
assert (
|
|
41
|
+
resource2.data is not None
|
|
42
|
+
and resource2.data.texts is not None
|
|
43
|
+
and "text" in resource2.data.texts
|
|
44
|
+
)
|
|
45
|
+
assert (
|
|
46
|
+
resource2.data is not None
|
|
47
|
+
and resource2.data.files is not None
|
|
48
|
+
and "file" in resource2.data.files
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
resource3 = knowledgebox["mykey1"]
|
|
52
|
+
assert resource3.id == resource2.id
|
|
53
|
+
|
|
54
|
+
assert len(knowledgebox) == 1
|
|
55
|
+
|
|
56
|
+
del knowledgebox["mykey1"]
|
|
57
|
+
|
|
58
|
+
assert len(knowledgebox) == 0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_create_resource_dict(knowledgebox: KnowledgeBox):
|
|
62
|
+
assert knowledgebox.get("mykey1") is None
|
|
63
|
+
|
|
64
|
+
resource_id = knowledgebox.upload(
|
|
65
|
+
key="mykey1",
|
|
66
|
+
binary=File(data=b"asd", filename="data"),
|
|
67
|
+
text="I'm Ramon",
|
|
68
|
+
labels=["labelset/positive"],
|
|
69
|
+
entities=[Entity(type="NAME", value="Ramon", positions=[(5, 9)])],
|
|
70
|
+
vectors={"base": [1.0, 2.0]},
|
|
71
|
+
)
|
|
72
|
+
resource2 = knowledgebox[resource_id]
|
|
73
|
+
assert (
|
|
74
|
+
resource2.data is not None
|
|
75
|
+
and resource2.data.texts is not None
|
|
76
|
+
and "text" in resource2.data.texts
|
|
77
|
+
)
|
|
78
|
+
assert (
|
|
79
|
+
resource2.data is not None
|
|
80
|
+
and resource2.data.files is not None
|
|
81
|
+
and "file" in resource2.data.files
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
resource3 = knowledgebox["mykey1"]
|
|
85
|
+
assert resource3.id == resource2.id
|
|
86
|
+
|
|
87
|
+
assert len(knowledgebox) == 1
|
|
88
|
+
|
|
89
|
+
del knowledgebox["mykey1"]
|
|
90
|
+
|
|
91
|
+
assert len(knowledgebox) == 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@pytest.mark.asyncio
|
|
95
|
+
async def test_create_resource_async(knowledgebox: KnowledgeBox):
|
|
96
|
+
assert await knowledgebox.async_get("mykey1") is None
|
|
97
|
+
|
|
98
|
+
resource_id = await knowledgebox.async_upload(
|
|
99
|
+
key="mykey1",
|
|
100
|
+
binary=File(data=b"asd", filename="data"),
|
|
101
|
+
text="asd",
|
|
102
|
+
labels=["labelset/positive"],
|
|
103
|
+
entities=[Entity(type="NAME", value="Ramon", positions=[(5, 9)])],
|
|
104
|
+
vectors=[Vector(value=[1.0, 0.2], vectorset="base")],
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
resource2 = await knowledgebox.async_get(resource_id)
|
|
108
|
+
assert (
|
|
109
|
+
resource2.data is not None
|
|
110
|
+
and resource2.data.texts is not None
|
|
111
|
+
and "text" in resource2.data.texts
|
|
112
|
+
)
|
|
113
|
+
assert (
|
|
114
|
+
resource2.data is not None
|
|
115
|
+
and resource2.data.files is not None
|
|
116
|
+
and "file" in resource2.data.files
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
resource3 = await knowledgebox.async_get("mykey1")
|
|
120
|
+
assert resource3.id == resource2.id
|
|
121
|
+
|
|
122
|
+
assert await knowledgebox.async_len() == 1
|
|
123
|
+
|
|
124
|
+
await knowledgebox.async_del("mykey1")
|
|
125
|
+
|
|
126
|
+
assert await knowledgebox.async_len() == 0
|
nucliadb_sdk/utils.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
2
|
+
#
|
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
5
|
+
#
|
|
6
|
+
# AGPL:
|
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
10
|
+
# License, or (at your option) any later version.
|
|
11
|
+
#
|
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
# GNU Affero General Public License for more details.
|
|
16
|
+
#
|
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
|
+
|
|
20
|
+
# europe-1.nuclia.cloud
|
|
21
|
+
# localhost:8080
|
|
22
|
+
|
|
23
|
+
from typing import List, Optional
|
|
24
|
+
from urllib.parse import urlparse
|
|
25
|
+
from uuid import uuid4
|
|
26
|
+
|
|
27
|
+
import requests
|
|
28
|
+
|
|
29
|
+
from nucliadb_models.resource import KnowledgeBoxList, KnowledgeBoxObj
|
|
30
|
+
from nucliadb_models.utils import SlugString
|
|
31
|
+
from nucliadb_sdk.client import Environment, NucliaDBClient
|
|
32
|
+
from nucliadb_sdk.knowledgebox import KnowledgeBox
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class InvalidHost(Exception):
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class KnowledgeBoxAlreadyExists(Exception):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def create_knowledge_box(
|
|
44
|
+
slug: Optional[str] = None,
|
|
45
|
+
nucliadb_base_url: Optional[str] = "http://localhost:8080",
|
|
46
|
+
):
|
|
47
|
+
url_obj = urlparse(nucliadb_base_url)
|
|
48
|
+
if url_obj.hostname and url_obj.hostname.endswith("nuclia.cloud"): # type: ignore
|
|
49
|
+
raise InvalidHost(
|
|
50
|
+
"You can not create a Knowledge Box via API, please use https://nuclia.cloud interface"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
if slug is None:
|
|
54
|
+
slug = uuid4().hex
|
|
55
|
+
|
|
56
|
+
api_path = f"{nucliadb_base_url}/api/v1"
|
|
57
|
+
response = requests.post(
|
|
58
|
+
f"{api_path}/kbs",
|
|
59
|
+
json={"slug": slug},
|
|
60
|
+
headers={"X-NUCLIADB-ROLES": "MANAGER"},
|
|
61
|
+
)
|
|
62
|
+
if response.status_code == 419:
|
|
63
|
+
raise KnowledgeBoxAlreadyExists()
|
|
64
|
+
|
|
65
|
+
assert response.status_code == 201
|
|
66
|
+
|
|
67
|
+
kb = KnowledgeBoxObj.parse_raw(response.content)
|
|
68
|
+
kbid = kb.uuid
|
|
69
|
+
|
|
70
|
+
url = f"{api_path}/kb/{kbid}"
|
|
71
|
+
client = NucliaDBClient(environment=Environment.OSS, url=url)
|
|
72
|
+
|
|
73
|
+
return KnowledgeBox(client)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def get_kb(
|
|
77
|
+
slug: str,
|
|
78
|
+
nucliadb_base_url: Optional[str] = "http://localhost:8080",
|
|
79
|
+
) -> Optional[KnowledgeBox]:
|
|
80
|
+
url_obj = urlparse(nucliadb_base_url)
|
|
81
|
+
if url_obj.hostname and url_obj.hostname.endswith("nuclia.cloud"): # type: ignore
|
|
82
|
+
raise InvalidHost(
|
|
83
|
+
"You can not create a Knowledge Box via API, please use https://nuclia.cloud interface"
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
api_path = f"{nucliadb_base_url}/api/v1"
|
|
87
|
+
response = requests.get(
|
|
88
|
+
f"{api_path}/kb/s/{slug}",
|
|
89
|
+
headers={"X-NUCLIADB-ROLES": "READER"},
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
if response.status_code == 404:
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
assert response.status_code == 200
|
|
96
|
+
|
|
97
|
+
kb = KnowledgeBoxObj.parse_raw(response.content)
|
|
98
|
+
kbid = kb.uuid
|
|
99
|
+
|
|
100
|
+
url = f"{api_path}/kb/{kbid}"
|
|
101
|
+
client = NucliaDBClient(environment=Environment.OSS, url=url)
|
|
102
|
+
|
|
103
|
+
return KnowledgeBox(client)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def get_or_create(
|
|
107
|
+
slug: str, nucliadb_base_url: Optional[str] = "http://localhost:8080"
|
|
108
|
+
):
|
|
109
|
+
kb = get_kb(slug, nucliadb_base_url)
|
|
110
|
+
if kb is None:
|
|
111
|
+
kb = create_knowledge_box(slug, nucliadb_base_url)
|
|
112
|
+
return kb
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def delete_kb(slug: str, nucliadb_base_url: Optional[str] = "http://localhost:8080"):
|
|
116
|
+
kb = get_kb(slug, nucliadb_base_url)
|
|
117
|
+
if kb is None or kb.client.url is None:
|
|
118
|
+
raise AttributeError("URL should not be none")
|
|
119
|
+
response = requests.delete(kb.client.url, headers={"X-NUCLIADB-ROLES": f"MANAGER"})
|
|
120
|
+
assert response.status_code == 200
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def list_kbs(
|
|
124
|
+
nucliadb_base_url: Optional[str] = "http://localhost:8080",
|
|
125
|
+
) -> List[Optional[SlugString]]:
|
|
126
|
+
response = requests.get(
|
|
127
|
+
f"{nucliadb_base_url}/kbs",
|
|
128
|
+
headers={"X-NUCLIADB-ROLES": "MANAGER"},
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
assert response.status_code == 200
|
|
132
|
+
kbs = KnowledgeBoxList.parse_raw(response.content)
|
|
133
|
+
return [kb.slug for kb in kbs.kbs]
|
nucliadb_sdk/vectors.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
2
|
+
#
|
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
5
|
+
#
|
|
6
|
+
# AGPL:
|
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
10
|
+
# License, or (at your option) any later version.
|
|
11
|
+
#
|
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
# GNU Affero General Public License for more details.
|
|
16
|
+
#
|
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
|
+
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from typing import Any, List, Optional, Tuple
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Vector:
|
|
26
|
+
value: List[float]
|
|
27
|
+
vectorset: str
|
|
28
|
+
key: Optional[str] = None
|
|
29
|
+
positions: Optional[Tuple[int, int]] = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
Vectors = List[Vector]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def convert_vector(vector: Any) -> List[float]:
|
|
36
|
+
if (
|
|
37
|
+
vector.__class__.__module__ == "numpy"
|
|
38
|
+
and vector.__class__.__name__ == "ndarray"
|
|
39
|
+
):
|
|
40
|
+
vector = vector.tolist()
|
|
41
|
+
|
|
42
|
+
if (
|
|
43
|
+
vector.__class__.__module__ == "tensorflow.python.framework.ops"
|
|
44
|
+
and vector.__class__.__name__ == "EagerTensor"
|
|
45
|
+
):
|
|
46
|
+
vector = vector.numpy().tolist()
|
|
47
|
+
|
|
48
|
+
if vector.__class__.__module__ == "torch" and vector.__class__.__name__ == "Tensor":
|
|
49
|
+
vector = vector.tolist()
|
|
50
|
+
|
|
51
|
+
return vector
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: nucliadb-sdk
|
|
3
|
+
Version: 0.0.0.post2
|
|
4
|
+
Summary: UNKNOWN
|
|
5
|
+
Home-page: https://nucliadb.com
|
|
6
|
+
Author: UNKNOWN
|
|
7
|
+
Author-email: UNKNOWN
|
|
8
|
+
License: BSD
|
|
9
|
+
Platform: UNKNOWN
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
13
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
|
|
14
|
+
Requires-Dist: httpx
|
|
15
|
+
Requires-Dist: nucliadb-models (==0.0.0-post2)
|
|
16
|
+
|
|
17
|
+
# NucliaDB SDK
|
|
18
|
+
|
|
19
|
+
NucliaDB Cloud and NucliaDB OSS sdk
|
|
20
|
+
|
|
21
|
+
Update and compile the nucliadb image locally before running the tests.
|
|
22
|
+
From nucliadb root dir:
|
|
23
|
+
`make build-nucliadb-local-withbinding`
|
|
24
|
+
|
|
25
|
+
|