nucliadb 6.8.1.post4983__py3-none-any.whl → 6.8.1.post5003__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of nucliadb might be problematic. Click here for more details.

@@ -0,0 +1,175 @@
1
+ # Copyright (C) 2021 Bosutech XXI S.L.
2
+ #
3
+ # nucliadb is offered under the AGPL v3.0 and as commercial software.
4
+ # For commercial licensing, contact us at info@nuclia.com.
5
+ #
6
+ # AGPL:
7
+ # This program is free software: you can redistribute it and/or modify
8
+ # it under the terms of the GNU Affero General Public License as
9
+ # published by the Free Software Foundation, either version 3 of the
10
+ # License, or (at your option) any later version.
11
+ #
12
+ # This program is distributed in the hope that it will be useful,
13
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ # GNU Affero General Public License for more details.
16
+ #
17
+ # You should have received a copy of the GNU Affero General Public License
18
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
19
+ #
20
+
21
+
22
+ from nucliadb.common.ids import FIELD_TYPE_STR_TO_NAME, FieldId
23
+ from nucliadb.common.models_utils import from_proto
24
+ from nucliadb.ingest.orm.resource import Resource
25
+ from nucliadb.search.search.hydrator import hydrate_field_text
26
+ from nucliadb_models import hydration as hydration_models
27
+ from nucliadb_models.common import FieldTypeName
28
+
29
+
30
+ def page_preview_id(page_number: int) -> str:
31
+ """Return the string page number for an specific page"""
32
+ return f"{page_number}"
33
+
34
+
35
+ async def hydrate_field(resource: Resource, field_id: FieldId, config: hydration_models.FieldHydration):
36
+ field_type = FIELD_TYPE_STR_TO_NAME[field_id.type]
37
+
38
+ if field_type == FieldTypeName.TEXT:
39
+ if not config.text is not None:
40
+ return
41
+ return await hydrate_text_field(resource, field_id, config.text)
42
+
43
+ elif field_type == FieldTypeName.FILE is not None:
44
+ if not config.file:
45
+ return
46
+ return await hydrate_file_field(resource, field_id, config.file)
47
+
48
+ elif field_type == FieldTypeName.LINK is not None:
49
+ if not config.link:
50
+ return
51
+ return await hydrate_link_field(resource, field_id, config.link)
52
+
53
+ elif field_type == FieldTypeName.CONVERSATION is not None:
54
+ if not config.conversation:
55
+ return
56
+ return await hydrate_conversation_field(resource, field_id, config.conversation)
57
+
58
+ elif field_type == FieldTypeName.GENERIC is not None:
59
+ if not config.generic:
60
+ return
61
+ return await hydrate_generic_field(resource, field_id, config.generic)
62
+
63
+ else: # pragma: no cover
64
+ # This is a trick so mypy generates an error if this branch can be reached,
65
+ # that is, if we are missing some ifs
66
+ _a: int = "a"
67
+
68
+
69
+ async def hydrate_text_field(
70
+ resource: Resource,
71
+ field_id: FieldId,
72
+ config: hydration_models.TextFieldHydration,
73
+ ) -> hydration_models.HydratedTextField:
74
+ hydrated = hydration_models.HydratedTextField(
75
+ id=field_id.full(),
76
+ resource=field_id.rid,
77
+ field_type=FieldTypeName.TEXT,
78
+ )
79
+
80
+ if config.extracted_text:
81
+ field_text = await hydrate_field_text(resource.kb.kbid, field_id)
82
+ if field_text is not None:
83
+ (_, text) = field_text
84
+ hydrated.extracted = hydration_models.FieldExtractedData(text=text)
85
+
86
+ return hydrated
87
+
88
+
89
+ async def hydrate_file_field(
90
+ resource: Resource,
91
+ field_id: FieldId,
92
+ config: hydration_models.FileFieldHydration,
93
+ ) -> hydration_models.HydratedFileField:
94
+ hydrated = hydration_models.HydratedFileField(
95
+ id=field_id.full(),
96
+ resource=field_id.rid,
97
+ field_type=FieldTypeName.FILE,
98
+ )
99
+
100
+ if config.value:
101
+ field = await resource.get_field(field_id.key, field_id.pb_type)
102
+ value = await field.get_value()
103
+ hydrated.value = from_proto.field_file(value)
104
+
105
+ if config.extracted_text:
106
+ field_text = await hydrate_field_text(resource.kb.kbid, field_id)
107
+ if field_text is not None:
108
+ (_, text) = field_text
109
+ hydrated.extracted = hydration_models.FieldExtractedData(text=text)
110
+
111
+ return hydrated
112
+
113
+
114
+ async def hydrate_link_field(
115
+ resource: Resource,
116
+ field_id: FieldId,
117
+ config: hydration_models.LinkFieldHydration,
118
+ ) -> hydration_models.HydratedLinkField:
119
+ hydrated = hydration_models.HydratedLinkField(
120
+ id=field_id.full(),
121
+ resource=field_id.rid,
122
+ field_type=FieldTypeName.LINK,
123
+ )
124
+
125
+ if config.value:
126
+ field = await resource.get_field(field_id.key, field_id.pb_type)
127
+ value = await field.get_value()
128
+ hydrated.value = from_proto.field_link(value)
129
+
130
+ if config.extracted_text:
131
+ field_text = await hydrate_field_text(resource.kb.kbid, field_id)
132
+ if field_text is not None:
133
+ (_, text) = field_text
134
+ hydrated.extracted = hydration_models.FieldExtractedData(text=text)
135
+
136
+ return hydrated
137
+
138
+
139
+ async def hydrate_conversation_field(
140
+ resource: Resource,
141
+ field_id: FieldId,
142
+ config: hydration_models.ConversationFieldHydration,
143
+ ) -> hydration_models.HydratedConversationField:
144
+ hydrated = hydration_models.HydratedConversationField(
145
+ id=field_id.full(),
146
+ resource=field_id.rid,
147
+ field_type=FieldTypeName.CONVERSATION,
148
+ )
149
+ # TODO: implement conversation fields
150
+ return hydrated
151
+
152
+
153
+ async def hydrate_generic_field(
154
+ resource: Resource,
155
+ field_id: FieldId,
156
+ config: hydration_models.GenericFieldHydration,
157
+ ) -> hydration_models.HydratedGenericField:
158
+ hydrated = hydration_models.HydratedGenericField(
159
+ id=field_id.full(),
160
+ resource=field_id.rid,
161
+ field_type=FieldTypeName.GENERIC,
162
+ )
163
+
164
+ if config.value:
165
+ field = await resource.get_field(field_id.key, field_id.pb_type)
166
+ value = await field.get_value()
167
+ hydrated.value = value
168
+
169
+ if config.extracted_text:
170
+ field_text = await hydrate_field_text(resource.kb.kbid, field_id)
171
+ if field_text is not None:
172
+ (_, text) = field_text
173
+ hydrated.extracted = hydration_models.FieldExtractedData(text=text)
174
+
175
+ return hydrated
@@ -0,0 +1,126 @@
1
+ # Copyright (C) 2021 Bosutech XXI S.L.
2
+ #
3
+ # nucliadb is offered under the AGPL v3.0 and as commercial software.
4
+ # For commercial licensing, contact us at info@nuclia.com.
5
+ #
6
+ # AGPL:
7
+ # This program is free software: you can redistribute it and/or modify
8
+ # it under the terms of the GNU Affero General Public License as
9
+ # published by the Free Software Foundation, either version 3 of the
10
+ # License, or (at your option) any later version.
11
+ #
12
+ # This program is distributed in the hope that it will be useful,
13
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ # GNU Affero General Public License for more details.
16
+ #
17
+ # You should have received a copy of the GNU Affero General Public License
18
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
19
+ #
20
+ import base64
21
+ from typing import Optional, cast
22
+
23
+ from nucliadb.common.ids import FIELD_TYPE_STR_TO_NAME, FieldId, ParagraphId
24
+ from nucliadb.ingest.fields.base import Field
25
+ from nucliadb.ingest.fields.file import File
26
+ from nucliadb.search import SERVICE_NAME
27
+ from nucliadb_models.common import FieldTypeName
28
+ from nucliadb_models.search import Image
29
+ from nucliadb_protos import resources_pb2
30
+ from nucliadb_utils.utilities import get_storage
31
+
32
+
33
+ async def paragraph_source_image(kbid: str, paragraph: resources_pb2.Paragraph) -> Optional[Image]:
34
+ """Certain paragraphs are extracted from images using techniques like OCR or
35
+ inception. If that's the case, return the original image for this paragraph.
36
+
37
+ """
38
+ source_image = paragraph.representation.reference_file
39
+
40
+ if paragraph.kind not in (
41
+ resources_pb2.Paragraph.TypeParagraph.OCR,
42
+ resources_pb2.Paragraph.TypeParagraph.INCEPTION,
43
+ ):
44
+ return None
45
+
46
+ field_id = ParagraphId.from_string(paragraph.key).field_id
47
+
48
+ # Paragraphs extracted from an image store its original image representation
49
+ # in the reference file. The path is incomplete though, as it's stored in
50
+ # the `generated` folder
51
+ image = await download_image(
52
+ kbid,
53
+ field_id,
54
+ f"generated/{source_image}",
55
+ # XXX: we assume all reference files are PNG images, but this actually
56
+ # depends on learning so it's a dangerous assumption. We should check it
57
+ # by ourselves
58
+ mime_type="image/png",
59
+ )
60
+ return image
61
+
62
+
63
+ async def download_image(
64
+ kbid: str, field_id: FieldId, image_path: str, *, mime_type: str
65
+ ) -> Optional[Image]:
66
+ storage = await get_storage(service_name=SERVICE_NAME)
67
+ sf = storage.file_extracted(
68
+ kbid,
69
+ field_id.rid,
70
+ field_id.type,
71
+ field_id.key,
72
+ image_path,
73
+ )
74
+ raw_image = (await storage.downloadbytes(sf.bucket, sf.key)).getvalue()
75
+ if not raw_image:
76
+ return None
77
+ return Image(content_type=mime_type, b64encoded=base64.b64encode(raw_image).decode())
78
+
79
+
80
+ async def download_page_preview(field: Field, page: int) -> Optional[Image]:
81
+ """Download a specific page preview for a field and return it as an Image.
82
+ As not all fields have previews, this function can return None.
83
+
84
+ Page previews are uploaded by learning and shared through a known path with.
85
+ nucliadb
86
+
87
+ """
88
+ field_type = FIELD_TYPE_STR_TO_NAME[field.type]
89
+
90
+ if field_type == FieldTypeName.FILE:
91
+ field = cast(File, field)
92
+ metadata = await field.get_file_extracted_data()
93
+
94
+ if metadata is None:
95
+ return None
96
+
97
+ assert page <= len(metadata.file_pages_previews.positions), (
98
+ f"paragraph page number {page} should be less or equal to the total file pages previews {len(metadata.file_pages_previews.positions)}"
99
+ )
100
+ image = await download_image(
101
+ field.kbid,
102
+ field.field_id,
103
+ f"generated/extracted_images_{page}.png",
104
+ mime_type="image/png",
105
+ )
106
+
107
+ elif field_type == FieldTypeName.LINK:
108
+ # TODO: in case of links, we want to return the link preview, that is a
109
+ # link converted to PDF and screenshotted
110
+ # REVIEW: link preview is an image or a PDF?
111
+ image = None
112
+
113
+ elif (
114
+ field_type == FieldTypeName.TEXT
115
+ or field_type == FieldTypeName.CONVERSATION
116
+ or field_type == FieldTypeName.GENERIC
117
+ ):
118
+ # these fields don't have previews
119
+ image = None
120
+
121
+ else: # pragma: no cover
122
+ # This is a trick so mypy generates an error if this branch can be reached,
123
+ # that is, if we are missing some ifs
124
+ _a: int = "a"
125
+
126
+ return image
@@ -0,0 +1,305 @@
1
+ # Copyright (C) 2021 Bosutech XXI S.L.
2
+ #
3
+ # nucliadb is offered under the AGPL v3.0 and as commercial software.
4
+ # For commercial licensing, contact us at info@nuclia.com.
5
+ #
6
+ # AGPL:
7
+ # This program is free software: you can redistribute it and/or modify
8
+ # it under the terms of the GNU Affero General Public License as
9
+ # published by the Free Software Foundation, either version 3 of the
10
+ # License, or (at your option) any later version.
11
+ #
12
+ # This program is distributed in the hope that it will be useful,
13
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ # GNU Affero General Public License for more details.
16
+ #
17
+ # You should have received a copy of the GNU Affero General Public License
18
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
19
+ #
20
+ import asyncio
21
+ from dataclasses import dataclass
22
+ from typing import Optional, Union
23
+
24
+ from nucliadb.common.ids import FieldId, ParagraphId
25
+ from nucliadb.ingest.fields.base import Field
26
+ from nucliadb.ingest.orm.resource import Resource
27
+ from nucliadb.search.search import paragraphs
28
+ from nucliadb.search.search.hydrator.fields import page_preview_id
29
+ from nucliadb.search.search.hydrator.images import paragraph_source_image
30
+ from nucliadb_models import hydration as hydration_models
31
+ from nucliadb_protos import resources_pb2
32
+ from nucliadb_protos.resources_pb2 import FieldComputedMetadata
33
+
34
+
35
+ class ParagraphIndex:
36
+ """Small helper class to cache field paragraphs and its relations and be
37
+ used as an index.
38
+
39
+ """
40
+
41
+ NEXT = "next"
42
+ PREVIOUS = "previous"
43
+ PARENTS = "parents"
44
+ SIBLINGS = "siblings"
45
+ REPLACEMENTS = "replacements"
46
+
47
+ def __init__(self, field_id: FieldId) -> None:
48
+ self.field_id = field_id
49
+ self.paragraphs: dict[str, resources_pb2.Paragraph] = {}
50
+ self.neighbours: dict[tuple[str, str], str] = {}
51
+ self.related: dict[tuple[str, str], list[str]] = {}
52
+ self._lock = asyncio.Lock()
53
+ self._built = False
54
+
55
+ async def build(self, field: Field):
56
+ """Build the index if it hasn't been built yet.
57
+
58
+ This function is async-safe, multiple concurrent tasks can ask for a
59
+ built and it'll only be done once
60
+ """
61
+ if self._built:
62
+ return
63
+
64
+ async with self._lock:
65
+ # double check we haven't built the index meanwhile we waited for the
66
+ # lock
67
+ if self._built:
68
+ return
69
+
70
+ field_metadata = await field.get_field_metadata()
71
+
72
+ if field_metadata is None:
73
+ # field metadata may be still processing. As we want to provide a
74
+ # consistent view, even if it can appear meanwhile we hydrate, we
75
+ # consider we don't have it. We mark the index as built and any
76
+ # paragraph will be found for this field
77
+ self._built = True
78
+ return None
79
+
80
+ # REVIEW: this is a CPU-bound code, we may consider running this in an
81
+ # executor to not block the loop
82
+ self._build(field_metadata)
83
+ self._built = True
84
+
85
+ def _build(self, field_metadata: FieldComputedMetadata):
86
+ self.paragraphs.clear()
87
+ self.neighbours.clear()
88
+ self.related.clear()
89
+
90
+ if self.field_id.subfield_id is None:
91
+ field_paragraphs = field_metadata.metadata.paragraphs
92
+ else:
93
+ field_paragraphs = field_metadata.split_metadata[self.field_id.subfield_id].paragraphs
94
+
95
+ previous = None
96
+ for paragraph in field_paragraphs:
97
+ paragraph_id = self.field_id.paragraph_id(paragraph.start, paragraph.end).full()
98
+ self.paragraphs[paragraph_id] = paragraph
99
+
100
+ if previous is not None:
101
+ self.neighbours[(previous, ParagraphIndex.NEXT)] = paragraph_id
102
+ self.neighbours[(paragraph_id, ParagraphIndex.PREVIOUS)] = previous
103
+ previous = paragraph_id
104
+
105
+ self.related[(paragraph_id, ParagraphIndex.PARENTS)] = [
106
+ parent for parent in paragraph.relations.parents
107
+ ]
108
+ self.related[(paragraph_id, ParagraphIndex.SIBLINGS)] = [
109
+ sibling for sibling in paragraph.relations.siblings
110
+ ]
111
+ self.related[(paragraph_id, ParagraphIndex.REPLACEMENTS)] = [
112
+ replacement for replacement in paragraph.relations.replacements
113
+ ]
114
+
115
+ def get(self, paragraph_id: Union[str, ParagraphId]) -> Optional[resources_pb2.Paragraph]:
116
+ paragraph_id = str(paragraph_id)
117
+ return self.paragraphs.get(paragraph_id)
118
+
119
+ def previous(self, paragraph_id: Union[str, ParagraphId]) -> Optional[str]:
120
+ paragraph_id = str(paragraph_id)
121
+ return self.neighbours.get((paragraph_id, ParagraphIndex.PREVIOUS))
122
+
123
+ def next(self, paragraph_id: Union[str, ParagraphId]) -> Optional[str]:
124
+ paragraph_id = str(paragraph_id)
125
+ return self.neighbours.get((paragraph_id, ParagraphIndex.NEXT))
126
+
127
+ def n_previous(self, paragraph_id: Union[str, ParagraphId], count: int = 1) -> list[str]:
128
+ assert count >= 1, f"can't find negative previous {count}"
129
+ paragraph_id = str(paragraph_id)
130
+ previous: list[str] = []
131
+ current_id = paragraph_id
132
+ for _ in range(count):
133
+ previous_id = self.previous(current_id)
134
+ if previous_id is None:
135
+ # we've reached the first paragraph
136
+ break
137
+ previous.insert(0, previous_id)
138
+ current_id = previous_id
139
+ return previous
140
+
141
+ def n_next(self, paragraph_id: Union[str, ParagraphId], count: int = 1) -> list[str]:
142
+ assert count >= 1, f"can't find negative nexts {count}"
143
+ paragraph_id = str(paragraph_id)
144
+ nexts = []
145
+ current_id = paragraph_id
146
+ for _ in range(count):
147
+ next_id = self.next(current_id)
148
+ if next_id is None:
149
+ # we've reached the last paragraph
150
+ break
151
+ current_id = next_id
152
+ nexts.append(next_id)
153
+ return nexts
154
+
155
+ def parents(self, paragraph_id: Union[str, ParagraphId]) -> list[str]:
156
+ paragraph_id = str(paragraph_id)
157
+ return self.related.get((paragraph_id, ParagraphIndex.PARENTS), [])
158
+
159
+ def siblings(self, paragraph_id: Union[str, ParagraphId]) -> list[str]:
160
+ paragraph_id = str(paragraph_id)
161
+ return self.related.get((paragraph_id, ParagraphIndex.SIBLINGS), [])
162
+
163
+ def replacements(self, paragraph_id: Union[str, ParagraphId]) -> list[str]:
164
+ paragraph_id = str(paragraph_id)
165
+ return self.related.get((paragraph_id, ParagraphIndex.REPLACEMENTS), [])
166
+
167
+
168
+ @dataclass
169
+ class ExtraParagraphHydration:
170
+ field_page: Optional[int]
171
+ field_table_page: Optional[int]
172
+ related_paragraph_ids: list[ParagraphId]
173
+
174
+
175
+ async def hydrate_paragraph(
176
+ resource: Resource,
177
+ field: Field,
178
+ paragraph_id: ParagraphId,
179
+ config: hydration_models.ParagraphHydration,
180
+ field_paragraphs_index: ParagraphIndex,
181
+ ) -> tuple[hydration_models.HydratedParagraph, ExtraParagraphHydration]:
182
+ """Hydrate a paragraph and return the extra hydration to built a coherent
183
+ hydration around this paragraph.
184
+
185
+ Although the resource and field exist, the paragraph doesn't necessarily
186
+ need to be a real one in the paragraph metadata, it can be made-up to
187
+ include more or less text than the originally extracted.
188
+
189
+ """
190
+ kbid = resource.kb.kbid
191
+
192
+ hydrated = hydration_models.HydratedParagraph(
193
+ id=paragraph_id.full(),
194
+ field=paragraph_id.field_id.full(),
195
+ resource=paragraph_id.rid,
196
+ )
197
+ extra_hydration = ExtraParagraphHydration(
198
+ field_page=None, field_table_page=None, related_paragraph_ids=[]
199
+ )
200
+
201
+ if config.text:
202
+ text = await paragraphs.get_paragraph_text(kbid=kbid, paragraph_id=paragraph_id)
203
+ hydrated.text = text
204
+
205
+ requires_paragraph_metadata = config.image or config.table or config.page or config.related
206
+ if requires_paragraph_metadata:
207
+ await field_paragraphs_index.build(field)
208
+ paragraph = field_paragraphs_index.get(paragraph_id)
209
+ if paragraph is not None:
210
+ # otherwise, this is a fake paragraph. We can't hydrate anything else here
211
+
212
+ if config.related:
213
+ hydrated.related, related_ids = await related_paragraphs_refs(
214
+ paragraph_id, field_paragraphs_index, config.related
215
+ )
216
+ extra_hydration.related_paragraph_ids = related_ids
217
+
218
+ if config.image:
219
+ hydrated.image = hydration_models.HydratedParagraphImage()
220
+
221
+ if config.image.source_image:
222
+ hydrated.image.source_image = await paragraph_source_image(kbid, paragraph)
223
+
224
+ if config.page:
225
+ if hydrated.page is None:
226
+ hydrated.page = hydration_models.HydratedParagraphPage()
227
+
228
+ if config.page.page_with_visual:
229
+ if paragraph.page.page_with_visual:
230
+ # Paragraphs can be found on pages with visual content. In this
231
+ # case, we want to return the preview of the paragraph page as
232
+ # an image
233
+ page_number = paragraph.page.page
234
+ # TODO: what should I do if I later find there's no page in the DB?
235
+ hydrated.page.page_preview_ref = page_preview_id(page_number)
236
+ extra_hydration.field_page = page_number
237
+
238
+ if config.table:
239
+ if hydrated.table is None:
240
+ hydrated.table = hydration_models.HydratedParagraphTable()
241
+
242
+ if config.table.table_page_preview:
243
+ if paragraph.representation.is_a_table:
244
+ # When a paragraph comes with a table and table hydration is
245
+ # enabled, we want to return the image representing that table.
246
+ # Ideally we should hydrate the paragraph reference_file, but
247
+ # table screenshots are not always perfect so we prefer to use
248
+ # the page preview. If at some point the table images are good
249
+ # enough, it'd be better to use those
250
+ page_number = paragraph.page.page
251
+ hydrated.table.page_preview_ref = page_preview_id(page_number)
252
+ extra_hydration.field_table_page = page_number
253
+
254
+ return hydrated, extra_hydration
255
+
256
+
257
+ async def related_paragraphs_refs(
258
+ paragraph_id: ParagraphId,
259
+ index: ParagraphIndex,
260
+ config: hydration_models.RelatedParagraphHydration,
261
+ ) -> tuple[hydration_models.RelatedParagraphRefs, list[ParagraphId]]:
262
+ """Compute the related paragraph references for a specific `paragraph_id`
263
+ and return them with the plain list of unique related paragraphs (to
264
+ facilitate work to the caller).
265
+
266
+ """
267
+ hydrated = hydration_models.RelatedParagraphRefs()
268
+ related = set()
269
+
270
+ if config.neighbours:
271
+ hydrated.neighbours = hydration_models.RelatedNeighbourParagraphRefs()
272
+
273
+ if config.neighbours.before is not None:
274
+ hydrated.neighbours.before = []
275
+ if config.neighbours.before > 0:
276
+ for previous_id in index.n_previous(paragraph_id, config.neighbours.before):
277
+ hydrated.neighbours.before.insert(0, previous_id)
278
+ related.add(ParagraphId.from_string(previous_id))
279
+
280
+ if config.neighbours.after is not None:
281
+ hydrated.neighbours.after = []
282
+ if config.neighbours.after > 0:
283
+ for next_id in index.n_next(paragraph_id, config.neighbours.after):
284
+ hydrated.neighbours.after.append(next_id)
285
+ related.add(ParagraphId.from_string(next_id))
286
+
287
+ if config.parents:
288
+ hydrated.parents = []
289
+ for parent_id in index.parents(paragraph_id):
290
+ hydrated.parents.append(parent_id)
291
+ related.add(ParagraphId.from_string(parent_id))
292
+
293
+ if config.siblings:
294
+ hydrated.siblings = []
295
+ for sibling_id in index.siblings(paragraph_id):
296
+ hydrated.siblings.append(sibling_id)
297
+ related.add(ParagraphId.from_string(sibling_id))
298
+
299
+ if config.replacements:
300
+ hydrated.replacements = []
301
+ for replacement_id in index.replacements(paragraph_id):
302
+ hydrated.replacements.append(replacement_id)
303
+ related.add(ParagraphId.from_string(replacement_id))
304
+
305
+ return hydrated, list(related)
@@ -0,0 +1,56 @@
1
+ #!/usr/bin/env python3
2
+
3
+ # Copyright (C) 2021 Bosutech XXI S.L.
4
+ #
5
+ # nucliadb is offered under the AGPL v3.0 and as commercial software.
6
+ # For commercial licensing, contact us at info@nuclia.com.
7
+ #
8
+ # AGPL:
9
+ # This program is free software: you can redistribute it and/or modify
10
+ # it under the terms of the GNU Affero General Public License as
11
+ # published by the Free Software Foundation, either version 3 of the
12
+ # License, or (at your option) any later version.
13
+ #
14
+ # This program is distributed in the hope that it will be useful,
15
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
16
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17
+ # GNU Affero General Public License for more details.
18
+ #
19
+ # You should have received a copy of the GNU Affero General Public License
20
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
21
+ #
22
+
23
+
24
+ from nucliadb.common.models_utils import from_proto
25
+ from nucliadb.ingest.orm.resource import Resource
26
+ from nucliadb_models import hydration as hydration_models
27
+ from nucliadb_models.security import ResourceSecurity
28
+
29
+
30
+ async def hydrate_resource(
31
+ resource: Resource, rid: str, config: hydration_models.ResourceHydration
32
+ ) -> hydration_models.HydratedResource:
33
+ basic = await resource.get_basic()
34
+
35
+ slug = basic.slug
36
+ hydrated = hydration_models.HydratedResource(id=rid, slug=slug)
37
+
38
+ if config.title:
39
+ hydrated.title = basic.title
40
+ if config.summary:
41
+ hydrated.summary = basic.summary
42
+
43
+ if config.security:
44
+ security = await resource.get_security()
45
+ hydrated.security = ResourceSecurity(access_groups=[])
46
+ if security is not None:
47
+ for group_id in security.access_groups:
48
+ hydrated.security.access_groups.append(group_id)
49
+
50
+ if config.origin:
51
+ origin = await resource.get_origin()
52
+ if origin is not None:
53
+ # TODO: we want a better hydration than proto to JSON
54
+ hydrated.origin = from_proto.origin(origin)
55
+
56
+ return hydrated
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nucliadb
3
- Version: 6.8.1.post4983
3
+ Version: 6.8.1.post5003
4
4
  Summary: NucliaDB
5
5
  Author-email: Nuclia <nucliadb@nuclia.com>
6
6
  License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
19
19
  Classifier: Programming Language :: Python :: 3 :: Only
20
20
  Requires-Python: <4,>=3.9
21
21
  Description-Content-Type: text/markdown
22
- Requires-Dist: nucliadb-telemetry[all]>=6.8.1.post4983
23
- Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.8.1.post4983
24
- Requires-Dist: nucliadb-protos>=6.8.1.post4983
25
- Requires-Dist: nucliadb-models>=6.8.1.post4983
26
- Requires-Dist: nidx-protos>=6.8.1.post4983
22
+ Requires-Dist: nucliadb-telemetry[all]>=6.8.1.post5003
23
+ Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.8.1.post5003
24
+ Requires-Dist: nucliadb-protos>=6.8.1.post5003
25
+ Requires-Dist: nucliadb-models>=6.8.1.post5003
26
+ Requires-Dist: nidx-protos>=6.8.1.post5003
27
27
  Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
28
28
  Requires-Dist: nuclia-models>=0.47.0
29
29
  Requires-Dist: uvicorn[standard]