nucliadb 6.8.1.post4981__py3-none-any.whl → 6.8.1.post4988__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of nucliadb might be problematic. Click here for more details.
- nucliadb/common/ids.py +77 -55
- nucliadb/ingest/fields/base.py +9 -0
- nucliadb/ingest/orm/resource.py +10 -1
- nucliadb/reader/api/v1/learning_config.py +4 -9
- nucliadb/search/api/v1/__init__.py +1 -0
- nucliadb/search/api/v1/hydrate.py +328 -0
- nucliadb/search/search/cache.py +0 -19
- nucliadb/search/search/{hydrator.py → hydrator/__init__.py} +3 -2
- nucliadb/search/search/hydrator/fields.py +175 -0
- nucliadb/search/search/hydrator/images.py +126 -0
- nucliadb/search/search/hydrator/paragraphs.py +305 -0
- nucliadb/search/search/hydrator/resources.py +56 -0
- nucliadb/writer/api/v1/learning_config.py +5 -4
- {nucliadb-6.8.1.post4981.dist-info → nucliadb-6.8.1.post4988.dist-info}/METADATA +6 -6
- {nucliadb-6.8.1.post4981.dist-info → nucliadb-6.8.1.post4988.dist-info}/RECORD +18 -13
- {nucliadb-6.8.1.post4981.dist-info → nucliadb-6.8.1.post4988.dist-info}/WHEEL +0 -0
- {nucliadb-6.8.1.post4981.dist-info → nucliadb-6.8.1.post4988.dist-info}/entry_points.txt +0 -0
- {nucliadb-6.8.1.post4981.dist-info → nucliadb-6.8.1.post4988.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
2
|
+
#
|
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
5
|
+
#
|
|
6
|
+
# AGPL:
|
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
10
|
+
# License, or (at your option) any later version.
|
|
11
|
+
#
|
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
# GNU Affero General Public License for more details.
|
|
16
|
+
#
|
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
|
+
#
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
from nucliadb.common.ids import FIELD_TYPE_STR_TO_NAME, FieldId
|
|
23
|
+
from nucliadb.common.models_utils import from_proto
|
|
24
|
+
from nucliadb.ingest.orm.resource import Resource
|
|
25
|
+
from nucliadb.search.search.hydrator import hydrate_field_text
|
|
26
|
+
from nucliadb_models import hydration as hydration_models
|
|
27
|
+
from nucliadb_models.common import FieldTypeName
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def page_preview_id(page_number: int) -> str:
|
|
31
|
+
"""Return the string page number for an specific page"""
|
|
32
|
+
return f"{page_number}"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def hydrate_field(resource: Resource, field_id: FieldId, config: hydration_models.FieldHydration):
|
|
36
|
+
field_type = FIELD_TYPE_STR_TO_NAME[field_id.type]
|
|
37
|
+
|
|
38
|
+
if field_type == FieldTypeName.TEXT:
|
|
39
|
+
if not config.text is not None:
|
|
40
|
+
return
|
|
41
|
+
return await hydrate_text_field(resource, field_id, config.text)
|
|
42
|
+
|
|
43
|
+
elif field_type == FieldTypeName.FILE is not None:
|
|
44
|
+
if not config.file:
|
|
45
|
+
return
|
|
46
|
+
return await hydrate_file_field(resource, field_id, config.file)
|
|
47
|
+
|
|
48
|
+
elif field_type == FieldTypeName.LINK is not None:
|
|
49
|
+
if not config.link:
|
|
50
|
+
return
|
|
51
|
+
return await hydrate_link_field(resource, field_id, config.link)
|
|
52
|
+
|
|
53
|
+
elif field_type == FieldTypeName.CONVERSATION is not None:
|
|
54
|
+
if not config.conversation:
|
|
55
|
+
return
|
|
56
|
+
return await hydrate_conversation_field(resource, field_id, config.conversation)
|
|
57
|
+
|
|
58
|
+
elif field_type == FieldTypeName.GENERIC is not None:
|
|
59
|
+
if not config.generic:
|
|
60
|
+
return
|
|
61
|
+
return await hydrate_generic_field(resource, field_id, config.generic)
|
|
62
|
+
|
|
63
|
+
else: # pragma: no cover
|
|
64
|
+
# This is a trick so mypy generates an error if this branch can be reached,
|
|
65
|
+
# that is, if we are missing some ifs
|
|
66
|
+
_a: int = "a"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
async def hydrate_text_field(
|
|
70
|
+
resource: Resource,
|
|
71
|
+
field_id: FieldId,
|
|
72
|
+
config: hydration_models.TextFieldHydration,
|
|
73
|
+
) -> hydration_models.HydratedTextField:
|
|
74
|
+
hydrated = hydration_models.HydratedTextField(
|
|
75
|
+
id=field_id.full(),
|
|
76
|
+
resource=field_id.rid,
|
|
77
|
+
field_type=FieldTypeName.TEXT,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if config.extracted_text:
|
|
81
|
+
field_text = await hydrate_field_text(resource.kb.kbid, field_id)
|
|
82
|
+
if field_text is not None:
|
|
83
|
+
(_, text) = field_text
|
|
84
|
+
hydrated.extracted = hydration_models.FieldExtractedData(text=text)
|
|
85
|
+
|
|
86
|
+
return hydrated
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
async def hydrate_file_field(
|
|
90
|
+
resource: Resource,
|
|
91
|
+
field_id: FieldId,
|
|
92
|
+
config: hydration_models.FileFieldHydration,
|
|
93
|
+
) -> hydration_models.HydratedFileField:
|
|
94
|
+
hydrated = hydration_models.HydratedFileField(
|
|
95
|
+
id=field_id.full(),
|
|
96
|
+
resource=field_id.rid,
|
|
97
|
+
field_type=FieldTypeName.FILE,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
if config.value:
|
|
101
|
+
field = await resource.get_field(field_id.key, field_id.pb_type)
|
|
102
|
+
value = await field.get_value()
|
|
103
|
+
hydrated.value = from_proto.field_file(value)
|
|
104
|
+
|
|
105
|
+
if config.extracted_text:
|
|
106
|
+
field_text = await hydrate_field_text(resource.kb.kbid, field_id)
|
|
107
|
+
if field_text is not None:
|
|
108
|
+
(_, text) = field_text
|
|
109
|
+
hydrated.extracted = hydration_models.FieldExtractedData(text=text)
|
|
110
|
+
|
|
111
|
+
return hydrated
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
async def hydrate_link_field(
|
|
115
|
+
resource: Resource,
|
|
116
|
+
field_id: FieldId,
|
|
117
|
+
config: hydration_models.LinkFieldHydration,
|
|
118
|
+
) -> hydration_models.HydratedLinkField:
|
|
119
|
+
hydrated = hydration_models.HydratedLinkField(
|
|
120
|
+
id=field_id.full(),
|
|
121
|
+
resource=field_id.rid,
|
|
122
|
+
field_type=FieldTypeName.LINK,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
if config.value:
|
|
126
|
+
field = await resource.get_field(field_id.key, field_id.pb_type)
|
|
127
|
+
value = await field.get_value()
|
|
128
|
+
hydrated.value = from_proto.field_link(value)
|
|
129
|
+
|
|
130
|
+
if config.extracted_text:
|
|
131
|
+
field_text = await hydrate_field_text(resource.kb.kbid, field_id)
|
|
132
|
+
if field_text is not None:
|
|
133
|
+
(_, text) = field_text
|
|
134
|
+
hydrated.extracted = hydration_models.FieldExtractedData(text=text)
|
|
135
|
+
|
|
136
|
+
return hydrated
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def hydrate_conversation_field(
|
|
140
|
+
resource: Resource,
|
|
141
|
+
field_id: FieldId,
|
|
142
|
+
config: hydration_models.ConversationFieldHydration,
|
|
143
|
+
) -> hydration_models.HydratedConversationField:
|
|
144
|
+
hydrated = hydration_models.HydratedConversationField(
|
|
145
|
+
id=field_id.full(),
|
|
146
|
+
resource=field_id.rid,
|
|
147
|
+
field_type=FieldTypeName.CONVERSATION,
|
|
148
|
+
)
|
|
149
|
+
# TODO: implement conversation fields
|
|
150
|
+
return hydrated
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
async def hydrate_generic_field(
|
|
154
|
+
resource: Resource,
|
|
155
|
+
field_id: FieldId,
|
|
156
|
+
config: hydration_models.GenericFieldHydration,
|
|
157
|
+
) -> hydration_models.HydratedGenericField:
|
|
158
|
+
hydrated = hydration_models.HydratedGenericField(
|
|
159
|
+
id=field_id.full(),
|
|
160
|
+
resource=field_id.rid,
|
|
161
|
+
field_type=FieldTypeName.GENERIC,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
if config.value:
|
|
165
|
+
field = await resource.get_field(field_id.key, field_id.pb_type)
|
|
166
|
+
value = await field.get_value()
|
|
167
|
+
hydrated.value = value
|
|
168
|
+
|
|
169
|
+
if config.extracted_text:
|
|
170
|
+
field_text = await hydrate_field_text(resource.kb.kbid, field_id)
|
|
171
|
+
if field_text is not None:
|
|
172
|
+
(_, text) = field_text
|
|
173
|
+
hydrated.extracted = hydration_models.FieldExtractedData(text=text)
|
|
174
|
+
|
|
175
|
+
return hydrated
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
2
|
+
#
|
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
5
|
+
#
|
|
6
|
+
# AGPL:
|
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
10
|
+
# License, or (at your option) any later version.
|
|
11
|
+
#
|
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
# GNU Affero General Public License for more details.
|
|
16
|
+
#
|
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
|
+
#
|
|
20
|
+
import base64
|
|
21
|
+
from typing import Optional, cast
|
|
22
|
+
|
|
23
|
+
from nucliadb.common.ids import FIELD_TYPE_STR_TO_NAME, FieldId, ParagraphId
|
|
24
|
+
from nucliadb.ingest.fields.base import Field
|
|
25
|
+
from nucliadb.ingest.fields.file import File
|
|
26
|
+
from nucliadb.search import SERVICE_NAME
|
|
27
|
+
from nucliadb_models.common import FieldTypeName
|
|
28
|
+
from nucliadb_models.search import Image
|
|
29
|
+
from nucliadb_protos import resources_pb2
|
|
30
|
+
from nucliadb_utils.utilities import get_storage
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
async def paragraph_source_image(kbid: str, paragraph: resources_pb2.Paragraph) -> Optional[Image]:
|
|
34
|
+
"""Certain paragraphs are extracted from images using techniques like OCR or
|
|
35
|
+
inception. If that's the case, return the original image for this paragraph.
|
|
36
|
+
|
|
37
|
+
"""
|
|
38
|
+
source_image = paragraph.representation.reference_file
|
|
39
|
+
|
|
40
|
+
if paragraph.kind not in (
|
|
41
|
+
resources_pb2.Paragraph.TypeParagraph.OCR,
|
|
42
|
+
resources_pb2.Paragraph.TypeParagraph.INCEPTION,
|
|
43
|
+
):
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
field_id = ParagraphId.from_string(paragraph.key).field_id
|
|
47
|
+
|
|
48
|
+
# Paragraphs extracted from an image store its original image representation
|
|
49
|
+
# in the reference file. The path is incomplete though, as it's stored in
|
|
50
|
+
# the `generated` folder
|
|
51
|
+
image = await download_image(
|
|
52
|
+
kbid,
|
|
53
|
+
field_id,
|
|
54
|
+
f"generated/{source_image}",
|
|
55
|
+
# XXX: we assume all reference files are PNG images, but this actually
|
|
56
|
+
# depends on learning so it's a dangerous assumption. We should check it
|
|
57
|
+
# by ourselves
|
|
58
|
+
mime_type="image/png",
|
|
59
|
+
)
|
|
60
|
+
return image
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def download_image(
|
|
64
|
+
kbid: str, field_id: FieldId, image_path: str, *, mime_type: str
|
|
65
|
+
) -> Optional[Image]:
|
|
66
|
+
storage = await get_storage(service_name=SERVICE_NAME)
|
|
67
|
+
sf = storage.file_extracted(
|
|
68
|
+
kbid,
|
|
69
|
+
field_id.rid,
|
|
70
|
+
field_id.type,
|
|
71
|
+
field_id.key,
|
|
72
|
+
image_path,
|
|
73
|
+
)
|
|
74
|
+
raw_image = (await storage.downloadbytes(sf.bucket, sf.key)).getvalue()
|
|
75
|
+
if not raw_image:
|
|
76
|
+
return None
|
|
77
|
+
return Image(content_type=mime_type, b64encoded=base64.b64encode(raw_image).decode())
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
async def download_page_preview(field: Field, page: int) -> Optional[Image]:
|
|
81
|
+
"""Download a specific page preview for a field and return it as an Image.
|
|
82
|
+
As not all fields have previews, this function can return None.
|
|
83
|
+
|
|
84
|
+
Page previews are uploaded by learning and shared through a known path with.
|
|
85
|
+
nucliadb
|
|
86
|
+
|
|
87
|
+
"""
|
|
88
|
+
field_type = FIELD_TYPE_STR_TO_NAME[field.type]
|
|
89
|
+
|
|
90
|
+
if field_type == FieldTypeName.FILE:
|
|
91
|
+
field = cast(File, field)
|
|
92
|
+
metadata = await field.get_file_extracted_data()
|
|
93
|
+
|
|
94
|
+
if metadata is None:
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
assert page <= len(metadata.file_pages_previews.positions), (
|
|
98
|
+
f"paragraph page number {page} should be less or equal to the total file pages previews {len(metadata.file_pages_previews.positions)}"
|
|
99
|
+
)
|
|
100
|
+
image = await download_image(
|
|
101
|
+
field.kbid,
|
|
102
|
+
field.field_id,
|
|
103
|
+
f"generated/extracted_images_{page}.png",
|
|
104
|
+
mime_type="image/png",
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
elif field_type == FieldTypeName.LINK:
|
|
108
|
+
# TODO: in case of links, we want to return the link preview, that is a
|
|
109
|
+
# link converted to PDF and screenshotted
|
|
110
|
+
# REVIEW: link preview is an image or a PDF?
|
|
111
|
+
image = None
|
|
112
|
+
|
|
113
|
+
elif (
|
|
114
|
+
field_type == FieldTypeName.TEXT
|
|
115
|
+
or field_type == FieldTypeName.CONVERSATION
|
|
116
|
+
or field_type == FieldTypeName.GENERIC
|
|
117
|
+
):
|
|
118
|
+
# these fields don't have previews
|
|
119
|
+
image = None
|
|
120
|
+
|
|
121
|
+
else: # pragma: no cover
|
|
122
|
+
# This is a trick so mypy generates an error if this branch can be reached,
|
|
123
|
+
# that is, if we are missing some ifs
|
|
124
|
+
_a: int = "a"
|
|
125
|
+
|
|
126
|
+
return image
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
2
|
+
#
|
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
5
|
+
#
|
|
6
|
+
# AGPL:
|
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
10
|
+
# License, or (at your option) any later version.
|
|
11
|
+
#
|
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
15
|
+
# GNU Affero General Public License for more details.
|
|
16
|
+
#
|
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
|
+
#
|
|
20
|
+
import asyncio
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import Optional, Union
|
|
23
|
+
|
|
24
|
+
from nucliadb.common.ids import FieldId, ParagraphId
|
|
25
|
+
from nucliadb.ingest.fields.base import Field
|
|
26
|
+
from nucliadb.ingest.orm.resource import Resource
|
|
27
|
+
from nucliadb.search.search import paragraphs
|
|
28
|
+
from nucliadb.search.search.hydrator.fields import page_preview_id
|
|
29
|
+
from nucliadb.search.search.hydrator.images import paragraph_source_image
|
|
30
|
+
from nucliadb_models import hydration as hydration_models
|
|
31
|
+
from nucliadb_protos import resources_pb2
|
|
32
|
+
from nucliadb_protos.resources_pb2 import FieldComputedMetadata
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ParagraphIndex:
|
|
36
|
+
"""Small helper class to cache field paragraphs and its relations and be
|
|
37
|
+
used as an index.
|
|
38
|
+
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
NEXT = "next"
|
|
42
|
+
PREVIOUS = "previous"
|
|
43
|
+
PARENTS = "parents"
|
|
44
|
+
SIBLINGS = "siblings"
|
|
45
|
+
REPLACEMENTS = "replacements"
|
|
46
|
+
|
|
47
|
+
def __init__(self, field_id: FieldId) -> None:
|
|
48
|
+
self.field_id = field_id
|
|
49
|
+
self.paragraphs: dict[str, resources_pb2.Paragraph] = {}
|
|
50
|
+
self.neighbours: dict[tuple[str, str], str] = {}
|
|
51
|
+
self.related: dict[tuple[str, str], list[str]] = {}
|
|
52
|
+
self._lock = asyncio.Lock()
|
|
53
|
+
self._built = False
|
|
54
|
+
|
|
55
|
+
async def build(self, field: Field):
|
|
56
|
+
"""Build the index if it hasn't been built yet.
|
|
57
|
+
|
|
58
|
+
This function is async-safe, multiple concurrent tasks can ask for a
|
|
59
|
+
built and it'll only be done once
|
|
60
|
+
"""
|
|
61
|
+
if self._built:
|
|
62
|
+
return
|
|
63
|
+
|
|
64
|
+
async with self._lock:
|
|
65
|
+
# double check we haven't built the index meanwhile we waited for the
|
|
66
|
+
# lock
|
|
67
|
+
if self._built:
|
|
68
|
+
return
|
|
69
|
+
|
|
70
|
+
field_metadata = await field.get_field_metadata()
|
|
71
|
+
|
|
72
|
+
if field_metadata is None:
|
|
73
|
+
# field metadata may be still processing. As we want to provide a
|
|
74
|
+
# consistent view, even if it can appear meanwhile we hydrate, we
|
|
75
|
+
# consider we don't have it. We mark the index as built and any
|
|
76
|
+
# paragraph will be found for this field
|
|
77
|
+
self._built = True
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
# REVIEW: this is a CPU-bound code, we may consider running this in an
|
|
81
|
+
# executor to not block the loop
|
|
82
|
+
self._build(field_metadata)
|
|
83
|
+
self._built = True
|
|
84
|
+
|
|
85
|
+
def _build(self, field_metadata: FieldComputedMetadata):
|
|
86
|
+
self.paragraphs.clear()
|
|
87
|
+
self.neighbours.clear()
|
|
88
|
+
self.related.clear()
|
|
89
|
+
|
|
90
|
+
if self.field_id.subfield_id is None:
|
|
91
|
+
field_paragraphs = field_metadata.metadata.paragraphs
|
|
92
|
+
else:
|
|
93
|
+
field_paragraphs = field_metadata.split_metadata[self.field_id.subfield_id].paragraphs
|
|
94
|
+
|
|
95
|
+
previous = None
|
|
96
|
+
for paragraph in field_paragraphs:
|
|
97
|
+
paragraph_id = self.field_id.paragraph_id(paragraph.start, paragraph.end).full()
|
|
98
|
+
self.paragraphs[paragraph_id] = paragraph
|
|
99
|
+
|
|
100
|
+
if previous is not None:
|
|
101
|
+
self.neighbours[(previous, ParagraphIndex.NEXT)] = paragraph_id
|
|
102
|
+
self.neighbours[(paragraph_id, ParagraphIndex.PREVIOUS)] = previous
|
|
103
|
+
previous = paragraph_id
|
|
104
|
+
|
|
105
|
+
self.related[(paragraph_id, ParagraphIndex.PARENTS)] = [
|
|
106
|
+
parent for parent in paragraph.relations.parents
|
|
107
|
+
]
|
|
108
|
+
self.related[(paragraph_id, ParagraphIndex.SIBLINGS)] = [
|
|
109
|
+
sibling for sibling in paragraph.relations.siblings
|
|
110
|
+
]
|
|
111
|
+
self.related[(paragraph_id, ParagraphIndex.REPLACEMENTS)] = [
|
|
112
|
+
replacement for replacement in paragraph.relations.replacements
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
def get(self, paragraph_id: Union[str, ParagraphId]) -> Optional[resources_pb2.Paragraph]:
|
|
116
|
+
paragraph_id = str(paragraph_id)
|
|
117
|
+
return self.paragraphs.get(paragraph_id)
|
|
118
|
+
|
|
119
|
+
def previous(self, paragraph_id: Union[str, ParagraphId]) -> Optional[str]:
|
|
120
|
+
paragraph_id = str(paragraph_id)
|
|
121
|
+
return self.neighbours.get((paragraph_id, ParagraphIndex.PREVIOUS))
|
|
122
|
+
|
|
123
|
+
def next(self, paragraph_id: Union[str, ParagraphId]) -> Optional[str]:
|
|
124
|
+
paragraph_id = str(paragraph_id)
|
|
125
|
+
return self.neighbours.get((paragraph_id, ParagraphIndex.NEXT))
|
|
126
|
+
|
|
127
|
+
def n_previous(self, paragraph_id: Union[str, ParagraphId], count: int = 1) -> list[str]:
|
|
128
|
+
assert count >= 1, f"can't find negative previous {count}"
|
|
129
|
+
paragraph_id = str(paragraph_id)
|
|
130
|
+
previous: list[str] = []
|
|
131
|
+
current_id = paragraph_id
|
|
132
|
+
for _ in range(count):
|
|
133
|
+
previous_id = self.previous(current_id)
|
|
134
|
+
if previous_id is None:
|
|
135
|
+
# we've reached the first paragraph
|
|
136
|
+
break
|
|
137
|
+
previous.insert(0, previous_id)
|
|
138
|
+
current_id = previous_id
|
|
139
|
+
return previous
|
|
140
|
+
|
|
141
|
+
def n_next(self, paragraph_id: Union[str, ParagraphId], count: int = 1) -> list[str]:
|
|
142
|
+
assert count >= 1, f"can't find negative nexts {count}"
|
|
143
|
+
paragraph_id = str(paragraph_id)
|
|
144
|
+
nexts = []
|
|
145
|
+
current_id = paragraph_id
|
|
146
|
+
for _ in range(count):
|
|
147
|
+
next_id = self.next(current_id)
|
|
148
|
+
if next_id is None:
|
|
149
|
+
# we've reached the last paragraph
|
|
150
|
+
break
|
|
151
|
+
current_id = next_id
|
|
152
|
+
nexts.append(next_id)
|
|
153
|
+
return nexts
|
|
154
|
+
|
|
155
|
+
def parents(self, paragraph_id: Union[str, ParagraphId]) -> list[str]:
|
|
156
|
+
paragraph_id = str(paragraph_id)
|
|
157
|
+
return self.related.get((paragraph_id, ParagraphIndex.PARENTS), [])
|
|
158
|
+
|
|
159
|
+
def siblings(self, paragraph_id: Union[str, ParagraphId]) -> list[str]:
|
|
160
|
+
paragraph_id = str(paragraph_id)
|
|
161
|
+
return self.related.get((paragraph_id, ParagraphIndex.SIBLINGS), [])
|
|
162
|
+
|
|
163
|
+
def replacements(self, paragraph_id: Union[str, ParagraphId]) -> list[str]:
|
|
164
|
+
paragraph_id = str(paragraph_id)
|
|
165
|
+
return self.related.get((paragraph_id, ParagraphIndex.REPLACEMENTS), [])
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@dataclass
|
|
169
|
+
class ExtraParagraphHydration:
|
|
170
|
+
field_page: Optional[int]
|
|
171
|
+
field_table_page: Optional[int]
|
|
172
|
+
related_paragraph_ids: list[ParagraphId]
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
async def hydrate_paragraph(
|
|
176
|
+
resource: Resource,
|
|
177
|
+
field: Field,
|
|
178
|
+
paragraph_id: ParagraphId,
|
|
179
|
+
config: hydration_models.ParagraphHydration,
|
|
180
|
+
field_paragraphs_index: ParagraphIndex,
|
|
181
|
+
) -> tuple[hydration_models.HydratedParagraph, ExtraParagraphHydration]:
|
|
182
|
+
"""Hydrate a paragraph and return the extra hydration to built a coherent
|
|
183
|
+
hydration around this paragraph.
|
|
184
|
+
|
|
185
|
+
Although the resource and field exist, the paragraph doesn't necessarily
|
|
186
|
+
need to be a real one in the paragraph metadata, it can be made-up to
|
|
187
|
+
include more or less text than the originally extracted.
|
|
188
|
+
|
|
189
|
+
"""
|
|
190
|
+
kbid = resource.kb.kbid
|
|
191
|
+
|
|
192
|
+
hydrated = hydration_models.HydratedParagraph(
|
|
193
|
+
id=paragraph_id.full(),
|
|
194
|
+
field=paragraph_id.field_id.full(),
|
|
195
|
+
resource=paragraph_id.rid,
|
|
196
|
+
)
|
|
197
|
+
extra_hydration = ExtraParagraphHydration(
|
|
198
|
+
field_page=None, field_table_page=None, related_paragraph_ids=[]
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
if config.text:
|
|
202
|
+
text = await paragraphs.get_paragraph_text(kbid=kbid, paragraph_id=paragraph_id)
|
|
203
|
+
hydrated.text = text
|
|
204
|
+
|
|
205
|
+
requires_paragraph_metadata = config.image or config.table or config.page or config.related
|
|
206
|
+
if requires_paragraph_metadata:
|
|
207
|
+
await field_paragraphs_index.build(field)
|
|
208
|
+
paragraph = field_paragraphs_index.get(paragraph_id)
|
|
209
|
+
if paragraph is not None:
|
|
210
|
+
# otherwise, this is a fake paragraph. We can't hydrate anything else here
|
|
211
|
+
|
|
212
|
+
if config.related:
|
|
213
|
+
hydrated.related, related_ids = await related_paragraphs_refs(
|
|
214
|
+
paragraph_id, field_paragraphs_index, config.related
|
|
215
|
+
)
|
|
216
|
+
extra_hydration.related_paragraph_ids = related_ids
|
|
217
|
+
|
|
218
|
+
if config.image:
|
|
219
|
+
hydrated.image = hydration_models.HydratedParagraphImage()
|
|
220
|
+
|
|
221
|
+
if config.image.source_image:
|
|
222
|
+
hydrated.image.source_image = await paragraph_source_image(kbid, paragraph)
|
|
223
|
+
|
|
224
|
+
if config.page:
|
|
225
|
+
if hydrated.page is None:
|
|
226
|
+
hydrated.page = hydration_models.HydratedParagraphPage()
|
|
227
|
+
|
|
228
|
+
if config.page.page_with_visual:
|
|
229
|
+
if paragraph.page.page_with_visual:
|
|
230
|
+
# Paragraphs can be found on pages with visual content. In this
|
|
231
|
+
# case, we want to return the preview of the paragraph page as
|
|
232
|
+
# an image
|
|
233
|
+
page_number = paragraph.page.page
|
|
234
|
+
# TODO: what should I do if I later find there's no page in the DB?
|
|
235
|
+
hydrated.page.page_preview_ref = page_preview_id(page_number)
|
|
236
|
+
extra_hydration.field_page = page_number
|
|
237
|
+
|
|
238
|
+
if config.table:
|
|
239
|
+
if hydrated.table is None:
|
|
240
|
+
hydrated.table = hydration_models.HydratedParagraphTable()
|
|
241
|
+
|
|
242
|
+
if config.table.table_page_preview:
|
|
243
|
+
if paragraph.representation.is_a_table:
|
|
244
|
+
# When a paragraph comes with a table and table hydration is
|
|
245
|
+
# enabled, we want to return the image representing that table.
|
|
246
|
+
# Ideally we should hydrate the paragraph reference_file, but
|
|
247
|
+
# table screenshots are not always perfect so we prefer to use
|
|
248
|
+
# the page preview. If at some point the table images are good
|
|
249
|
+
# enough, it'd be better to use those
|
|
250
|
+
page_number = paragraph.page.page
|
|
251
|
+
hydrated.table.page_preview_ref = page_preview_id(page_number)
|
|
252
|
+
extra_hydration.field_table_page = page_number
|
|
253
|
+
|
|
254
|
+
return hydrated, extra_hydration
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
async def related_paragraphs_refs(
|
|
258
|
+
paragraph_id: ParagraphId,
|
|
259
|
+
index: ParagraphIndex,
|
|
260
|
+
config: hydration_models.RelatedParagraphHydration,
|
|
261
|
+
) -> tuple[hydration_models.RelatedParagraphRefs, list[ParagraphId]]:
|
|
262
|
+
"""Compute the related paragraph references for a specific `paragraph_id`
|
|
263
|
+
and return them with the plain list of unique related paragraphs (to
|
|
264
|
+
facilitate work to the caller).
|
|
265
|
+
|
|
266
|
+
"""
|
|
267
|
+
hydrated = hydration_models.RelatedParagraphRefs()
|
|
268
|
+
related = set()
|
|
269
|
+
|
|
270
|
+
if config.neighbours:
|
|
271
|
+
hydrated.neighbours = hydration_models.RelatedNeighbourParagraphRefs()
|
|
272
|
+
|
|
273
|
+
if config.neighbours.before is not None:
|
|
274
|
+
hydrated.neighbours.before = []
|
|
275
|
+
if config.neighbours.before > 0:
|
|
276
|
+
for previous_id in index.n_previous(paragraph_id, config.neighbours.before):
|
|
277
|
+
hydrated.neighbours.before.insert(0, previous_id)
|
|
278
|
+
related.add(ParagraphId.from_string(previous_id))
|
|
279
|
+
|
|
280
|
+
if config.neighbours.after is not None:
|
|
281
|
+
hydrated.neighbours.after = []
|
|
282
|
+
if config.neighbours.after > 0:
|
|
283
|
+
for next_id in index.n_next(paragraph_id, config.neighbours.after):
|
|
284
|
+
hydrated.neighbours.after.append(next_id)
|
|
285
|
+
related.add(ParagraphId.from_string(next_id))
|
|
286
|
+
|
|
287
|
+
if config.parents:
|
|
288
|
+
hydrated.parents = []
|
|
289
|
+
for parent_id in index.parents(paragraph_id):
|
|
290
|
+
hydrated.parents.append(parent_id)
|
|
291
|
+
related.add(ParagraphId.from_string(parent_id))
|
|
292
|
+
|
|
293
|
+
if config.siblings:
|
|
294
|
+
hydrated.siblings = []
|
|
295
|
+
for sibling_id in index.siblings(paragraph_id):
|
|
296
|
+
hydrated.siblings.append(sibling_id)
|
|
297
|
+
related.add(ParagraphId.from_string(sibling_id))
|
|
298
|
+
|
|
299
|
+
if config.replacements:
|
|
300
|
+
hydrated.replacements = []
|
|
301
|
+
for replacement_id in index.replacements(paragraph_id):
|
|
302
|
+
hydrated.replacements.append(replacement_id)
|
|
303
|
+
related.add(ParagraphId.from_string(replacement_id))
|
|
304
|
+
|
|
305
|
+
return hydrated, list(related)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
|
4
|
+
#
|
|
5
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
|
6
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
|
7
|
+
#
|
|
8
|
+
# AGPL:
|
|
9
|
+
# This program is free software: you can redistribute it and/or modify
|
|
10
|
+
# it under the terms of the GNU Affero General Public License as
|
|
11
|
+
# published by the Free Software Foundation, either version 3 of the
|
|
12
|
+
# License, or (at your option) any later version.
|
|
13
|
+
#
|
|
14
|
+
# This program is distributed in the hope that it will be useful,
|
|
15
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
16
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
17
|
+
# GNU Affero General Public License for more details.
|
|
18
|
+
#
|
|
19
|
+
# You should have received a copy of the GNU Affero General Public License
|
|
20
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
21
|
+
#
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
from nucliadb.common.models_utils import from_proto
|
|
25
|
+
from nucliadb.ingest.orm.resource import Resource
|
|
26
|
+
from nucliadb_models import hydration as hydration_models
|
|
27
|
+
from nucliadb_models.security import ResourceSecurity
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def hydrate_resource(
|
|
31
|
+
resource: Resource, rid: str, config: hydration_models.ResourceHydration
|
|
32
|
+
) -> hydration_models.HydratedResource:
|
|
33
|
+
basic = await resource.get_basic()
|
|
34
|
+
|
|
35
|
+
slug = basic.slug
|
|
36
|
+
hydrated = hydration_models.HydratedResource(id=rid, slug=slug)
|
|
37
|
+
|
|
38
|
+
if config.title:
|
|
39
|
+
hydrated.title = basic.title
|
|
40
|
+
if config.summary:
|
|
41
|
+
hydrated.summary = basic.summary
|
|
42
|
+
|
|
43
|
+
if config.security:
|
|
44
|
+
security = await resource.get_security()
|
|
45
|
+
hydrated.security = ResourceSecurity(access_groups=[])
|
|
46
|
+
if security is not None:
|
|
47
|
+
for group_id in security.access_groups:
|
|
48
|
+
hydrated.security.access_groups.append(group_id)
|
|
49
|
+
|
|
50
|
+
if config.origin:
|
|
51
|
+
origin = await resource.get_origin()
|
|
52
|
+
if origin is not None:
|
|
53
|
+
# TODO: we want a better hydration than proto to JSON
|
|
54
|
+
hydrated.origin = from_proto.origin(origin)
|
|
55
|
+
|
|
56
|
+
return hydrated
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# You should have received a copy of the GNU Affero General Public License
|
|
18
18
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
19
19
|
#
|
|
20
|
-
from fastapi import Request
|
|
20
|
+
from fastapi import Header, Request
|
|
21
21
|
from fastapi_versioning import version
|
|
22
22
|
from nuclia_models.config.proto import ExtractConfig, SplitConfiguration
|
|
23
23
|
|
|
@@ -55,10 +55,11 @@ async def set_configuration(
|
|
|
55
55
|
@requires_one([NucliaDBRoles.MANAGER, NucliaDBRoles.WRITER])
|
|
56
56
|
@version(1)
|
|
57
57
|
async def patch_configuration(
|
|
58
|
-
request: Request,
|
|
59
|
-
kbid: str,
|
|
58
|
+
request: Request, kbid: str, x_nucliadb_account: str = Header(default="", include_in_schema=False)
|
|
60
59
|
):
|
|
61
|
-
return await learning_config_proxy(
|
|
60
|
+
return await learning_config_proxy(
|
|
61
|
+
request, "PATCH", f"/config/{kbid}", headers={"account-id": x_nucliadb_account}
|
|
62
|
+
)
|
|
62
63
|
|
|
63
64
|
|
|
64
65
|
@api.post(
|