hyperforge-nucliadb-agentic 1.0.0.post64__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hyperforge_nucliadb_agentic/__init__.py +3 -0
- hyperforge_nucliadb_agentic/agent.py +1642 -0
- hyperforge_nucliadb_agentic/ask/__init__.py +5 -0
- hyperforge_nucliadb_agentic/ask/audit.py +439 -0
- hyperforge_nucliadb_agentic/ask/exceptions.py +50 -0
- hyperforge_nucliadb_agentic/ask/lifespan.py +21 -0
- hyperforge_nucliadb_agentic/ask/model.py +1299 -0
- hyperforge_nucliadb_agentic/ask/predict.py +431 -0
- hyperforge_nucliadb_agentic/ask/predict_models.py +78 -0
- hyperforge_nucliadb_agentic/ask/search/__init__.py +0 -0
- hyperforge_nucliadb_agentic/ask/search/ask.py +1182 -0
- hyperforge_nucliadb_agentic/ask/search/graph_strategy.py +1138 -0
- hyperforge_nucliadb_agentic/ask/search/highlight.py +93 -0
- hyperforge_nucliadb_agentic/ask/search/hydrator.py +29 -0
- hyperforge_nucliadb_agentic/ask/search/metrics.py +112 -0
- hyperforge_nucliadb_agentic/ask/search/parsers/__init__.py +0 -0
- hyperforge_nucliadb_agentic/ask/search/parsers/ask.py +70 -0
- hyperforge_nucliadb_agentic/ask/search/parsers/fetcher.py +192 -0
- hyperforge_nucliadb_agentic/ask/search/parsers/find.py +729 -0
- hyperforge_nucliadb_agentic/ask/search/prompt.py +1298 -0
- hyperforge_nucliadb_agentic/ask/search/rank_fusion.py +157 -0
- hyperforge_nucliadb_agentic/ask/search/rerankers.py +161 -0
- hyperforge_nucliadb_agentic/ask/search/retrieval.py +750 -0
- hyperforge_nucliadb_agentic/ask/search/rpc.py +192 -0
- hyperforge_nucliadb_agentic/ask/settings.py +9 -0
- hyperforge_nucliadb_agentic/ask/utils/ids.py +188 -0
- hyperforge_nucliadb_agentic/ask/utils/proto.py +6 -0
- hyperforge_nucliadb_agentic/ask/utils/responses.py +6 -0
- hyperforge_nucliadb_agentic/ask/utils/text_blocks.py +51 -0
- hyperforge_nucliadb_agentic/config.py +47 -0
- hyperforge_nucliadb_agentic/internal_driver.py +87 -0
- hyperforge_nucliadb_agentic/py.typed +0 -0
- hyperforge_nucliadb_agentic-1.0.0.post64.dist-info/METADATA +24 -0
- hyperforge_nucliadb_agentic-1.0.0.post64.dist-info/RECORD +35 -0
- hyperforge_nucliadb_agentic-1.0.0.post64.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
from nucliadb_models.augment import AugmentRequest, AugmentResponse
|
|
5
|
+
from nucliadb_models.configuration import SearchConfiguration
|
|
6
|
+
from nucliadb_models.graph.requests import GraphNodesSearchRequest, GraphSearchRequest
|
|
7
|
+
from nucliadb_models.graph.responses import (
|
|
8
|
+
GraphNodesSearchResponse,
|
|
9
|
+
GraphSearchResponse,
|
|
10
|
+
)
|
|
11
|
+
from nucliadb_models.labels import KnowledgeBoxLabels
|
|
12
|
+
from nucliadb_models.retrieval import RetrievalRequest, RetrievalResponse
|
|
13
|
+
from nucliadb_models.search import (
|
|
14
|
+
FindRequest,
|
|
15
|
+
KnowledgeboxFindResults,
|
|
16
|
+
NucliaDBClientType,
|
|
17
|
+
)
|
|
18
|
+
from nucliadb_sdk import NucliaDBAsync
|
|
19
|
+
from nucliadb_sdk.v2.exceptions import NotFoundError, UnknownError
|
|
20
|
+
from pydantic import TypeAdapter
|
|
21
|
+
from typing_extensions import assert_never
|
|
22
|
+
|
|
23
|
+
from hyperforge_nucliadb_agentic.ask import logger
|
|
24
|
+
from hyperforge_nucliadb_agentic.ask.exceptions import (
|
|
25
|
+
KnowledgeBoxNotFound,
|
|
26
|
+
NucliaDBError,
|
|
27
|
+
)
|
|
28
|
+
from hyperforge_nucliadb_agentic.ask.model import Image
|
|
29
|
+
from hyperforge_nucliadb_agentic.ask.settings import settings
|
|
30
|
+
from hyperforge_nucliadb_agentic.ask.utils.ids import FieldId
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
async def get_resource_uuid_from_slug(
|
|
34
|
+
reader_sdk: NucliaDBAsync, kbid: str, slug: str
|
|
35
|
+
) -> str | None:
|
|
36
|
+
try:
|
|
37
|
+
resource = await reader_sdk.get_resource_by_slug(kbid=kbid, slug=slug)
|
|
38
|
+
except NotFoundError:
|
|
39
|
+
return None
|
|
40
|
+
else:
|
|
41
|
+
return resource.id
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def get_search_configuration(
|
|
45
|
+
reader_sdk: NucliaDBAsync, kbid: str, name: str
|
|
46
|
+
) -> SearchConfiguration | None:
|
|
47
|
+
resp = await reader_sdk.session.get(f"/v1/kb/{kbid}/search_configurations/{name}")
|
|
48
|
+
|
|
49
|
+
if resp.status_code == 404:
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
if resp.status_code != 200:
|
|
53
|
+
raise Exception(
|
|
54
|
+
f"/search_configurations/{name} call failed: {resp.status_code} {resp.content.decode()}"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
config: SearchConfiguration = TypeAdapter(SearchConfiguration).validate_json(
|
|
58
|
+
resp.content
|
|
59
|
+
)
|
|
60
|
+
return config
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def find(
|
|
64
|
+
search_sdk: NucliaDBAsync,
|
|
65
|
+
kbid: str,
|
|
66
|
+
item: FindRequest,
|
|
67
|
+
x_ndb_client: NucliaDBClientType,
|
|
68
|
+
x_nucliadb_user: str,
|
|
69
|
+
x_forwarded_for: str,
|
|
70
|
+
) -> tuple[KnowledgeboxFindResults, bool]:
|
|
71
|
+
"""RPC to /find endpoint making it look as an internal call."""
|
|
72
|
+
|
|
73
|
+
resp = await search_sdk.session.post(
|
|
74
|
+
f"/v1/kb/{kbid}/find",
|
|
75
|
+
headers={
|
|
76
|
+
"x-ndb-client": x_ndb_client,
|
|
77
|
+
"x-nucliadb-user": x_nucliadb_user,
|
|
78
|
+
"x-forwarded-for": x_forwarded_for,
|
|
79
|
+
},
|
|
80
|
+
json=item.model_dump(),
|
|
81
|
+
)
|
|
82
|
+
if resp.status_code == 200:
|
|
83
|
+
incomplete = False
|
|
84
|
+
elif resp.status_code == 206:
|
|
85
|
+
incomplete = True
|
|
86
|
+
elif resp.status_code == 404:
|
|
87
|
+
raise KnowledgeBoxNotFound()
|
|
88
|
+
else:
|
|
89
|
+
raise Exception(
|
|
90
|
+
f"/find call failed: {resp.status_code} {resp.content.decode()}"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
find_results = KnowledgeboxFindResults.model_validate(resp.json())
|
|
94
|
+
|
|
95
|
+
return find_results, incomplete
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def retrieve(
|
|
99
|
+
search_sdk: NucliaDBAsync,
|
|
100
|
+
kbid: str,
|
|
101
|
+
item: RetrievalRequest,
|
|
102
|
+
) -> RetrievalResponse:
|
|
103
|
+
try:
|
|
104
|
+
retrieved = await search_sdk.retrieve(kbid=kbid, content=item)
|
|
105
|
+
except NotFoundError as exc:
|
|
106
|
+
raise KnowledgeBoxNotFound() from exc
|
|
107
|
+
except UnknownError as exc:
|
|
108
|
+
logger.warning(
|
|
109
|
+
"/retrieve RPC to NucliaDB failed", extra={"kbid": kbid}, exc_info=True
|
|
110
|
+
)
|
|
111
|
+
raise NucliaDBError() from exc
|
|
112
|
+
return retrieved
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
async def augment(
|
|
116
|
+
search_sdk: NucliaDBAsync, kbid: str, item: AugmentRequest
|
|
117
|
+
) -> AugmentResponse:
|
|
118
|
+
try:
|
|
119
|
+
augmented = await search_sdk.augment(kbid=kbid, content=item)
|
|
120
|
+
except NotFoundError as exc:
|
|
121
|
+
raise KnowledgeBoxNotFound() from exc
|
|
122
|
+
except UnknownError as exc:
|
|
123
|
+
logger.warning(
|
|
124
|
+
"/augment RPC to NucliaDB failed", extra={"kbid": kbid}, exc_info=True
|
|
125
|
+
)
|
|
126
|
+
raise NucliaDBError() from exc
|
|
127
|
+
return augmented
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
async def graph_paths(
|
|
131
|
+
search_sdk: NucliaDBAsync, kbid: str, item: GraphSearchRequest
|
|
132
|
+
) -> GraphSearchResponse:
|
|
133
|
+
paths = await search_sdk.graph_search(kbid=kbid, content=item)
|
|
134
|
+
return paths
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
async def graph_nodes(
|
|
138
|
+
search_sdk: NucliaDBAsync, kbid: str, item: GraphNodesSearchRequest
|
|
139
|
+
) -> GraphNodesSearchResponse:
|
|
140
|
+
nodes = await search_sdk.graph_nodes(kbid=kbid, content=item)
|
|
141
|
+
return nodes
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
async def labelsets(reader_sdk: NucliaDBAsync, kbid: str) -> KnowledgeBoxLabels:
|
|
145
|
+
labelsets = await reader_sdk.get_labelsets(kbid=kbid)
|
|
146
|
+
return labelsets
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
async def download_image(
|
|
150
|
+
reader_sdk: NucliaDBAsync,
|
|
151
|
+
kbid: str,
|
|
152
|
+
field_id: FieldId,
|
|
153
|
+
path: str,
|
|
154
|
+
*,
|
|
155
|
+
mime_type: str,
|
|
156
|
+
) -> Image | None:
|
|
157
|
+
async with (
|
|
158
|
+
reader_sdk.session.stream(
|
|
159
|
+
"GET",
|
|
160
|
+
f"/v1/kb/{kbid}/resource/{field_id.rid}/{field_id.type_name.value}/{field_id.key}/download/extracted/{path}",
|
|
161
|
+
) as resp,
|
|
162
|
+
):
|
|
163
|
+
if resp.status_code == 404:
|
|
164
|
+
return None
|
|
165
|
+
|
|
166
|
+
data = await resp.aread()
|
|
167
|
+
return Image(
|
|
168
|
+
b64encoded=base64.b64encode(data).decode(),
|
|
169
|
+
content_type=mime_type,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
__SDK: dict[Literal["reader"] | Literal["search"], NucliaDBAsync] = {}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def get_sdk(service: Literal["reader"] | Literal["search"]) -> NucliaDBAsync:
|
|
177
|
+
if service in __SDK:
|
|
178
|
+
return __SDK[service]
|
|
179
|
+
|
|
180
|
+
if service == "reader":
|
|
181
|
+
service_address = settings.nucliadb_reader_address
|
|
182
|
+
elif service == "search":
|
|
183
|
+
service_address = settings.nucliadb_search_address
|
|
184
|
+
else:
|
|
185
|
+
assert_never(service)
|
|
186
|
+
|
|
187
|
+
sdk = NucliaDBAsync(
|
|
188
|
+
url=service_address,
|
|
189
|
+
headers={"x-nucliadb-roles": "READER"},
|
|
190
|
+
)
|
|
191
|
+
__SDK[service] = sdk
|
|
192
|
+
return sdk
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from pydantic_settings import BaseSettings
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Settings(BaseSettings):
|
|
5
|
+
nucliadb_reader_address: str = "http://reader.nucliadb.svc.cluster.local:8080/api"
|
|
6
|
+
nucliadb_search_address: str = "http://search.nucliadb.svc.cluster.local:8080/api"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
settings = Settings()
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from nucliadb_models.common import FieldTypeName
|
|
4
|
+
from nucliadb_protos.resources_pb2 import FieldType
|
|
5
|
+
|
|
6
|
+
FIELD_TYPE_STR_TO_PB: dict[str, FieldType.ValueType] = {
|
|
7
|
+
"t": FieldType.TEXT,
|
|
8
|
+
"f": FieldType.FILE,
|
|
9
|
+
"u": FieldType.LINK,
|
|
10
|
+
"a": FieldType.GENERIC,
|
|
11
|
+
"c": FieldType.CONVERSATION,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
FIELD_TYPE_PB_TO_STR = {v: k for k, v in FIELD_TYPE_STR_TO_PB.items()}
|
|
15
|
+
|
|
16
|
+
FIELD_TYPE_NAME_TO_STR = {
|
|
17
|
+
FieldTypeName.TEXT: "t",
|
|
18
|
+
FieldTypeName.FILE: "f",
|
|
19
|
+
FieldTypeName.LINK: "u",
|
|
20
|
+
FieldTypeName.GENERIC: "a",
|
|
21
|
+
FieldTypeName.CONVERSATION: "c",
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
FIELD_TYPE_STR_TO_NAME = {v: k for k, v in FIELD_TYPE_NAME_TO_STR.items()}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class FieldId:
|
|
29
|
+
"""
|
|
30
|
+
Field ids are used to identify fields in resources. They usually have the following format:
|
|
31
|
+
|
|
32
|
+
`rid/field_type/field_key`
|
|
33
|
+
|
|
34
|
+
where field type is one of: `t`, `f`, `u`, `a`, `c` (text, file, link, generic, conversation)
|
|
35
|
+
and field_key is an identifier for that field type on the resource, usually chosen by the user.
|
|
36
|
+
|
|
37
|
+
In some cases, fields can have subfields, for example, in conversations, where each part of the
|
|
38
|
+
conversation is a subfield. In those cases, the id has the following format:
|
|
39
|
+
|
|
40
|
+
`rid/field_type/field_key/subfield_id`
|
|
41
|
+
|
|
42
|
+
Examples:
|
|
43
|
+
|
|
44
|
+
>>> FieldId(rid="rid", type="u", key="my-link")
|
|
45
|
+
FieldID("rid/u/my-link")
|
|
46
|
+
>>> FieldId.from_string("rid/u/my-link")
|
|
47
|
+
FieldID("rid/u/my-link")
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
rid: str
|
|
51
|
+
type: str
|
|
52
|
+
key: str
|
|
53
|
+
# also knwon as `split`, this indicates a part of a field in, for example, conversations
|
|
54
|
+
subfield_id: str | None = None
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_string(cls, value: str) -> "FieldId":
|
|
58
|
+
"""
|
|
59
|
+
Parse a FieldId from a string
|
|
60
|
+
Example:
|
|
61
|
+
>>> fid = FieldId.from_string("rid/u/foo")
|
|
62
|
+
>>> fid
|
|
63
|
+
FieldId("rid/u/foo")
|
|
64
|
+
>>> fid.type
|
|
65
|
+
'u'
|
|
66
|
+
>>> fid.key
|
|
67
|
+
'foo'
|
|
68
|
+
>>> FieldId.from_string("rid/u/foo/subfield_id").subfield_id
|
|
69
|
+
'subfield_id'
|
|
70
|
+
"""
|
|
71
|
+
parts = value.split("/")
|
|
72
|
+
if len(parts) == 3:
|
|
73
|
+
rid, _type, key = parts
|
|
74
|
+
_type = cls._parse_field_type(_type)
|
|
75
|
+
return cls(rid=rid, type=_type, key=key)
|
|
76
|
+
elif len(parts) == 4:
|
|
77
|
+
rid, _type, key, subfield_id = parts
|
|
78
|
+
_type = cls._parse_field_type(_type)
|
|
79
|
+
return cls(
|
|
80
|
+
rid=rid,
|
|
81
|
+
type=_type,
|
|
82
|
+
key=key,
|
|
83
|
+
subfield_id=subfield_id,
|
|
84
|
+
)
|
|
85
|
+
else:
|
|
86
|
+
raise ValueError(f"Invalid FieldId: {value}")
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def from_pb(
|
|
90
|
+
cls,
|
|
91
|
+
rid: str,
|
|
92
|
+
field_type: FieldType.ValueType,
|
|
93
|
+
key: str,
|
|
94
|
+
subfield_id: str | None = None,
|
|
95
|
+
) -> "FieldId":
|
|
96
|
+
return cls(
|
|
97
|
+
rid=rid,
|
|
98
|
+
type=FIELD_TYPE_PB_TO_STR[field_type],
|
|
99
|
+
key=key,
|
|
100
|
+
subfield_id=subfield_id,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def pb_type(self) -> FieldType.ValueType:
|
|
105
|
+
return FIELD_TYPE_STR_TO_PB[self.type]
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def type_name(self) -> FieldTypeName:
|
|
109
|
+
return FIELD_TYPE_STR_TO_NAME[self.type]
|
|
110
|
+
|
|
111
|
+
def full(self) -> str:
|
|
112
|
+
if self.subfield_id is None:
|
|
113
|
+
return f"{self.rid}/{self.type}/{self.key}"
|
|
114
|
+
else:
|
|
115
|
+
return f"{self.rid}/{self.type}/{self.key}/{self.subfield_id}"
|
|
116
|
+
|
|
117
|
+
def full_without_subfield(self) -> str:
|
|
118
|
+
return f"{self.rid}/{self.type}/{self.key}"
|
|
119
|
+
|
|
120
|
+
def short_without_subfield(self) -> str:
|
|
121
|
+
return f"/{self.type}/{self.key}"
|
|
122
|
+
|
|
123
|
+
def paragraph_id(self, paragraph_start: int, paragraph_end: int) -> "ParagraphId":
|
|
124
|
+
"""Generate a ParagraphId from the current field given its start and
|
|
125
|
+
end.
|
|
126
|
+
|
|
127
|
+
"""
|
|
128
|
+
return ParagraphId(
|
|
129
|
+
field_id=self,
|
|
130
|
+
paragraph_start=paragraph_start,
|
|
131
|
+
paragraph_end=paragraph_end,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
def __str__(self) -> str:
|
|
135
|
+
return self.full()
|
|
136
|
+
|
|
137
|
+
def __repr__(self) -> str:
|
|
138
|
+
return f"FieldId({self.full()})"
|
|
139
|
+
|
|
140
|
+
def __hash__(self) -> int:
|
|
141
|
+
return hash(self.full())
|
|
142
|
+
|
|
143
|
+
@staticmethod
|
|
144
|
+
def _parse_field_type(_type: str) -> str:
|
|
145
|
+
if _type not in FIELD_TYPE_STR_TO_PB:
|
|
146
|
+
# Try to parse the enum value
|
|
147
|
+
# XXX: This is to support field types that are integer values of FieldType
|
|
148
|
+
# Which is how legacy processor relations reported the paragraph_id
|
|
149
|
+
try:
|
|
150
|
+
type_pb = FieldType.ValueType(int(_type))
|
|
151
|
+
except ValueError:
|
|
152
|
+
raise ValueError(f"Invalid FieldId: {_type}")
|
|
153
|
+
if type_pb in FIELD_TYPE_PB_TO_STR:
|
|
154
|
+
return FIELD_TYPE_PB_TO_STR[type_pb]
|
|
155
|
+
else:
|
|
156
|
+
raise ValueError(f"Invalid FieldId: {_type}")
|
|
157
|
+
return _type
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass
|
|
161
|
+
class ParagraphId:
|
|
162
|
+
field_id: FieldId
|
|
163
|
+
paragraph_start: int
|
|
164
|
+
paragraph_end: int
|
|
165
|
+
|
|
166
|
+
@classmethod
|
|
167
|
+
def from_string(cls, value: str) -> "ParagraphId":
|
|
168
|
+
parts = value.split("/")
|
|
169
|
+
paragraph_range = parts[-1]
|
|
170
|
+
start, end = map(int, paragraph_range.split("-"))
|
|
171
|
+
field_id = FieldId.from_string("/".join(parts[:-1]))
|
|
172
|
+
return cls(field_id=field_id, paragraph_start=start, paragraph_end=end)
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def rid(self) -> str:
|
|
176
|
+
return self.field_id.rid
|
|
177
|
+
|
|
178
|
+
def full(self) -> str:
|
|
179
|
+
return f"{self.field_id.full()}/{self.paragraph_start}-{self.paragraph_end}"
|
|
180
|
+
|
|
181
|
+
def __str__(self) -> str:
|
|
182
|
+
return self.full()
|
|
183
|
+
|
|
184
|
+
def __repr__(self) -> str:
|
|
185
|
+
return f"ParagraphId({self.full()})"
|
|
186
|
+
|
|
187
|
+
def __hash__(self) -> int:
|
|
188
|
+
return hash(self.full())
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from hyperforge_nucliadb_agentic.ask.utils.ids import ParagraphId
|
|
2
|
+
from nucliadb_models.retrieval import Score
|
|
3
|
+
from nucliadb_models.search import SCORE_TYPE, Relations, TextPosition
|
|
4
|
+
from nucliadb_protos import resources_pb2
|
|
5
|
+
from pydantic import BaseModel
|
|
6
|
+
|
|
7
|
+
# /k/ocr
|
|
8
|
+
_OCR_LABEL = f"/k/{resources_pb2.Paragraph.TypeParagraph.Name(resources_pb2.Paragraph.TypeParagraph.OCR).lower()}"
|
|
9
|
+
# /k/inception
|
|
10
|
+
_INCEPTION_LABEL = f"/k/{resources_pb2.Paragraph.TypeParagraph.Name(resources_pb2.Paragraph.TypeParagraph.OCR).lower()}"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ScoredTextBlock(BaseModel):
|
|
14
|
+
paragraph_id: ParagraphId
|
|
15
|
+
score_type: SCORE_TYPE
|
|
16
|
+
|
|
17
|
+
scores: list[Score]
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def score(self) -> float:
|
|
21
|
+
return self.current_score.score
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def current_score(self) -> Score:
|
|
25
|
+
assert len(self.scores) > 0, "text block matches must be scored"
|
|
26
|
+
return self.scores[-1]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class TextBlockMatch(ScoredTextBlock):
|
|
30
|
+
"""
|
|
31
|
+
Model a text block/paragraph retrieved from an external index with all the information
|
|
32
|
+
needed in order to later hydrate retrieval results.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
position: TextPosition
|
|
36
|
+
order: int
|
|
37
|
+
page_with_visual: bool = False
|
|
38
|
+
fuzzy_search: bool
|
|
39
|
+
is_a_table: bool = False
|
|
40
|
+
representation_file: str | None = None
|
|
41
|
+
paragraph_labels: list[str] = []
|
|
42
|
+
field_labels: list[str] = []
|
|
43
|
+
text: str | None = None
|
|
44
|
+
relevant_relations: Relations | None = None
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def is_an_image(self) -> bool:
|
|
48
|
+
return (
|
|
49
|
+
_OCR_LABEL in self.paragraph_labels
|
|
50
|
+
or _INCEPTION_LABEL in self.paragraph_labels
|
|
51
|
+
)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from typing import List, Literal, Optional, Tuple
|
|
2
|
+
|
|
3
|
+
from hyperforge.context.agent import ContextAgentConfig
|
|
4
|
+
from hyperforge.utils import WidgetType
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
from pydantic.config import ConfigDict
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class NucliaDBAgentConfig(ContextAgentConfig):
|
|
10
|
+
model_config = ConfigDict(title="Knowledge Box Agent")
|
|
11
|
+
module: Literal["nucliadb_agent"] = "nucliadb_agent"
|
|
12
|
+
sources: List[str] = Field(
|
|
13
|
+
default_factory=list,
|
|
14
|
+
json_schema_extra={
|
|
15
|
+
"show_in_node": True,
|
|
16
|
+
},
|
|
17
|
+
)
|
|
18
|
+
generative_model: str = Field(
|
|
19
|
+
default="chatgpt-azure-4o-mini",
|
|
20
|
+
title="Generative model",
|
|
21
|
+
description="Model used to generate answers",
|
|
22
|
+
json_schema_extra={"widget": WidgetType.MODEL_SELECT},
|
|
23
|
+
)
|
|
24
|
+
published_functions: Optional[Tuple[str, ...]] = Field(
|
|
25
|
+
default=(
|
|
26
|
+
"search_by_title",
|
|
27
|
+
"ask_labels_list",
|
|
28
|
+
"ask_by_title",
|
|
29
|
+
"ask_agent",
|
|
30
|
+
"ask_labels",
|
|
31
|
+
"facets_count",
|
|
32
|
+
"facets_search",
|
|
33
|
+
"catalog_search",
|
|
34
|
+
"all_images_by_title",
|
|
35
|
+
"search_images",
|
|
36
|
+
),
|
|
37
|
+
title="Published functions",
|
|
38
|
+
description="List of functions published by this agent to be used by other agents in the chain",
|
|
39
|
+
json_schema_extra={
|
|
40
|
+
"widget": WidgetType.NOT_SHOWN,
|
|
41
|
+
},
|
|
42
|
+
)
|
|
43
|
+
generate_inner_answer: bool = Field(
|
|
44
|
+
default=True,
|
|
45
|
+
title="Generate inner answer",
|
|
46
|
+
description="If true, the agent will generate an inner answer to be used by other agents in the chain. If false, the agent will only generate a reasoning and a list of resources.",
|
|
47
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Internal NucliaDB driver for nucliadb_agentic_api.
|
|
3
|
+
|
|
4
|
+
Registered as provider 'nucliadb_internal'. Creates two NucliaDBAsync clients
|
|
5
|
+
(reader + search) and exposes a proxy that delegates to the correct one.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
from hyperforge.configure import driver
|
|
12
|
+
from hyperforge.driver import DriverConfig
|
|
13
|
+
from hyperforge_nucliadb.driver import NucliaDBDriver, manager_connect
|
|
14
|
+
from hyperforge_nucliadb.driver_config import NucliaDBConnection
|
|
15
|
+
from nucliadb_sdk.v2 import NucliaDBAsync
|
|
16
|
+
from pydantic.config import ConfigDict
|
|
17
|
+
|
|
18
|
+
# Methods served by the search component; everything else goes to reader.
|
|
19
|
+
_SEARCH_METHODS = {
|
|
20
|
+
"ask",
|
|
21
|
+
"find",
|
|
22
|
+
"search",
|
|
23
|
+
"catalog",
|
|
24
|
+
"catalog_facets",
|
|
25
|
+
"graph_nodes",
|
|
26
|
+
"graph_relations",
|
|
27
|
+
"graph_search",
|
|
28
|
+
"summarize",
|
|
29
|
+
"retrieve",
|
|
30
|
+
"augment",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class NucliaDBProxy:
|
|
35
|
+
"""Holds two NucliaDBAsync clients and dispatches method calls to the correct one."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, reader: NucliaDBAsync, search: NucliaDBAsync):
|
|
38
|
+
self._reader = reader
|
|
39
|
+
self._search = search
|
|
40
|
+
|
|
41
|
+
def __getattr__(self, name: str):
|
|
42
|
+
if name in _SEARCH_METHODS:
|
|
43
|
+
return getattr(self._search, name)
|
|
44
|
+
return getattr(self._reader, name)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class InternalNucliaDBConnection(NucliaDBConnection):
|
|
48
|
+
"""Connection config for internal cluster access. Provider is nucliadb_internal."""
|
|
49
|
+
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class InternalNucliaDBConfig(DriverConfig[InternalNucliaDBConnection]):
|
|
54
|
+
model_config = ConfigDict(title="Internal Knowledge Box")
|
|
55
|
+
provider: Literal["nucliadb_internal"]
|
|
56
|
+
config: InternalNucliaDBConnection
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@driver(
|
|
60
|
+
id="nucliadb_internal",
|
|
61
|
+
title="Internal KnowledgeBox Source",
|
|
62
|
+
description="Internal cluster source for KnowledgeBox. Routes to reader/search components.",
|
|
63
|
+
config_schema=InternalNucliaDBConfig,
|
|
64
|
+
)
|
|
65
|
+
class InternalNucliaDBDriver(NucliaDBDriver):
|
|
66
|
+
driver: NucliaDBProxy
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
async def init(cls, driver: InternalNucliaDBConfig):
|
|
70
|
+
base_url = os.environ.get("INTERNAL_NUCLIADB_URL")
|
|
71
|
+
if not base_url:
|
|
72
|
+
raise RuntimeError(
|
|
73
|
+
"INTERNAL_NUCLIADB_URL must be set to use the nucliadb_internal driver"
|
|
74
|
+
)
|
|
75
|
+
reader_url = base_url.format(component="reader")
|
|
76
|
+
search_url = base_url.format(component="search")
|
|
77
|
+
headers = {"X-NUCLIADB-ROLES": "READER"}
|
|
78
|
+
reader_ndb = NucliaDBAsync(url=reader_url, headers=headers)
|
|
79
|
+
search_ndb = NucliaDBAsync(url=search_url, headers=headers)
|
|
80
|
+
return cls(
|
|
81
|
+
provider=driver.provider,
|
|
82
|
+
name=driver.name,
|
|
83
|
+
config=driver.config,
|
|
84
|
+
driver=NucliaDBProxy(reader_ndb, search_ndb),
|
|
85
|
+
manager=await manager_connect(driver.config),
|
|
86
|
+
_synonyms=None,
|
|
87
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hyperforge_nucliadb_agentic
|
|
3
|
+
Version: 1.0.0.post64
|
|
4
|
+
Summary: NucliaDB Hyperforge agent
|
|
5
|
+
Author: Nuclia
|
|
6
|
+
Author-email: Nuclia <nucliadb@nuclia.com>
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
Classifier: Programming Language :: Python
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
14
|
+
Requires-Dist: hyperforge>=1.0.0.post76
|
|
15
|
+
Requires-Dist: lingua-language-detector
|
|
16
|
+
Requires-Dist: nucliadb-sdk
|
|
17
|
+
Requires-Python: >=3.10, <4
|
|
18
|
+
Project-URL: Homepage, https://progress.com
|
|
19
|
+
Project-URL: Repository, https://github.com/nuclia/nucliadb_agentic_api
|
|
20
|
+
Project-URL: Changelog, https://github.com/nuclia/nucliadb_agentic_api/blob/main/agents/nucliadb/CHANGELOG.md
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# NucliaDB Agentic Hyperforge agents
|
|
24
|
+
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
hyperforge_nucliadb_agentic/__init__.py,sha256=78Naois_RaK3UZ_g9M_tAke18pWA-CiTSaO8ZPHiDSo,62
|
|
2
|
+
hyperforge_nucliadb_agentic/agent.py,sha256=8_3Xmb3qEdnI8tzvGBqcmONSWB34rBLGmH1MTZGXOh8,65404
|
|
3
|
+
hyperforge_nucliadb_agentic/ask/__init__.py,sha256=LFMSFtFFoLQMCLkeYgwXap8gQTGDkMpRDdMbUdobx1o,104
|
|
4
|
+
hyperforge_nucliadb_agentic/ask/audit.py,sha256=NeBk9zpTwxtE3bQIq1UM6VBrebkZNofawyLzaaUDoaA,15670
|
|
5
|
+
hyperforge_nucliadb_agentic/ask/exceptions.py,sha256=MGd8zIB0z9OIQOWVzfUuHWyp4BYGcUG2TiV-4L6aZOI,1171
|
|
6
|
+
hyperforge_nucliadb_agentic/ask/lifespan.py,sha256=RMEtoO_1Stjo7bLgPUA0sBA0IF_UsO6_mCB36SfItUc,521
|
|
7
|
+
hyperforge_nucliadb_agentic/ask/model.py,sha256=FqoYMsEg2elpYY7TTYlp9ONAhxdytfjOQIZhu0gRkWs,52131
|
|
8
|
+
hyperforge_nucliadb_agentic/ask/predict.py,sha256=tfJPC0WLkSwJCfvEHg-VvGQa0dt6zmP33kO5wjKL2YA,14864
|
|
9
|
+
hyperforge_nucliadb_agentic/ask/predict_models.py,sha256=QPOWZ579ShKo4pCIqBtkB8tjYosHl7usTVeAQPUGNvs,3041
|
|
10
|
+
hyperforge_nucliadb_agentic/ask/search/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
hyperforge_nucliadb_agentic/ask/search/ask.py,sha256=Mn1rbFTnmOtkD9-nGYrC-m_nUMhYeKP9y07vSNlSzqU,43260
|
|
12
|
+
hyperforge_nucliadb_agentic/ask/search/graph_strategy.py,sha256=OP2aHVjL_BlD1zC-usUbZ7TdUfb71DARJuD75r6Ys4g,39968
|
|
13
|
+
hyperforge_nucliadb_agentic/ask/search/highlight.py,sha256=09PmwrScPusa8Ar26-yilnhFX-ibAw3wpjqCTkBEpJI,2775
|
|
14
|
+
hyperforge_nucliadb_agentic/ask/search/hydrator.py,sha256=tyX4VmJ7N8eG0EbgId96D3ITf-3TcA5fiZwpxOccJcE,851
|
|
15
|
+
hyperforge_nucliadb_agentic/ask/search/metrics.py,sha256=mtJO8v_Dfk-j1r1dwcDpUzPwLHcP7oc1z5A1HJwGov4,2954
|
|
16
|
+
hyperforge_nucliadb_agentic/ask/search/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
hyperforge_nucliadb_agentic/ask/search/parsers/ask.py,sha256=CfXd-mbOjm-Llf_hz1SghTrUqmhTXmKYIzsyY53YQWU,2056
|
|
18
|
+
hyperforge_nucliadb_agentic/ask/search/parsers/fetcher.py,sha256=ZO4DvmXdNZdzeqv-4Pw0ORuUSHUSaoisAs8y0Am1NDs,7377
|
|
19
|
+
hyperforge_nucliadb_agentic/ask/search/parsers/find.py,sha256=u-s7akTC1890WyNfCj4jBVI68ncDMyGrz0hx7SYUEho,25726
|
|
20
|
+
hyperforge_nucliadb_agentic/ask/search/prompt.py,sha256=ADa1AwQqVKJ4oDBE1xWtulHl4VjhyGRpbEhvqqy6SHU,49051
|
|
21
|
+
hyperforge_nucliadb_agentic/ask/search/rank_fusion.py,sha256=XJgflFgnmtHxtRG1xQMcobiFkPWce6J8yS4iKTHsa8M,5065
|
|
22
|
+
hyperforge_nucliadb_agentic/ask/search/rerankers.py,sha256=9mRPHaR_Ki3mK7hTMJzxsyG8TUgoMa9csvZtbakI1dg,4435
|
|
23
|
+
hyperforge_nucliadb_agentic/ask/search/retrieval.py,sha256=vjgdjahRUr3G9qH2pp9csJT6vOOyLsbXdUvyWxSyv30,26498
|
|
24
|
+
hyperforge_nucliadb_agentic/ask/search/rpc.py,sha256=x82OJPpghQIgTk3CW0lpLEWix-09vYSqPUJAv9BfRwI,5653
|
|
25
|
+
hyperforge_nucliadb_agentic/ask/settings.py,sha256=0UiZKc6OiLwZbKaf9fYvujy5nng9Lr5cAej6m50Sh68,273
|
|
26
|
+
hyperforge_nucliadb_agentic/ask/utils/ids.py,sha256=LiC78oQ4BE7TlF7sK_45a2Tc0899qEWRdJWE2Zi6lfE,5581
|
|
27
|
+
hyperforge_nucliadb_agentic/ask/utils/proto.py,sha256=l245LXONx9elbppnbo7sHgaevOefPUUe1DAdh_GU1dg,209
|
|
28
|
+
hyperforge_nucliadb_agentic/ask/utils/responses.py,sha256=5-mbQuWIYUSRmMqDuDZaW83vtpOKce9J_Yq-knzy-JE,215
|
|
29
|
+
hyperforge_nucliadb_agentic/ask/utils/text_blocks.py,sha256=WGXFo4qL0L_ZS2CMU_6m8kASnLrBB7wC8i_mHfvzq0U,1580
|
|
30
|
+
hyperforge_nucliadb_agentic/config.py,sha256=UkLnvA8_GbQAtpIqCC8t-UIy6v_zoKI6UL6oaEnLgJ0,1661
|
|
31
|
+
hyperforge_nucliadb_agentic/internal_driver.py,sha256=uDValbgHE9U2z46UObce5gVXT2u8ITerimNRXt0q6kM,2774
|
|
32
|
+
hyperforge_nucliadb_agentic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
|
+
hyperforge_nucliadb_agentic-1.0.0.post64.dist-info/WHEEL,sha256=YR4QUxGCbsyoOnWBVTv-p1I4yc3seKGPev46mvmc8Cg,81
|
|
34
|
+
hyperforge_nucliadb_agentic-1.0.0.post64.dist-info/METADATA,sha256=IjxiIF8ccYsga1IIE-FC-NDr_nZEH8zLARf9HsKnDLQ,968
|
|
35
|
+
hyperforge_nucliadb_agentic-1.0.0.post64.dist-info/RECORD,,
|