nucliadb-utils 4.0.0.post534__py3-none-any.whl → 4.0.0.post536__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nucliadb_utils/audit/audit.py +8 -0
- nucliadb_utils/audit/basic.py +8 -0
- nucliadb_utils/audit/stream.py +86 -0
- nucliadb_utils/nuclia_usage/__init__.py +19 -0
- nucliadb_utils/nuclia_usage/protos/__init__.py +19 -0
- nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.py +71 -0
- nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.py +23 -0
- nucliadb_utils/nuclia_usage/utils/__init__.py +19 -0
- nucliadb_utils/nuclia_usage/utils/kb_usage_report.py +121 -0
- {nucliadb_utils-4.0.0.post534.dist-info → nucliadb_utils-4.0.0.post536.dist-info}/METADATA +3 -3
- {nucliadb_utils-4.0.0.post534.dist-info → nucliadb_utils-4.0.0.post536.dist-info}/RECORD +14 -8
- {nucliadb_utils-4.0.0.post534.dist-info → nucliadb_utils-4.0.0.post536.dist-info}/WHEEL +0 -0
- {nucliadb_utils-4.0.0.post534.dist-info → nucliadb_utils-4.0.0.post536.dist-info}/top_level.txt +0 -0
- {nucliadb_utils-4.0.0.post534.dist-info → nucliadb_utils-4.0.0.post536.dist-info}/zip-safe +0 -0
nucliadb_utils/audit/audit.py
CHANGED
nucliadb_utils/audit/basic.py
CHANGED
@@ -55,6 +55,14 @@ class BasicAuditStorage(AuditStorage):
|
|
55
55
|
f"AUDIT {audit_type} {kbid} {user} {origin} {rid} {audit_fields} {kb_counter}"
|
56
56
|
)
|
57
57
|
|
58
|
+
def report_resources(
|
59
|
+
self,
|
60
|
+
*,
|
61
|
+
kbid: str,
|
62
|
+
resources: int,
|
63
|
+
):
|
64
|
+
logger.debug(f"REPORT RESOURCES {kbid} {resources}")
|
65
|
+
|
58
66
|
async def visited(self, kbid: str, uuid: str, user: str, origin: str):
|
59
67
|
logger.debug(f"VISITED {kbid} {uuid} {user} {origin}")
|
60
68
|
|
nucliadb_utils/audit/stream.py
CHANGED
@@ -30,6 +30,7 @@ from nucliadb_protos.audit_pb2 import (
|
|
30
30
|
AuditKBCounter,
|
31
31
|
AuditRequest,
|
32
32
|
ChatContext,
|
33
|
+
ClientType,
|
33
34
|
)
|
34
35
|
from nucliadb_protos.nodereader_pb2 import SearchRequest
|
35
36
|
from nucliadb_protos.resources_pb2 import FieldID
|
@@ -38,6 +39,19 @@ from opentelemetry.trace import format_trace_id, get_current_span
|
|
38
39
|
from nucliadb_utils import logger
|
39
40
|
from nucliadb_utils.audit.audit import AuditStorage
|
40
41
|
from nucliadb_utils.nats import get_traced_jetstream
|
42
|
+
from nucliadb_utils.nuclia_usage.protos.kb_usage_pb2 import (
|
43
|
+
ClientType as ClientTypeKbUsage,
|
44
|
+
)
|
45
|
+
from nucliadb_utils.nuclia_usage.protos.kb_usage_pb2 import (
|
46
|
+
KBSource,
|
47
|
+
Search,
|
48
|
+
SearchType,
|
49
|
+
Service,
|
50
|
+
Storage,
|
51
|
+
)
|
52
|
+
from nucliadb_utils.nuclia_usage.utils.kb_usage_report import KbUsageReportUtility
|
53
|
+
|
54
|
+
KB_USAGE_STREAM_AUDIT = "kb-usage.audit"
|
41
55
|
|
42
56
|
|
43
57
|
class StreamAuditStorage(AuditStorage):
|
@@ -99,9 +113,17 @@ class StreamAuditStorage(AuditStorage):
|
|
99
113
|
|
100
114
|
self.js = get_traced_jetstream(self.nc, self.service)
|
101
115
|
self.task = asyncio.create_task(self.run())
|
116
|
+
|
117
|
+
self.kb_usage_utility = KbUsageReportUtility(
|
118
|
+
nats_stream=self.js, nats_subject=KB_USAGE_STREAM_AUDIT
|
119
|
+
)
|
120
|
+
await self.kb_usage_utility.initialize()
|
121
|
+
|
102
122
|
self.initialized = True
|
103
123
|
|
104
124
|
async def finalize(self):
|
125
|
+
await self.kb_usage_utility.finalize()
|
126
|
+
|
105
127
|
if self.task is not None:
|
106
128
|
self.task.cancel()
|
107
129
|
if self.nc:
|
@@ -178,10 +200,34 @@ class StreamAuditStorage(AuditStorage):
|
|
178
200
|
if kb_counter:
|
179
201
|
auditrequest.kb_counter.CopyFrom(kb_counter)
|
180
202
|
|
203
|
+
self.kb_usage_utility.send_kb_usage(
|
204
|
+
service=Service.NUCLIA_DB,
|
205
|
+
account_id=None,
|
206
|
+
kb_id=kbid,
|
207
|
+
kb_source=KBSource.HOSTED,
|
208
|
+
storage=Storage(
|
209
|
+
paragraphs=kb_counter.paragraphs, fields=kb_counter.fields
|
210
|
+
),
|
211
|
+
)
|
212
|
+
|
181
213
|
auditrequest.trace_id = get_trace_id()
|
182
214
|
|
183
215
|
await self.send(auditrequest)
|
184
216
|
|
217
|
+
def report_resources(
|
218
|
+
self,
|
219
|
+
*,
|
220
|
+
kbid: str,
|
221
|
+
resources: int,
|
222
|
+
):
|
223
|
+
self.kb_usage_utility.send_kb_usage(
|
224
|
+
service=Service.NUCLIA_DB,
|
225
|
+
account_id=None,
|
226
|
+
kb_id=kbid,
|
227
|
+
kb_source=KBSource.HOSTED,
|
228
|
+
storage=Storage(resources=resources),
|
229
|
+
)
|
230
|
+
|
185
231
|
async def visited(self, kbid: str, uuid: str, user: str, origin: str):
|
186
232
|
auditrequest = AuditRequest()
|
187
233
|
auditrequest.origin = origin
|
@@ -204,6 +250,14 @@ class StreamAuditStorage(AuditStorage):
|
|
204
250
|
auditrequest.trace_id = get_trace_id()
|
205
251
|
await self.send(auditrequest)
|
206
252
|
|
253
|
+
self.kb_usage_utility.send_kb_usage(
|
254
|
+
service=Service.NUCLIA_DB,
|
255
|
+
account_id=None,
|
256
|
+
kb_id=kbid,
|
257
|
+
kb_source=KBSource.HOSTED,
|
258
|
+
storage=Storage(paragraphs=0, fields=0, resources=0),
|
259
|
+
)
|
260
|
+
|
207
261
|
async def search(
|
208
262
|
self,
|
209
263
|
kbid: str,
|
@@ -229,6 +283,22 @@ class StreamAuditStorage(AuditStorage):
|
|
229
283
|
auditrequest.trace_id = get_trace_id()
|
230
284
|
await self.send(auditrequest)
|
231
285
|
|
286
|
+
self.kb_usage_utility.send_kb_usage(
|
287
|
+
service=Service.NUCLIA_DB,
|
288
|
+
account_id=None,
|
289
|
+
kb_id=kbid,
|
290
|
+
kb_source=KBSource.HOSTED,
|
291
|
+
# TODO unify AuditRequest client type and Nuclia Usage client type
|
292
|
+
searches=[
|
293
|
+
Search(
|
294
|
+
client=ClientTypeKbUsage.Value(ClientType.Name(client_type)), # type: ignore
|
295
|
+
type=SearchType.SEARCH,
|
296
|
+
tokens=2000,
|
297
|
+
num_searches=1,
|
298
|
+
)
|
299
|
+
],
|
300
|
+
)
|
301
|
+
|
232
302
|
async def suggest(
|
233
303
|
self,
|
234
304
|
kbid: str,
|
@@ -249,6 +319,22 @@ class StreamAuditStorage(AuditStorage):
|
|
249
319
|
|
250
320
|
await self.send(auditrequest)
|
251
321
|
|
322
|
+
self.kb_usage_utility.send_kb_usage(
|
323
|
+
service=Service.NUCLIA_DB,
|
324
|
+
account_id=None,
|
325
|
+
kb_id=kbid,
|
326
|
+
kb_source=KBSource.HOSTED,
|
327
|
+
# TODO unify AuditRequest client type and Nuclia Usage client type
|
328
|
+
searches=[
|
329
|
+
Search(
|
330
|
+
client=ClientTypeKbUsage.Value(ClientType.Name(client_type)), # type: ignore
|
331
|
+
type=SearchType.SUGGEST,
|
332
|
+
tokens=0,
|
333
|
+
num_searches=1,
|
334
|
+
)
|
335
|
+
],
|
336
|
+
)
|
337
|
+
|
252
338
|
async def chat(
|
253
339
|
self,
|
254
340
|
kbid: str,
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
+
#
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
+
#
|
6
|
+
# AGPL:
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
10
|
+
# License, or (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU Affero General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
#
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
+
#
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
+
#
|
6
|
+
# AGPL:
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
10
|
+
# License, or (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU Affero General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
#
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
+
#
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
+
#
|
6
|
+
# AGPL:
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
10
|
+
# License, or (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU Affero General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
#
|
20
|
+
# flake8: noqa
|
21
|
+
# -*- coding: utf-8 -*-
|
22
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
23
|
+
# source: protos/kb_usage.proto
|
24
|
+
# Protobuf Python Version: 4.25.1
|
25
|
+
"""Generated protocol buffer code."""
|
26
|
+
from google.protobuf import descriptor as _descriptor
|
27
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
28
|
+
from google.protobuf import symbol_database as _symbol_database
|
29
|
+
from google.protobuf.internal import builder as _builder
|
30
|
+
|
31
|
+
# @@protoc_insertion_point(imports)
|
32
|
+
|
33
|
+
_sym_db = _symbol_database.Default()
|
34
|
+
|
35
|
+
|
36
|
+
from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
|
37
|
+
|
38
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
|
39
|
+
b'\n\x15protos/kb_usage.proto\x12\x08kb_usage\x1a\x1fgoogle/protobuf/timestamp.proto"\xf2\x01\n\x07Process\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12\x1c\n\x14slow_processing_time\x18\x02 \x01(\x02\x12\x1b\n\x13pre_processing_time\x18\x03 \x01(\x02\x12\r\n\x05\x62ytes\x18\x04 \x01(\x04\x12\r\n\x05\x63hars\x18\x05 \x01(\r\x12\x15\n\rmedia_seconds\x18\x06 \x01(\r\x12\r\n\x05pages\x18\x07 \x01(\r\x12\x12\n\nparagraphs\x18\x08 \x01(\r\x12\x17\n\x0bmedia_files\x18\t \x01(\rB\x02\x18\x01\x12\x15\n\rnum_processed\x18\n \x01(\r"w\n\x07Storage\x12\x17\n\nparagraphs\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x13\n\x06\x66ields\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x16\n\tresources\x18\x03 \x01(\x04H\x02\x88\x01\x01\x42\r\n\x0b_paragraphsB\t\n\x07_fieldsB\x0c\n\n_resources"x\n\x06Search\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12"\n\x04type\x18\x02 \x01(\x0e\x32\x14.kb_usage.SearchType\x12\x0e\n\x06tokens\x18\x03 \x01(\r\x12\x14\n\x0cnum_searches\x18\x04 \x01(\r"\xa7\x01\n\x07Predict\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12#\n\x04type\x18\x02 \x01(\x0e\x32\x15.kb_usage.PredictType\x12\r\n\x05model\x18\x03 \x01(\t\x12\r\n\x05input\x18\x04 \x01(\r\x12\x0e\n\x06output\x18\x05 \x01(\r\x12\r\n\x05image\x18\x06 \x01(\r\x12\x14\n\x0cnum_predicts\x18\x07 \x01(\r"\xed\x02\n\x07KbUsage\x12"\n\x07service\x18\x01 \x01(\x0e\x32\x11.kb_usage.Service\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x17\n\naccount_id\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05kb_id\x18\x04 \x01(\tH\x01\x88\x01\x01\x12%\n\tkb_source\x18\x05 \x01(\x0e\x32\x12.kb_usage.KBSource\x12$\n\tprocesses\x18\x06 \x03(\x0b\x32\x11.kb_usage.Process\x12#\n\x08predicts\x18\x07 \x03(\x0b\x32\x11.kb_usage.Predict\x12"\n\x08searches\x18\x08 \x03(\x0b\x32\x10.kb_usage.Search\x12\'\n\x07storage\x18\t \x01(\x0b\x32\x11.kb_usage.StorageH\x02\x88\x01\x01\x42\r\n\x0b_account_idB\x08\n\x06_kb_idB\n\n\x08_storage"9\n\x11KbUsageAggregated\x12$\n\tkb_usages\x18\x01 \x03(\x0b\x32\x11.kb_usage.KbUsage*"\n\x08KBSource\x12\n\n\x06HOSTED\x10\x00\x12\n\n\x06ONPREM\x10\x01*5\n\x07Service\x12\x0b\n\x07PREDICT\x10\x00\x12\x0e\n\nPROCESSING\x10\x01\x12\r\n\tNUCLIA_DB\x10\x02*%\n\nSearchType\x12\n\n\x06SEARCH\x10\x00\x12\x0b\n\x07SUGGEST\x10\x01*\xa0\x01\n\x0bPredictType\x12\x0c\n\x08SENTENCE\x10\x00\x12\t\n\x05TOKEN\x10\x01\x12\x13\n\x0fQUESTION_ANSWER\x10\x02\x12\x0c\n\x08REPHRASE\x10\x03\x12\r\n\tSUMMARIZE\x10\x04\x12\x12\n\x0e\x45XTRACT_TABLES\x10\x05\x12\n\n\x06RERANK\x10\x06\x12\r\n\tRELATIONS\x10\x07\x12\n\n\x06SPEECH\x10\x08\x12\x0b\n\x07\x43\x41PTION\x10\t*j\n\nClientType\x12\x07\n\x03\x41PI\x10\x00\x12\x07\n\x03WEB\x10\x01\x12\n\n\x06WIDGET\x10\x02\x12\x0b\n\x07\x44\x45SKTOP\x10\x03\x12\r\n\tDASHBOARD\x10\x04\x12\x14\n\x10\x43HROME_EXTENSION\x10\x05\x12\x0c\n\x08INTERNAL\x10\x06\x62\x06proto3'
|
40
|
+
)
|
41
|
+
|
42
|
+
_globals = globals()
|
43
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
44
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "protos.kb_usage_pb2", _globals)
|
45
|
+
if _descriptor._USE_C_DESCRIPTORS == False:
|
46
|
+
DESCRIPTOR._options = None
|
47
|
+
_globals["_PROCESS"].fields_by_name["media_files"]._options = None
|
48
|
+
_globals["_PROCESS"].fields_by_name["media_files"]._serialized_options = b"\030\001"
|
49
|
+
_globals["_KBSOURCE"]._serialized_start = 1153
|
50
|
+
_globals["_KBSOURCE"]._serialized_end = 1187
|
51
|
+
_globals["_SERVICE"]._serialized_start = 1189
|
52
|
+
_globals["_SERVICE"]._serialized_end = 1242
|
53
|
+
_globals["_SEARCHTYPE"]._serialized_start = 1244
|
54
|
+
_globals["_SEARCHTYPE"]._serialized_end = 1281
|
55
|
+
_globals["_PREDICTTYPE"]._serialized_start = 1284
|
56
|
+
_globals["_PREDICTTYPE"]._serialized_end = 1444
|
57
|
+
_globals["_CLIENTTYPE"]._serialized_start = 1446
|
58
|
+
_globals["_CLIENTTYPE"]._serialized_end = 1552
|
59
|
+
_globals["_PROCESS"]._serialized_start = 69
|
60
|
+
_globals["_PROCESS"]._serialized_end = 311
|
61
|
+
_globals["_STORAGE"]._serialized_start = 313
|
62
|
+
_globals["_STORAGE"]._serialized_end = 432
|
63
|
+
_globals["_SEARCH"]._serialized_start = 434
|
64
|
+
_globals["_SEARCH"]._serialized_end = 554
|
65
|
+
_globals["_PREDICT"]._serialized_start = 557
|
66
|
+
_globals["_PREDICT"]._serialized_end = 724
|
67
|
+
_globals["_KBUSAGE"]._serialized_start = 727
|
68
|
+
_globals["_KBUSAGE"]._serialized_end = 1092
|
69
|
+
_globals["_KBUSAGEAGGREGATED"]._serialized_start = 1094
|
70
|
+
_globals["_KBUSAGEAGGREGATED"]._serialized_end = 1151
|
71
|
+
# @@protoc_insertion_point(module_scope)
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
+
#
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
+
#
|
6
|
+
# AGPL:
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
10
|
+
# License, or (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU Affero General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
#
|
20
|
+
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
21
|
+
# flake8: noqa
|
22
|
+
"""Client and server classes corresponding to protobuf-defined services."""
|
23
|
+
import grpc
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
+
#
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
+
#
|
6
|
+
# AGPL:
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
10
|
+
# License, or (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU Affero General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
#
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
+
#
|
3
|
+
# nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
+
# For commercial licensing, contact us at info@nuclia.com.
|
5
|
+
#
|
6
|
+
# AGPL:
|
7
|
+
# This program is free software: you can redistribute it and/or modify
|
8
|
+
# it under the terms of the GNU Affero General Public License as
|
9
|
+
# published by the Free Software Foundation, either version 3 of the
|
10
|
+
# License, or (at your option) any later version.
|
11
|
+
#
|
12
|
+
# This program is distributed in the hope that it will be useful,
|
13
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
# GNU Affero General Public License for more details.
|
16
|
+
#
|
17
|
+
# You should have received a copy of the GNU Affero General Public License
|
18
|
+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
|
20
|
+
import asyncio
|
21
|
+
import logging
|
22
|
+
from collections.abc import Iterable
|
23
|
+
from contextlib import suppress
|
24
|
+
from datetime import datetime, timezone
|
25
|
+
from typing import Optional
|
26
|
+
|
27
|
+
from nats.js.client import JetStreamContext
|
28
|
+
|
29
|
+
from nucliadb_utils.nuclia_usage.protos.kb_usage_pb2 import (
|
30
|
+
KBSource,
|
31
|
+
KbUsage,
|
32
|
+
Predict,
|
33
|
+
Process,
|
34
|
+
Search,
|
35
|
+
Service,
|
36
|
+
Storage,
|
37
|
+
)
|
38
|
+
|
39
|
+
logger = logging.getLogger(__name__)
|
40
|
+
|
41
|
+
|
42
|
+
class KbUsageReportUtility:
|
43
|
+
queue: asyncio.Queue
|
44
|
+
lock: asyncio.Lock
|
45
|
+
|
46
|
+
def __init__(
|
47
|
+
self,
|
48
|
+
nats_stream: JetStreamContext,
|
49
|
+
nats_subject: str,
|
50
|
+
max_queue_size: int = 100,
|
51
|
+
):
|
52
|
+
self.nats_stream = nats_stream
|
53
|
+
self.nats_subject = nats_subject
|
54
|
+
self.queue = asyncio.Queue(max_queue_size)
|
55
|
+
self.task = None
|
56
|
+
|
57
|
+
async def initialize(self):
|
58
|
+
if self.task is None:
|
59
|
+
self.task = asyncio.create_task(self.run())
|
60
|
+
|
61
|
+
async def finalize(self):
|
62
|
+
if self.task is not None:
|
63
|
+
self.task.cancel()
|
64
|
+
with suppress(asyncio.CancelledError, asyncio.exceptions.TimeoutError):
|
65
|
+
await asyncio.wait_for(self.task, timeout=2)
|
66
|
+
|
67
|
+
async def run(self) -> None:
|
68
|
+
while True:
|
69
|
+
message: KbUsage = await self.queue.get()
|
70
|
+
try:
|
71
|
+
await self._send(message)
|
72
|
+
except Exception:
|
73
|
+
logger.exception("Could not send KbUsage message")
|
74
|
+
finally:
|
75
|
+
self.queue.task_done()
|
76
|
+
|
77
|
+
def send(self, message: KbUsage):
|
78
|
+
try:
|
79
|
+
self.queue.put_nowait(message)
|
80
|
+
except asyncio.QueueFull:
|
81
|
+
logger.warning("KbUsage utility queue is full, dropping message")
|
82
|
+
|
83
|
+
async def _send(self, message: KbUsage) -> int:
|
84
|
+
res = await self.nats_stream.publish(
|
85
|
+
self.nats_subject,
|
86
|
+
message.SerializeToString(),
|
87
|
+
)
|
88
|
+
return res.seq
|
89
|
+
|
90
|
+
def send_kb_usage(
|
91
|
+
self,
|
92
|
+
service: Service,
|
93
|
+
account_id: Optional[str],
|
94
|
+
kb_id: Optional[str],
|
95
|
+
kb_source: KBSource,
|
96
|
+
processes: Iterable[Process] = (),
|
97
|
+
predicts: Iterable[Predict] = (),
|
98
|
+
searches: Iterable[Search] = (),
|
99
|
+
storage: Optional[Storage] = None,
|
100
|
+
):
|
101
|
+
usage = KbUsage()
|
102
|
+
usage.service = service # type: ignore
|
103
|
+
usage.timestamp.FromDatetime(datetime.now(tz=timezone.utc))
|
104
|
+
if account_id is not None:
|
105
|
+
usage.account_id = account_id
|
106
|
+
if kb_id is not None:
|
107
|
+
usage.kb_id = kb_id
|
108
|
+
usage.kb_source = kb_source # type: ignore
|
109
|
+
|
110
|
+
usage.processes.extend(processes)
|
111
|
+
usage.predicts.extend(predicts)
|
112
|
+
usage.searches.extend(searches)
|
113
|
+
if storage is not None:
|
114
|
+
if storage.HasField("fields"):
|
115
|
+
usage.storage.fields = storage.fields
|
116
|
+
if storage.HasField("paragraphs"):
|
117
|
+
usage.storage.paragraphs = storage.paragraphs
|
118
|
+
if storage.HasField("resources"):
|
119
|
+
usage.storage.resources = storage.resources
|
120
|
+
|
121
|
+
self.send(usage)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: nucliadb-utils
|
3
|
-
Version: 4.0.0.
|
3
|
+
Version: 4.0.0.post536
|
4
4
|
Home-page: https://nuclia.com
|
5
5
|
License: BSD
|
6
6
|
Classifier: Development Status :: 4 - Beta
|
@@ -22,8 +22,8 @@ Requires-Dist: PyNaCl
|
|
22
22
|
Requires-Dist: pyjwt >=2.4.0
|
23
23
|
Requires-Dist: memorylru >=1.1.2
|
24
24
|
Requires-Dist: mrflagly
|
25
|
-
Requires-Dist: nucliadb-protos >=4.0.0.
|
26
|
-
Requires-Dist: nucliadb-telemetry >=4.0.0.
|
25
|
+
Requires-Dist: nucliadb-protos >=4.0.0.post536
|
26
|
+
Requires-Dist: nucliadb-telemetry >=4.0.0.post536
|
27
27
|
Provides-Extra: cache
|
28
28
|
Requires-Dist: redis >=4.3.4 ; extra == 'cache'
|
29
29
|
Requires-Dist: orjson >=3.6.7 ; extra == 'cache'
|
@@ -18,9 +18,9 @@ nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
|
|
18
18
|
nucliadb_utils/transaction.py,sha256=CQpsuF-E2omh4gGMxXCn0dv7vL9ctxooWpSgWGbGfBA,7212
|
19
19
|
nucliadb_utils/utilities.py,sha256=s5MVXDj4DTtc1VPFBRxMjud3HB0xkadyQ0f7QQLb0NM,14178
|
20
20
|
nucliadb_utils/audit/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
|
21
|
-
nucliadb_utils/audit/audit.py,sha256=
|
22
|
-
nucliadb_utils/audit/basic.py,sha256=
|
23
|
-
nucliadb_utils/audit/stream.py,sha256=
|
21
|
+
nucliadb_utils/audit/audit.py,sha256=fmEVb6ahKrkGAY-GEy4_L4ccmcGM5YKl-Vs05260_cg,2834
|
22
|
+
nucliadb_utils/audit/basic.py,sha256=8yL7HI9MnykSt7j4QbUeRBbBsTKFIIX6hppJ3ADVLdM,3430
|
23
|
+
nucliadb_utils/audit/stream.py,sha256=oGblTZMiTFFaXIf5684gbizspg6LxX6j3JdKTzAtMwA,12038
|
24
24
|
nucliadb_utils/cache/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
|
25
25
|
nucliadb_utils/cache/exceptions.py,sha256=Zu-O_-0-yctOEgoDGI92gPzWfBMRrpiAyESA62ld6MA,975
|
26
26
|
nucliadb_utils/cache/nats.py,sha256=h0x894QNLkkvyOcmqH8ncC-d0AiHm9SjDgvPPiwxYao,7089
|
@@ -30,6 +30,12 @@ nucliadb_utils/fastapi/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu
|
|
30
30
|
nucliadb_utils/fastapi/openapi.py,sha256=b0pLuri0QuzQd0elDyOVXM42YYmES_cmT-jEfsQ1G6Y,1737
|
31
31
|
nucliadb_utils/fastapi/run.py,sha256=n6vOX64QqF1I5n4UlKnpm_ZJ24rmwfRGi-J9YMGpZzA,3631
|
32
32
|
nucliadb_utils/fastapi/versioning.py,sha256=uOVdKl8GzBt8L1YnU5GjBsNeDdFV2yrjjYFEGqUXRgQ,3774
|
33
|
+
nucliadb_utils/nuclia_usage/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
|
34
|
+
nucliadb_utils/nuclia_usage/protos/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
|
35
|
+
nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.py,sha256=7jBWh5aFkNOU6rF_rdp-twZONgrJe3ztJdqKvRWcMM4,5934
|
36
|
+
nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.py,sha256=dhop8WwjplPfORYPYb9HtcS9gHMzqXPJQGqXYRjV-6M,1008
|
37
|
+
nucliadb_utils/nuclia_usage/utils/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
|
38
|
+
nucliadb_utils/nuclia_usage/utils/kb_usage_report.py,sha256=E1eUSFXBVNzQP9Q2rWj9y3koCO5S7iKwckny_AoLKuk,3870
|
33
39
|
nucliadb_utils/storages/__init__.py,sha256=5Qc8AUWiJv9_JbGCBpAn88AIJhwDlm0OPQpg2ZdRL4U,872
|
34
40
|
nucliadb_utils/storages/exceptions.py,sha256=n6aBOyurWMo8mXd1XY6Psgno4VfXJ9TRbxCy67c08-g,2417
|
35
41
|
nucliadb_utils/storages/gcs.py,sha256=-M-abojobXGwZaS1kuFXRBxwashsAaTXMStjx3QpX4U,27076
|
@@ -63,8 +69,8 @@ nucliadb_utils/tests/unit/storages/test_aws.py,sha256=GCsB_jwCUNV3Ogt8TZZEmNKAHv
|
|
63
69
|
nucliadb_utils/tests/unit/storages/test_gcs.py,sha256=2XzJwgNpfjVGjtE-QdZhu3ayuT1EMEXINdM-_SatPCY,3554
|
64
70
|
nucliadb_utils/tests/unit/storages/test_pg.py,sha256=sJfUttMSzq8W1XYolAUcMxl_R5HcEzb5fpCklPeMJiY,17000
|
65
71
|
nucliadb_utils/tests/unit/storages/test_storage.py,sha256=CwX4wO21og1pdw9IqnK4sFxBfKY3sPhQTd56AS7-Me8,6666
|
66
|
-
nucliadb_utils-4.0.0.
|
67
|
-
nucliadb_utils-4.0.0.
|
68
|
-
nucliadb_utils-4.0.0.
|
69
|
-
nucliadb_utils-4.0.0.
|
70
|
-
nucliadb_utils-4.0.0.
|
72
|
+
nucliadb_utils-4.0.0.post536.dist-info/METADATA,sha256=zGUTuozAM-3Mr0zLlNEEHZivVR7HV4OvtwQ4Q7XK7Ss,1979
|
73
|
+
nucliadb_utils-4.0.0.post536.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
74
|
+
nucliadb_utils-4.0.0.post536.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
|
75
|
+
nucliadb_utils-4.0.0.post536.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
76
|
+
nucliadb_utils-4.0.0.post536.dist-info/RECORD,,
|
File without changes
|
{nucliadb_utils-4.0.0.post534.dist-info → nucliadb_utils-4.0.0.post536.dist-info}/top_level.txt
RENAMED
File without changes
|
File without changes
|