nucliadb-utils 5.0.1.post1080__py3-none-any.whl → 5.0.1.post1100__py3-none-any.whl

nucliadb_utils/aiopynecone/client.py

@@ -28,7 +28,7 @@ from typing import Any, AsyncGenerator, Optional
 import backoff
 import httpx
 
-from nucliadb_telemetry.metrics import Observer
+from nucliadb_telemetry.metrics import INF, Histogram, Observer
 from nucliadb_utils.aiopynecone.exceptions import (
     PineconeAPIError,
     PineconeRateLimitError,
@@ -46,6 +46,25 @@ from nucliadb_utils.aiopynecone.models import (
 
 logger = logging.getLogger(__name__)
 
+upsert_batch_size_histogram = Histogram(
+    "pinecone_upsert_batch_size",
+    buckets=[10.0, 100.0, 200.0, 500.0, 1000.0, 5000.0, INF],
+)
+upsert_batch_count_histogram = Histogram(
+    "pinecone_upsert_batch_count",
+    buckets=[0.0, 1.0, 2.0, 3.0, 5.0, 10.0, 15.0, 20.0, 30.0, 50.0, INF],
+)
+
+delete_batch_size_histogram = Histogram(
+    "pinecone_delete_batch_size",
+    buckets=[1.0, 5.0, 10.0, 20.0, 50.0, 100.0, 150.0, INF],
+)
+
+delete_batch_count_histogram = Histogram(
+    "pinecone_delete_batch_count",
+    buckets=[0.0, 1.0, 2.0, 3.0, 5.0, 10.0, 15.0, 20.0, 30.0, 50.0, INF],
+)
+
 
 pinecone_observer = Observer(
     "pinecone_client",
@@ -57,7 +76,6 @@ pinecone_observer = Observer(
 
 DEFAULT_TIMEOUT = 30
 CONTROL_PLANE_BASE_URL = "https://api.pinecone.io/"
-INDEX_HOST_BASE_URL = "https://{index_host}/"
 BASE_API_HEADERS = {
     "Content-Type": "application/json",
     "Accept": "application/json",
@@ -198,6 +216,7 @@ class DataPlane:
         if len(vectors) == 0:
             # Nothing to upsert.
             return
+        upsert_batch_size_histogram.observe(len(vectors))
         headers = {"Api-Key": self.api_key}
         payload = UpsertRequest(vectors=vectors)
         post_kwargs: dict[str, Any] = {
@@ -261,7 +280,10 @@ class DataPlane:
         for batch in batchify(vectors, batch_size):
             tasks.append(asyncio.create_task(_upsert_batch(batch)))
 
-        await asyncio.gather(*tasks)
+        upsert_batch_count_histogram.observe(len(tasks))
+
+        if len(tasks) > 0:
+            await asyncio.gather(*tasks)
 
     @backoff.on_exception(
         backoff.expo,
@@ -280,8 +302,16 @@ class DataPlane:
         """
         if len(ids) > MAX_DELETE_BATCH_SIZE:
             raise ValueError(f"Maximum number of ids in a single request is {MAX_DELETE_BATCH_SIZE}.")
+        if len(ids) == 0:  # pragma: no cover
+            return
 
+        delete_batch_size_histogram.observe(len(ids))
         headers = {"Api-Key": self.api_key}
+
+        # This is a temporary log info to hunt down a bug.
+        rids = {vid.split("/")[0] for vid in ids}
+        logger.info(f"Deleting vectors from resources: {list(rids)}")
+
         payload = {"ids": ids}
         post_kwargs: dict[str, Any] = {
             "headers": headers,
@@ -428,7 +458,10 @@ class DataPlane:
         async for batch in async_batchify(async_iterable, batch_size):
             tasks.append(asyncio.create_task(_delete_batch(batch)))
 
-        await asyncio.gather(*tasks)
+        delete_batch_count_histogram.observe(len(tasks))
+
+        if len(tasks) > 0:
+            await asyncio.gather(*tasks)
 
     @backoff.on_exception(
         backoff.expo,
@@ -516,8 +549,12 @@ class PineconeSession:
         if session is not None:
             return session
 
+        base_url = index_host
+        if not index_host.startswith("https://"):
+            base_url = f"https://{index_host}/"
+
         session = httpx.AsyncClient(
-            base_url=INDEX_HOST_BASE_URL.format(index_host=index_host),
+            base_url=base_url,
             headers=BASE_API_HEADERS,
             timeout=DEFAULT_TIMEOUT,
         )

nucliadb_utils/audit/audit.py

@@ -94,7 +94,7 @@ class AuditStorage:
     ):
         raise NotImplementedError
 
-    def report_fields_and_paragraphs(self, kbid: str, paragraphs: int, fields: int):
+    def report_storage(self, kbid: str, paragraphs: int, fields: int, bytes: int):
         raise NotImplementedError
 
     def report_resources(

nucliadb_utils/audit/basic.py

@@ -95,7 +95,7 @@ class BasicAuditStorage(AuditStorage):
     ):
         logger.debug(f"CHAT {kbid} {user} {origin}")
 
-    def report_fields_and_paragraphs(self, kbid: str, paragraphs: int, fields: int):
+    def report_storage(self, kbid: str, paragraphs: int, fields: int, bytes: int):
         logger.debug(f"FIELDS & PARAGRAPHS {kbid} {paragraphs} {fields}")
 
     def report_resources(

nucliadb_utils/audit/stream.py

@@ -280,13 +280,13 @@ class StreamAuditStorage(AuditStorage):
 
         self.send(auditrequest)
 
-    def report_fields_and_paragraphs(self, kbid: str, paragraphs: int, fields: int):
+    def report_storage(self, kbid: str, paragraphs: int, fields: int, bytes: int):
         self.kb_usage_utility.send_kb_usage(
             service=Service.NUCLIA_DB,
             account_id=None,
             kb_id=kbid,
             kb_source=KBSource.HOSTED,
-            storage=Storage(paragraphs=paragraphs, fields=fields),
+            storage=Storage(paragraphs=paragraphs, fields=fields, bytes=bytes),
         )
 
     def report_resources(

nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.py

@@ -27,7 +27,6 @@ from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
 from google.protobuf import symbol_database as _symbol_database
 from google.protobuf.internal import builder as _builder
-
 # @@protoc_insertion_point(imports)
 
 _sym_db = _symbol_database.Default()
@@ -35,37 +34,36 @@ _sym_db = _symbol_database.Default()
 
 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x15protos/kb_usage.proto\x12\x08kb_usage\x1a\x1fgoogle/protobuf/timestamp.proto"\xf2\x01\n\x07Process\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12\x1c\n\x14slow_processing_time\x18\x02 \x01(\x02\x12\x1b\n\x13pre_processing_time\x18\x03 \x01(\x02\x12\r\n\x05\x62ytes\x18\x04 \x01(\x04\x12\r\n\x05\x63hars\x18\x05 \x01(\r\x12\x15\n\rmedia_seconds\x18\x06 \x01(\r\x12\r\n\x05pages\x18\x07 \x01(\r\x12\x12\n\nparagraphs\x18\x08 \x01(\r\x12\x17\n\x0bmedia_files\x18\t \x01(\rB\x02\x18\x01\x12\x15\n\rnum_processed\x18\n \x01(\r"w\n\x07Storage\x12\x17\n\nparagraphs\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x13\n\x06\x66ields\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x16\n\tresources\x18\x03 \x01(\x04H\x02\x88\x01\x01\x42\r\n\x0b_paragraphsB\t\n\x07_fieldsB\x0c\n\n_resources"x\n\x06Search\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12"\n\x04type\x18\x02 \x01(\x0e\x32\x14.kb_usage.SearchType\x12\x0e\n\x06tokens\x18\x03 \x01(\r\x12\x14\n\x0cnum_searches\x18\x04 \x01(\r"\xa7\x01\n\x07Predict\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12#\n\x04type\x18\x02 \x01(\x0e\x32\x15.kb_usage.PredictType\x12\r\n\x05model\x18\x03 \x01(\t\x12\r\n\x05input\x18\x04 \x01(\r\x12\x0e\n\x06output\x18\x05 \x01(\r\x12\r\n\x05image\x18\x06 \x01(\r\x12\x14\n\x0cnum_predicts\x18\x07 \x01(\r"\xed\x02\n\x07KbUsage\x12"\n\x07service\x18\x01 \x01(\x0e\x32\x11.kb_usage.Service\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x17\n\naccount_id\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05kb_id\x18\x04 \x01(\tH\x01\x88\x01\x01\x12%\n\tkb_source\x18\x05 \x01(\x0e\x32\x12.kb_usage.KBSource\x12$\n\tprocesses\x18\x06 \x03(\x0b\x32\x11.kb_usage.Process\x12#\n\x08predicts\x18\x07 \x03(\x0b\x32\x11.kb_usage.Predict\x12"\n\x08searches\x18\x08 \x03(\x0b\x32\x10.kb_usage.Search\x12\'\n\x07storage\x18\t \x01(\x0b\x32\x11.kb_usage.StorageH\x02\x88\x01\x01\x42\r\n\x0b_account_idB\x08\n\x06_kb_idB\n\n\x08_storage"9\n\x11KbUsageAggregated\x12$\n\tkb_usages\x18\x01 \x03(\x0b\x32\x11.kb_usage.KbUsage*"\n\x08KBSource\x12\n\n\x06HOSTED\x10\x00\x12\n\n\x06ONPREM\x10\x01*5\n\x07Service\x12\x0b\n\x07PREDICT\x10\x00\x12\x0e\n\nPROCESSING\x10\x01\x12\r\n\tNUCLIA_DB\x10\x02*%\n\nSearchType\x12\n\n\x06SEARCH\x10\x00\x12\x0b\n\x07SUGGEST\x10\x01*\xa0\x01\n\x0bPredictType\x12\x0c\n\x08SENTENCE\x10\x00\x12\t\n\x05TOKEN\x10\x01\x12\x13\n\x0fQUESTION_ANSWER\x10\x02\x12\x0c\n\x08REPHRASE\x10\x03\x12\r\n\tSUMMARIZE\x10\x04\x12\x12\n\x0e\x45XTRACT_TABLES\x10\x05\x12\n\n\x06RERANK\x10\x06\x12\r\n\tRELATIONS\x10\x07\x12\n\n\x06SPEECH\x10\x08\x12\x0b\n\x07\x43\x41PTION\x10\t*j\n\nClientType\x12\x07\n\x03\x41PI\x10\x00\x12\x07\n\x03WEB\x10\x01\x12\n\n\x06WIDGET\x10\x02\x12\x0b\n\x07\x44\x45SKTOP\x10\x03\x12\r\n\tDASHBOARD\x10\x04\x12\x14\n\x10\x43HROME_EXTENSION\x10\x05\x12\x0c\n\x08INTERNAL\x10\x06\x62\x06proto3'
-)
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15protos/kb_usage.proto\x12\x08kb_usage\x1a\x1fgoogle/protobuf/timestamp.proto\"\xf2\x01\n\x07Process\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12\x1c\n\x14slow_processing_time\x18\x02 \x01(\x02\x12\x1b\n\x13pre_processing_time\x18\x03 \x01(\x02\x12\r\n\x05\x62ytes\x18\x04 \x01(\x04\x12\r\n\x05\x63hars\x18\x05 \x01(\r\x12\x15\n\rmedia_seconds\x18\x06 \x01(\r\x12\r\n\x05pages\x18\x07 \x01(\r\x12\x12\n\nparagraphs\x18\x08 \x01(\r\x12\x17\n\x0bmedia_files\x18\t \x01(\rB\x02\x18\x01\x12\x15\n\rnum_processed\x18\n \x01(\r\"\x95\x01\n\x07Storage\x12\x17\n\nparagraphs\x18\x01 \x01(\x04H\x00\x88\x01\x01\x12\x13\n\x06\x66ields\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x16\n\tresources\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12\x12\n\x05\x62ytes\x18\x04 \x01(\x04H\x03\x88\x01\x01\x42\r\n\x0b_paragraphsB\t\n\x07_fieldsB\x0c\n\n_resourcesB\x08\n\x06_bytes\"x\n\x06Search\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12\"\n\x04type\x18\x02 \x01(\x0e\x32\x14.kb_usage.SearchType\x12\x0e\n\x06tokens\x18\x03 \x01(\r\x12\x14\n\x0cnum_searches\x18\x04 \x01(\r\"\xa7\x01\n\x07Predict\x12$\n\x06\x63lient\x18\x01 \x01(\x0e\x32\x14.kb_usage.ClientType\x12#\n\x04type\x18\x02 \x01(\x0e\x32\x15.kb_usage.PredictType\x12\r\n\x05model\x18\x03 \x01(\t\x12\r\n\x05input\x18\x04 \x01(\r\x12\x0e\n\x06output\x18\x05 \x01(\r\x12\r\n\x05image\x18\x06 \x01(\r\x12\x14\n\x0cnum_predicts\x18\x07 \x01(\r\"\xed\x02\n\x07KbUsage\x12\"\n\x07service\x18\x01 \x01(\x0e\x32\x11.kb_usage.Service\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x17\n\naccount_id\x18\x03 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05kb_id\x18\x04 \x01(\tH\x01\x88\x01\x01\x12%\n\tkb_source\x18\x05 \x01(\x0e\x32\x12.kb_usage.KBSource\x12$\n\tprocesses\x18\x06 \x03(\x0b\x32\x11.kb_usage.Process\x12#\n\x08predicts\x18\x07 \x03(\x0b\x32\x11.kb_usage.Predict\x12\"\n\x08searches\x18\x08 \x03(\x0b\x32\x10.kb_usage.Search\x12\'\n\x07storage\x18\t \x01(\x0b\x32\x11.kb_usage.StorageH\x02\x88\x01\x01\x42\r\n\x0b_account_idB\x08\n\x06_kb_idB\n\n\x08_storage\"9\n\x11KbUsageAggregated\x12$\n\tkb_usages\x18\x01 \x03(\x0b\x32\x11.kb_usage.KbUsage*\"\n\x08KBSource\x12\n\n\x06HOSTED\x10\x00\x12\n\n\x06ONPREM\x10\x01*5\n\x07Service\x12\x0b\n\x07PREDICT\x10\x00\x12\x0e\n\nPROCESSING\x10\x01\x12\r\n\tNUCLIA_DB\x10\x02*%\n\nSearchType\x12\n\n\x06SEARCH\x10\x00\x12\x0b\n\x07SUGGEST\x10\x01*\xa0\x01\n\x0bPredictType\x12\x0c\n\x08SENTENCE\x10\x00\x12\t\n\x05TOKEN\x10\x01\x12\x13\n\x0fQUESTION_ANSWER\x10\x02\x12\x0c\n\x08REPHRASE\x10\x03\x12\r\n\tSUMMARIZE\x10\x04\x12\x12\n\x0e\x45XTRACT_TABLES\x10\x05\x12\n\n\x06RERANK\x10\x06\x12\r\n\tRELATIONS\x10\x07\x12\n\n\x06SPEECH\x10\x08\x12\x0b\n\x07\x43\x41PTION\x10\t*j\n\nClientType\x12\x07\n\x03\x41PI\x10\x00\x12\x07\n\x03WEB\x10\x01\x12\n\n\x06WIDGET\x10\x02\x12\x0b\n\x07\x44\x45SKTOP\x10\x03\x12\r\n\tDASHBOARD\x10\x04\x12\x14\n\x10\x43HROME_EXTENSION\x10\x05\x12\x0c\n\x08INTERNAL\x10\x06\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "protos.kb_usage_pb2", _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.kb_usage_pb2', _globals)
 if _descriptor._USE_C_DESCRIPTORS == False:
-    DESCRIPTOR._options = None
-    _globals["_PROCESS"].fields_by_name["media_files"]._options = None
-    _globals["_PROCESS"].fields_by_name["media_files"]._serialized_options = b"\030\001"
-    _globals["_KBSOURCE"]._serialized_start = 1153
-    _globals["_KBSOURCE"]._serialized_end = 1187
-    _globals["_SERVICE"]._serialized_start = 1189
-    _globals["_SERVICE"]._serialized_end = 1242
-    _globals["_SEARCHTYPE"]._serialized_start = 1244
-    _globals["_SEARCHTYPE"]._serialized_end = 1281
-    _globals["_PREDICTTYPE"]._serialized_start = 1284
-    _globals["_PREDICTTYPE"]._serialized_end = 1444
-    _globals["_CLIENTTYPE"]._serialized_start = 1446
-    _globals["_CLIENTTYPE"]._serialized_end = 1552
-    _globals["_PROCESS"]._serialized_start = 69
-    _globals["_PROCESS"]._serialized_end = 311
-    _globals["_STORAGE"]._serialized_start = 313
-    _globals["_STORAGE"]._serialized_end = 432
-    _globals["_SEARCH"]._serialized_start = 434
-    _globals["_SEARCH"]._serialized_end = 554
-    _globals["_PREDICT"]._serialized_start = 557
-    _globals["_PREDICT"]._serialized_end = 724
-    _globals["_KBUSAGE"]._serialized_start = 727
-    _globals["_KBUSAGE"]._serialized_end = 1092
-    _globals["_KBUSAGEAGGREGATED"]._serialized_start = 1094
-    _globals["_KBUSAGEAGGREGATED"]._serialized_end = 1151
+  DESCRIPTOR._options = None
+  _globals['_PROCESS'].fields_by_name['media_files']._options = None
+  _globals['_PROCESS'].fields_by_name['media_files']._serialized_options = b'\030\001'
+  _globals['_KBSOURCE']._serialized_start=1184
+  _globals['_KBSOURCE']._serialized_end=1218
+  _globals['_SERVICE']._serialized_start=1220
+  _globals['_SERVICE']._serialized_end=1273
+  _globals['_SEARCHTYPE']._serialized_start=1275
+  _globals['_SEARCHTYPE']._serialized_end=1312
+  _globals['_PREDICTTYPE']._serialized_start=1315
+  _globals['_PREDICTTYPE']._serialized_end=1475
+  _globals['_CLIENTTYPE']._serialized_start=1477
+  _globals['_CLIENTTYPE']._serialized_end=1583
+  _globals['_PROCESS']._serialized_start=69
+  _globals['_PROCESS']._serialized_end=311
+  _globals['_STORAGE']._serialized_start=314
+  _globals['_STORAGE']._serialized_end=463
+  _globals['_SEARCH']._serialized_start=465
+  _globals['_SEARCH']._serialized_end=585
+  _globals['_PREDICT']._serialized_start=588
+  _globals['_PREDICT']._serialized_end=755
+  _globals['_KBUSAGE']._serialized_start=758
+  _globals['_KBUSAGE']._serialized_end=1123
+  _globals['_KBUSAGEAGGREGATED']._serialized_start=1125
+  _globals['_KBUSAGEAGGREGATED']._serialized_end=1182
 # @@protoc_insertion_point(module_scope)

nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.pyi

@@ -43,10 +43,7 @@ class _KBSource:
     ValueType = typing.NewType("ValueType", builtins.int)
     V: typing_extensions.TypeAlias = ValueType
 
-class _KBSourceEnumTypeWrapper(
-    google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_KBSource.ValueType],
-    builtins.type,
-):
+class _KBSourceEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_KBSource.ValueType], builtins.type):
     DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
     HOSTED: _KBSource.ValueType  # 0
     ONPREM: _KBSource.ValueType  # 1
@@ -61,10 +58,7 @@ class _Service:
     ValueType = typing.NewType("ValueType", builtins.int)
     V: typing_extensions.TypeAlias = ValueType
 
-class _ServiceEnumTypeWrapper(
-    google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_Service.ValueType],
-    builtins.type,
-):
+class _ServiceEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_Service.ValueType], builtins.type):
     DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
     PREDICT: _Service.ValueType  # 0
     PROCESSING: _Service.ValueType  # 1
@@ -81,10 +75,7 @@ class _SearchType:
     ValueType = typing.NewType("ValueType", builtins.int)
     V: typing_extensions.TypeAlias = ValueType
 
-class _SearchTypeEnumTypeWrapper(
-    google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_SearchType.ValueType],
-    builtins.type,
-):
+class _SearchTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_SearchType.ValueType], builtins.type):
     DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
     SEARCH: _SearchType.ValueType  # 0
     SUGGEST: _SearchType.ValueType  # 1
@@ -99,10 +90,7 @@ class _PredictType:
     ValueType = typing.NewType("ValueType", builtins.int)
     V: typing_extensions.TypeAlias = ValueType
 
-class _PredictTypeEnumTypeWrapper(
-    google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_PredictType.ValueType],
-    builtins.type,
-):
+class _PredictTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_PredictType.ValueType], builtins.type):
     DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
     SENTENCE: _PredictType.ValueType  # 0
     TOKEN: _PredictType.ValueType  # 1
@@ -133,10 +121,7 @@ class _ClientType:
     ValueType = typing.NewType("ValueType", builtins.int)
     V: typing_extensions.TypeAlias = ValueType
 
-class _ClientTypeEnumTypeWrapper(
-    google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ClientType.ValueType],
-    builtins.type,
-):
+class _ClientTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ClientType.ValueType], builtins.type):
     DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
     API: _ClientType.ValueType  # 0
     WEB: _ClientType.ValueType  # 1
@@ -195,31 +180,7 @@ class Process(google.protobuf.message.Message):
         media_files: builtins.int = ...,
         num_processed: builtins.int = ...,
     ) -> None: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "bytes",
-            b"bytes",
-            "chars",
-            b"chars",
-            "client",
-            b"client",
-            "media_files",
-            b"media_files",
-            "media_seconds",
-            b"media_seconds",
-            "num_processed",
-            b"num_processed",
-            "pages",
-            b"pages",
-            "paragraphs",
-            b"paragraphs",
-            "pre_processing_time",
-            b"pre_processing_time",
-            "slow_processing_time",
-            b"slow_processing_time",
-        ],
-    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["bytes", b"bytes", "chars", b"chars", "client", b"client", "media_files", b"media_files", "media_seconds", b"media_seconds", "num_processed", b"num_processed", "pages", b"pages", "paragraphs", b"paragraphs", "pre_processing_time", b"pre_processing_time", "slow_processing_time", b"slow_processing_time"]) -> None: ...
 
 global___Process = Process
 
@@ -230,62 +191,29 @@ class Storage(google.protobuf.message.Message):
     PARAGRAPHS_FIELD_NUMBER: builtins.int
     FIELDS_FIELD_NUMBER: builtins.int
     RESOURCES_FIELD_NUMBER: builtins.int
+    BYTES_FIELD_NUMBER: builtins.int
     paragraphs: builtins.int
     fields: builtins.int
     resources: builtins.int
+    bytes: builtins.int
     def __init__(
         self,
         *,
         paragraphs: builtins.int | None = ...,
         fields: builtins.int | None = ...,
         resources: builtins.int | None = ...,
+        bytes: builtins.int | None = ...,
     ) -> None: ...
-    def HasField(
-        self,
-        field_name: typing_extensions.Literal[
-            "_fields",
-            b"_fields",
-            "_paragraphs",
-            b"_paragraphs",
-            "_resources",
-            b"_resources",
-            "fields",
-            b"fields",
-            "paragraphs",
-            b"paragraphs",
-            "resources",
-            b"resources",
-        ],
-    ) -> builtins.bool: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "_fields",
-            b"_fields",
-            "_paragraphs",
-            b"_paragraphs",
-            "_resources",
-            b"_resources",
-            "fields",
-            b"fields",
-            "paragraphs",
-            b"paragraphs",
-            "resources",
-            b"resources",
-        ],
-    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["_bytes", b"_bytes", "_fields", b"_fields", "_paragraphs", b"_paragraphs", "_resources", b"_resources", "bytes", b"bytes", "fields", b"fields", "paragraphs", b"paragraphs", "resources", b"resources"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["_bytes", b"_bytes", "_fields", b"_fields", "_paragraphs", b"_paragraphs", "_resources", b"_resources", "bytes", b"bytes", "fields", b"fields", "paragraphs", b"paragraphs", "resources", b"resources"]) -> None: ...
+    @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_bytes", b"_bytes"]) -> typing_extensions.Literal["bytes"] | None: ...
     @typing.overload
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["_fields", b"_fields"]
-    ) -> typing_extensions.Literal["fields"] | None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_fields", b"_fields"]) -> typing_extensions.Literal["fields"] | None: ...
     @typing.overload
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["_paragraphs", b"_paragraphs"]
-    ) -> typing_extensions.Literal["paragraphs"] | None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_paragraphs", b"_paragraphs"]) -> typing_extensions.Literal["paragraphs"] | None: ...
     @typing.overload
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["_resources", b"_resources"]
-    ) -> typing_extensions.Literal["resources"] | None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_resources", b"_resources"]) -> typing_extensions.Literal["resources"] | None: ...
 
 global___Storage = Storage
 
@@ -309,19 +237,7 @@ class Search(google.protobuf.message.Message):
         tokens: builtins.int = ...,
         num_searches: builtins.int = ...,
     ) -> None: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "client",
-            b"client",
-            "num_searches",
-            b"num_searches",
-            "tokens",
-            b"tokens",
-            "type",
-            b"type",
-        ],
-    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["client", b"client", "num_searches", b"num_searches", "tokens", b"tokens", "type", b"type"]) -> None: ...
 
 global___Search = Search
 
@@ -354,25 +270,7 @@ class Predict(google.protobuf.message.Message):
         image: builtins.int = ...,
         num_predicts: builtins.int = ...,
     ) -> None: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "client",
-            b"client",
-            "image",
-            b"image",
-            "input",
-            b"input",
-            "model",
-            b"model",
-            "num_predicts",
-            b"num_predicts",
-            "output",
-            b"output",
-            "type",
-            b"type",
-        ],
-    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["client", b"client", "image", b"image", "input", b"input", "model", b"model", "num_predicts", b"num_predicts", "output", b"output", "type", b"type"]) -> None: ...
 
 global___Predict = Predict
 
@@ -397,25 +295,12 @@ class KbUsage(google.protobuf.message.Message):
     kb_id: builtins.str
     kb_source: global___KBSource.ValueType
     @property
-    def processes(
-        self,
-    ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
-        global___Process
-    ]:
+    def processes(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Process]:
         """Data"""
-
     @property
-    def predicts(
-        self,
-    ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
-        global___Predict
-    ]: ...
+    def predicts(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Predict]: ...
     @property
-    def searches(
-        self,
-    ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
-        global___Search
-    ]: ...
+    def searches(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Search]: ...
     @property
     def storage(self) -> global___Storage: ...
     def __init__(
@@ -431,66 +316,14 @@ class KbUsage(google.protobuf.message.Message):
         searches: collections.abc.Iterable[global___Search] | None = ...,
         storage: global___Storage | None = ...,
     ) -> None: ...
-    def HasField(
-        self,
-        field_name: typing_extensions.Literal[
-            "_account_id",
-            b"_account_id",
-            "_kb_id",
-            b"_kb_id",
-            "_storage",
-            b"_storage",
-            "account_id",
-            b"account_id",
-            "kb_id",
-            b"kb_id",
-            "storage",
-            b"storage",
-            "timestamp",
-            b"timestamp",
-        ],
-    ) -> builtins.bool: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "_account_id",
-            b"_account_id",
-            "_kb_id",
-            b"_kb_id",
-            "_storage",
-            b"_storage",
-            "account_id",
-            b"account_id",
-            "kb_id",
-            b"kb_id",
-            "kb_source",
-            b"kb_source",
-            "predicts",
-            b"predicts",
-            "processes",
-            b"processes",
-            "searches",
-            b"searches",
-            "service",
-            b"service",
-            "storage",
-            b"storage",
-            "timestamp",
-            b"timestamp",
-        ],
-    ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["_account_id", b"_account_id", "_kb_id", b"_kb_id", "_storage", b"_storage", "account_id", b"account_id", "kb_id", b"kb_id", "storage", b"storage", "timestamp", b"timestamp"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["_account_id", b"_account_id", "_kb_id", b"_kb_id", "_storage", b"_storage", "account_id", b"account_id", "kb_id", b"kb_id", "kb_source", b"kb_source", "predicts", b"predicts", "processes", b"processes", "searches", b"searches", "service", b"service", "storage", b"storage", "timestamp", b"timestamp"]) -> None: ...
     @typing.overload
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["_account_id", b"_account_id"]
-    ) -> typing_extensions.Literal["account_id"] | None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_account_id", b"_account_id"]) -> typing_extensions.Literal["account_id"] | None: ...
     @typing.overload
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["_kb_id", b"_kb_id"]
-    ) -> typing_extensions.Literal["kb_id"] | None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_kb_id", b"_kb_id"]) -> typing_extensions.Literal["kb_id"] | None: ...
     @typing.overload
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["_storage", b"_storage"]
-    ) -> typing_extensions.Literal["storage"] | None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_storage", b"_storage"]) -> typing_extensions.Literal["storage"] | None: ...
 
 global___KbUsage = KbUsage
 
@@ -500,18 +333,12 @@ class KbUsageAggregated(google.protobuf.message.Message):
 
     KB_USAGES_FIELD_NUMBER: builtins.int
     @property
-    def kb_usages(
-        self,
-    ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
-        global___KbUsage
-    ]: ...
+    def kb_usages(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___KbUsage]: ...
     def __init__(
         self,
         *,
         kb_usages: collections.abc.Iterable[global___KbUsage] | None = ...,
     ) -> None: ...
-    def ClearField(
-        self, field_name: typing_extensions.Literal["kb_usages", b"kb_usages"]
-    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["kb_usages", b"kb_usages"]) -> None: ...
 
 global___KbUsageAggregated = KbUsageAggregated

nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
 # flake8: noqa
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc

nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.pyi

@@ -17,6 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+# flake8: noqa
 """
 @generated by mypy-protobuf.  Do not edit manually!
 isort:skip_file

nucliadb_utils/nuclia_usage/utils/kb_usage_report.py

@@ -110,11 +110,7 @@ class KbUsageReportUtility:
         usage.predicts.extend(predicts)
         usage.searches.extend(searches)
         if storage is not None:
-            if storage.HasField("fields"):
-                usage.storage.fields = storage.fields
-            if storage.HasField("paragraphs"):
-                usage.storage.paragraphs = storage.paragraphs
-            if storage.HasField("resources"):
-                usage.storage.resources = storage.resources
+            for field, value in storage.ListFields():
+                setattr(usage.storage, field.name, value)
 
         self.send(usage)

{nucliadb_utils-5.0.1.post1080.dist-info → nucliadb_utils-5.0.1.post1100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nucliadb_utils
-Version: 5.0.1.post1080
+Version: 5.0.1.post1100
 Home-page: https://nuclia.com
 License: BSD
 Classifier: Development Status :: 4 - Beta
@@ -24,8 +24,8 @@ Requires-Dist: PyNaCl
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: memorylru>=1.1.2
 Requires-Dist: mrflagly>=0.2.9
-Requires-Dist: nucliadb-protos>=5.0.1.post1080
-Requires-Dist: nucliadb-telemetry>=5.0.1.post1080
+Requires-Dist: nucliadb-protos>=5.0.1.post1100
+Requires-Dist: nucliadb-telemetry>=5.0.1.post1100
 Provides-Extra: cache
 Requires-Dist: redis>=4.3.4; extra == "cache"
 Requires-Dist: orjson>=3.6.7; extra == "cache"

{nucliadb_utils-5.0.1.post1080.dist-info → nucliadb_utils-5.0.1.post1100.dist-info}/RECORD

@@ -18,13 +18,13 @@ nucliadb_utils/store.py,sha256=kQ35HemE0v4_Qg6xVqNIJi8vSFAYQtwI3rDtMsNy62Y,890
 nucliadb_utils/transaction.py,sha256=mwcI3aIHAvU5KOGqd_Uz_d1XQzXhk_-NWY8NqU1lfb0,7307
 nucliadb_utils/utilities.py,sha256=idajCm_4Sojh7b3HTkP0fTfG2Mb6PIB9xtMmcfB7Nl0,15758
 nucliadb_utils/aiopynecone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb_utils/aiopynecone/client.py,sha256=EFAQbNp-OHZ55vyXWzdzzp_59FGacGs_nfM2sMBIG8A,21153
+nucliadb_utils/aiopynecone/client.py,sha256=3DGJuHXO9Vm83atMLD4fd2FTz8SAr8RvPCGU_bXH9Ho,22333
 nucliadb_utils/aiopynecone/exceptions.py,sha256=EEE0XoGs1zIB5yOJ_fy6yoG4uIb4cWIawYdJeNe4eDo,3012
 nucliadb_utils/aiopynecone/models.py,sha256=ketK2IYLWiwFZ76rnJmwfcuopFJrCAtCUszdTSurm_Q,3236
 nucliadb_utils/audit/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb_utils/audit/audit.py,sha256=2OyAQ3sUfFfVqBgaA3xiAby3eSIwpey2wg7Ctf0wDxc,2968
-nucliadb_utils/audit/basic.py,sha256=NC3bEuEKS1EV2v7HuH0zt8YobK_xdpM45L9SBsR-GFU,3579
-nucliadb_utils/audit/stream.py,sha256=Zysj70rwxCziJtptDCc4fHoACHyC5FVjkG47NaKx214,13661
+nucliadb_utils/audit/audit.py,sha256=Tbi8poUP6BKZy4ZsaFaeTaQ1K5ys8GXym5ps389-XHE,2966
+nucliadb_utils/audit/basic.py,sha256=1dIWwRWXCb_q3TZObHIMJYOllnbPxullpuqEFaWSUR8,3577
+nucliadb_utils/audit/stream.py,sha256=VpwSrfPU95sqBUgMiWBX0cOC6yjuBXuh5f3uRGFJdwc,13672
 nucliadb_utils/cache/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb_utils/cache/exceptions.py,sha256=Zu-O_-0-yctOEgoDGI92gPzWfBMRrpiAyESA62ld6MA,975
 nucliadb_utils/cache/nats.py,sha256=-AjCfkFgKVdJUlGR0hT9JDSNkPVFg4S6w9eW-ZIcXPM,7037
@@ -38,12 +38,12 @@ nucliadb_utils/fastapi/run.py,sha256=n6vOX64QqF1I5n4UlKnpm_ZJ24rmwfRGi-J9YMGpZzA
 nucliadb_utils/fastapi/versioning.py,sha256=pwiwuesJW1jElUUI3pI5kcxkigfGBvI64IL6QCBEWS8,3805
 nucliadb_utils/nuclia_usage/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb_utils/nuclia_usage/protos/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.py,sha256=7jBWh5aFkNOU6rF_rdp-twZONgrJe3ztJdqKvRWcMM4,5934
-nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.pyi,sha256=xhyc3jJBh0KZuWcgmIbwSkWYfoUKHDC8Fqa7GBjwkmE,15983
-nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.py,sha256=dhop8WwjplPfORYPYb9HtcS9gHMzqXPJQGqXYRjV-6M,1008
-nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.pyi,sha256=6RIsZ2934iodEckflpBStgLKEkFhKfNmZ72UKg2Bwb4,911
+nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.py,sha256=9pDy1QE4E6Z_q7x86_8FwHHLJYDSgmvt8LRD7NJiy88,5925
+nucliadb_utils/nuclia_usage/protos/kb_usage_pb2.pyi,sha256=lA1BiWVOQWioPHPxrfzKZSrj-zyfOlO_H2IEYYdyWtU,14567
+nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.py,sha256=3XT17o832unN2OTBPXho5jj8mm0_8ChSr6m5Jds7SF8,1008
+nucliadb_utils/nuclia_usage/protos/kb_usage_pb2_grpc.pyi,sha256=sXLWrl7G0vtMXs49h-efrwKWc6A_TU420zF9xBMv_ng,926
 nucliadb_utils/nuclia_usage/utils/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb_utils/nuclia_usage/utils/kb_usage_report.py,sha256=P1fKvrcYdSPNJ1ycd0ArSXOqhUcvz1BaQhxAWHo0kcc,3847
+nucliadb_utils/nuclia_usage/utils/kb_usage_report.py,sha256=lTr9CMBpdk34KtkH5K8vFxJRtF6xZpslVMNg1SD03n8,3647
 nucliadb_utils/storages/__init__.py,sha256=5Qc8AUWiJv9_JbGCBpAn88AIJhwDlm0OPQpg2ZdRL4U,872
 nucliadb_utils/storages/azure.py,sha256=egMDwLNIGSQyVevuySt2AswzFdNAcih05BbRg3-p8IU,16015
 nucliadb_utils/storages/exceptions.py,sha256=mm_wX4YRtp7u7enkk_4pMSlX5AQQuFbq4xLmupVDt3Y,2502
@@ -64,8 +64,8 @@ nucliadb_utils/tests/indexing.py,sha256=YW2QhkhO9Q_8A4kKWJaWSvXvyQ_AiAwY1VylcfVQ
 nucliadb_utils/tests/local.py,sha256=7nuP8EFUAiA8ZH50R1iPV9EUXBySQxOanVm3Zht_e0g,1835
 nucliadb_utils/tests/nats.py,sha256=xqpww4jZjTKY9oPGlJdDJG67L3FIBQsa9qDHxILR8r8,7687
 nucliadb_utils/tests/s3.py,sha256=IdMxK_cNdSHLvO1u8BwsKFzD87Hk1MVPDZ57zx6h-rA,3656
-nucliadb_utils-5.0.1.post1080.dist-info/METADATA,sha256=UdBKpI_eCpz0A581-DSLn3D3jYo4EhFZIivDOeRTsdU,2071
-nucliadb_utils-5.0.1.post1080.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-nucliadb_utils-5.0.1.post1080.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
-nucliadb_utils-5.0.1.post1080.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb_utils-5.0.1.post1080.dist-info/RECORD,,
+nucliadb_utils-5.0.1.post1100.dist-info/METADATA,sha256=n4X2-hojpwchiuLbHiT8yF4J-rCuefDxRzkOQWPdek8,2071
+nucliadb_utils-5.0.1.post1100.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+nucliadb_utils-5.0.1.post1100.dist-info/top_level.txt,sha256=fE3vJtALTfgh7bcAWcNhcfXkNPp_eVVpbKK-2IYua3E,15
+nucliadb_utils-5.0.1.post1100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb_utils-5.0.1.post1100.dist-info/RECORD,,