nucliadb 6.3.5.post3995__py3-none-any.whl → 6.3.5.post3997__py3-none-any.whl

nucliadb/ingest/orm/brain_v2.py

@@ -24,6 +24,7 @@ from typing import Optional
 
 from nucliadb.common import ids
 from nucliadb.ingest import logger
+from nucliadb.ingest.orm.metrics import brain_observer as observer
 from nucliadb.ingest.orm.utils import compute_paragraph_key
 from nucliadb_models.labels import BASE_LABELS, LABEL_HIDDEN, flatten_resource_labels
 from nucliadb_models.metadata import ResourceProcessingStatus
@@ -74,7 +75,8 @@ class ResourceBrainV2:
         self.brain: PBBrainResource = PBBrainResource(resource=ResourceID(uuid=rid))
         self.labels: dict[str, set[str]] = deepcopy(BASE_LABELS)
 
-    def generate_resource_indexing_metadata(
+    @observer.wrap({"type": "generate_resource_metadata"})
+    def generate_resource_metadata(
         self,
         basic: Basic,
         user_relations: Relations,
@@ -89,7 +91,8 @@ class ResourceBrainV2:
         if security is not None:
             self._set_resource_security(security)
 
-    def generate_texts_index_message(
+    @observer.wrap({"type": "generate_texts"})
+    def generate_texts(
         self,
         field_key: str,
         extracted_text: ExtractedText,
@@ -112,6 +115,7 @@ class ResourceBrainV2:
             basic_user_metadata,
         )
 
+    @observer.wrap({"type": "apply_field_text"})
     def apply_field_text(
         self,
         field_key: str,
@@ -131,6 +135,7 @@ class ResourceBrainV2:
             full_field_id = ids.FieldId(rid=self.rid, type=ftype, key=fkey).full()
             self.brain.texts_to_delete.append(full_field_id)
 
+    @observer.wrap({"type": "apply_field_labels"})
     def apply_field_labels(
         self,
         field_key: str,
@@ -200,7 +205,8 @@ class ResourceBrainV2:
 
         self.brain.texts[field_key].labels.extend(flatten_resource_labels(labels))
 
-    def generate_paragraphs_index_message(
+    @observer.wrap({"type": "generate_paragraphs"})
+    def generate_paragraphs(
         self,
         field_key: str,
         field_computed_metadata: FieldComputedMetadata,
@@ -228,6 +234,7 @@ class ResourceBrainV2:
             skip_paragraphs=skip_index,
         )
 
+    @observer.wrap({"type": "apply_field_paragraphs"})
     def apply_field_paragraphs(
         self,
         field_key: str,
@@ -371,7 +378,8 @@ class ResourceBrainV2:
                     pc.valid.setdefault(paragraph_key, []).append(classif_label)
         return pc
 
-    def generate_relations_index_message(
+    @observer.wrap({"type": "generate_relations"})
+    def generate_relations(
         self,
         field_key: str,
         field_computed_metadata: Optional[FieldComputedMetadata],
@@ -477,7 +485,8 @@ class ResourceBrainV2:
         self.brain.sentences_to_delete.append(full_field_id)
         self.brain.relation_fields_to_delete.append(field_key)
 
-    def generate_vectors_index_message(
+    @observer.wrap({"type": "generate_vectors"})
+    def generate_vectors(
         self,
         field_id: str,
         vo: utils_pb2.VectorObject,
@@ -547,6 +556,7 @@ class ResourceBrainV2:
             full_field_id = ids.FieldId(rid=self.rid, type=fid.type, key=fid.key).full()
             self.brain.vector_prefixes_to_delete[vectorset].items.append(full_field_id)
 
+    @observer.wrap({"type": "apply_field_vector"})
     def _apply_field_vector(
         self,
         field_id: str,
@@ -764,6 +774,7 @@ class ParagraphPages:
         self.positions = positions
         self._materialized = self._materialize_page_numbers(positions)
 
+    @observer.wrap({"type": "materialize_page_numbers"})
     def _materialize_page_numbers(self, positions: FilePagePositions) -> list[int]:
         page_numbers_by_index = []
         for page_number, (page_start, page_end) in positions.items():

nucliadb/ingest/orm/index_message.py

@@ -26,6 +26,7 @@ from nucliadb.common import datamanagers
 from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
 from nucliadb.ingest.fields.file import File
 from nucliadb.ingest.orm.brain_v2 import ResourceBrainV2 as ResourceBrain
+from nucliadb.ingest.orm.metrics import index_message_observer as observer
 from nucliadb.ingest.orm.resource import Resource, get_file_page_positions
 from nucliadb_protos.knowledgebox_pb2 import VectorSetConfig
 from nucliadb_protos.noderesources_pb2 import Resource as IndexMessage
@@ -40,6 +41,7 @@ class IndexMessageBuilder:
         self.resource = resource
         self.brain = ResourceBrain(resource.uuid)
 
+    @observer.wrap({"type": "resource_data"})
     async def _apply_resource_index_data(self, brain: ResourceBrain) -> None:
         # Set the metadata at the resource level
         basic = await self.resource.get_basic()
@@ -48,7 +50,7 @@ class IndexMessageBuilder:
         origin = await self.resource.get_origin()
         security = await self.resource.get_security()
         await asyncio.to_thread(
-            brain.generate_resource_indexing_metadata,
+            brain.generate_resource_metadata,
             basic,
             user_relations,
             origin,
@@ -56,6 +58,7 @@ class IndexMessageBuilder:
             security,
         )
 
+    @observer.wrap({"type": "field_data"})
     async def _apply_field_index_data(
         self,
         brain: ResourceBrain,
@@ -87,7 +90,7 @@ class IndexMessageBuilder:
                 except FieldAuthorNotFound:
                     field_author = None
                 await asyncio.to_thread(
-                    brain.generate_texts_index_message,
+                    brain.generate_texts,
                     self.resource.generate_field_id(fieldid),
                     extracted_text,
                     field_computed_metadata,
@@ -108,7 +111,7 @@ class IndexMessageBuilder:
                     await get_file_page_positions(field) if isinstance(field, File) else None
                 )
                 await asyncio.to_thread(
-                    brain.generate_paragraphs_index_message,
+                    brain.generate_paragraphs,
                     self.resource.generate_field_id(fieldid),
                     field_computed_metadata,
                     extracted_text,
@@ -127,7 +130,7 @@ class IndexMessageBuilder:
                 if vo is not None:
                     dimension = vectorset_config.vectorset_index_config.vector_dimension
                     await asyncio.to_thread(
-                        brain.generate_vectors_index_message,
+                        brain.generate_vectors,
                         self.resource.generate_field_id(fieldid),
                         vo,
                         vectorset=vectorset_config.vectorset_id,
@@ -136,7 +139,7 @@ class IndexMessageBuilder:
                     )
         if relations:
             await asyncio.to_thread(
-                brain.generate_relations_index_message,
+                brain.generate_relations,
                 self.resource.generate_field_id(fieldid),
                 field_computed_metadata,
                 basic.usermetadata,
@@ -151,6 +154,7 @@ class IndexMessageBuilder:
         for field_id in field_ids:
             brain.delete_field(self.resource.generate_field_id(field_id))
 
+    @observer.wrap({"type": "writer_bm"})
     async def for_writer_bm(
         self,
         messages: list[BrokerMessage],
@@ -192,6 +196,7 @@ class IndexMessageBuilder:
             )
         return self.brain.brain
 
+    @observer.wrap({"type": "processor_bm"})
     async def for_processor_bm(
         self,
         messages: list[BrokerMessage],
@@ -223,6 +228,7 @@ class IndexMessageBuilder:
             )
         return self.brain.brain
 
+    @observer.wrap({"type": "full"})
     async def full(self, reindex: bool) -> IndexMessage:
         await self._apply_resource_index_data(self.brain)
         basic = await self.get_basic()

nucliadb/ingest/orm/metrics.py

@@ -22,6 +22,17 @@ from nucliadb_telemetry import metrics
 
 processor_observer = metrics.Observer(
     "nucliadb_ingest_processor",
-    labels={"type": "", "source": ""},
+    labels={"type": ""},
     error_mappings={"kb_conflict": KnowledgeBoxConflict},
 )
+
+
+index_message_observer = metrics.Observer(
+    "index_message_builder",
+    labels={"type": ""},
+)
+
+brain_observer = metrics.Observer(
+    "brain",
+    labels={"type": ""},
+)

nucliadb/ingest/orm/processor/processor.py

@@ -462,6 +462,7 @@ class Processor:
                 source=source,
             )
 
+    @processor_observer.wrap({"type": "generate_index_message_v2"})
     async def generate_index_message_v2(
         self,
         resource: Resource,
@@ -471,14 +472,13 @@ class Processor:
         builder = IndexMessageBuilder(resource)
         message_source = messages_source(messages)
         if message_source == nodewriter_pb2.IndexMessageSource.WRITER:
-            with processor_observer({"type": "generate_index_message", "source": "writer"}):
-                return await builder.for_writer_bm(messages, resource_created)
+            return await builder.for_writer_bm(messages, resource_created)
         elif message_source == nodewriter_pb2.IndexMessageSource.PROCESSOR:
-            with processor_observer({"type": "generate_index_message", "source": "processor"}):
-                return await builder.for_processor_bm(messages)
+            return await builder.for_processor_bm(messages)
         else:  # pragma: no cover
             raise InvalidBrokerMessage(f"Unknown broker message source: {message_source}")
 
+    @processor_observer.wrap({"type": "generate_index_message_v1"})
     async def generate_index_message_v1(
         self,
         resource: Resource,

{nucliadb-6.3.5.post3995.dist-info → nucliadb-6.3.5.post3997.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.3.5.post3995
+Version: 6.3.5.post3997
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3995
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3995
-Requires-Dist: nucliadb-protos>=6.3.5.post3995
-Requires-Dist: nucliadb-models>=6.3.5.post3995
-Requires-Dist: nidx-protos>=6.3.5.post3995
+Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3997
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3997
+Requires-Dist: nucliadb-protos>=6.3.5.post3997
+Requires-Dist: nucliadb-models>=6.3.5.post3997
+Requires-Dist: nidx-protos>=6.3.5.post3997
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.3.5.post3995.dist-info → nucliadb-6.3.5.post3997.dist-info}/RECORD

@@ -142,20 +142,20 @@ nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJ
 nucliadb/ingest/fields/text.py,sha256=tFvSQJAe0W7ePpp2_WDfLiE2yglR1OTU0Zht9acvOFw,1594
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/orm/brain.py,sha256=8nXdxgI3zYn6DGnCq5ciq3PA7ouhcTW5dSgHaxAO6xg,29074
-nucliadb/ingest/orm/brain_v2.py,sha256=3ejtH58X9Hkhvg2m7wrp2eEyIQybKSMHzoJBDtkL0b8,33065
+nucliadb/ingest/orm/brain_v2.py,sha256=XEOfvjpnvSKNrAOtbO4vt9n_PWVbzOhB-seHs76uY0M,33588
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=a-aYuKBUQhxDKFtXOzTAkLlY_t2JiTfaptw2vt3AQDQ,14915
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/index_message.py,sha256=3Okq-POmGnxpgzysU1bvgj5skRLSDBUpP6dnTYqNA18,15821
+nucliadb/ingest/orm/index_message.py,sha256=fFNYRZTH45fm6IZ9tHNwa4KNgV8KxzwS5uuklRe65ww,16044
 nucliadb/ingest/orm/knowledgebox.py,sha256=Bfb4-MIQWlaJrQAUDbgs_iIsXCYjS7s5YiiGl_Jb4jo,23887
-nucliadb/ingest/orm/metrics.py,sha256=z-xVOJWeWXZFPIzRg_NB8nlbkdJFs_myEwLROdTNp24,1110
+nucliadb/ingest/orm/metrics.py,sha256=OiuggTh-n3kZHA2G73NEUdIlh8c3yFrbusI88DK-Mko,1273
 nucliadb/ingest/orm/resource.py,sha256=GjxcEPuu8bM06Uea7_yJk0UFvOfiZNP9i_G4V-4D8_U,46845
 nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,2693
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=H-OCRz0RuTUb80LZBxDowLA9V7ECv1DWiXlnzKW5XGI,3103
-nucliadb/ingest/orm/processor/processor.py,sha256=flw2U1OB6il9mP7h6fCY_dZX-Jo6XemWq1dAwtH7pMs,33202
+nucliadb/ingest/orm/processor/processor.py,sha256=q2iBJJ_5SV_bxA3t5MrbV70iQhir94aFbjZjnYJzEAQ,33141
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=MME_G_ERxzJR6JW_hfE2qcfXpmpH1kdG-S0a-M0qRm8,2043
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -360,8 +360,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.5.post3995.dist-info/METADATA,sha256=WMPXEMAoMggoPDt4NkDQBnl5014KHnBKxU7AEAueHsk,4301
-nucliadb-6.3.5.post3995.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-nucliadb-6.3.5.post3995.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.5.post3995.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.5.post3995.dist-info/RECORD,,
+nucliadb-6.3.5.post3997.dist-info/METADATA,sha256=K-G5B3YkJzhIf9IiEVxZT9t41hvtsJymVAn5KZheMVY,4301
+nucliadb-6.3.5.post3997.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+nucliadb-6.3.5.post3997.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.5.post3997.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.5.post3997.dist-info/RECORD,,