nucliadb 6.2.1.post2777__py3-none-any.whl → 6.2.1.post2803__py3-none-any.whl

nucliadb/ingest/orm/knowledgebox.py

@@ -146,6 +146,27 @@ class KnowledgeBox:
                 kb_shards.actual = -1
 
                 vs_external_indexes = []
+
+                # HACK! Currently, we share responsibility of deciding where to
+                # store extracted vectors with processing. Depending on whether
+                # it sends the vectorset id or not (in the extracted vectors
+                # wrapper) nucliadb will store the extracted vectors in a different place
+                #
+                # Right now, processing behaviour is not setting vectorset ids
+                # if there's only one semantic model configured. This is done
+                # for Bw/c with KBs previous to vectorsets.
+                #
+                # We now hardcode this assumption here, so we can annotate each
+                # vectorset with a mark and don't depend on processing
+                # information to decide the storage key. Once this is done we'll
+                # be able to force processing to always send vectorset ids and
+                # remove that bw/c behavior
+                #
+                if len(semantic_models) == 1:
+                    storage_key_kind = knowledgebox_pb2.VectorSetConfig.StorageKeyKind.LEGACY
+                else:
+                    storage_key_kind = knowledgebox_pb2.VectorSetConfig.StorageKeyKind.VECTORSET_PREFIX
+
                 for vectorset_id, semantic_model in semantic_models.items():  # type: ignore
                     # if this KB uses a matryoshka model, we can choose a different
                     # dimension
@@ -172,6 +193,7 @@ class KnowledgeBox:
                             vector_dimension=dimension,
                         ),
                         matryoshka_dimensions=semantic_model.matryoshka_dimensions,
+                        storage_key_kind=storage_key_kind,
                     )
                     await datamanagers.vectorsets.set(txn, kbid=kbid, config=vectorset_config)
 

nucliadb/ingest/orm/resource.py

@@ -504,7 +504,6 @@ class Resource:
     @processor_observer.wrap({"type": "apply_fields"})
     async def apply_fields(self, message: BrokerMessage):
         message_updated_fields = []
-
         for field, text in message.texts.items():
             fid = FieldID(field_type=FieldType.TEXT, field=field)
             await self.set_field(fid.field_type, fid.field, text)

nucliadb/ingest/processing.py

@@ -261,6 +261,7 @@ class ProcessingEngine:
             "content_type": file.file.content_type,
             "password": file.password,
             "language": file.language,
+            "extract_strategy": file.extract_strategy,
         }
         return jwt.encode(payload, self.nuclia_jwt_key, algorithm="HS256")
 
@@ -278,6 +279,8 @@ class ProcessingEngine:
         headers["X-LANGUAGE"] = file.language
         headers["X-FILENAME"] = base64.b64encode(file.file.filename.encode()).decode()  # type: ignore
         headers["X-MD5"] = file.file.md5
+        if file.extract_strategy is not None:
+            headers["X-EXTRACT-STRATEGY"] = file.extract_strategy
         headers["CONTENT_TYPE"] = file.file.content_type
         headers["CONTENT-LENGTH"] = str(len(file.file.payload))  # type: ignore
         headers["X-STF-NUAKEY"] = f"Bearer {self.nuclia_service_account}"
@@ -317,6 +320,7 @@ class ProcessingEngine:
             "content_type": file_field.file.content_type,
             "language": file_field.language,
             "password": file_field.password,
+            "extract_strategy": file_field.extract_strategy,
         }
         return jwt.encode(payload, self.nuclia_jwt_key, algorithm="HS256")
 
@@ -341,6 +345,8 @@ class ProcessingEngine:
             headers["CONTENT-TYPE"] = file.file.content_type
             if file.file.size:
                 headers["CONTENT-LENGTH"] = str(file.file.size)
+            if file.extract_strategy != "":
+                headers["X-EXTRACT-STRATEGY"] = file.extract_strategy
             headers["X-STF-NUAKEY"] = f"Bearer {self.nuclia_service_account}"
 
             iterator = storage.downloadbytescf_iterator(file.file)

nucliadb/writer/api/v1/upload.py

@@ -23,7 +23,7 @@ import uuid
 from datetime import datetime
 from hashlib import md5
 from io import BytesIO
-from typing import Optional
+from typing import Annotated, Optional
 
 from fastapi import HTTPException
 from fastapi.params import Header
@@ -82,6 +82,10 @@ TUS_HEADERS = {
     "Tus-Extension": "creation-defer-length",
 }
 
+ExtractStrategyHeader = Header(
+    description="Extract strategy to use when uploading a file. If not provided, the default strategy will be used.",
+)
+
 
 @api.options(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}/file/{{field}}/{TUSUPLOAD}/{{upload_id}}",
@@ -142,9 +146,12 @@ async def tus_post_rslug_prefix(
     rslug: str,
     field: FieldIdString,
     item: Optional[CreateResourcePayload] = None,
+    x_extract_strategy: Annotated[Optional[str], ExtractStrategyHeader] = None,
 ) -> Response:
     rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
-    return await _tus_post(request, kbid, item, path_rid=rid, field_id=field)
+    return await _tus_post(
+        request, kbid, item, path_rid=rid, field_id=field, extract_strategy=x_extract_strategy
+    )
 
 
 @api.post(
@@ -161,8 +168,11 @@ async def tus_post_rid_prefix(
     path_rid: str,
     field: FieldIdString,
     item: Optional[CreateResourcePayload] = None,
+    x_extract_strategy: Annotated[Optional[str], ExtractStrategyHeader] = None,
 ) -> Response:
-    return await _tus_post(request, kbid, item, path_rid=path_rid, field_id=field)
+    return await _tus_post(
+        request, kbid, item, path_rid=path_rid, field_id=field, extract_strategy=x_extract_strategy
+    )
 
 
 @api.post(
@@ -177,8 +187,9 @@ async def tus_post(
     request: Request,
     kbid: str,
     item: Optional[CreateResourcePayload] = None,
+    x_extract_strategy: Annotated[Optional[str], ExtractStrategyHeader] = None,
 ) -> Response:
-    return await _tus_post(request, kbid, item)
+    return await _tus_post(request, kbid, item, extract_strategy=x_extract_strategy)
 
 
 # called by one the three POST above - there are defined distinctly to produce clean API doc
@@ -188,6 +199,7 @@ async def _tus_post(
     item: Optional[CreateResourcePayload] = None,
     path_rid: Optional[str] = None,
     field_id: Optional[str] = None,
+    extract_strategy: Optional[str] = None,
 ) -> Response:
     """
     An empty POST request is used to create a new upload resource.
@@ -285,6 +297,7 @@ async def _tus_post(
         deferred_length=deferred_length,
         offset=0,
         item=creation_payload,
+        extract_strategy=extract_strategy,
     )
 
     if size is not None:
@@ -569,6 +582,7 @@ async def _tus_patch(
                 request=request,
                 bucket=storage_manager.storage.get_bucket_name(kbid),
                 item=creation_payload,
+                extract_strategy=dm.get("extract_strategy") or None,
             )
         except LimitsExceededError as exc:
             raise HTTPException(status_code=exc.status_code, detail=exc.detail)
@@ -606,6 +620,7 @@ async def upload_rslug_prefix(
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
+    x_extract_strategy: Annotated[Optional[str], ExtractStrategyHeader] = None,
 ) -> ResourceFileUploaded:
     rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
     return await _upload(
@@ -617,6 +632,7 @@ async def upload_rslug_prefix(
         x_password=x_password,
         x_language=x_language,
         x_md5=x_md5,
+        x_extract_strategy=x_extract_strategy,
     )
 
 
@@ -638,6 +654,7 @@ async def upload_rid_prefix(
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
+    x_extract_strategy: Annotated[Optional[str], ExtractStrategyHeader] = None,
 ) -> ResourceFileUploaded:
     return await _upload(
         request,
@@ -648,6 +665,7 @@ async def upload_rid_prefix(
         x_password=x_password,
         x_language=x_language,
         x_md5=x_md5,
+        x_extract_strategy=x_extract_strategy,
     )
 
 
@@ -667,6 +685,7 @@ async def upload(
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
+    x_extract_strategy: Annotated[Optional[str], ExtractStrategyHeader] = None,
 ) -> ResourceFileUploaded:
     return await _upload(
         request,
@@ -675,6 +694,7 @@ async def upload(
         x_password=x_password,
         x_language=x_language,
         x_md5=x_md5,
+        x_extract_strategy=x_extract_strategy,
     )
 
 
@@ -688,6 +708,7 @@ async def _upload(
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
+    x_extract_strategy: Optional[str] = None,
 ) -> ResourceFileUploaded:
     if path_rid is not None:
         await validate_rid_exists_or_raise_error(kbid, path_rid)
@@ -781,6 +802,7 @@ async def _upload(
             path=path,
             request=request,
             bucket=storage_manager.storage.get_bucket_name(kbid),
+            extract_strategy=x_extract_strategy,
         )
     except LimitsExceededError as exc:
         raise HTTPException(status_code=exc.status_code, detail=exc.detail)
@@ -840,6 +862,7 @@ async def store_file_on_nuclia_db(
     language: Optional[str] = None,
     md5: Optional[str] = None,
     item: Optional[CreateResourcePayload] = None,
+    extract_strategy: Optional[str] = None,
 ) -> Optional[int]:
     # File is on NucliaDB Storage at path
     partitioning = get_partitioning()
@@ -921,6 +944,8 @@ async def store_file_on_nuclia_db(
             file_field.language = language
         if password:
             file_field.password = password
+        if extract_strategy is not None:
+            file_field.extract_strategy = extract_strategy
 
         writer.files[field].CopyFrom(file_field)
         # Do not store passwords on maindb

nucliadb/writer/resource/field.py

@@ -50,6 +50,7 @@ async def extract_file_field_from_pb(field_pb: resources_pb2.FieldFile) -> str:
             language=field_pb.language,
             password=field_pb.password,
             file=models.File(payload=None, uri=field_pb.file.uri),
+            extract_strategy=field_pb.extract_strategy,
         )
         return processing.convert_external_filefield_to_str(file_field)
     else:
@@ -171,6 +172,8 @@ def parse_text_field(
     writer: BrokerMessage,
     toprocess: PushPayload,
 ) -> None:
+    if text_field.extract_strategy is not None:
+        writer.texts[key].extract_strategy = text_field.extract_strategy
     writer.texts[key].body = text_field.body
     writer.texts[key].format = resources_pb2.FieldText.Format.Value(text_field.format.value)
     etw = resources_pb2.ExtractedTextWrapper()
@@ -181,6 +184,7 @@ def parse_text_field(
     toprocess.textfield[key] = models.Text(
         body=text_field.body,
         format=getattr(models.PushTextFormat, text_field.format.value),
+        extract_strategy=text_field.extract_strategy,
     )
 
 
@@ -213,6 +217,8 @@ async def parse_internal_file_field(
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
+    if file_field.extract_strategy is not None:
+        writer.files[key].extract_strategy = file_field.extract_strategy
 
     processing = get_processing()
 
@@ -248,6 +254,8 @@ def parse_external_file_field(
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
+    if file_field.extract_strategy is not None:
+        writer.files[key].extract_strategy = file_field.extract_strategy
     uri = file_field.file.uri
     writer.files[key].url = uri  # type: ignore
     writer.files[key].file.uri = uri  # type: ignore
@@ -290,6 +298,9 @@ def parse_link_field(
     if link_field.xpath is not None:
         writer.links[key].xpath = link_field.xpath
 
+    if link_field.extract_strategy is not None:
+        writer.links[key].extract_strategy = link_field.extract_strategy
+
     toprocess.linkfield[key] = models.LinkUpload(
         link=link_field.uri,
         headers=link_field.headers or {},
@@ -297,6 +308,7 @@ def parse_link_field(
         localstorage=link_field.localstorage or {},
         css_selector=link_field.css_selector,
         xpath=link_field.xpath,
+        extract_strategy=link_field.extract_strategy,
     )
 
 
@@ -310,7 +322,6 @@ async def parse_conversation_field(
 ) -> None:
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
-
     field_value = resources_pb2.Conversation()
     convs = models.PushConversation()
     for message in conversation_field.messages:

{nucliadb-6.2.1.post2777.dist-info → nucliadb-6.2.1.post2803.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post2777
+Version: 6.2.1.post2803
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2777
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2777
-Requires-Dist: nucliadb-protos>=6.2.1.post2777
-Requires-Dist: nucliadb-models>=6.2.1.post2777
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2803
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2803
+Requires-Dist: nucliadb-protos>=6.2.1.post2803
+Requires-Dist: nucliadb-models>=6.2.1.post2803
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: nuclia-models>=0.24.2

{nucliadb-6.2.1.post2777.dist-info → nucliadb-6.2.1.post2803.dist-info}/RECORD

@@ -111,7 +111,7 @@ nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1
 nucliadb/ingest/app.py,sha256=L8MDbURnSdD6yI4yCnSbmnNccsnha-zJEkKSaGk1xMg,7612
 nucliadb/ingest/cache.py,sha256=w7jMMzamOmQ7gwXna6Dqm6isRNBVv6l5BTBlTxaYWjE,1005
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
-nucliadb/ingest/processing.py,sha256=0iWEgJVdcOHudwY8Uz9vXRdDoUznM3_WGmviY8FQWT0,20276
+nucliadb/ingest/processing.py,sha256=gg1DqbMFwqdOsmCSGsZc2abRdYz86xOZJun9vrHOCzs,20618
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/ingest/serialize.py,sha256=l2cIIHgo0rgkbaYvAY5slzjr8keVo_3Vb3B6DI120IY,15272
 nucliadb/ingest/settings.py,sha256=0B-wQNa8FLqtNcQgRzh-fuIuGptM816XHcbH1NQKfmE,3050
@@ -138,9 +138,9 @@ nucliadb/ingest/orm/brain.py,sha256=Hzq-3aarKaUCiUoa8H83unRUfduRE9TsQH1dEq0mvZY,
 nucliadb/ingest/orm/broker_message.py,sha256=JYYUJIZEL_EqovQuw6u-FmEkjyoYlxIXJq9hFekOiks,6441
 nucliadb/ingest/orm/entities.py,sha256=2PslT1FZ6yCvJtjR0UpKTSzxJrtS-C_gZx4ZTWHunTc,15759
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/knowledgebox.py,sha256=dBetjoJBYT6JuGmMHiqjcfJeD8qJrK3MQt9X03IrHRA,23228
+nucliadb/ingest/orm/knowledgebox.py,sha256=UpWJrVaVfCtk8R4qfSR6h6vzwOKXa8Teuwkna5QSljE,24508
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
-nucliadb/ingest/orm/resource.py,sha256=uqX50Cp74BaysV9duMuRvnAkeWBqNelYszXBiyTc0uE,55728
+nucliadb/ingest/orm/resource.py,sha256=FIC7Tzef_0hoN92tFwN8PUi8IHlUkyifPJxsA1_XumM,55727
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
@@ -320,11 +320,11 @@ nucliadb/writer/api/v1/router.py,sha256=RjuoWLpZer6Kl2BW_wznpNo6XL3BOpdTGqXZCn3Q
 nucliadb/writer/api/v1/services.py,sha256=U8OGxhA1tdt-wxw2uDAjFpwFXFEXSDTfBe1iV5nfmx8,9897
 nucliadb/writer/api/v1/slug.py,sha256=xlVBDBpRi9bNulpBHZwhyftVvulfE0zFm1XZIWl-AKY,2389
 nucliadb/writer/api/v1/transaction.py,sha256=d2Vbgnkk_-FLGSTt3vfldwiJIUf0XoyD0wP1jQNz_DY,2430
-nucliadb/writer/api/v1/upload.py,sha256=8M_Hwt25sRnQvk8w71QqCBMZYgUZ9xRnLu6-fIaO9Ow,32065
+nucliadb/writer/api/v1/upload.py,sha256=q2_YmQBz6MyFfX4xPHNLMwO0BvHdCFOewi9dlGg0Cq8,33384
 nucliadb/writer/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/resource/audit.py,sha256=FvxMZPzrNHtd31HgpZEvxzwAkbxJTZRhPLqRYYJi3tA,1426
 nucliadb/writer/resource/basic.py,sha256=l9zD-Qiq4eUkHezMf0w1Ksx2izKYLYuNoMIlXcNxxpM,11163
-nucliadb/writer/resource/field.py,sha256=N2TDSMxOhCvrrUicFZMXfkCP_4zq9nQrubGAKom5PYg,13953
+nucliadb/writer/resource/field.py,sha256=sh4BrewJrPKAajA7jx3dcnm3UB0L7FKbLueBWnpxq9I,14601
 nucliadb/writer/resource/origin.py,sha256=pvhUDdU0mlWPUcpoQi4LDUJaRtfjzVVrA8XcGVI_N8k,2021
 nucliadb/writer/tus/__init__.py,sha256=huWpKnDnjsrKlBBJk30ta5vamlA-4x0TbPs_2Up8hyM,5443
 nucliadb/writer/tus/azure.py,sha256=XhWAlWTM0vmXcXtuEPYjjeEhuZjiZXZu8q9WsJ7omFE,4107
@@ -335,9 +335,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post2777.dist-info/METADATA,sha256=oVR3xzLObP57xg2Fi4Xhy4MvhyP-pdGWYAh3LE0fR1w,4689
-nucliadb-6.2.1.post2777.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post2777.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post2777.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post2777.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post2777.dist-info/RECORD,,
+nucliadb-6.2.1.post2803.dist-info/METADATA,sha256=NZ6FUiG4RGSNyA919ez19edmW1geeUPg9AIFVSRrt4k,4689
+nucliadb-6.2.1.post2803.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post2803.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post2803.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post2803.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post2803.dist-info/RECORD,,