nucliadb 6.2.1.post3253__py3-none-any.whl → 6.2.1.post3256__py3-none-any.whl

nucliadb/search/search/query_parser/models.py

@@ -62,9 +62,6 @@ class Reranker(BaseModel): ...
 class NoopReranker(Reranker): ...
 
 
-class MultiMatchBoosterReranker(Reranker): ...
-
-
 class PredictReranker(Reranker):
     window: int = Field(le=200)
 

nucliadb/search/search/query_parser/parser.py

@@ -31,7 +31,6 @@ from nucliadb.search.search.query_parser.models import (
     CatalogFilters,
     CatalogQuery,
     DateTimeFilter,
-    MultiMatchBoosterReranker,
     NoopReranker,
     PredictReranker,
     RankFusion,
@@ -123,9 +122,6 @@ class _FindParser:
             if self.item.reranker == search_models.RerankerName.NOOP:
                 reranking = NoopReranker()
 
-            elif self.item.reranker == search_models.RerankerName.MULTI_MATCH_BOOSTER:
-                reranking = MultiMatchBoosterReranker()
-
             elif self.item.reranker == search_models.RerankerName.PREDICT_RERANKER:
                 # for predict rearnker, by default, we want a x2 factor with a
                 # top of 200 results

nucliadb/search/search/rerankers.py

@@ -169,58 +169,17 @@ class PredictReranker(Reranker):
         return best
 
 
-class MultiMatchBoosterReranker(Reranker):
-    """This reranker gives more value to items that come from different indices"""
-
-    @property
-    def window(self) -> Optional[int]:
-        return None
-
-    @reranker_observer.wrap({"type": "multi_match_booster"})
-    async def _rerank(self, items: list[RerankableItem], options: RerankingOptions) -> list[RankedItem]:
-        """Given a list of rerankable items, boost matches that appear multiple
-        times. The returned list can be smaller than the initial, as repeated
-        matches are deduplicated.
-        """
-        reranked_by_id = {}
-        for item in items:
-            if item.id not in reranked_by_id:
-                reranked_by_id[item.id] = RankedItem(
-                    id=item.id,
-                    score=item.score,
-                    score_type=item.score_type,
-                )
-            else:
-                # it's a mutiple match, boost the score
-                if reranked_by_id[item.id].score < item.score:
-                    # previous implementation noted that we are using vector
-                    # score x2 when we find a multiple match. However, this may
-                    # not be true, as the same paragraph could come in any
-                    # position in the rank fusioned result list
-                    reranked_by_id[item.id].score = item.score * 2
-
-                reranked_by_id[item.id].score_type = SCORE_TYPE.BOTH
-
-        reranked = list(reranked_by_id.values())
-        sort_by_score(reranked)
-        return reranked
-
-
 def get_reranker(reranker: parser_models.Reranker) -> Reranker:
     algorithm: Reranker
 
     if isinstance(reranker, parser_models.NoopReranker):
         algorithm = NoopReranker()
 
-    elif isinstance(reranker, parser_models.MultiMatchBoosterReranker):
-        algorithm = MultiMatchBoosterReranker()
-
     elif isinstance(reranker, parser_models.PredictReranker):
         algorithm = PredictReranker(reranker.window)
 
     else:
-        logger.warning(f"Unknown reranker requested: {reranker}. Using default instead")
-        algorithm = MultiMatchBoosterReranker()
+        raise ValueError(f"Unknown reranker requested: {reranker}")
 
     return algorithm
 

nucliadb/writer/resource/basic.py

@@ -95,7 +95,8 @@ def parse_basic_modify(bm: BrokerMessage, item: ComingResourcePayload, toprocess
         if item.metadata.language:
             bm.basic.metadata.language = item.metadata.language
         if item.metadata.languages:
-            bm.basic.metadata.languages.extend(item.metadata.languages)
+            unique_languages = list(set(item.metadata.languages))
+            bm.basic.metadata.languages.extend(unique_languages)
 
     if item.fieldmetadata is not None:
         for fieldmetadata in item.fieldmetadata:

{nucliadb-6.2.1.post3253.dist-info → nucliadb-6.2.1.post3256.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post3253
+Version: 6.2.1.post3256
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3253
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3253
-Requires-Dist: nucliadb-protos>=6.2.1.post3253
-Requires-Dist: nucliadb-models>=6.2.1.post3253
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3256
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3256
+Requires-Dist: nucliadb-protos>=6.2.1.post3256
+Requires-Dist: nucliadb-models>=6.2.1.post3256
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.2.1.post3253.dist-info → nucliadb-6.2.1.post3256.dist-info}/RECORD

@@ -221,7 +221,7 @@ nucliadb/search/search/pgcatalog.py,sha256=IaNK4dAxdXs38PoIkTdgqMDuZDjeiOtcXn3Le
 nucliadb/search/search/predict_proxy.py,sha256=xBlh6kjuQpWRq7KsBx4pEl2PtnwljjQIiYMaTWpcCSA,3015
 nucliadb/search/search/query.py,sha256=AlhRw4Mick4Oab5HsKHaQpBXsVc_UUY5IpkUIwsFfU8,30577
 nucliadb/search/search/rank_fusion.py,sha256=tRGo_KlsFsVx1CQEy1iqQ6f0T1Dq1kf0axDXHuuzvvM,6946
-nucliadb/search/search/rerankers.py,sha256=0kAHES9X_FKkP7KSN9NRETFmRPKzwrFAo_54MbyvM7Q,9051
+nucliadb/search/search/rerankers.py,sha256=3vep4EOVNeDJGsMdx-1g6Ar4ZGJG3IHym3HkxnbwtAQ,7321
 nucliadb/search/search/shards.py,sha256=JSRSrHgHcF4sXyuZZoJdMfK0v_LHpoSRf1lCr5-K5ko,2742
 nucliadb/search/search/summarize.py,sha256=ksmYPubEQvAQgfPdZHfzB_rR19B2ci4IYZ6jLdHxZo8,4996
 nucliadb/search/search/utils.py,sha256=iF2tbBA56gRMJH1TlE2hMrqeXqjoeOPt4KgRdp2m9Ek,3313
@@ -234,8 +234,8 @@ nucliadb/search/search/chat/query.py,sha256=rBssR6MPSx8h2DASRMTLODaz9oGE5tNVVVeD
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=szAOXUZ27oNY-OSa9t2hQ5HHkQQC0EX1FZz_LluJHJE,1224
 nucliadb/search/search/query_parser/fetcher.py,sha256=jhr__J0KmAzjdsTTadWQmD9qf6lZvqlKAfZdYjZH_UY,15742
-nucliadb/search/search/query_parser/models.py,sha256=-VlCDXUCgOroAZw1Leqhj2VMgRv_CD2w40PXXOBLaUM,2332
-nucliadb/search/search/query_parser/parser.py,sha256=JC6koS9Np1PzCfEk1Xy6mpP1HmovS_vIxxA9u-kwzos,6498
+nucliadb/search/search/query_parser/models.py,sha256=2iWuTcH24RDF8xokgXr0j5qbMoURQ1TFyqJIYs16LqU,2283
+nucliadb/search/search/query_parser/parser.py,sha256=m6meq5QQO_ofdtbrvEORsZLjxURWfRR0dINrgDXmYRg,6323
 nucliadb/standalone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/standalone/api_router.py,sha256=4-g-eEq27nL6vKCLRCoV0Pxf-L273N-eHeEX2vI9qgg,6215
 nucliadb/standalone/app.py,sha256=mAApNK_iVsQgJyd-mtwCeZq5csSimwnXmlQGH9a70pE,5586
@@ -319,7 +319,7 @@ nucliadb/writer/api/v1/upload.py,sha256=VOeqNTrZx1_z8iaKjM7p8fVlVcIYMtnQNK1dm72c
 nucliadb/writer/api/v1/vectorsets.py,sha256=mESaXkkI9f-jWWMW61ZZgv7E5YWXKemyc6vwT0lFXns,6747
 nucliadb/writer/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/resource/audit.py,sha256=FvxMZPzrNHtd31HgpZEvxzwAkbxJTZRhPLqRYYJi3tA,1426
-nucliadb/writer/resource/basic.py,sha256=l9zD-Qiq4eUkHezMf0w1Ksx2izKYLYuNoMIlXcNxxpM,11163
+nucliadb/writer/resource/basic.py,sha256=wo5VDyp9VBekoJaQAbhE7uPFkpB5auAoRnsdrh3Ny3s,11222
 nucliadb/writer/resource/field.py,sha256=HsOERELyAsb9e0dx2IkSQ9lk0SThALFRcDKCVBw8ifU,15478
 nucliadb/writer/resource/origin.py,sha256=pvhUDdU0mlWPUcpoQi4LDUJaRtfjzVVrA8XcGVI_N8k,2021
 nucliadb/writer/tus/__init__.py,sha256=huWpKnDnjsrKlBBJk30ta5vamlA-4x0TbPs_2Up8hyM,5443
@@ -331,9 +331,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post3253.dist-info/METADATA,sha256=MVwir1nyj-58meH0zdQmizihGvR_1k_EIKxl3NxEZpg,4603
-nucliadb-6.2.1.post3253.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post3253.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post3253.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post3253.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post3253.dist-info/RECORD,,
+nucliadb-6.2.1.post3256.dist-info/METADATA,sha256=HVAp6MZqKVjVZ9e5gpnJqSro67IFGw9tyJ8Ay3oYHzk,4603
+nucliadb-6.2.1.post3256.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post3256.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post3256.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post3256.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post3256.dist-info/RECORD,,