nucliadb 6.4.2.post4389__py3-none-any.whl → 6.5.0.post4404__py3-none-any.whl

nucliadb/common/exceptions.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class InvalidQueryError(Exception):
+    """Raised when parsing a query containing an invalid parameter"""
+
+    def __init__(self, param: str, reason: str):
+        self.param = param
+        self.reason = reason
+        super().__init__(f"Invalid query. Error in {param}: {reason}")

nucliadb/{search/search/query_parser → common}/filter_expression.py

@@ -23,8 +23,8 @@ from typing import Union
 from nidx_protos.nodereader_pb2 import FilterExpression as PBFilterExpression
 
 from nucliadb.common import datamanagers
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.common.ids import FIELD_TYPE_NAME_TO_STR
-from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb_models.filters import (
     And,
     DateCreated,

nucliadb/search/api/v1/catalog.py

@@ -26,6 +26,7 @@ from fastapi_versioning import version
 from pydantic import ValidationError
 
 from nucliadb.common.datamanagers.exceptions import KnowledgeBoxNotFound
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.common.maindb.pg import PGDriver
 from nucliadb.common.maindb.utils import get_driver
 from nucliadb.models.responses import HTTPClientError
@@ -33,7 +34,6 @@ from nucliadb.search import logger
 from nucliadb.search.api.v1.router import KB_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
 from nucliadb.search.search import cache
-from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb.search.search.merge import fetch_resources
 from nucliadb.search.search.pgcatalog import pgcatalog_search
 from nucliadb.search.search.query_parser.parsers import parse_catalog

nucliadb/search/api/v1/find.py

@@ -27,12 +27,12 @@ from pydantic import ValidationError
 
 from nucliadb.common import datamanagers
 from nucliadb.common.datamanagers.exceptions import KnowledgeBoxNotFound
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.search import predict
 from nucliadb.search.api.v1.router import KB_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
 from nucliadb.search.search import cache
-from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb.search.search.find import find
 from nucliadb.search.search.metrics import Metrics
 from nucliadb.search.search.utils import maybe_log_request_payload, min_score_from_query_params

nucliadb/search/api/v1/resource/search.py

@@ -24,12 +24,12 @@ from fastapi import Header, Request, Response
 from fastapi_versioning import version
 from pydantic import ValidationError
 
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.search.api.v1.router import KB_PREFIX, RESOURCE_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
 from nucliadb.search.requesters.utils import Method, nidx_query
 from nucliadb.search.search import cache
-from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb.search.search.merge import merge_paragraphs_results
 from nucliadb.search.search.query import paragraph_query_to_pb
 from nucliadb_models.filters import FilterExpression

nucliadb/search/api/v1/search.py

@@ -27,6 +27,7 @@ from fastapi_versioning import version
 from pydantic import ValidationError
 
 from nucliadb.common.datamanagers.exceptions import KnowledgeBoxNotFound
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.common.models_utils import to_proto
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.search import predict
@@ -34,7 +35,6 @@ from nucliadb.search.api.v1.router import KB_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
 from nucliadb.search.requesters.utils import Method, nidx_query
 from nucliadb.search.search import cache
-from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb.search.search.merge import merge_results
 from nucliadb.search.search.query_parser.parsers.search import parse_search
 from nucliadb.search.search.query_parser.parsers.unit_retrieval import legacy_convert_retrieval_to_proto

nucliadb/search/api/v1/suggest.py

@@ -25,12 +25,12 @@ from fastapi import Header, Request, Response
 from fastapi_versioning import version
 from pydantic import ValidationError
 
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.search.api.v1.router import KB_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
 from nucliadb.search.requesters.utils import Method, nidx_query
 from nucliadb.search.search import cache
-from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb.search.search.merge import merge_suggest_results
 from nucliadb.search.search.query import suggest_query_to_pb
 from nucliadb.search.search.utils import filter_hidden_resources

nucliadb/search/search/chat/ask.py

@@ -33,6 +33,7 @@ from nuclia_models.predict.generative_responses import (
 from pydantic_core import ValidationError
 
 from nucliadb.common.datamanagers.exceptions import KnowledgeBoxNotFound
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.common.external_index_providers.base import ScoredTextBlock
 from nucliadb.common.ids import ParagraphId
 from nucliadb.models.responses import HTTPClientError
@@ -59,7 +60,6 @@ from nucliadb.search.search.chat.query import (
 )
 from nucliadb.search.search.exceptions import (
     IncompleteFindResultsError,
-    InvalidQueryError,
 )
 from nucliadb.search.search.graph_strategy import get_graph_results
 from nucliadb.search.search.metrics import AskMetrics, Metrics

nucliadb/search/search/exceptions.py

@@ -17,8 +17,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-from nucliadb.search.search.query_parser.exceptions import InvalidQueryError as InvalidQueryError
-
 
 class IncompleteFindResultsError(Exception):
     pass

nucliadb/search/search/filters.py

@@ -20,12 +20,11 @@
 from collections.abc import Iterator
 from typing import Any, Optional, Union
 
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb_models.labels import translate_alias_to_system_label
 from nucliadb_models.search import Filter
 from nucliadb_protos import knowledgebox_pb2
 
-from .exceptions import InvalidQueryError
-
 ENTITY_PREFIX = "/e/"
 CLASSIFICATION_LABEL_PREFIX = "/l/"
 

nucliadb/search/search/query.py

@@ -24,6 +24,8 @@ from nidx_protos import nodereader_pb2
 from nidx_protos.noderesources_pb2 import Resource
 
 from nucliadb.common import datamanagers
+from nucliadb.common.exceptions import InvalidQueryError
+from nucliadb.common.filter_expression import add_and_expression, parse_expression
 from nucliadb.search.search.filters import (
     translate_label,
 )
@@ -38,8 +40,6 @@ from nucliadb_models.search import (
 )
 from nucliadb_protos import utils_pb2
 
-from .exceptions import InvalidQueryError
-from .query_parser.filter_expression import add_and_expression, parse_expression
 from .query_parser.old_filters import OldFilterParams, parse_old_filters
 
 

nucliadb/search/search/query_parser/exceptions.py

@@ -21,12 +21,3 @@
 
 class InternalParserError(ValueError):
     """Raised when parsing fails due to some internal error"""
-
-
-class InvalidQueryError(Exception):
-    """Raised when parsing a query containing an invalid parameter"""
-
-    def __init__(self, param: str, reason: str):
-        self.param = param
-        self.reason = reason
-        super().__init__(f"Invalid query. Error in {param}: {reason}")

nucliadb/search/search/query_parser/fetcher.py

@@ -24,13 +24,13 @@ from async_lru import alru_cache
 from typing_extensions import TypeIs
 
 from nucliadb.common import datamanagers
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.common.maindb.utils import get_driver
 from nucliadb.search import logger
 from nucliadb.search.predict import SendToPredictError, convert_relations
 from nucliadb.search.search.metrics import (
     query_parse_dependency_observer,
 )
-from nucliadb.search.search.query_parser.exceptions import InvalidQueryError
 from nucliadb.search.utilities import get_predict
 from nucliadb_models.internal.predict import QueryInfo
 from nucliadb_models.search import (

nucliadb/search/search/query_parser/old_filters.py

@@ -24,13 +24,13 @@ from typing import Optional, Union
 
 from nidx_protos.nodereader_pb2 import FilterExpression
 
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.search.search.filters import translate_label
 from nucliadb_models.search import (
     Filter,
 )
 from nucliadb_protos import knowledgebox_pb2
 
-from .exceptions import InvalidQueryError
 from .fetcher import Fetcher
 
 

nucliadb/search/search/query_parser/parsers/catalog.py

@@ -19,9 +19,9 @@
 #
 
 from nucliadb.common import datamanagers
-from nucliadb.search.search.exceptions import InvalidQueryError
+from nucliadb.common.exceptions import InvalidQueryError
+from nucliadb.common.filter_expression import FacetFilterTypes, facet_from_filter
 from nucliadb.search.search.filters import translate_label
-from nucliadb.search.search.query_parser.filter_expression import FacetFilterTypes, facet_from_filter
 from nucliadb.search.search.query_parser.models import (
     CatalogExpression,
     CatalogQuery,

nucliadb/search/search/query_parser/parsers/common.py

@@ -21,8 +21,8 @@ import re
 import string
 from typing import Optional, Union
 
+from nucliadb.common.exceptions import InvalidQueryError
 from nucliadb.search import logger
-from nucliadb.search.search.query_parser.exceptions import InvalidQueryError
 from nucliadb.search.search.query_parser.fetcher import Fetcher
 from nucliadb.search.search.query_parser.models import (
     KeywordQuery,

nucliadb/search/search/query_parser/parsers/find.py

@@ -23,12 +23,13 @@ from typing import Optional
 from nidx_protos import nodereader_pb2
 from pydantic import ValidationError
 
+from nucliadb.common.exceptions import InvalidQueryError
+from nucliadb.common.filter_expression import parse_expression
 from nucliadb.common.models_utils.from_proto import RelationNodeTypeMap
 from nucliadb.search.search.metrics import query_parser_observer
 from nucliadb.search.search.query import expand_entities
-from nucliadb.search.search.query_parser.exceptions import InternalParserError, InvalidQueryError
+from nucliadb.search.search.query_parser.exceptions import InternalParserError
 from nucliadb.search.search.query_parser.fetcher import Fetcher
-from nucliadb.search.search.query_parser.filter_expression import parse_expression
 from nucliadb.search.search.query_parser.models import (
     Filters,
     GraphQuery,

nucliadb/search/search/query_parser/parsers/graph.py

@@ -22,8 +22,8 @@ from typing import Optional, Union
 
 from nidx_protos import nodereader_pb2
 
+from nucliadb.common.filter_expression import add_and_expression, parse_expression
 from nucliadb.common.models_utils.from_proto import RelationNodeTypeMap, RelationTypeMap
-from nucliadb.search.search.query_parser.filter_expression import add_and_expression, parse_expression
 from nucliadb.search.search.query_parser.models import GraphRetrieval
 from nucliadb.search.search.utils import filter_hidden_resources
 from nucliadb_models.graph import requests as graph_requests

nucliadb/search/search/query_parser/parsers/search.py

@@ -21,11 +21,11 @@ from typing import Optional
 
 from nidx_protos import nodereader_pb2
 
+from nucliadb.common.exceptions import InvalidQueryError
+from nucliadb.common.filter_expression import parse_expression
 from nucliadb.search.search.metrics import query_parser_observer
 from nucliadb.search.search.query import expand_entities
-from nucliadb.search.search.query_parser.exceptions import InvalidQueryError
 from nucliadb.search.search.query_parser.fetcher import Fetcher
-from nucliadb.search.search.query_parser.filter_expression import parse_expression
 from nucliadb.search.search.query_parser.models import (
     Filters,
     ParsedQuery,

nucliadb/search/search/query_parser/parsers/unit_retrieval.py

@@ -22,10 +22,10 @@ from typing import Optional
 from nidx_protos import nodereader_pb2
 from nidx_protos.nodereader_pb2 import SearchRequest
 
+from nucliadb.common.filter_expression import add_and_expression
 from nucliadb.search.search.filters import translate_label
 from nucliadb.search.search.metrics import node_features, query_parser_observer
 from nucliadb.search.search.query import apply_entities_filter, get_sort_field_proto
-from nucliadb.search.search.query_parser.filter_expression import add_and_expression
 from nucliadb.search.search.query_parser.models import ParsedQuery, PredictReranker, UnitRetrieval
 from nucliadb.search.search.query_parser.parsers.graph import parse_path_query
 from nucliadb_models.labels import LABEL_HIDDEN, translate_system_to_alias_label

nucliadb/train/api/utils.py

@@ -22,7 +22,6 @@
 from typing import Optional
 
 from nucliadb.train.utils import get_shard_manager
-from nucliadb_protos.dataset_pb2 import TrainSet
 
 
 async def get_kb_partitions(kbid: str, prefix: Optional[str] = None):
@@ -35,9 +34,3 @@ async def get_kb_partitions(kbid: str, prefix: Optional[str] = None):
         if shard.shard.startswith(prefix):
             valid_shards.append(shard.shard)
     return valid_shards
-
-
-def get_train(trainset: bytes) -> TrainSet:
-    train = TrainSet()
-    train.ParseFromString(trainset)
-    return train

nucliadb/train/api/v1/shards.py

@@ -17,16 +17,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import json
+from typing import Optional
 
-
+import google.protobuf.message
+import pydantic
 from fastapi import HTTPException, Request
 from fastapi.responses import StreamingResponse
 from fastapi_versioning import version
 
-from nucliadb.train.api.utils import get_kb_partitions, get_train
+from nucliadb.train.api.utils import get_kb_partitions
 from nucliadb.train.api.v1.router import KB_PREFIX, api
 from nucliadb.train.generator import generate_train_data
+from nucliadb_models.filters import FilterExpression
 from nucliadb_models.resource import NucliaDBRoles
+from nucliadb_models.trainset import TrainSet as TrainSetModel
+from nucliadb_protos.dataset_pb2 import TaskType, TrainSet
 from nucliadb_utils.authentication import requires_one
 
 
@@ -43,14 +49,35 @@ async def object_get_response(
     kbid: str,
     shard: str,
 ) -> StreamingResponse:
-    item: bytes = await request.body()
-    trainset = get_train(item)
-    all_keys = await get_kb_partitions(kbid, shard)
-
-    if len(all_keys) == 0:
-        raise HTTPException(status_code=404)
-
+    partitions = await get_kb_partitions(kbid, shard)
+    if shard not in partitions:
+        raise HTTPException(status_code=404, detail=f"Partition {shard} not found")
+    trainset, filter_expression = await get_trainset(request)
     return StreamingResponse(
-        generate_train_data(kbid, shard, trainset),
+        generate_train_data(kbid, shard, trainset, filter_expression),
         media_type="application/octet-stream",
     )
+
+
+async def get_trainset(request: Request) -> tuple[TrainSet, Optional[FilterExpression]]:
+    if request.headers.get("Content-Type") == "application/json":
+        try:
+            trainset_model = TrainSetModel.model_validate(await request.json())
+        except (pydantic.ValidationError, json.JSONDecodeError, ValueError) as err:
+            raise HTTPException(status_code=422, detail=str(err))
+        trainset_pb = TrainSet(
+            type=TaskType.ValueType(trainset_model.type.value),
+            batch_size=trainset_model.batch_size,
+            exclude_text=trainset_model.exclude_text,
+        )
+        filter_expression = trainset_model.filter_expression
+    else:
+        # Legacy version of the endpoint where the encoded TrainSet protobuf is passed as request body.
+        trainset_pb = TrainSet()
+        try:
+            trainset_pb.ParseFromString(await request.body())
+        except google.protobuf.message.DecodeError as err:
+            raise HTTPException(status_code=422, detail=str(err))
+        # Filter expressions not supported on legacy version of the endpoint
+        filter_expression = None
+    return trainset_pb, filter_expression

nucliadb/train/generator.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import AsyncIterator, Optional
+from typing import AsyncIterator, Callable, Optional
 
 from fastapi import HTTPException
 
@@ -48,10 +48,15 @@ from nucliadb.train.generators.token_classifier import (
 from nucliadb.train.settings import settings
 from nucliadb.train.types import TrainBatch
 from nucliadb.train.utils import get_shard_manager
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import TaskType, TrainSet
 
+BatchGenerator = Callable[[str, TrainSet, str, Optional[FilterExpression]], AsyncIterator[TrainBatch]]
 
-async def generate_train_data(kbid: str, shard: str, trainset: TrainSet):
+
+async def generate_train_data(
+    kbid: str, shard: str, trainset: TrainSet, filter_expression: Optional[FilterExpression] = None
+):
     # Get the data structure to generate data
     shard_manager = get_shard_manager()
     shard_replica_id = await shard_manager.get_shard_id(kbid, shard)
@@ -59,25 +64,25 @@ async def generate_train_data(kbid: str, shard: str, trainset: TrainSet):
     if trainset.batch_size == 0:
         trainset.batch_size = 50
 
-    batch_generator: Optional[AsyncIterator[TrainBatch]] = None
+    batch_generator: Optional[BatchGenerator] = None
 
     if trainset.type == TaskType.FIELD_CLASSIFICATION:
-        batch_generator = field_classification_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = field_classification_batch_generator
     elif trainset.type == TaskType.IMAGE_CLASSIFICATION:
-        batch_generator = image_classification_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = image_classification_batch_generator
     elif trainset.type == TaskType.PARAGRAPH_CLASSIFICATION:
-        batch_generator = paragraph_classification_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = paragraph_classification_batch_generator
     elif trainset.type == TaskType.TOKEN_CLASSIFICATION:
-        batch_generator = token_classification_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = token_classification_batch_generator
     elif trainset.type == TaskType.SENTENCE_CLASSIFICATION:
-        batch_generator = sentence_classification_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = sentence_classification_batch_generator
     elif trainset.type == TaskType.PARAGRAPH_STREAMING:
-        batch_generator = paragraph_streaming_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = paragraph_streaming_batch_generator
 
     elif trainset.type == TaskType.QUESTION_ANSWER_STREAMING:
-        batch_generator = question_answer_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = question_answer_batch_generator
     elif trainset.type == TaskType.FIELD_STREAMING:
-        batch_generator = field_streaming_batch_generator(kbid, trainset, shard_replica_id)
+        batch_generator = field_streaming_batch_generator
 
     if batch_generator is None:
         raise HTTPException(
@@ -88,7 +93,7 @@ async def generate_train_data(kbid: str, shard: str, trainset: TrainSet):
     # This cache size is an arbitrary number, once we have a metric in place and
     # we analyze memory consumption, we can adjust it with more knoweldge
     with resource_cache(size=settings.resource_cache_size):
-        async for item in batch_generator:
+        async for item in batch_generator(kbid, trainset, shard_replica_id, filter_expression):
             payload = item.SerializeToString()
             yield len(payload).to_bytes(4, byteorder="big", signed=False)
             yield payload

nucliadb/train/generators/field_classifier.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from nidx_protos.nodereader_pb2 import StreamRequest
 
@@ -26,6 +26,7 @@ from nucliadb.common.ids import FIELD_TYPE_STR_TO_PB
 from nucliadb.common.nidx import get_nidx_searcher_client
 from nucliadb.train import logger
 from nucliadb.train.generators.utils import batchify, get_resource_from_cache_or_db
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import (
     FieldClassificationBatch,
     Label,
@@ -38,6 +39,7 @@ def field_classification_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[FieldClassificationBatch, None]:
     generator = generate_field_classification_payloads(kbid, trainset, shard_replica_id)
     batch_generator = batchify(generator, trainset.batch_size, FieldClassificationBatch)

nucliadb/train/generators/field_streaming.py

@@ -23,11 +23,15 @@ from typing import AsyncGenerator, AsyncIterable, Optional
 
 from nidx_protos.nodereader_pb2 import DocumentItem, StreamRequest
 
+from nucliadb.common.filter_expression import parse_expression
 from nucliadb.common.ids import FIELD_TYPE_STR_TO_PB
 from nucliadb.common.nidx import get_nidx_searcher_client
 from nucliadb.train import logger
 from nucliadb.train.generators.utils import batchify, get_resource_from_cache_or_db
 from nucliadb.train.settings import settings
+from nucliadb_models.filters import (
+    FilterExpression,
+)
 from nucliadb_protos.dataset_pb2 import (
     FieldSplitData,
     FieldStreamingBatch,
@@ -41,32 +45,23 @@ def field_streaming_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[FieldStreamingBatch, None]:
-    generator = generate_field_streaming_payloads(kbid, trainset, shard_replica_id)
+    generator = generate_field_streaming_payloads(kbid, trainset, shard_replica_id, filter_expression)
     batch_generator = batchify(generator, trainset.batch_size, FieldStreamingBatch)
     return batch_generator
 
 
 async def generate_field_streaming_payloads(
-    kbid: str,
-    trainset: TrainSet,
-    shard_replica_id: str,
+    kbid: str, trainset: TrainSet, shard_replica_id: str, filter_expression: Optional[FilterExpression]
 ) -> AsyncGenerator[FieldSplitData, None]:
     request = StreamRequest()
     request.shard_id.id = shard_replica_id
 
-    for label in trainset.filter.labels:
-        request.filter.labels.append(f"/l/{label}")
-    for path in trainset.filter.paths:
-        request.filter.labels.append(f"/p/{path}")
-    for metadata in trainset.filter.metadata:
-        request.filter.labels.append(f"/m/{metadata}")
-    for entity in trainset.filter.entities:
-        request.filter.labels.append(f"/e/{entity}")
-    for field in trainset.filter.fields:
-        request.filter.labels.append(f"/f/{field}")
-    for status in trainset.filter.status:
-        request.filter.labels.append(f"/n/s/{status}")
+    if filter_expression:
+        await parse_filter_expression(kbid, request, filter_expression)
+    else:
+        parse_legacy_filters(request, trainset)
 
     resources = set()
     fields = set()
@@ -107,6 +102,30 @@ async def generate_field_streaming_payloads(
     )
 
 
+async def parse_filter_expression(
+    kbid: str, request: StreamRequest, filter_expression: FilterExpression
+):
+    if filter_expression.field:
+        expr = await parse_expression(filter_expression.field, kbid)
+        if expr:
+            request.filter_expression.CopyFrom(expr)
+
+
+def parse_legacy_filters(request: StreamRequest, trainset: TrainSet):
+    for label in trainset.filter.labels:
+        request.filter.labels.append(f"/l/{label}")
+    for path in trainset.filter.paths:
+        request.filter.labels.append(f"/p/{path}")
+    for metadata in trainset.filter.metadata:
+        request.filter.labels.append(f"/m/{metadata}")
+    for entity in trainset.filter.entities:
+        request.filter.labels.append(f"/e/{entity}")
+    for field in trainset.filter.fields:
+        request.filter.labels.append(f"/f/{field}")
+    for status in trainset.filter.status:
+        request.filter.labels.append(f"/n/s/{status}")
+
+
 async def iter_field_split_data(
     request: StreamRequest, kbid: str, trainset: TrainSet, max_parallel: int = 5
 ) -> AsyncIterable[FieldSplitData]:

nucliadb/train/generators/image_classifier.py

@@ -18,9 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from nucliadb.train.generators.utils import batchify
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import (
     ImageClassification,
     ImageClassificationBatch,
@@ -32,6 +33,7 @@ def image_classification_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[ImageClassificationBatch, None]:
     generator = generate_image_classification_payloads(kbid, trainset, shard_replica_id)
     batch_generator = batchify(generator, trainset.batch_size, ImageClassificationBatch)

nucliadb/train/generators/paragraph_classifier.py

@@ -18,13 +18,14 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from fastapi import HTTPException
 from nidx_protos.nodereader_pb2 import StreamRequest
 
 from nucliadb.common.nidx import get_nidx_searcher_client
 from nucliadb.train.generators.utils import batchify, get_paragraph
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import (
     Label,
     ParagraphClassificationBatch,
@@ -37,6 +38,7 @@ def paragraph_classification_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[ParagraphClassificationBatch, None]:
     if len(trainset.filter.labels) != 1:
         raise HTTPException(

nucliadb/train/generators/paragraph_streaming.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from nidx_protos.nodereader_pb2 import StreamRequest
 
@@ -26,6 +26,7 @@ from nucliadb.common.ids import FIELD_TYPE_STR_TO_PB
 from nucliadb.common.nidx import get_nidx_searcher_client
 from nucliadb.train import logger
 from nucliadb.train.generators.utils import batchify, get_resource_from_cache_or_db
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import (
     ParagraphStreamingBatch,
     ParagraphStreamItem,
@@ -37,6 +38,7 @@ def paragraph_streaming_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[ParagraphStreamingBatch, None]:
     generator = generate_paragraph_streaming_payloads(kbid, trainset, shard_replica_id)
     batch_generator = batchify(generator, trainset.batch_size, ParagraphStreamingBatch)

nucliadb/train/generators/question_answer_streaming.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from nidx_protos.nodereader_pb2 import StreamRequest
 
@@ -30,6 +30,7 @@ from nucliadb.train.generators.utils import (
     get_paragraph,
     get_resource_from_cache_or_db,
 )
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import (
     QuestionAnswerStreamingBatch,
     QuestionAnswerStreamItem,
@@ -46,6 +47,7 @@ def question_answer_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[QuestionAnswerStreamingBatch, None]:
     generator = generate_question_answer_streaming_payloads(kbid, trainset, shard_replica_id)
     batch_generator = batchify(generator, trainset.batch_size, QuestionAnswerStreamingBatch)

nucliadb/train/generators/sentence_classifier.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Optional
 
 from fastapi import HTTPException
 from nidx_protos.nodereader_pb2 import StreamRequest
@@ -27,6 +27,7 @@ from nucliadb.common.ids import FIELD_TYPE_STR_TO_PB
 from nucliadb.common.nidx import get_nidx_searcher_client
 from nucliadb.train import logger
 from nucliadb.train.generators.utils import batchify, get_resource_from_cache_or_db
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import (
     Label,
     MultipleTextSameLabels,
@@ -39,6 +40,7 @@ def sentence_classification_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[SentenceClassificationBatch, None]:
     if len(trainset.filter.labels) == 0:
         raise HTTPException(

nucliadb/train/generators/token_classifier.py

@@ -19,7 +19,7 @@
 #
 
 from collections import OrderedDict
-from typing import AsyncGenerator, cast
+from typing import AsyncGenerator, Optional, cast
 
 from nidx_protos.nodereader_pb2 import StreamFilter, StreamRequest
 
@@ -27,6 +27,7 @@ from nucliadb.common.ids import FIELD_TYPE_STR_TO_PB
 from nucliadb.common.nidx import get_nidx_searcher_client
 from nucliadb.train import logger
 from nucliadb.train.generators.utils import batchify, get_resource_from_cache_or_db
+from nucliadb_models.filters import FilterExpression
 from nucliadb_protos.dataset_pb2 import (
     TokenClassificationBatch,
     TokensClassification,
@@ -42,6 +43,7 @@ def token_classification_batch_generator(
     kbid: str,
     trainset: TrainSet,
     shard_replica_id: str,
+    filter_expression: Optional[FilterExpression],
 ) -> AsyncGenerator[TokenClassificationBatch, None]:
     generator = generate_token_classification_payloads(kbid, trainset, shard_replica_id)
     batch_generator = batchify(generator, trainset.batch_size, TokenClassificationBatch)

{nucliadb-6.4.2.post4389.dist-info → nucliadb-6.5.0.post4404.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.4.2.post4389
+Version: 6.5.0.post4404
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.4.2.post4389
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.2.post4389
-Requires-Dist: nucliadb-protos>=6.4.2.post4389
-Requires-Dist: nucliadb-models>=6.4.2.post4389
-Requires-Dist: nidx-protos>=6.4.2.post4389
+Requires-Dist: nucliadb-telemetry[all]>=6.5.0.post4404
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.5.0.post4404
+Requires-Dist: nucliadb-protos>=6.5.0.post4404
+Requires-Dist: nucliadb-models>=6.5.0.post4404
+Requires-Dist: nidx-protos>=6.5.0.post4404
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.4.2.post4389.dist-info → nucliadb-6.5.0.post4404.dist-info}/RECORD

@@ -57,6 +57,8 @@ nucliadb/common/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,8
 nucliadb/common/cache.py,sha256=NM69CVvNjlh58jiVUF1JeYPmBO7_L4rB3tffxK0k_vI,6549
 nucliadb/common/constants.py,sha256=QpigxJh_CtD85Evy0PtV5cVq6x0U_f9xfIcXz1ymkUg,869
 nucliadb/common/counters.py,sha256=8lOi3A2HeLDDlcNaS2QT1SfD3350VPBjiY3FkmHH1V8,977
+nucliadb/common/exceptions.py,sha256=_PJk_NfAhZBFBvmgAfvsJKZ9KuRt5Y1cNsH3-cXE07w,1120
+nucliadb/common/filter_expression.py,sha256=aRbGfg2pci7sUo1KgSqcm72Eu-3Ordy8knnW6DYId5o,6568
 nucliadb/common/ids.py,sha256=4QjoIofes_vtKj2HsFWZf8VVIVWXxdkYtLpx1n618Us,8239
 nucliadb/common/locking.py,sha256=RL0CabZVPzxHZyUjYeUyLvsJTm7W3J9o4fEgsY_ufNc,5896
 nucliadb/common/nidx.py,sha256=3EeQGjM_gxK0l_Rb54fspFWVNnzUiKF-_GMxTiiDC8Q,9116
@@ -213,30 +215,30 @@ nucliadb/search/utilities.py,sha256=9SsRDw0rJVXVoLBfF7rBb6q080h-thZc7u8uRcTiBeY,
 nucliadb/search/api/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/api/v1/__init__.py,sha256=DH16OYnw9jQ38OpKlmdXeoq2j40ZPXZRtGvClKOkMhw,1239
 nucliadb/search/api/v1/ask.py,sha256=b4tz33HNsfT5DXv_2DMc_jirnFsHuobreWkbAKkzj5o,5337
-nucliadb/search/api/v1/catalog.py,sha256=W0cPWuC27Y4bO7Ifl1VQp8OPYfF5gv5yeWZBsuJMxUU,7721
+nucliadb/search/api/v1/catalog.py,sha256=mVAPPf6CXimVOsBpbhPo63KXf8eXps--cifZOEQAIyk,7714
 nucliadb/search/api/v1/feedback.py,sha256=kNLc4dHz2SXHzV0PwC1WiRAwY88fDptPcP-kO0q-FrQ,2620
-nucliadb/search/api/v1/find.py,sha256=C4sTGFRS9tQFF8v1zhnHQvnExJoGDYi78bZTRfwhGrc,10831
+nucliadb/search/api/v1/find.py,sha256=iMjyq4y0JOMC_x1B8kUfVdkCoc9G9Ark58kPLLY4HDw,10824
 nucliadb/search/api/v1/graph.py,sha256=gthqxCOn9biE6D6s93jRGLglk0ono8U7OyS390kWiI8,4178
 nucliadb/search/api/v1/knowledgebox.py,sha256=e9xeLPUqnQTx33i4A8xuV93ENvtJGrpjPlLRbGJtAI8,8415
 nucliadb/search/api/v1/predict_proxy.py,sha256=Q03ZTvWp7Sq0x71t5Br4LHxTiYsRd6-GCb4YuKqhynM,3131
 nucliadb/search/api/v1/router.py,sha256=mtT07rBZcVfpa49doaw9b1tj3sdi3qLH0gn9Io6NYM0,988
-nucliadb/search/api/v1/search.py,sha256=bp2JfBO_wiPl7vG3-MXJfqdFfIGwJM3L25UqqGWj4V4,12304
-nucliadb/search/api/v1/suggest.py,sha256=GJ7DveD6c9_h0m6NbI7IAvfO2j82TtrGuLg6UF3GBh4,6350
+nucliadb/search/api/v1/search.py,sha256=eqlrvRE7IlMpunNwD1RJwt6RgMV01sIDJLgxxE7CFcE,12297
+nucliadb/search/api/v1/suggest.py,sha256=gaJE60r8-z6TVO05mQRKBITwXn2_ofM3B4-OtpOgZEk,6343
 nucliadb/search/api/v1/summarize.py,sha256=VAHJvE6V3xUgEBfqNKhgoxmDqCvh30RnrEIBVhMcNLU,2499
 nucliadb/search/api/v1/utils.py,sha256=5Ve-frn7LAE2jqAgB85F8RSeqxDlyA08--gS-AdOLS4,1434
 nucliadb/search/api/v1/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/api/v1/resource/ask.py,sha256=nsVzBSanSSlf0Ody6LSTjdEy75Vg283_YhbkAtWEjh8,3637
 nucliadb/search/api/v1/resource/ingestion_agents.py,sha256=AZ5_cH1jbf7d5wh_gz6EHLEKAzEOMrQZwEZAu1Q_3FE,4846
-nucliadb/search/api/v1/resource/search.py,sha256=Gnn4CY5NO4AK5ZWwrSIRJqBDm16u8k0XtpUwDXEBeYY,4930
+nucliadb/search/api/v1/resource/search.py,sha256=PZR7fs5oYD0RKqKoD38NZMAnOJzBv35NB2YOr2xy1ck,4923
 nucliadb/search/api/v1/resource/utils.py,sha256=-NjZqAQtFEXKpIh8ui5S26ItnJ5rzmmG0BHxGSS9QPw,1141
 nucliadb/search/requesters/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/search/requesters/utils.py,sha256=Ne5fweSWk9hettQKyUZAMZrw_MTjPE5W_EVqj4p5XiI,6109
 nucliadb/search/search/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/cache.py,sha256=-6l3i2Qi8ig2SM_FCgOLIaQ48XVj7L5ctd5PdQRY5mY,4458
 nucliadb/search/search/cut.py,sha256=ytY0_GY7ocNjfxTb4aosxEp4ZfhQNDP--JkhEMGD298,1153
-nucliadb/search/search/exceptions.py,sha256=klGLgAGGrXcSGix_W6418ZBMqDchAIGjN77ofkOScEI,1039
+nucliadb/search/search/exceptions.py,sha256=q6IKlajYRGLx_AVc2DI6gIZLpOY7ydf4EevMr5_2Krw,940
 nucliadb/search/search/fetch.py,sha256=eiljOKim-4OOEZn-3fyVZSYxztCH156BXYdqlIwVdN4,6181
-nucliadb/search/search/filters.py,sha256=1MkHlJjAQqoRCj7e5cEzK2HvBxGLE17I_omsjiklbtw,6476
+nucliadb/search/search/filters.py,sha256=vZnbf3BjYuDkEQcBeLX_GDkq3Ahbbb7pLJ6DJU9z-QE,6490
 nucliadb/search/search/find.py,sha256=ZocoQNN28OHOmMaroGVFCnce3YHPZbFb1-9jxLNHSFM,7805
 nucliadb/search/search/find_merge.py,sha256=c-7IlfjfdmWAvQOyM7IO3bKS1EQpnR4oi6pN6mwrQKw,19815
 nucliadb/search/search/graph_merge.py,sha256=y5V7X-BhjHsKDXE69tzQLIIKGm4XuaFrZXw0odcHVNM,3402
@@ -248,32 +250,31 @@ nucliadb/search/search/metrics.py,sha256=3I6IN0qDSmqIvUaWJmT3rt-Jyjs6LcvnKI8ZqCi
 nucliadb/search/search/paragraphs.py,sha256=pNAEiYqJGGUVcEf7xf-PFMVqz0PX4Qb-WNG-_zPGN2o,7799
 nucliadb/search/search/pgcatalog.py,sha256=s_J98fsX_RuFXwpejpkGqG-tD9ELuzz4YQ6U3ew5h2g,9313
 nucliadb/search/search/predict_proxy.py,sha256=JwgBeEg1j4LnCjPCvTUrnmOd9LceJAt3iAu4m9cmJBo,3390
-nucliadb/search/search/query.py,sha256=-gvKsyGmKYpsoEVzKkq3HJUMcs_3LD3TYUueOcJsTec,11511
+nucliadb/search/search/query.py,sha256=0qIQdt548L3jtKOyKo06aGJ73SLBxAW3N38_Hc1M3Uw,11528
 nucliadb/search/search/rank_fusion.py,sha256=xZtXhbmKb_56gs73u6KkFm2efvTATOSMmpOV2wrAIqE,9613
 nucliadb/search/search/rerankers.py,sha256=E2J1QdKAojqbhHM3KAyaOXKf6tJyETUxKs4tf_BEyqk,7472
 nucliadb/search/search/shards.py,sha256=mc5DK-MoCv9AFhlXlOFHbPvetcyNDzTFOJ5rimK8PC8,2636
 nucliadb/search/search/summarize.py,sha256=ksmYPubEQvAQgfPdZHfzB_rR19B2ci4IYZ6jLdHxZo8,4996
 nucliadb/search/search/utils.py,sha256=ajRIXfdTF67dBVahQCXW-rSv6gJpUMPt3QhJrWqArTQ,2175
 nucliadb/search/search/chat/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/search/search/chat/ask.py,sha256=aaNj0MeAbx9dyeKpQJdm3VsHMq9OmcCESxahbgSxvCk,37805
+nucliadb/search/search/chat/ask.py,sha256=GFxUh6KvqbidXmtvzgA7trVwF9xNPLcPDCD4IlqvTmI,37839
 nucliadb/search/search/chat/exceptions.py,sha256=Siy4GXW2L7oPhIR86H3WHBhE9lkV4A4YaAszuGGUf54,1356
 nucliadb/search/search/chat/images.py,sha256=PA8VWxT5_HUGfW1ULhKTK46UBsVyINtWWqEM1ulzX1E,3095
 nucliadb/search/search/chat/prompt.py,sha256=e8C7_MPr6Cn3nJHA4hWpeW3629KVI1ZUQA_wZf9Kiu4,48503
 nucliadb/search/search/chat/query.py,sha256=3jMPNbiFEOoS0ydMOPYkSx1qVlvAv51npzadWXDwkMs,16650
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/search/search/query_parser/exceptions.py,sha256=szAOXUZ27oNY-OSa9t2hQ5HHkQQC0EX1FZz_LluJHJE,1224
-nucliadb/search/search/query_parser/fetcher.py,sha256=SkvBRDfSKmuz-QygNKLAU4AhZhhDo1dnOZmt1zA28RA,16851
-nucliadb/search/search/query_parser/filter_expression.py,sha256=fZI8qFRF3h2sa01gwPmDlA5c16mq7ShMOBk-rDaA_fE,6575
+nucliadb/search/search/query_parser/exceptions.py,sha256=sVl9gRNzhE-s480LBBVkiXzNRbKhYRQN5F3it5tNNp8,939
+nucliadb/search/search/query_parser/fetcher.py,sha256=nP4EySj2BvH10QgCvgzvp13Nf22wwfHsdLbDoPlH2cQ,16831
 nucliadb/search/search/query_parser/models.py,sha256=k9cCjTpndP9ynr8A9J8MBmDYmjLBKL1UM4L0GXVuJw0,5031
-nucliadb/search/search/query_parser/old_filters.py,sha256=0NKjRdzAn2bH6veG0M-xM9BNKEYwa4U6WXtZzJAWRvo,9068
+nucliadb/search/search/query_parser/old_filters.py,sha256=HircRqYEac_90bNCtFIJZ2RKA90kjbpNOQcp_ArBqR0,9083
 nucliadb/search/search/query_parser/parsers/__init__.py,sha256=ySCNSdbesLXGZyR88919njulA6UE10_3PhqMG_Yj1o4,1034
 nucliadb/search/search/query_parser/parsers/ask.py,sha256=eTz8wS-EJHuAagR384h6TT64itymFZRpfZJGX8r6aZM,2771
-nucliadb/search/search/query_parser/parsers/catalog.py,sha256=XdBiTweGTQkj8m_V_i2xbwp7P5pPO8K1Tud692XKhMw,7149
-nucliadb/search/search/query_parser/parsers/common.py,sha256=o3028wUnK78lOmFK0jtmpvx2Y1Jh_atBYBoO5VD-qJ4,6359
-nucliadb/search/search/query_parser/parsers/find.py,sha256=Fo4lXOnCbP0AKEc1mKLNINJBv63B4DPlix0vlhyesck,12717
-nucliadb/search/search/query_parser/parsers/graph.py,sha256=lDRJO_JvOe7yytNgXZyMogyPMgB5xc8obNY2kqz3yGU,9405
-nucliadb/search/search/query_parser/parsers/search.py,sha256=yEebeMOXJza7HMK3TdIPO6UGQbe79maSDg-GgohQIMk,10517
-nucliadb/search/search/query_parser/parsers/unit_retrieval.py,sha256=rW3YHDWLkI2Hhznl_1oOMhC01bwZMAjv-Wu3iHPIaiU,11475
+nucliadb/search/search/query_parser/parsers/catalog.py,sha256=e89kh3nGV9JT9wjdWB8JbC2HPydn0rVk7WsKBo6q3gw,7122
+nucliadb/search/search/query_parser/parsers/common.py,sha256=mJMPOKurBK7-A7s3oNlPLxHP_yIn4j5Uw8rh_OQtzS4,6339
+nucliadb/search/search/query_parser/parsers/find.py,sha256=lHVspg-i_eWXvu7BT9WfuFVGVKYhr380y4tDX5yfTD4,12735
+nucliadb/search/search/query_parser/parsers/graph.py,sha256=zyqdUg5Afmhb2_-hvj9FUCaoLh026MUP1fgY2j-lD7c,9385
+nucliadb/search/search/query_parser/parsers/search.py,sha256=huhz3lk6y4n7fcHU2XB-90Q34sXIwP5-cwp2rqFtPh8,10477
+nucliadb/search/search/query_parser/parsers/unit_retrieval.py,sha256=xiOQ7_X6MkcZs3W_0DjdVfyk-G1AY6RBx3oG5hsq7ig,11455
 nucliadb/standalone/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/standalone/api_router.py,sha256=zRSMlaRVHUDGTYA3zC03UV_aLLn-ch-kaeWn1tEjTXw,4338
 nucliadb/standalone/app.py,sha256=mAApNK_iVsQgJyd-mtwCeZq5csSimwnXmlQGH9a70pE,5586
@@ -301,7 +302,7 @@ nucliadb/tests/config.py,sha256=JN_Jhgj-fwM9_8IeO9pwxr6C1PiwRDrXxm67Y38rU30,2080
 nucliadb/tests/vectors.py,sha256=CcNKx-E8LPpyvRyljbmb-Tn_wST9Juw2CBoogWrKiTk,62843
 nucliadb/train/__init__.py,sha256=NVwe5yULoHXb80itIJT8YJYEz2xbiOPQ7_OMys6XJw8,1301
 nucliadb/train/app.py,sha256=z6xlGVVVaJmZZmLPIVTgkjD-wIz5b0NYlXAQp7hBHYw,2652
-nucliadb/train/generator.py,sha256=UHcvBtzNIGtqFR_Rnn3zH4yQkp0koXiC2hFFTgKgRco,4155
+nucliadb/train/generator.py,sha256=fwFYal7VsV0EP7J_g3IOJ-WLpjwqrVo0gEP7vxIlxGs,4152
 nucliadb/train/lifecycle.py,sha256=3HadM4GRsYb2m-v4jtdr9C-KBEBx8GlrJDArPYi3SWQ,1960
 nucliadb/train/models.py,sha256=BmgmMjDsu_1Ih5JDAqo6whhume90q0ASJcDP9dkMQm8,1198
 nucliadb/train/nodes.py,sha256=6QD9ZnAacPyFCzs1jxJzsL3CkVNyADvsfnPjjrvn-NU,5610
@@ -315,20 +316,20 @@ nucliadb/train/upload.py,sha256=fTjH1KEL-0ogf3LV0T6ODO0QdPGwdZShSUtFUCAcUlA,3256
 nucliadb/train/uploader.py,sha256=xdLGz1ToDue9Q_M8A-_KYkO-V6fWKYOZQ6IGM4FuwWA,6424
 nucliadb/train/utils.py,sha256=OxQ8No19nxOxUhwNYDICNt8n2H-gowkAgu2Vt3Hatzk,3163
 nucliadb/train/api/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/train/api/utils.py,sha256=xxTZml7yGstWwnEtjEtdus4iZXX9G1WtK6Z6Bf2f59Y,1478
+nucliadb/train/api/utils.py,sha256=1E5h1dS3dLppXD4k6qH6jmaY0WYa_ChaNHiTW9y336Q,1307
 nucliadb/train/api/v1/__init__.py,sha256=P4vCIv93r_Cq1WFDDNjy_Wg7zBkzx0S4euXwfPy1LA4,928
 nucliadb/train/api/v1/router.py,sha256=ukdxn5q1oMar6NSPobgJczWsSxLCHw6DYKlb3zwCiSo,910
-nucliadb/train/api/v1/shards.py,sha256=GJRnQe8P-7_VTIN1oxVmxlrDA08qVN7opEZdbF4WxDk,1892
+nucliadb/train/api/v1/shards.py,sha256=olgjQFKeYSSCbpYhglCRq6Q9WsM9hxOZgk6mOBb4g0o,3367
 nucliadb/train/api/v1/trainset.py,sha256=kpnpDgiMWr1FKHZJgwH7hue5kzilA8-i9X0YHlNeHuU,2113
 nucliadb/train/generators/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/train/generators/field_classifier.py,sha256=xUA10o9CtBtilbP3uc-8Wn_zQ0oK3BrqYGqZgxh4ZLk,3428
-nucliadb/train/generators/field_streaming.py,sha256=fq4XxHf5fPLccPjO722tA9Xcw6smmceVqSme0fY2_NA,7631
-nucliadb/train/generators/image_classifier.py,sha256=BDXgyd5TGZRnzDnVRvp-qsRCuoTbTYwui3JiDIjuiDc,1736
-nucliadb/train/generators/paragraph_classifier.py,sha256=4sH3IQc7yJrlDs1C76SxFzL9N5mXWRZzJzoiF7y4dSQ,2703
-nucliadb/train/generators/paragraph_streaming.py,sha256=1xsc_IqP-1M0TzYTqu5qCvWBNp_J3Kyvnx8HVbToXmQ,3532
-nucliadb/train/generators/question_answer_streaming.py,sha256=yZZD0GpuHdV-BT4O8CV1sYVDk8ri8yGPVhfjSp3FTBQ,5626
-nucliadb/train/generators/sentence_classifier.py,sha256=bp-UeIbZ0vm4ujbTbZnqdI0JWijuMi53cmeLbP7dofI,5063
-nucliadb/train/generators/token_classifier.py,sha256=DdyMbrpxIVGWdTcz3SEN_3HwxKffUV3JGyTZzlCET8c,9503
+nucliadb/train/generators/field_classifier.py,sha256=UcA5snqLNjIHw0VBzXo9ZtSua6o7wBU3tV9_d5qWpRA,3542
+nucliadb/train/generators/field_streaming.py,sha256=p0xu39D5gaSQc-LagKwpgsVaxm2ULTkWZDPi-Ad1lHc,8378
+nucliadb/train/generators/image_classifier.py,sha256=46YShcl7nf1_iLXZklWTAFvUNIIPulBwHlc9Y1ZJHZU,1850
+nucliadb/train/generators/paragraph_classifier.py,sha256=Jk3B8a2zkAf-2-59RECiNmNP1Nz7f0-hfnS4rIE69xA,2817
+nucliadb/train/generators/paragraph_streaming.py,sha256=axGNYjOTgxGsOcAAvCz_rTYzZCdZf0f1s-Hzn-VvIX0,3646
+nucliadb/train/generators/question_answer_streaming.py,sha256=YXjWQc_SJ_TyXXJRg8tGkC9IeIzbRDcn0spQw9AlRNY,5740
+nucliadb/train/generators/sentence_classifier.py,sha256=4JBGtInnWUQrM9wBd-P7Z2lTK1Dka6U3PPDqnLbGnFM,5177
+nucliadb/train/generators/token_classifier.py,sha256=T8JOVR1vv5g7rn7HtcQcIZ3O5TFqh15uv5rOselLBVo,9617
 nucliadb/train/generators/utils.py,sha256=ZNwvEVPZr-eP0MW3ABN7a11hPQKaa0NdVaRcgBcTp5w,3601
 nucliadb/writer/__init__.py,sha256=S298mrZL3vr62OrBqi97mdLxgR5cReMlRJgnaQHZV7s,1304
 nucliadb/writer/app.py,sha256=ABBO8-u4pDAa61b3mCdD0TFhuHAYcxMkgpZSGgWARuE,2736
@@ -368,8 +369,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.4.2.post4389.dist-info/METADATA,sha256=aeKKQWQ-B_KzWiVM1ZY8kXfO6wa0uJTQI0OPUiaNQPk,4152
-nucliadb-6.4.2.post4389.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.4.2.post4389.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.4.2.post4389.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.4.2.post4389.dist-info/RECORD,,
+nucliadb-6.5.0.post4404.dist-info/METADATA,sha256=5sqpnNr9UzsK7TlG4IxnM1bXMwocq2mCwi-dDOBvb4M,4152
+nucliadb-6.5.0.post4404.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.5.0.post4404.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.5.0.post4404.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.5.0.post4404.dist-info/RECORD,,