nucliadb 6.3.5.post3914__py3-none-any.whl → 6.3.5.post3922__py3-none-any.whl

nucliadb/common/models_utils/from_proto.py

@@ -85,15 +85,6 @@ def user_field_metadata(message: resources_pb2.UserFieldMetadata) -> UserFieldMe
         including_default_value_fields=True,
         use_integers_for_enums=True,
     )
-    value["selections"] = [
-        MessageToDict(
-            selections,
-            preserving_proto_field_name=True,
-            including_default_value_fields=True,
-            use_integers_for_enums=True,
-        )
-        for selections in message.page_selections
-    ]
     value["field"]["field_type"] = field_type_name(value["field"]["field_type"]).value
     return UserFieldMetadata(**value)
 

nucliadb/ingest/orm/brain.py

@@ -606,20 +606,6 @@ class ResourceBrain:
             )
 
         if basic_user_fieldmetadata is not None:
-            for token in basic_user_fieldmetadata.token:
-                if token.cancelled_by_user is False:
-                    labels["e"].add(f"{token.klass}/{token.token}")
-                    relation_node_entity = RelationNode(
-                        value=token.token,
-                        ntype=RelationNode.NodeType.ENTITY,
-                        subtype=token.klass,
-                    )
-                    rel = Relation(
-                        relation=Relation.ENTITY,
-                        source=relation_node_resource,
-                        to=relation_node_entity,
-                    )
-                    self.brain.field_relations[field_key].relations.append(IndexRelation(relation=rel))
             for paragraph_annotation in basic_user_fieldmetadata.paragraphs:
                 for classification in paragraph_annotation.classifications:
                     if not classification.cancelled_by_user:

nucliadb/train/generators/image_classifier.py

@@ -18,27 +18,15 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-import json
-from typing import Any, AsyncGenerator
+from typing import AsyncGenerator
 
 from nucliadb.common.cluster.base import AbstractIndexNode
-from nucliadb.common.ids import FIELD_TYPE_STR_TO_PB
-from nucliadb.ingest.fields.base import Field
-from nucliadb.ingest.orm.resource import Resource
-from nucliadb.train import logger
-from nucliadb.train.generators.utils import batchify, get_resource_from_cache_or_db
+from nucliadb.train.generators.utils import batchify
 from nucliadb_protos.dataset_pb2 import (
     ImageClassification,
     ImageClassificationBatch,
     TrainSet,
 )
-from nucliadb_protos.nodereader_pb2 import StreamRequest
-from nucliadb_protos.resources_pb2 import FieldType, PageStructure, VisualSelection
-
-VISUALLY_ANNOTABLE_FIELDS = {FieldType.FILE, FieldType.LINK}
-
-# PAWLS JSON format
-PawlsPayload = dict[str, Any]
 
 
 def image_classification_batch_generator(
@@ -58,125 +46,8 @@ async def generate_image_classification_payloads(
     node: AbstractIndexNode,
     shard_replica_id: str,
 ) -> AsyncGenerator[ImageClassification, None]:
-    request = StreamRequest()
-    request.shard_id.id = shard_replica_id
-    async for item in node.stream_get_fields(request):
-        rid = item.uuid
-        resource = await get_resource_from_cache_or_db(kbid, rid)
-        if resource is None:
-            logger.error(f"Resource {rid} does not exist on DB")
-            return
-
-        _, field_type_key, field_key = item.field.split("/")
-        field_type = FIELD_TYPE_STR_TO_PB[field_type_key]
-
-        if field_type not in VISUALLY_ANNOTABLE_FIELDS:
-            continue
-
-        field = await resource.get_field(field_key, field_type, load=True)
-
-        page_selections = await get_page_selections(resource, field)
-        if len(page_selections) == 0:
-            # Generating a payload without annotations makes no sense
-            continue
-
-        page_structure = await get_page_structure(field)
-
-        for page, (page_uri, ps) in enumerate(page_structure):
-            pawls_payload = {
-                "width": ps.page.width,
-                "height": ps.page.height,
-                "tokens": [
-                    {
-                        "x": token.x,
-                        "y": token.y,
-                        "width": token.width,
-                        "height": token.height,
-                        "text": token.text,
-                        "line": token.line,
-                    }
-                    for token in ps.tokens
-                ],
-                "annotations": [
-                    {
-                        "page": page,
-                        "label": {
-                            "text": selection.label,
-                        },
-                        "bounds": {
-                            "top": selection.top,
-                            "left": selection.left,
-                            "right": selection.right,
-                            "bottom": selection.bottom,
-                        },
-                        "tokens": [
-                            {
-                                "pageIndex": page,
-                                "tokenIndex": token_id,
-                            }
-                            for token_id in selection.token_ids
-                        ],
-                    }
-                    for selection in page_selections[page]
-                ],
-            }
-
-            ic = ImageClassification()
-            ic.page_uri = page_uri
-            ic.selections = json.dumps(pawls_payload)
-
-            yield ic
-
-
-async def get_page_selections(resource: Resource, field: Field) -> dict[int, list[VisualSelection]]:
-    page_selections: dict[int, list[VisualSelection]] = {}
-    basic = await resource.get_basic()
-    if basic is None or basic.fieldmetadata is None:
-        return page_selections
-
-    # We assume only one fieldmetadata per field as it's implemented in
-    # resource ingestion
-    for fieldmetadata in basic.fieldmetadata:
-        if (
-            fieldmetadata.field.field == field.id
-            and fieldmetadata.field.field_type == FIELD_TYPE_STR_TO_PB[field.type]
-        ):
-            for selection in fieldmetadata.page_selections:
-                page_selections[selection.page] = selection.visual  # type: ignore
-            break
-
-    return page_selections
-
-
-async def get_page_structure(field: Field) -> list[tuple[str, PageStructure]]:
-    page_structures: list[tuple[str, PageStructure]] = []
-    field_type = FIELD_TYPE_STR_TO_PB[field.type]
-    if field_type == FieldType.FILE:
-        fed = await field.get_file_extracted_data()  # type: ignore
-        if fed is None:
-            return page_structures
-
-        fp = fed.file_pages_previews
-        if len(fp.pages) != len(fp.structures):
-            field_path = f"/kb/{field.kbid}/resource/{field.resource.uuid}/file/{field.id}"
-            logger.warning(
-                f"File extracted data has a different number of pages and structures! ({field_path})"
-            )
-            return page_structures
-        page_structures.extend(
-            [
-                # we expect this two field to have the same length, if not,
-                # something went wrong while processing
-                (fp.pages[i].uri, fp.structures[i])
-                for i in range(len(fp.pages))
-            ]
-        )
-
-    elif field_type == FieldType.LINK:
-        led = await field.get_link_extracted_data()  # type: ignore
-        if led is None:
-            return page_structures
-
-        page_structures.append((led.link_image.uri, led.pdf_structure))
-
-    return page_structures
+    # NOTE: image classifications are no longer supported, as the page selection annotations were removed
+    # from the API.
+    if False:
+        yield
+    return

nucliadb/train/generators/token_classifier.py

@@ -109,33 +109,6 @@ async def get_field_text(
     ] = {}  # Dict of entity group , with entity and list of positions in field
     split_ners[MAIN] = {}
 
-    basic_data = await orm_resource.get_basic()
-    invalid_tokens_split: dict[str, list[tuple[str, str, int, int]]] = {}
-    # Check user definition of entities
-    if basic_data is not None:
-        for userfieldmetadata in basic_data.fieldmetadata:
-            if (
-                userfieldmetadata.field.field == field
-                and userfieldmetadata.field.field_type == field_type_int
-            ):
-                for token in userfieldmetadata.token:
-                    if token.klass in valid_entity_groups:
-                        if token.cancelled_by_user:
-                            if token.split in (None, ""):
-                                split = MAIN
-                            else:
-                                split = token.split
-                            invalid_tokens_split[split].append(
-                                (token.klass, token.token, token.start, token.end)
-                            )
-                        else:
-                            if token.split in (None, ""):
-                                split = MAIN
-                            else:
-                                split = token.split
-                            split_ners[split].setdefault(token.klass, {}).setdefault(token.token, [])
-                            split_ners[split][token.klass][token.token].append((token.start, token.end))
-
     field_metadata = await field_obj.get_field_metadata()
     # Check computed definition of entities
     if field_metadata is not None:
@@ -189,17 +162,6 @@ async def get_field_text(
                     for position in positions.position:
                         split_ners[split][entity_group][entity].append((position.start, position.end))
 
-    for split, invalid_tokens in invalid_tokens_split.items():
-        for token.klass, token.token, token.start, token.end in invalid_tokens:
-            if token.klass in split_ners.get(split, {}):
-                if token.token in split_ners.get(split, {}).get(token.klass, {}):
-                    if (token.start, token.end) in split_ners[split][token.klass][token.token]:
-                        split_ners[split][token.klass][token.token].remove((token.start, token.end))
-                        if len(split_ners[split][token.klass][token.token]) == 0:
-                            del split_ners[split][token.klass][token.token]
-                        if len(split_ners[split][token.klass]) == 0:
-                            del split_ners[split][token.klass]
-
     ordered_positions: dict[str, POSITION_DICT] = {}
     for split, ners in split_ners.items():
         split_positions: dict[tuple[int, int], tuple[str, str]] = {}

nucliadb/writer/resource/basic.py

@@ -52,11 +52,8 @@ from nucliadb_protos.resources_pb2 import (
     FieldComputedMetadataWrapper,
     FieldType,
     Metadata,
-    PageSelections,
     Paragraph,
-    TokenSplit,
     UserFieldMetadata,
-    VisualSelection,
 )
 from nucliadb_protos.resources_pb2 import ParagraphAnnotation as PBParagraphAnnotation
 from nucliadb_protos.resources_pb2 import (
@@ -102,16 +99,6 @@ def parse_basic_modify(bm: BrokerMessage, item: ComingResourcePayload, toprocess
     if item.fieldmetadata is not None:
         for fieldmetadata in item.fieldmetadata:
             userfieldmetadata = UserFieldMetadata()
-            for token in fieldmetadata.token:
-                userfieldmetadata.token.append(
-                    TokenSplit(
-                        token=token.token,
-                        klass=token.klass,
-                        start=token.start,
-                        end=token.end,
-                        cancelled_by_user=token.cancelled_by_user,
-                    )
-                )
             for paragraph in fieldmetadata.paragraphs:
                 validate_classifications(paragraph)
                 paragraphpb = PBParagraphAnnotation(key=paragraph.key)
@@ -125,24 +112,6 @@ def parse_basic_modify(bm: BrokerMessage, item: ComingResourcePayload, toprocess
                     )
                 userfieldmetadata.paragraphs.append(paragraphpb)
 
-            for page_selections in fieldmetadata.selections:
-                page_selections_pb = PageSelections()
-                page_selections_pb.page = page_selections.page
-                page_selections_pb.visual.extend(
-                    [
-                        VisualSelection(
-                            label=visual_selection.label,
-                            top=visual_selection.top,
-                            left=visual_selection.left,
-                            right=visual_selection.right,
-                            bottom=visual_selection.bottom,
-                            token_ids=visual_selection.token_ids,
-                        )
-                        for visual_selection in page_selections.visual
-                    ]
-                )
-                userfieldmetadata.page_selections.append(page_selections_pb)
-
             for qa_annotation in fieldmetadata.question_answers:
                 qa_annotation_pb = build_question_answer_annotation_pb(qa_annotation)
                 userfieldmetadata.question_answers.append(qa_annotation_pb)

{nucliadb-6.3.5.post3914.dist-info → nucliadb-6.3.5.post3922.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.3.5.post3914
+Version: 6.3.5.post3922
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3914
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3914
-Requires-Dist: nucliadb-protos>=6.3.5.post3914
-Requires-Dist: nucliadb-models>=6.3.5.post3914
-Requires-Dist: nidx-protos>=6.3.5.post3914
+Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3922
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3922
+Requires-Dist: nucliadb-protos>=6.3.5.post3922
+Requires-Dist: nucliadb-models>=6.3.5.post3922
+Requires-Dist: nidx-protos>=6.3.5.post3922
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.3.5.post3914.dist-info → nucliadb-6.3.5.post3922.dist-info}/RECORD

@@ -105,7 +105,7 @@ nucliadb/common/maindb/local.py,sha256=uE9DIQX1yCNHNN8Tx4fPgSiuTtWpQhlfWkMJ8QZPa
 nucliadb/common/maindb/pg.py,sha256=FNq2clckJYj4Te-1svjQblqGoAF5OwJ5nwz2JtxD0d4,13645
 nucliadb/common/maindb/utils.py,sha256=zWLs82rWEVhpc1dYvdqTZiAcjZroB6Oo5MQaxMeFuKk,3301
 nucliadb/common/models_utils/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/common/models_utils/from_proto.py,sha256=yYn5vg4SKgB57RKOmeqzfD8VfmjKSSr4UNhw-Vvb4vs,15891
+nucliadb/common/models_utils/from_proto.py,sha256=zoQrqVsL5nfcvbuPPHoWHEt7UkTi-9uUd8f6ZqQnvU4,15614
 nucliadb/common/models_utils/to_proto.py,sha256=97JvOR_3odu50YvzLa2CERfEN3w_QPmAVcCJwJB5m5A,2438
 nucliadb/export_import/__init__.py,sha256=y-Is0Bxa8TMV6UiOW0deC_D3U465P65CQ5RjBjIWnow,932
 nucliadb/export_import/datamanager.py,sha256=xL8b0xvk45q6wx1l7J32JgPmpyjnF6fKiJi2F2B_UQY,6968
@@ -141,7 +141,7 @@ nucliadb/ingest/fields/generic.py,sha256=elgtqv15aJUq3zY7X_g0bli_2BpcwPArVvzhe54
 nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJs,4172
 nucliadb/ingest/fields/text.py,sha256=tFvSQJAe0W7ePpp2_WDfLiE2yglR1OTU0Zht9acvOFw,1594
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/orm/brain.py,sha256=A8H1J7Bo95sNzDgYr0_UNoemQhWOFEFz9UlYfs6ug-8,29407
+nucliadb/ingest/orm/brain.py,sha256=S08SVgHukdBs4RGlFSu69xIT51YR6fcVolhJ_E3cR9w,28686
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=a-aYuKBUQhxDKFtXOzTAkLlY_t2JiTfaptw2vt3AQDQ,14915
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
@@ -310,12 +310,12 @@ nucliadb/train/api/v1/trainset.py,sha256=kpnpDgiMWr1FKHZJgwH7hue5kzilA8-i9X0YHlN
 nucliadb/train/generators/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/train/generators/field_classifier.py,sha256=yatj7U-LHRN5xTR6XsYz_3acIAUKTcpkNZcZaSY8MtE,3482
 nucliadb/train/generators/field_streaming.py,sha256=kjwg4VNiROVqVDN--mRd4ylLw55Sg2VYxKRDdbmpYSM,5398
-nucliadb/train/generators/image_classifier.py,sha256=yqdBKIHiumeDSTggbxLFiUPanjMtbWGEaWo4mg2OcqA,6704
+nucliadb/train/generators/image_classifier.py,sha256=B4P88JfpjMcAZIPzlSOYaGseq5NgfssEr_Ecvlprr3g,1859
 nucliadb/train/generators/paragraph_classifier.py,sha256=0pOZYcT1cAmG7gjSD1HIUaMM5T3Ag-96iUTXRhiV8MI,2761
 nucliadb/train/generators/paragraph_streaming.py,sha256=dsM7a5hBd2iokvFuxnZhQeko4Jad6djyP2p3tevku8A,3586
 nucliadb/train/generators/question_answer_streaming.py,sha256=P7-de4W4yW2mgEQ82fF2OZVyx6QJHXezY52qDciDcmw,5680
 nucliadb/train/generators/sentence_classifier.py,sha256=DuvXfnWvLhklYR_qFGk2LqUyl2JE7CMVFwuHaPyC9Ys,5121
-nucliadb/train/generators/token_classifier.py,sha256=Vl14aaWoqrgYPijmvM62OjxDdANbpcbEZSZq2X2KhEo,11697
+nucliadb/train/generators/token_classifier.py,sha256=0848GqoXh8ywU82cPUrkzOM53-lZ1MVCw--8yDABigY,9557
 nucliadb/train/generators/utils.py,sha256=1uSELmM4CpKy9jWp6j_u7_n_KR-udRNkes4UmPMOCcI,3907
 nucliadb/writer/__init__.py,sha256=S298mrZL3vr62OrBqi97mdLxgR5cReMlRJgnaQHZV7s,1304
 nucliadb/writer/app.py,sha256=ABBO8-u4pDAa61b3mCdD0TFhuHAYcxMkgpZSGgWARuE,2736
@@ -344,7 +344,7 @@ nucliadb/writer/api/v1/upload.py,sha256=hLMHXSaqEOE-vjKjhIupgdx8klJc3mVQp_oMwx5N
 nucliadb/writer/api/v1/vectorsets.py,sha256=F3iMViL5G95_Tns4aO2SOA0DwAzxK2_P8MXxtd_XLRE,6973
 nucliadb/writer/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/resource/audit.py,sha256=FvxMZPzrNHtd31HgpZEvxzwAkbxJTZRhPLqRYYJi3tA,1426
-nucliadb/writer/resource/basic.py,sha256=_zdAr110C7rtEzOKoBRMzPjAnQ0pAtRfGjB8qCzodvI,11767
+nucliadb/writer/resource/basic.py,sha256=P2VXXXLKs43_Cd7Uvrcd-JTeuOJuUGu1Jpx8eujGi7Q,10451
 nucliadb/writer/resource/field.py,sha256=e5QGkR5ZDT1VUQgMXK7v6GGXJ2eek6jxGA0nPqjq_g4,20241
 nucliadb/writer/resource/origin.py,sha256=pvhUDdU0mlWPUcpoQi4LDUJaRtfjzVVrA8XcGVI_N8k,2021
 nucliadb/writer/tus/__init__.py,sha256=huWpKnDnjsrKlBBJk30ta5vamlA-4x0TbPs_2Up8hyM,5443
@@ -356,8 +356,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.5.post3914.dist-info/METADATA,sha256=_kbS1OuZddK3euMgqL96FHsWqVreZJPHIazpqsxSIzI,4301
-nucliadb-6.3.5.post3914.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-nucliadb-6.3.5.post3914.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.5.post3914.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.5.post3914.dist-info/RECORD,,
+nucliadb-6.3.5.post3922.dist-info/METADATA,sha256=blA0HPryUhZB_Z8QgNTv8DaGx3OSrJLDuagaaN_Ojdk,4301
+nucliadb-6.3.5.post3922.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+nucliadb-6.3.5.post3922.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.5.post3922.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.5.post3922.dist-info/RECORD,,