nucliadb 6.7.2.post4874__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

nucliadb/writer/resource/field.py

@@ -19,15 +19,15 @@
 #
 import dataclasses
 from datetime import datetime
-from typing import Optional, Union
 
+from fastapi import HTTPException
 from google.protobuf.json_format import MessageToDict
 
 import nucliadb_models as models
 from nucliadb.common import datamanagers
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.common.models_utils import from_proto, to_proto
-from nucliadb.ingest.fields.conversation import Conversation
+from nucliadb.ingest.fields.conversation import MAX_CONVERSATION_MESSAGES, Conversation
 from nucliadb.ingest.orm.resource import Resource as ORMResource
 from nucliadb.models.internal import processing as processing_models
 from nucliadb.models.internal.processing import ClassificationLabel, PushConversation, PushPayload
@@ -87,7 +87,7 @@ async def extract_file_field(
     resource: ORMResource,
     toprocess: PushPayload,
     resource_classifications: ResourceClassifications,
-    password: Optional[str] = None,
+    password: str | None = None,
 ):
     field_type = resources_pb2.FieldType.FILE
     field = await resource.get_field(field_id, field_type)
@@ -182,7 +182,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
 async def parse_fields(
     writer: BrokerMessage,
     toprocess: PushPayload,
-    item: Union[CreateResourcePayload, UpdateResourcePayload],
+    item: CreateResourcePayload | UpdateResourcePayload,
     kbid: str,
     uuid: str,
     x_skip_store: bool,
@@ -227,6 +227,7 @@ async def parse_fields(
             kbid,
             uuid,
             resource_classifications,
+            replace_field=True,
         )
 
 
@@ -430,11 +431,15 @@ async def parse_conversation_field(
     kbid: str,
     uuid: str,
     resource_classifications: ResourceClassifications,
+    replace_field: bool,
 ) -> None:
+    if not replace_field:
+        # Appending messages to conversation
+        await _conversation_append_checks(kbid, uuid, key, conversation_field)
     classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.CONVERSATION)
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
-    field_value = resources_pb2.Conversation()
+    field_value = resources_pb2.Conversation(replace_field=replace_field)
     convs = processing_models.PushConversation()
     for message in conversation_field.messages:
         cm = resources_pb2.Message()
@@ -543,3 +548,36 @@ async def get_stored_resource_classifications(
             classif = ClassificationLabel(labelset=f_classif.labelset, label=f_classif.label)
             rc.field_level.setdefault(fid, set()).add(classif)
     return rc
+
+
+async def _conversation_append_checks(
+    kbid: str, rid: str, field_id: str, input: models.InputConversationField
+):
+    async with datamanagers.with_ro_transaction() as txn:
+        resource_obj = await ORMResource.get(txn, kbid=kbid, rid=rid)
+        if resource_obj is None:
+            return
+        conv: Conversation = await resource_obj.get_field(
+            field_id, resources_pb2.FieldType.CONVERSATION, load=False
+        )
+
+        # Make sure that the max number of messages is not exceeded
+        current_message_count = (await conv.get_metadata()).total
+        if (
+            MAX_CONVERSATION_MESSAGES is not None
+            and (len(input.messages) + current_message_count) > MAX_CONVERSATION_MESSAGES
+        ):
+            raise HTTPException(
+                status_code=422,
+                detail=f"Conversation fields cannot have more than {MAX_CONVERSATION_MESSAGES} messages.",
+            )
+
+        # Make sure input messages use unique idents
+        existing_message_ids = set((await conv.get_splits_metadata()).metadata.keys())
+        input_message_ids = {message.ident for message in input.messages}
+        intersection = input_message_ids.intersection(existing_message_ids)
+        if intersection != set():
+            raise HTTPException(
+                status_code=422,
+                detail=f"Message identifiers must be unique field={field_id}: {list(intersection)[:50]}",
+            )

nucliadb/writer/resource/origin.py

@@ -42,6 +42,13 @@ def parse_origin(origin: Origin, origin_payload: InputOrigin):
         origin.metadata.update(origin_payload.metadata)
     if origin_payload.path:
         origin.path = origin_payload.path
+    if origin_payload.sync_metadata is not None:
+        origin.sync_metadata.CopyFrom(
+            resources_pb2.SyncMetadata(
+                file_id=origin_payload.sync_metadata.file_id,
+                auth_provider=origin_payload.sync_metadata.auth_provider,
+            )
+        )
     origin.source = Origin.Source.API
 
 

nucliadb/writer/settings.py

@@ -17,15 +17,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Optional
 
 from pydantic_settings import BaseSettings
 
 
 class Settings(BaseSettings):
     dm_enabled: bool = True
-    dm_redis_host: Optional[str] = None
-    dm_redis_port: Optional[int] = None
+    dm_redis_host: str | None = None
+    dm_redis_port: int | None = None
 
 
 settings = Settings()

nucliadb/writer/tus/__init__.py

@@ -18,7 +18,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from dataclasses import dataclass
-from typing import Optional
 
 from nucliadb.writer.settings import settings as writer_settings
 from nucliadb.writer.tus.dm import FileDataManager, RedisFileDataManagerFactory
@@ -37,8 +36,8 @@ class TusStorageDriver:
     manager: FileStorageManager
 
 
-DRIVER: Optional[TusStorageDriver] = None
-REDIS_FILE_DATA_MANAGER_FACTORY: Optional[RedisFileDataManagerFactory] = None
+DRIVER: TusStorageDriver | None = None
+REDIS_FILE_DATA_MANAGER_FACTORY: RedisFileDataManagerFactory | None = None
 
 
 async def initialize():

nucliadb/writer/tus/azure.py

@@ -19,15 +19,12 @@
 #
 from __future__ import annotations
 
-from typing import Optional
-
 from nucliadb.writer import logger
 from nucliadb.writer.tus.dm import FileDataManager
 from nucliadb.writer.tus.storage import BlobStore, FileStorageManager
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_utils.storages import CHUNK_SIZE
 from nucliadb_utils.storages.azure import AzureObjectStore
-from nucliadb_utils.storages.exceptions import ObjectNotFoundError
 from nucliadb_utils.storages.utils import ObjectMetadata
 
 
@@ -41,7 +38,7 @@ class AzureBlobStore(BlobStore):
             logger.exception("Error closing AzureBlobStore")
         self._object_store = None
 
-    async def initialize(self, account_url: str, connection_string: Optional[str] = None):
+    async def initialize(self, account_url: str, connection_string: str | None = None):
         self.bucket = "nucliadb-{kbid}"
         self.source = CloudFile.Source.AZURE
         self._object_store = AzureObjectStore(account_url, connection_string=connection_string)
@@ -63,7 +60,7 @@ class AzureBlobStore(BlobStore):
 class AzureFileStorageManager(FileStorageManager):
     storage: AzureBlobStore
     chunk_size = CHUNK_SIZE
-    min_upload_size = None
+    min_upload_size = CHUNK_SIZE
 
     @property
     def object_store(self) -> AzureObjectStore:
@@ -87,7 +84,7 @@ class AzureFileStorageManager(FileStorageManager):
         bucket = self.storage.get_bucket_name(kbid)
         try:
             await self.object_store.delete(bucket, uri)
-        except ObjectNotFoundError:
+        except KeyError:
             logger.warning(
                 "Attempt to delete an upload but not found",
                 extra={"uri": uri, "kbid": kbid, "bucket": bucket},
@@ -108,4 +105,5 @@ class AzureFileStorageManager(FileStorageManager):
         return path
 
     def validate_intermediate_chunk(self, uploaded_bytes: int):
-        pass
+        if uploaded_bytes < self.min_upload_size:
+            raise ValueError(f"Intermediate chunks cannot be smaller than {self.min_upload_size} bytes")

nucliadb/writer/tus/dm.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import time
-from typing import Any, Optional
+from typing import Any
 
 import backoff
 import orjson
@@ -43,7 +43,7 @@ DATA: dict[str, Any] = {}
 
 
 class FileDataManager:
-    _data: Optional[dict[str, Any]] = None
+    _data: dict[str, Any] | None = None
     _loaded = False
     key = None
     _ttl = 60 * 50 * 5  # 5 minutes should be plenty of time between activity
@@ -63,7 +63,7 @@ class FileDataManager:
         if self._data and "last_activity" in self._data:
             # check for another active upload, fail if we're screwing with
             # someone else
-            last_activity: Optional[int] = self._data.get("last_activity")
+            last_activity: int | None = self._data.get("last_activity")
             if last_activity and (time.time() - last_activity) < self._ttl:
                 if request.headers and request.headers.get("tus-override-upload", "0") != "1":
                     raise HTTPPreconditionFailed(

nucliadb/writer/tus/exceptions.py

@@ -17,7 +17,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Optional
 
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
@@ -27,11 +26,11 @@ class InvalidTUSMetadata(Exception):
 
 
 class HTTPException(StarletteHTTPException):
-    _status_code: Optional[int] = None
+    _status_code: int | None = None
 
-    def __init__(self, detail: Optional[str] = None):
+    def __init__(self, detail: str | None = None):
         if self._status_code:
-            super(HTTPException, self).__init__(status_code=self._status_code, detail=detail)
+            super().__init__(status_code=self._status_code, detail=detail)
         else:
             raise AttributeError("Status code not defined")
 

nucliadb/writer/tus/gcs.py

@@ -28,7 +28,6 @@ import tempfile
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
-from typing import Optional
 from urllib.parse import quote_plus
 
 import aiohttp
@@ -74,16 +73,22 @@ RETRIABLE_EXCEPTIONS = (
 
 
 class GCloudBlobStore(BlobStore):
-    session: Optional[aiohttp.ClientSession] = None
+    _session: aiohttp.ClientSession | None = None
     loop = None
     upload_url: str
     object_base_url: str
-    json_credentials: Optional[str]
+    json_credentials: str | None
     bucket: str
     location: str
     project: str
     executor = ThreadPoolExecutor(max_workers=5)
 
+    @property
+    def session(self) -> aiohttp.ClientSession:
+        if self._session is None:  # pragma: no cover
+            raise AttributeError("Session not initialized")
+        return self._session
+
     async def get_access_headers(self):
         if self._credentials is None:
             return {}
@@ -106,8 +111,9 @@ class GCloudBlobStore(BlobStore):
             return access_token.access_token
 
     async def finalize(self):
-        if self.session is not None:
-            await self.session.close()
+        if self._session is not None:
+            await self._session.close()
+            self._session = None
 
     async def initialize(
         self,
@@ -116,7 +122,7 @@ class GCloudBlobStore(BlobStore):
         project: str,
         bucket_labels,
         object_base_url: str,
-        json_credentials: Optional[str],
+        json_credentials: str | None,
     ):
         self.bucket = bucket
         self.source = CloudFile.Source.GCS
@@ -124,7 +130,7 @@ class GCloudBlobStore(BlobStore):
         self.project = project
         self.bucket_labels = bucket_labels
         self.object_base_url = object_base_url + "/storage/v1/b"
-        self.upload_url = object_base_url + "/upload/storage/v1/b/{bucket}/o?uploadType=resumable"  # noqa
+        self.upload_url = object_base_url + "/upload/storage/v1/b/{bucket}/o?uploadType=resumable"
         self.json_credentials = json_credentials
         self._credentials = None
 
@@ -143,12 +149,9 @@ class GCloudBlobStore(BlobStore):
                 self._credentials = None
 
         loop = asyncio.get_event_loop()
-        self.session = aiohttp.ClientSession(loop=loop, timeout=TIMEOUT)
+        self._session = aiohttp.ClientSession(loop=loop, timeout=TIMEOUT)
 
     async def check_exists(self, bucket_name: str):
-        if self.session is None:
-            raise AttributeError()
-
         headers = await self.get_access_headers()
         # Using object access url instead of bucket access to avoid
         # giving admin permission to the SA, needed to GET a bucket
@@ -163,8 +166,6 @@ class GCloudBlobStore(BlobStore):
         return False
 
     async def create_bucket(self, bucket_name: str):
-        if self.session is None:
-            raise AttributeError()
         headers = await self.get_access_headers()
         url = f"{self.object_base_url}?project={self.project}"
 
@@ -199,10 +200,6 @@ class GCloudFileStorageManager(FileStorageManager):
         _resumable_uri : uri to resumable upload
         _uri : finished uploaded image
         """
-
-        if self.storage.session is None:
-            raise AttributeError()
-
         upload_file_id = dm.get("upload_file_id")
         if upload_file_id is not None:
             await self.delete_upload(upload_file_id, kbid)
@@ -287,8 +284,6 @@ class GCloudFileStorageManager(FileStorageManager):
 
     @backoff.on_exception(backoff.expo, RETRIABLE_EXCEPTIONS, jitter=backoff.random_jitter, max_tries=4)
     async def _append(self, dm: FileDataManager, data, offset):
-        if self.storage.session is None:
-            raise AttributeError()
         if dm.size:
             size = str(dm.size)
         else:
@@ -315,7 +310,7 @@ class GCloudFileStorageManager(FileStorageManager):
             },
             data=data,
         ) as call:
-            text = await call.text()  # noqa
+            text = await call.text()
             if call.status not in [200, 201, 308]:
                 raise GoogleCloudException(f"{call.status}: {text}")
             return call
@@ -353,8 +348,6 @@ class GCloudFileStorageManager(FileStorageManager):
     @backoff.on_exception(backoff.expo, RETRIABLE_EXCEPTIONS, jitter=backoff.random_jitter, max_tries=4)
     async def finish(self, dm: FileDataManager):
         if dm.size == 0:
-            if self.storage.session is None:
-                raise AttributeError()
             # In case of empty file, we need to send a PUT request with empty body
             # and Content-Range header set to "bytes */0"
             headers = {

nucliadb/writer/tus/s3.py

@@ -22,7 +22,6 @@ from __future__ import annotations
 import base64
 import uuid
 from contextlib import AsyncExitStack
-from typing import Optional
 
 import aiobotocore  # type: ignore
 import aiohttp
@@ -195,8 +194,8 @@ class S3BlobStore(BlobStore):
         endpoint_url,
         region_name,
         bucket,
-        bucket_tags: Optional[dict[str, str]] = None,
-        kms_key_id: Optional[str] = None,
+        bucket_tags: dict[str, str] | None = None,
+        kms_key_id: str | None = None,
     ):
         self.bucket = bucket
         self.bucket_tags = bucket_tags

nucliadb/writer/tus/storage.py

@@ -19,7 +19,7 @@
 #
 from __future__ import annotations
 
-from typing import AsyncIterator, Optional
+from collections.abc import AsyncIterator
 
 from nucliadb.writer.tus.dm import FileDataManager
 from nucliadb_protos.resources_pb2 import CloudFile
@@ -47,13 +47,13 @@ class BlobStore:
 
 class FileStorageManager:
     chunk_size: int
-    min_upload_size: Optional[int] = None
+    min_upload_size: int | None = None
 
     def __init__(self, storage: BlobStore):
         self.storage = storage
 
     def iter_data(
-        self, uri: str, kbid: str, headers: Optional[dict[str, str]] = None
+        self, uri: str, kbid: str, headers: dict[str, str] | None = None
     ) -> AsyncIterator[bytes]:
         raise NotImplementedError()
 

{nucliadb-6.7.2.post4874.dist-info → nucliadb-6.10.0.post5705.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.7.2.post4874
+Version: 6.10.0.post5705
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -12,20 +12,19 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: <4,>=3.9
+Requires-Python: <4,>=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4874
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4874
-Requires-Dist: nucliadb-protos>=6.7.2.post4874
-Requires-Dist: nucliadb-models>=6.7.2.post4874
-Requires-Dist: nidx-protos>=6.7.2.post4874
+Requires-Dist: nucliadb-telemetry[all]>=6.10.0.post5705
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.10.0.post5705
+Requires-Dist: nucliadb-protos[grpc]>=6.10.0.post5705
+Requires-Dist: nucliadb-models>=6.10.0.post5705
+Requires-Dist: nidx-protos[grpc]>=6.10.0.post5705
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
-Requires-Dist: nuclia-models>=0.46.0
+Requires-Dist: nuclia-models>=0.50.0
 Requires-Dist: uvicorn[standard]
 Requires-Dist: argdantic
 Requires-Dist: aiohttp>=3.11.11
@@ -35,7 +34,7 @@ Requires-Dist: aiofiles>=0.8.0
 Requires-Dist: psutil>=5.9.7
 Requires-Dist: types-psutil>=5.9.5.17
 Requires-Dist: types-aiofiles>=0.8.3
-Requires-Dist: protobuf<6,>=5
+Requires-Dist: protobuf>=5
 Requires-Dist: types-protobuf<6,>=5
 Requires-Dist: grpcio>=1.71.0
 Requires-Dist: grpcio-health-checking>=1.71.0
@@ -57,7 +56,7 @@ Requires-Dist: jwcrypto>=1.5.6
 Requires-Dist: pyyaml>=5.1
 Requires-Dist: fastapi-versioning>=0.10.0
 Requires-Dist: fastapi>=0.95.2
-Requires-Dist: sentry-sdk>=2.8.0
+Requires-Dist: sentry-sdk[fastapi]>=2.8.0
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: mmh3>=3.0.0
 Requires-Dist: httpx>=0.23.0