nucliadb 6.7.2.post4862__py3-none-any.whl → 6.9.2.post5282__py3-none-any.whl

nucliadb/writer/api/v1/knowledgebox.py

@@ -36,10 +36,6 @@ from nucliadb.writer import logger
 from nucliadb.writer.api.utils import only_for_onprem
 from nucliadb.writer.api.v1.router import KB_PREFIX, KBS_PREFIX, api
 from nucliadb.writer.utilities import get_processing
-from nucliadb_models.external_index_providers import (
-    ExternalIndexProviderType,
-    PineconeServerlessCloud,
-)
 from nucliadb_models.resource import (
     KnowledgeBoxConfig,
     KnowledgeBoxObj,
@@ -118,20 +114,6 @@ async def create_kb(item: KnowledgeBoxConfig) -> tuple[str, str]:
     external_index_provider = knowledgebox_pb2.CreateExternalIndexProviderMetadata(
         type=knowledgebox_pb2.ExternalIndexProviderType.UNSET,
     )
-    if (
-        item.external_index_provider
-        and item.external_index_provider.type == ExternalIndexProviderType.PINECONE
-    ):
-        pinecone_api_key = item.external_index_provider.api_key
-        serverless_pb = to_pinecone_serverless_cloud_pb(item.external_index_provider.serverless_cloud)
-        external_index_provider = knowledgebox_pb2.CreateExternalIndexProviderMetadata(
-            type=knowledgebox_pb2.ExternalIndexProviderType.PINECONE,
-            pinecone_config=knowledgebox_pb2.CreatePineconeConfig(
-                api_key=pinecone_api_key,
-                serverless_cloud=serverless_pb,
-            ),
-        )
-
     try:
         (kbid, slug) = await KnowledgeBox.create(
             driver,
@@ -165,8 +147,6 @@ async def create_kb(item: KnowledgeBoxConfig) -> tuple[str, str]:
 @requires(NucliaDBRoles.MANAGER)
 @version(1)
 async def update_kb(request: Request, kbid: str, item: KnowledgeBoxConfig) -> KnowledgeBoxObjID:
-    driver = get_driver()
-    config = None
     if (
         item.slug
         or item.title
@@ -174,29 +154,24 @@ async def update_kb(request: Request, kbid: str, item: KnowledgeBoxConfig) -> Kn
         or item.hidden_resources_enabled
         or item.hidden_resources_hide_on_creation
     ):
-        config = knowledgebox_pb2.KnowledgeBoxConfig(
-            slug=item.slug or "",
-            title=item.title or "",
-            description=item.description or "",
-            hidden_resources_enabled=item.hidden_resources_enabled,
-            hidden_resources_hide_on_creation=item.hidden_resources_hide_on_creation,
-        )
-    try:
-        async with driver.transaction() as txn:
+        try:
+            driver = get_driver()
             await KnowledgeBox.update(
-                txn,
-                uuid=kbid,
+                driver,
+                kbid=kbid,
                 slug=item.slug,
-                config=config,
+                title=item.title,
+                description=item.description,
+                hidden_resources_enabled=item.hidden_resources_enabled,
+                hidden_resources_hide_on_creation=item.hidden_resources_hide_on_creation,
             )
-            await txn.commit()
-    except datamanagers.exceptions.KnowledgeBoxNotFound:
-        raise HTTPException(status_code=404, detail="Knowledge box does not exist")
-    except Exception as exc:
-        logger.exception("Could not update KB", exc_info=exc, extra={"kbid": kbid})
-        raise HTTPException(status_code=500, detail="Error updating knowledge box")
-    else:
-        return KnowledgeBoxObjID(uuid=kbid)
+        except datamanagers.exceptions.KnowledgeBoxNotFound:
+            raise HTTPException(status_code=404, detail="Knowledge box does not exist")
+        except Exception as exc:
+            logger.exception("Could not update KB", exc_info=exc, extra={"kbid": kbid})
+            raise HTTPException(status_code=500, detail="Error updating knowledge box")
+
+    return KnowledgeBoxObjID(uuid=kbid)
 
 
 @only_for_onprem
@@ -236,15 +211,3 @@ async def delete_kb(request: Request, kbid: str) -> KnowledgeBoxObj:
     asyncio.create_task(processing.delete_from_processing(kbid=kbid))
 
     return KnowledgeBoxObj(uuid=kbid)
-
-
-def to_pinecone_serverless_cloud_pb(
-    serverless: PineconeServerlessCloud,
-) -> knowledgebox_pb2.PineconeServerlessCloud.ValueType:
-    return {
-        PineconeServerlessCloud.AWS_EU_WEST_1: knowledgebox_pb2.PineconeServerlessCloud.AWS_EU_WEST_1,
-        PineconeServerlessCloud.AWS_US_EAST_1: knowledgebox_pb2.PineconeServerlessCloud.AWS_US_EAST_1,
-        PineconeServerlessCloud.AWS_US_WEST_2: knowledgebox_pb2.PineconeServerlessCloud.AWS_US_WEST_2,
-        PineconeServerlessCloud.AZURE_EASTUS2: knowledgebox_pb2.PineconeServerlessCloud.AZURE_EASTUS2,
-        PineconeServerlessCloud.GCP_US_CENTRAL1: knowledgebox_pb2.PineconeServerlessCloud.GCP_US_CENTRAL1,
-    }[serverless]

nucliadb/writer/api/v1/learning_config.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from fastapi import Request
+from fastapi import Header, Request
 from fastapi_versioning import version
 from nuclia_models.config.proto import ExtractConfig, SplitConfiguration
 
@@ -55,10 +55,11 @@ async def set_configuration(
 @requires_one([NucliaDBRoles.MANAGER, NucliaDBRoles.WRITER])
 @version(1)
 async def patch_configuration(
-    request: Request,
-    kbid: str,
+    request: Request, kbid: str, x_nucliadb_account: str = Header(default="", include_in_schema=False)
 ):
-    return await learning_config_proxy(request, "PATCH", f"/config/{kbid}")
+    return await learning_config_proxy(
+        request, "PATCH", f"/config/{kbid}", headers={"account-id": x_nucliadb_account}
+    )
 
 
 @api.post(

nucliadb/writer/api/v1/resource.py

@@ -373,7 +373,7 @@ async def update_resource_slug(
     rid: str,
     new_slug: str,
 ):
-    async with driver.transaction() as txn:
+    async with driver.rw_transaction() as txn:
         old_slug = await datamanagers.resources.modify_slug(txn, kbid=kbid, rid=rid, new_slug=new_slug)
         await txn.commit()
         return old_slug
@@ -460,7 +460,7 @@ async def _reprocess_resource(
     driver = get_driver()
 
     writer = BrokerMessage()
-    async with driver.transaction(read_only=True) as txn:
+    async with driver.ro_transaction() as txn:
         kb = KnowledgeBox(txn, storage, kbid)
 
         resource = await kb.get(rid)

nucliadb/writer/resource/field.py

@@ -21,13 +21,14 @@ import dataclasses
 from datetime import datetime
 from typing import Optional, Union
 
+from fastapi import HTTPException
 from google.protobuf.json_format import MessageToDict
 
 import nucliadb_models as models
 from nucliadb.common import datamanagers
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.common.models_utils import from_proto, to_proto
-from nucliadb.ingest.fields.conversation import Conversation
+from nucliadb.ingest.fields.conversation import MAX_CONVERSATION_MESSAGES, Conversation
 from nucliadb.ingest.orm.resource import Resource as ORMResource
 from nucliadb.models.internal import processing as processing_models
 from nucliadb.models.internal.processing import ClassificationLabel, PushConversation, PushPayload
@@ -227,6 +228,7 @@ async def parse_fields(
             kbid,
             uuid,
             resource_classifications,
+            replace_field=True,
         )
 
 
@@ -430,11 +432,15 @@ async def parse_conversation_field(
     kbid: str,
     uuid: str,
     resource_classifications: ResourceClassifications,
+    replace_field: bool,
 ) -> None:
+    if not replace_field:
+        # Appending messages to conversation
+        await _conversation_append_checks(kbid, uuid, key, conversation_field)
     classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.CONVERSATION)
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
-    field_value = resources_pb2.Conversation()
+    field_value = resources_pb2.Conversation(replace_field=replace_field)
     convs = processing_models.PushConversation()
     for message in conversation_field.messages:
         cm = resources_pb2.Message()
@@ -543,3 +549,33 @@ async def get_stored_resource_classifications(
             classif = ClassificationLabel(labelset=f_classif.labelset, label=f_classif.label)
             rc.field_level.setdefault(fid, set()).add(classif)
     return rc
+
+
+async def _conversation_append_checks(
+    kbid: str, rid: str, field_id: str, input: models.InputConversationField
+):
+    async with datamanagers.with_ro_transaction() as txn:
+        resource_obj = await datamanagers.resources.get_resource(txn, kbid=kbid, rid=rid)
+        if resource_obj is None:
+            return
+        conv: Conversation = await resource_obj.get_field(
+            field_id, resources_pb2.FieldType.CONVERSATION, load=False
+        )
+
+        # Make sure that the max number of messages is not exceeded
+        current_message_count = (await conv.get_metadata()).total
+        if len(input.messages) + current_message_count > MAX_CONVERSATION_MESSAGES:
+            raise HTTPException(
+                status_code=422,
+                detail=f"Conversation fields cannot have more than {MAX_CONVERSATION_MESSAGES} messages.",
+            )
+
+        # Make sure input messages use unique idents
+        existing_message_ids = set((await conv.get_splits_metadata()).metadata.keys())
+        input_message_ids = {message.ident for message in input.messages}
+        intersection = input_message_ids.intersection(existing_message_ids)
+        if intersection != set():
+            raise HTTPException(
+                status_code=422,
+                detail=f"Message identifiers must be unique field={field_id}: {list(intersection)[:50]}",
+            )

nucliadb/writer/tus/azure.py

@@ -27,7 +27,6 @@ from nucliadb.writer.tus.storage import BlobStore, FileStorageManager
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_utils.storages import CHUNK_SIZE
 from nucliadb_utils.storages.azure import AzureObjectStore
-from nucliadb_utils.storages.exceptions import ObjectNotFoundError
 from nucliadb_utils.storages.utils import ObjectMetadata
 
 
@@ -63,7 +62,7 @@ class AzureBlobStore(BlobStore):
 class AzureFileStorageManager(FileStorageManager):
     storage: AzureBlobStore
     chunk_size = CHUNK_SIZE
-    min_upload_size = None
+    min_upload_size = CHUNK_SIZE
 
     @property
     def object_store(self) -> AzureObjectStore:
@@ -87,7 +86,7 @@ class AzureFileStorageManager(FileStorageManager):
         bucket = self.storage.get_bucket_name(kbid)
         try:
             await self.object_store.delete(bucket, uri)
-        except ObjectNotFoundError:
+        except KeyError:
             logger.warning(
                 "Attempt to delete an upload but not found",
                 extra={"uri": uri, "kbid": kbid, "bucket": bucket},
@@ -108,4 +107,5 @@ class AzureFileStorageManager(FileStorageManager):
         return path
 
     def validate_intermediate_chunk(self, uploaded_bytes: int):
-        pass
+        if uploaded_bytes < self.min_upload_size:
+            raise ValueError(f"Intermediate chunks cannot be smaller than {self.min_upload_size} bytes")

nucliadb/writer/tus/gcs.py

@@ -74,7 +74,7 @@ RETRIABLE_EXCEPTIONS = (
 
 
 class GCloudBlobStore(BlobStore):
-    session: Optional[aiohttp.ClientSession] = None
+    _session: Optional[aiohttp.ClientSession] = None
     loop = None
     upload_url: str
     object_base_url: str
@@ -84,6 +84,12 @@ class GCloudBlobStore(BlobStore):
     project: str
     executor = ThreadPoolExecutor(max_workers=5)
 
+    @property
+    def session(self) -> aiohttp.ClientSession:
+        if self._session is None:  # pragma: no cover
+            raise AttributeError("Session not initialized")
+        return self._session
+
     async def get_access_headers(self):
         if self._credentials is None:
             return {}
@@ -106,8 +112,9 @@ class GCloudBlobStore(BlobStore):
             return access_token.access_token
 
     async def finalize(self):
-        if self.session is not None:
-            await self.session.close()
+        if self._session is not None:
+            await self._session.close()
+            self._session = None
 
     async def initialize(
         self,
@@ -143,12 +150,9 @@ class GCloudBlobStore(BlobStore):
                 self._credentials = None
 
         loop = asyncio.get_event_loop()
-        self.session = aiohttp.ClientSession(loop=loop, timeout=TIMEOUT)
+        self._session = aiohttp.ClientSession(loop=loop, timeout=TIMEOUT)
 
     async def check_exists(self, bucket_name: str):
-        if self.session is None:
-            raise AttributeError()
-
         headers = await self.get_access_headers()
         # Using object access url instead of bucket access to avoid
         # giving admin permission to the SA, needed to GET a bucket
@@ -163,8 +167,6 @@ class GCloudBlobStore(BlobStore):
         return False
 
     async def create_bucket(self, bucket_name: str):
-        if self.session is None:
-            raise AttributeError()
         headers = await self.get_access_headers()
         url = f"{self.object_base_url}?project={self.project}"
 
@@ -199,10 +201,6 @@ class GCloudFileStorageManager(FileStorageManager):
         _resumable_uri : uri to resumable upload
         _uri : finished uploaded image
         """
-
-        if self.storage.session is None:
-            raise AttributeError()
-
         upload_file_id = dm.get("upload_file_id")
         if upload_file_id is not None:
             await self.delete_upload(upload_file_id, kbid)
@@ -287,8 +285,6 @@ class GCloudFileStorageManager(FileStorageManager):
 
     @backoff.on_exception(backoff.expo, RETRIABLE_EXCEPTIONS, jitter=backoff.random_jitter, max_tries=4)
     async def _append(self, dm: FileDataManager, data, offset):
-        if self.storage.session is None:
-            raise AttributeError()
         if dm.size:
             size = str(dm.size)
         else:
@@ -353,8 +349,6 @@ class GCloudFileStorageManager(FileStorageManager):
     @backoff.on_exception(backoff.expo, RETRIABLE_EXCEPTIONS, jitter=backoff.random_jitter, max_tries=4)
     async def finish(self, dm: FileDataManager):
         if dm.size == 0:
-            if self.storage.session is None:
-                raise AttributeError()
             # In case of empty file, we need to send a PUT request with empty body
             # and Content-Range header set to "bytes */0"
             headers = {

{nucliadb-6.7.2.post4862.dist-info → nucliadb-6.9.2.post5282.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.7.2.post4862
+Version: 6.9.2.post5282
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -12,20 +12,19 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: <4,>=3.9
+Requires-Python: <4,>=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.7.2.post4862
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.2.post4862
-Requires-Dist: nucliadb-protos>=6.7.2.post4862
-Requires-Dist: nucliadb-models>=6.7.2.post4862
-Requires-Dist: nidx-protos>=6.7.2.post4862
+Requires-Dist: nucliadb-telemetry[all]>=6.9.2.post5282
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.9.2.post5282
+Requires-Dist: nucliadb-protos>=6.9.2.post5282
+Requires-Dist: nucliadb-models>=6.9.2.post5282
+Requires-Dist: nidx-protos>=6.9.2.post5282
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
-Requires-Dist: nuclia-models>=0.46.0
+Requires-Dist: nuclia-models>=0.50.0
 Requires-Dist: uvicorn[standard]
 Requires-Dist: argdantic
 Requires-Dist: aiohttp>=3.11.11
@@ -57,7 +56,7 @@ Requires-Dist: jwcrypto>=1.5.6
 Requires-Dist: pyyaml>=5.1
 Requires-Dist: fastapi-versioning>=0.10.0
 Requires-Dist: fastapi>=0.95.2
-Requires-Dist: sentry-sdk>=2.8.0
+Requires-Dist: sentry-sdk[fastapi]>=2.8.0
 Requires-Dist: pyjwt>=2.4.0
 Requires-Dist: mmh3>=3.0.0
 Requires-Dist: httpx>=0.23.0