extralit-server 0.2.0__tar.gz → 0.2.1__tar.gz

{extralit_server-0.2.0 → extralit_server-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: extralit-server
-Version: 0.2.0
+Version: 0.2.1
 Summary: Open-source tool for accurate & fast scientific literature data extraction with LLM and human-in-the-loop.
 Keywords: literature-review,data-annotation,artificial-intelligence,machine-learning,human-in-the-loop,mlops
 Home-page: https://www.argilla.io

{extralit_server-0.2.0 → extralit_server-0.2.1}/pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
     "packaging>=23.2",
     "minio>=7.2.7",
 ]
-version = "0.2.0"
+version = "0.2.1"
 
 [project.license]
 text = "Apache-2.0"
@@ -91,6 +91,7 @@ test = [
     "pytest>=7.4.4",
     "pytest-cov>=4.1.0",
     "pytest-mock>=3.12.0",
+    "pytest-xdist>=3.6.0",
     "pytest-asyncio~=0.21.1",
     "pytest-env>=1.1.3",
     "factory-boy~=3.2.1",
@@ -117,10 +118,10 @@ cmd = "pytest"
 env_file = ".env.test"
 
 [tool.pdm.scripts.build-server-image]
-shell = "cp -R dist docker/server && docker build -t extralit/extralit-server:local docker/server"
+shell = "cp -R dist docker/server && docker build -t extralit/argilla-server:local docker/server"
 
 [tool.pdm.scripts.build-quickstart-image]
-shell = "docker build --build-arg ARGILLA_VERSION=local -t extralit/extralit-quickstart:local docker/quickstart"
+shell = "docker build --build-arg ARGILLA_VERSION=local -t extralit/argilla-quickstart:local docker/quickstart"
 
 [tool.pytest.ini_options]
 log_format = "%(asctime)s %(name)s %(levelname)s %(message)s"

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/_version.py

@@ -13,4 +13,4 @@
 #  limitations under the License.
 
 # coding: utf-8
-__version__ = "0.2.0"
+__version__ = "0.2.1"

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/alembic/versions/3fc3c0839959_create_suggestions_table.py

@@ -28,7 +28,7 @@ down_revision = "8c574ada5e5f"
 branch_labels = None
 depends_on = None
 
-suggestion_type_enum = sa.Enum("model", "human", name="suggestion_type_enum")
+suggestion_type_enum = sa.Enum("model", "human", "selection", name="suggestion_type_enum")
 
 
 def upgrade() -> None:
@@ -47,7 +47,7 @@ def upgrade() -> None:
         sa.ForeignKeyConstraint(["question_id"], ["questions.id"], ondelete="CASCADE"),
         sa.ForeignKeyConstraint(["record_id"], ["records.id"], ondelete="CASCADE"),
         sa.PrimaryKeyConstraint("id"),
-        sa.UniqueConstraint("record_id", "question_id", "type", "agent", name="suggestion_record_id_question_id_uq"),
+        sa.UniqueConstraint("record_id", "question_id", name="suggestion_record_id_question_id_uq"),
     )
     op.create_index(op.f("ix_suggestions_question_id"), "suggestions", ["question_id"], unique=False)
     op.create_index(op.f("ix_suggestions_record_id"), "suggestions", ["record_id"], unique=False)

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/alembic/versions/7552df94427a_added_document_model_and_updated_record_.py

@@ -1,4 +1,4 @@
-"""Added Document model and updated Record model
+"""add Document model and update Record model
 
 Revision ID: 7552df94427a
 Revises: ca7293c38970
@@ -11,7 +11,7 @@ from sqlalchemy.dialects import postgresql
 
 
 revision = '7552df94427a'
-down_revision = 'ca7293c38970'  # set to latest revision identifiers on updates from from argilla-io/argilla upstream repo.
+down_revision = 'ca7293c38970'  # TODO set to latest revision identifiers on updates from from argilla-io/argilla upstream repo.
 branch_labels = None
 depends_on = None
 

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/apis/v0/handlers/authentication.py

@@ -12,7 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, HTTPException, status
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from argilla_server.contexts import accounts
@@ -29,6 +29,9 @@ router = APIRouter(tags=["Authentication"])
 async def create_access_token(
     db: AsyncSession = Depends(get_async_db), form: UserPasswordRequestForm = Depends()
 ) -> Token:
+    if not form.username or not form.password:
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Password cannot be empty")
+
     user = await accounts.authenticate_user(db, form.username, form.password)
     if not user:
         raise UnauthorizedError()

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/apis/v0/handlers/workspaces.py

@@ -22,7 +22,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 
 from argilla_server.contexts import accounts, files
 from argilla_server.database import get_async_db
-from argilla_server.errors import EntityAlreadyExistsError, EntityNotFoundError
+from argilla_server.errors import EntityAlreadyExistsError, EntityNotFoundError, GenericServerError
 from argilla_server.errors.future import NotUniqueError
 from argilla_server.policies import WorkspacePolicy, WorkspaceUserPolicy, authorize
 from argilla_server.pydantic_v1 import parse_obj_as
@@ -46,11 +46,11 @@ async def create_workspace(
     try:
         files.create_bucket(minio_client, workspace_create.name)
     except Exception as e:
-        raise e
+        raise GenericServerError(e)
 
     try:
         workspace = await accounts.create_workspace(db, workspace_create.dict())
-    except NotUniqueError:
+    except NotUniqueError as e:
         _LOGGER.error(f"Could not create workspace '{workspace_create.name}': {e}")
         raise EntityAlreadyExistsError(name=workspace_create.name, type=Workspace)
 

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/apis/v1/handlers/datasets/records.py

@@ -29,7 +29,7 @@ from argilla_server.database import get_async_db
 from argilla_server.enums import MetadataPropertyType, RecordSortField, ResponseStatus, ResponseStatusFilter, SortOrder
 from argilla_server.errors.future.base_errors import MISSING_VECTOR_ERROR_CODE
 from argilla_server.models import Dataset as DatasetModel
-from argilla_server.models import Record, User, Suggestion
+from argilla_server.models import Record, User, Suggestion, Response, Question
 from argilla_server.policies import DatasetPolicyV1, RecordPolicyV1, authorize, is_authorized
 from argilla_server.schemas.v1.datasets import Dataset
 from argilla_server.schemas.v1.records import (
@@ -128,7 +128,7 @@ async def _filter_records_using_search_engine(
         user=user,
         parsed_metadata=parsed_metadata,
         response_statuses=response_statuses,
-        sort_by_query_param=sort_by_query_param or LIST_DATASET_RECORDS_DEFAULT_SORT_BY,
+        sort_by_query_param=sort_by_query_param,
     )
 
     record_ids = [response.record_id for response in search_responses.items]
@@ -397,43 +397,59 @@ async def _get_vector_settings_by_name_or_raise(
     return vector_settings
 
 
-def convert_workspace_users_response_suggestions(
-        records: Records,
+def add_suggestions_from_responses(
+        records: List[Record],
         current_user: User, 
         workspace_users: List[User], 
         dataset: Dataset, 
-        ):
+    ) -> Records:
     user_id2name = {user.id: user.username for user in workspace_users.items}
-    question_name2id = {question.name: question for question in dataset.questions}
+    questions_name_map = {question.name: question for question in dataset.questions}
     
     for record in records:
         suggestion_responses = [response for response in record.responses \
                                 if response.user_id != current_user.id]
 
         for response in suggestion_responses:
-            if response.status == ResponseStatus.discarded: continue
-
-            for question_name, suggestion_value in response.values.items():
-                if question_name not in question_name2id or \
-                    not suggestion_value or not suggestion_value.get("value"): continue
-                question = question_name2id.get(question_name)
-
-                suggestion = Suggestion(
-                    id=response.id,
-                    question_id=question.id,
-                    type="human",
-                    value=suggestion_value.get("value"),
-                    agent=user_id2name.get(response.user_id),
-                    score=None,
-                    inserted_at=response.inserted_at,
-                    updated_at=response.updated_at
-                )
-                record.suggestions.append(suggestion)
+            suggestions = generate_suggestions_from_response(response, current_user, questions_name_map, user_id2name)
+            record.suggestions.extend(suggestions)
 
         if record.responses and record.responses[0].user_id == current_user.id:
             record.responses = [record.responses[0]]
         else:
             record.responses = []
+    
+    return records
+
+
+def generate_suggestions_from_response(
+        response: Response,
+        current_user: User,
+        questions_name_map: Dict[str, Question],
+        workspace_users_id2name: Dict[UUID, str]
+    ) -> List[Suggestion]:
+    suggestions = []
+    if response.user_id == current_user.id or response.status == ResponseStatus.discarded:
+        return suggestions
+
+    for question_name, suggestion_value in response.values.items():
+        if question_name not in questions_name_map or not suggestion_value or not suggestion_value.get("value"):
+            continue
+        question = questions_name_map.get(question_name)
+
+        suggestion = Suggestion(
+            id=response.id,
+            question_id=question.id,
+            type="human",
+            value=suggestion_value.get("value"),
+            agent=workspace_users_id2name.get(response.user_id),
+            score=None,
+            inserted_at=response.inserted_at,
+            updated_at=response.updated_at
+        )
+        suggestions.append(suggestion)
+    
+    return suggestions
 
 
 @router.get("/me/datasets/{dataset_id}/records", response_model=Records, response_model_exclude_unset=True)
@@ -469,7 +485,7 @@ async def list_current_user_dataset_records(
         user=current_user,
         response_statuses=response_statuses,
         include=include,
-        sort_by_query_param=sort_by_query_param or LIST_DATASET_RECORDS_DEFAULT_SORT_BY,
+        sort_by_query_param=sort_by_query_param,
         workspace_user_ids=workspace_user_ids,
     )
 
@@ -478,7 +494,7 @@ async def list_current_user_dataset_records(
         record.metadata_ = await _filter_record_metadata_for_user(record, current_user)
 
     if include and include.with_response_suggestions:
-        convert_workspace_users_response_suggestions(records, current_user, workspace_users, dataset)
+        records = add_suggestions_from_responses(records, current_user, workspace_users, dataset)
 
     return Records(items=records, total=total)
 
@@ -645,7 +661,7 @@ async def search_current_user_dataset_records(
         offset=offset,
         user=current_user,
         response_statuses=response_statuses,
-        sort_by_query_param=sort_by_query_param or LIST_DATASET_RECORDS_DEFAULT_SORT_BY,
+        sort_by_query_param=sort_by_query_param,
     )
 
     record_id_score_map: Dict[UUID, Dict[str, Union[float, SearchRecord, None]]] = {
@@ -663,7 +679,7 @@ async def search_current_user_dataset_records(
     )
 
     if include and include.with_response_suggestions:
-        convert_workspace_users_response_suggestions(records, current_user, workspace_users, dataset)
+        records = add_suggestions_from_responses(records, current_user, workspace_users, dataset)
 
     for record in records:
         record.dataset = dataset
@@ -713,7 +729,7 @@ async def search_dataset_records(
         offset=offset,
         parsed_metadata=metadata.metadata_parsed,
         response_statuses=response_statuses,
-        sort_by_query_param=sort_by_query_param or LIST_DATASET_RECORDS_DEFAULT_SORT_BY
+        sort_by_query_param=sort_by_query_param,
     )
 
     record_id_score_map = {

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/apis/v1/handlers/documents.py

@@ -5,7 +5,7 @@ from uuid import UUID
 from typing import TYPE_CHECKING, Optional, List, Union
 
 from argilla_server.schemas.v1.files import ObjectMetadata
-from fastapi import APIRouter, Body, Depends, HTTPException, UploadFile, Path, status, Security
+from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile, Path, status, Security
 from fastapi.responses import StreamingResponse
 from minio import Minio
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -17,7 +17,7 @@ from argilla_server.security import auth
 from argilla_server.policies import DocumentPolicy, authorize, is_authorized
 from argilla_server.models import User
 from argilla_server.contexts import accounts, datasets, files
-from argilla_server.schemas.v1.documents import DocumentCreate, DocumentDelete, DocumentListItem
+from argilla_server.schemas.v1.documents import DocumentCreateRequest, DocumentDeleteRequest, DocumentListItem
 
 if TYPE_CHECKING:
     from argilla_server.models import Document
@@ -26,7 +26,7 @@ _LOGGER = logging.getLogger("documents")
 
 router = APIRouter(tags=["documents"])
 
-async def check_existing_document(db: AsyncSession, document_create: DocumentCreate):
+async def check_existing_document(db: AsyncSession, document_create: DocumentCreateRequest):
     # Add conditions for non-empty attributes
     conditions = []
     if document_create.pmid:
@@ -37,6 +37,8 @@ async def check_existing_document(db: AsyncSession, document_create: DocumentCre
         conditions.append(Document.doi == document_create.doi)
     if document_create.id:
         conditions.append(Document.id == document_create.id)
+    if document_create.reference:
+        conditions.append(Document.reference == document_create.reference)
 
     if not conditions:
         return None
@@ -58,7 +60,8 @@ async def check_existing_document(db: AsyncSession, document_create: DocumentCre
 @router.post("/documents", status_code=status.HTTP_201_CREATED, response_model=UUID)
 async def upload_document(
     *,
-    document_create: DocumentCreate,
+    document_create: DocumentCreateRequest = Depends(),
+    file_data: UploadFile = File(None),
     db: AsyncSession = Depends(get_async_db),
     client: Minio = Depends(files.get_minio_client),
     current_user: User = Security(auth.get_current_user)
@@ -72,17 +75,33 @@ async def upload_document(
             detail=f"Workspace with id `{document_create.workspace_id}` not found",
         )
     
-    object_path = files.get_pdf_s3_object_path(document_create.id)
-    existing_files = files.list_objects(client, workspace.name, prefix=object_path, include_version=False)
-    if not existing_files.objects and document_create.file_data is not None:
-        file_data_bytes = base64.b64decode(document_create.file_data)
-        response = files.put_object(
-            client, bucket=workspace.name, object=object_path, data=file_data_bytes, 
-            size=len(file_data_bytes), content_type="application/pdf", 
-            metadata=document_create.dict(include={"file_name": True, "pmid": True, "doi": True}))
+    if file_data is not None:
+        object_path = files.get_pdf_s3_object_path(document_create.id)
+        existing_files = files.list_objects(client, workspace.name, prefix=object_path, include_version=False, recursive=False)
+        # file_data_bytes = base64.b64decode(file_data)
+        file_data_bytes = await file_data.read()
+
+        put_object = False
+    
+        if existing_files.objects:
+            new_file_hash = files.compute_hash(file_data_bytes)
+            existing_hashes = [existing_file.etag.strip('"') for existing_file in existing_files.objects if existing_file.etag is not None]
+            
+            if new_file_hash not in existing_hashes:
+                put_object = True
+        else:
+            put_object = True
         
-        if not document_create.url:
+        if put_object:
+            
+            response = files.put_object(
+                client, bucket=workspace.name, object=object_path, data=file_data_bytes, 
+                size=len(file_data_bytes), content_type="application/pdf", 
+                metadata=document_create.dict(include={"file_name": True, "pmid": True, "doi": True}))
+            
             document_create.url = files.get_s3_object_url(response.bucket_name, response.object_name)
+            if file_data.filename and not document_create.file_name:
+                document_create.file_name = file_data.filename
     
     existing_document = await check_existing_document(db, document_create)
     if existing_document is not None:
@@ -159,10 +178,10 @@ async def get_document_by_id(
     return DocumentListItem.from_orm(document)
 
 
-@router.delete("/documents/workspace/{workspace_id}", status_code=status.HTTP_200_OK)
+@router.delete("/documents/workspace/{workspace_id}", status_code=status.HTTP_200_OK, response_model=int, description="Delete all documents by workspace_id, or a specific document by id, pmid, doi, or url")
 async def delete_documents_by_workspace_id(*,
     workspace_id: Union[UUID, str],
-    document_delete: DocumentDelete = Body(None),
+    document_delete: DocumentDeleteRequest = Body(None),
     db: AsyncSession = Depends(get_async_db),
     client: Minio = Depends(files.get_minio_client),
     current_user: User = Security(auth.get_current_user)

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/apis/v1/handlers/files.py

@@ -39,11 +39,11 @@ async def get_file(
         )
     except S3Error as se:
         _LOGGER.error(f"Error getting object '{bucket}/{object}': {se}")
-        raise HTTPException(status_code=404, detail=f"No object at path '{bucket}/{object}' was found")
+        raise HTTPException(status_code=404, detail=f"No object at path '{bucket}/{object}' was found") from se
     
     except Exception as e:
         _LOGGER.error(f"Error getting object '{bucket}/{object}': {e}")
-        raise HTTPException(status_code=500, detail=str(e))
+        raise HTTPException(status_code=500, detail=str(e)) from e
 
     
 @router.post("/file/{bucket}/{object:path}", response_model=ObjectMetadata)
@@ -64,7 +64,7 @@ async def put_file(
                                     data=file.file, size=file.size, content_type=file.content_type)
         return response
     except S3Error as se:
-        raise HTTPException(status_code=500, detail="Internal server error")
+        raise HTTPException(status_code=500, detail="Internal server error") from se
     
 
 @router.get("/files/{bucket}/{prefix:path}", response_model=ListObjectsResponse)
@@ -73,6 +73,8 @@ async def list_objects(
     bucket: str,
     prefix: str,
     include_version=True,
+    recursive = True,
+    start_after: Optional[str] = None,
     client: Minio = Depends(files.get_minio_client),
     current_user: User = Security(auth.get_current_user),
     ):
@@ -80,10 +82,14 @@ async def list_objects(
     await authorize(current_user, FilePolicy.list(bucket))
 
     try:
-        objects = files.list_objects(client, bucket, prefix=prefix, include_version=include_version)
+        objects = files.list_objects(client, bucket, prefix=prefix, include_version=include_version, recursive=recursive, start_after=start_after)
         return objects
     except S3Error as se:
-        raise HTTPException(status_code=404, detail=f"No objects at prefix '{bucket}/{prefix}' were found")
+        _LOGGER.error(f"Error listing objects in '{bucket}/{prefix}': {se}")
+        if se.code == "NoSuchBucket":
+            raise HTTPException(status_code=404, detail=f"Bucket '{bucket}' not found, please run `rg.Workspace.create('{bucket}')` to create the S3 bucket.") from se
+        else:
+            raise HTTPException(status_code=404, detail=f"Cannot list objects as '{bucket}/{prefix}' is not found") from se
     except Exception as e:
         raise e
 
@@ -106,7 +112,7 @@ async def delete_files(
         files.delete_object(client, bucket, object, version_id=version_id)
         return {"message": "File deleted"}
     except S3Error as se:
-        raise HTTPException(status_code=500, detail="Internal server error")
+        raise HTTPException(status_code=500, detail="Internal server error") from se
     except Exception as e:
         raise e
 

extralit_server-0.2.1/src/argilla_server/apis/v1/handlers/models.py

@@ -0,0 +1,81 @@
+import logging
+from urllib.parse import urljoin
+
+import httpx
+from fastapi import APIRouter, Depends, HTTPException
+from starlette.requests import Request
+from starlette.responses import StreamingResponse
+
+from argilla_server.models import User
+from argilla_server.policies import _exists_workspace_user_by_user_and_workspace_name
+from argilla_server.security import auth
+from argilla_server.settings import settings
+from argilla_server.errors import UnauthorizedError, BadRequestError
+
+_LOGGER = logging.getLogger("models")
+
+router = APIRouter(tags=["models"])
+
+client = httpx.AsyncClient(timeout=10.0)
+
+@router.api_route("/models/{rest_of_path:path}", 
+                  methods=["GET", "POST", "PUT", "DELETE"], 
+                  response_class=StreamingResponse)
+async def proxy(request: Request, rest_of_path: str,
+                current_user: User = Depends(auth.get_current_user)):
+    url = urljoin(settings.extralit_url, rest_of_path)
+    params = dict(request.query_params)
+
+    print(f'PROXY {url} {params}', end='')
+    _LOGGER.info('PROXY %s %s', url, params)
+
+    if 'workspace' not in params or not params['workspace']:
+        raise BadRequestError("`workspace` is required in query parameters")
+
+    if current_user:
+        params['username'] = current_user.username
+
+        if current_user.role != "owner" and not await _exists_workspace_user_by_user_and_workspace_name(current_user, params['workspace']):
+            raise UnauthorizedError(f"{current_user.username} is not authorized to access workspace {params['workspace']}")
+
+    if request.method == "GET":
+        proxy_request = client.build_request("GET", url, params=params)
+    elif request.method == "POST":
+        data = await request.json()
+        print(data)
+        proxy_request = client.build_request("POST", url, json=data, params=params)
+    elif request.method == "PUT":
+        data = await request.json()
+        print(data)
+        proxy_request = client.build_request("PUT", url, data=data, params=params)
+    elif request.method == "DELETE":
+        proxy_request = client.build_request("DELETE", url, params=params)
+    else:
+        return {"message": "Method not supported"}
+
+    async def stream_response():
+        try:
+            response = await client.send(proxy_request, stream=True)
+            async for chunk in response.aiter_raw():
+                yield chunk
+        except httpx.ReadTimeout as exc:
+            _LOGGER.error("Request to %s timed out.", exc.request.url)
+            yield b"Request timed out."
+        except httpx.TimeoutException as exc:
+            _LOGGER.error("Request to %s timed out.", exc.request.url)
+            yield b"Request timed out."
+        except httpx.RequestError as exc:
+            _LOGGER.error("An error occurred while requesting %s: %s", exc.request.url, exc)
+            yield b"An error occurred while processing the request."
+
+    return StreamingResponse(stream_response(), media_type="text/event-stream")
+
+@router.on_event("startup")
+async def startup_event():
+    global client
+    if client is None:
+        client = httpx.AsyncClient(timeout=10.0)
+
+@router.on_event("shutdown")
+async def shutdown():
+    await client.aclose()

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/apis/v1/handlers/workspaces.py

@@ -21,7 +21,7 @@ from minio import Minio
 from argilla_server import models
 from argilla_server.contexts import accounts, datasets, files
 from argilla_server.database import get_async_db
-from argilla_server.errors import EntityAlreadyExistsError
+from argilla_server.errors import EntityAlreadyExistsError, GenericServerError
 from argilla_server.errors.future import NotUniqueError
 from argilla_server.models import User
 from argilla_server.policies import WorkspacePolicyV1, WorkspaceUserPolicyV1, authorize
@@ -58,9 +58,15 @@ async def create_workspace(
     db: AsyncSession = Depends(get_async_db),
     workspace_create: WorkspaceCreate,
     current_user: models.User = Security(auth.get_current_user),
+    minio_client: Minio = Depends(files.get_minio_client),
 ):
     await authorize(current_user, WorkspacePolicyV1.create)
 
+    try:
+        files.create_bucket(minio_client, workspace_create.name)
+    except Exception as e:
+        raise GenericServerError(e)
+
     try:
         workspace = await accounts.create_workspace(db, workspace_create.dict())
     except NotUniqueError as e:

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/cli/database/users/create.py

@@ -72,7 +72,9 @@ async def _create(
             typer.echo(f"User with username {username!r} already exists in database. Skipping...")
             return
 
-        if await accounts.get_user_by_api_key(session, api_key):
+        if not api_key:
+            api_key = None
+        elif await accounts.get_user_by_api_key(session, api_key):
             typer.echo(f"User with api_key {api_key!r} already exists in database. Skipping...")
             return
 

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/cli/search_engine/reindex.py

@@ -132,9 +132,17 @@ async def _reindex_datasets(db: AsyncSession, search_engine: SearchEngine, progr
     task = progress.add_task(f"reindexing feedback datasets...", total=await Reindexer.count_datasets(db))
 
     async for dataset in Reindexer.reindex_datasets(db, search_engine):
-        await _reindex_dataset_records(db, search_engine, progress, dataset)
-
-        progress.advance(task)
+        try:
+            await _reindex_dataset_records(db, search_engine, progress, dataset)
+        except Exception as e:
+            echo_in_panel(
+                f"Failed to reindex dataset `{dataset.name}`: {str(e)}",
+                title="Reindexing Error",
+                title_align="left",
+                success=False,
+            )
+        finally:
+            progress.advance(task)
 
 
 async def _reindex_dataset_records(

{extralit_server-0.2.0 → extralit_server-0.2.1}/src/argilla_server/contexts/datasets.py

@@ -81,7 +81,7 @@ from argilla_server.schemas.v1.vector_settings import (
 from argilla_server.schemas.v1.vector_settings import (
     VectorSettingsCreate,
 )
-from argilla_server.schemas.v1.documents import DocumentCreate, DocumentListItem
+from argilla_server.schemas.v1.documents import DocumentCreateRequest, DocumentListItem
 from argilla_server.schemas.v1.vectors import Vector as VectorSchema
 from argilla_server.search_engine import SearchEngine
 from argilla_server.validators.responses import (
@@ -807,6 +807,9 @@ async def _build_record_update(
 
     if record_update.suggestions is not None:
         params.pop("suggestions")
+        questions_ids = [suggestion.question_id for suggestion in record_update.suggestions]
+        if len(questions_ids) != len(set(questions_ids)):
+            raise ValueError("found duplicate suggestions question IDs")
         suggestions = await _build_record_suggestions(db, record, record_update.suggestions, caches["questions"])
 
     if record_update.vectors is not None:
@@ -1050,8 +1053,8 @@ async def update_response(
 ):
     ResponseUpdateValidator(response_update).validate_for(response.record)
 
-    if 'duration' in response.values:
-        if 'duration' in response_update.values:
+    if response.values and 'duration' in response.values:
+        if response_update.values and 'duration' in response_update.values:
             response_update.values['duration'].value = \
                 response.values['duration']['value'] + response_update.values['duration'].value
         else:
@@ -1080,14 +1083,6 @@ async def upsert_response(
 ) -> Response:
     ResponseUpsertValidator(response_upsert).validate_for(record)
 
-    if 'duration' in response.values:
-        if 'duration' in response_update.values:
-            response_update.values['duration'].value = \
-                response.values['duration']['value'] + response_update.values['duration'].value
-        else:
-            response_update.values['duration'] = ResponseValueUpdate(value=response.values['duration']['value'])
-
-
     schema = {
         "values": jsonable_encoder(response_upsert.values),
         "status": response_upsert.status,
@@ -1203,7 +1198,6 @@ async def get_suggestion_by_id(db: AsyncSession, suggestion_id: "UUID") -> Union
         .options(
             selectinload(Suggestion.record).selectinload(Record.dataset),
             selectinload(Suggestion.question),
-            selectinload(Suggestion.type),
         )
     )
 
@@ -1219,7 +1213,6 @@ async def list_suggestions_by_id_and_record_id(
         .options(
             selectinload(Suggestion.record).selectinload(Record.dataset),
             selectinload(Suggestion.question),
-            selectinload(Suggestion.type),
         )
     )
 
@@ -1243,7 +1236,7 @@ async def get_metadata_property_by_id(db: AsyncSession, metadata_property_id: UU
     return result.scalar_one_or_none()
 
 
-async def create_document(db: "AsyncSession", dataset_create: DocumentCreate) -> DocumentListItem:
+async def create_document(db: "AsyncSession", dataset_create: DocumentCreateRequest) -> DocumentListItem:
     document = await Document.create(
         db,
         id=dataset_create.id,