unique_toolkit 0.7.9__py3-none-any.whl → 1.33.3__py3-none-any.whl

unique_toolkit/content/functions.py

@@ -3,7 +3,9 @@ import os
 import re
 import tempfile
 from pathlib import Path
+from typing import Any
 
+import httpx
 import requests
 import unique_sdk
 
@@ -12,8 +14,12 @@ from unique_toolkit.content.constants import DEFAULT_SEARCH_LANGUAGE
 from unique_toolkit.content.schemas import (
     Content,
     ContentChunk,
+    ContentInfo,
     ContentRerankerConfig,
     ContentSearchType,
+    DeleteContentResponse,
+    FolderInfo,
+    PaginatedContentInfos,
 )
 from unique_toolkit.content.utils import map_contents, map_to_content_chunks
 
@@ -33,6 +39,7 @@ def search_content_chunks(
     chat_only: bool | None = None,
     metadata_filter: dict | None = None,
     content_ids: list[str] | None = None,
+    score_threshold: float | None = None,
 ) -> list[ContentChunk]:
     """
     Performs a synchronous search for content chunks in the knowledge base.
@@ -47,6 +54,7 @@ def search_content_chunks(
         chat_only (bool | None): Whether to search only in the current chat. Defaults to None.
         metadata_filter (dict | None): UniqueQL metadata filter. If unspecified/None, it tries to use the metadata filter from the event. Defaults to None.
         content_ids (list[str] | None): The content IDs to search. Defaults to None.
+        score_threshold (float | None): The minimum score threshold for results. Defaults to 0.
     Returns:
         list[ContentChunk]: The search results.
     """
@@ -72,6 +80,7 @@ def search_content_chunks(
             chatOnly=chat_only,
             metaDataFilter=metadata_filter,
             contentIds=content_ids,
+            scoreThreshold=score_threshold,
         )
         return map_to_content_chunks(searches)
     except Exception as e:
@@ -92,6 +101,7 @@ async def search_content_chunks_async(
     chat_only: bool | None = None,
     metadata_filter: dict | None = None,
     content_ids: list[str] | None = None,
+    score_threshold: float | None = None,
 ):
     """
     Performs an asynchronous search for content chunks in the knowledge base.
@@ -120,6 +130,7 @@ async def search_content_chunks_async(
             chatOnly=chat_only,
             metaDataFilter=metadata_filter,
             contentIds=content_ids,
+            scoreThreshold=score_threshold,
         )
         return map_to_content_chunks(searches)
     except Exception as e:
@@ -132,7 +143,8 @@ def search_contents(
     company_id: str,
     chat_id: str,
     where: dict,
-):
+    include_failed_content: bool = False,
+) -> list[Content]:
     """
     Performs an asynchronous search for content files in the knowledge base by filter.
 
@@ -155,6 +167,7 @@ def search_contents(
             chatId=chat_id,
             # TODO add type parameter in SDK
             where=where,  # type: ignore
+            includeFailedContent=include_failed_content,
         )
         return map_contents(contents)
     except Exception as e:
@@ -167,6 +180,7 @@ async def search_contents_async(
     company_id: str,
     chat_id: str,
     where: dict,
+    include_failed_content: bool = False,
 ):
     """Asynchronously searches for content in the knowledge base."""
     if where.get("contentId"):
@@ -178,6 +192,7 @@ async def search_contents_async(
             company_id=company_id,
             chatId=chat_id,
             where=where,  # type: ignore
+            includeFailedContent=include_failed_content,
         )
         return map_contents(contents)
     except Exception as e:
@@ -204,6 +219,73 @@ def _upsert_content(
     )  # type: ignore
 
 
+async def _upsert_content_async(
+    user_id: str,
+    company_id: str,
+    input_data: dict,
+    scope_id: str | None = None,
+    chat_id: str | None = None,
+    file_url: str | None = None,
+):
+    return await unique_sdk.Content.upsert_async(
+        user_id=user_id,
+        company_id=company_id,
+        input=input_data,  # type: ignore
+        scopeId=scope_id,
+        chatId=chat_id,
+        fileUrl=file_url,
+    )
+
+
+async def upload_content_from_bytes_async(
+    user_id: str,
+    company_id: str,
+    content: bytes,
+    content_name: str,
+    mime_type: str,
+    scope_id: str | None = None,
+    chat_id: str | None = None,
+    skip_ingestion: bool = False,
+    ingestion_config: unique_sdk.Content.IngestionConfig | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> Content:
+    """
+    Asynchronously uploads content to the knowledge base.
+
+    Args:
+        user_id (str): The user ID.
+        company_id (str): The company ID.
+        content (bytes): The content to upload.
+        content_name (str): The name of the content.
+        mime_type (str): The MIME type of the content.
+        scope_id (str | None): The scope ID. Defaults to None.
+        chat_id (str | None): The chat ID. Defaults to None.
+        skip_ingestion (bool): Whether to skip ingestion. Defaults to False.
+        ingestion_config (unique_sdk.Content.IngestionConfig | None): The ingestion configuration. Defaults to None.
+        metadata ( dict[str, Any] | None): The metadata for the content. Defaults to None.
+
+    Returns:
+        Content: The uploaded content.
+    """
+
+    try:
+        return await _trigger_upload_content_async(
+            user_id=user_id,
+            company_id=company_id,
+            content=content,
+            content_name=content_name,
+            mime_type=mime_type,
+            scope_id=scope_id,
+            chat_id=chat_id,
+            skip_ingestion=skip_ingestion,
+            ingestion_config=ingestion_config,
+            metadata=metadata,
+        )
+    except Exception as e:
+        logger.error(f"Error while uploading content: {e}")
+        raise e
+
+
 def upload_content_from_bytes(
     user_id: str,
     company_id: str,
@@ -213,7 +295,9 @@ def upload_content_from_bytes(
     scope_id: str | None = None,
     chat_id: str | None = None,
     skip_ingestion: bool = False,
-):
+    ingestion_config: unique_sdk.Content.IngestionConfig | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> Content:
     """
     Uploads content to the knowledge base.
 
@@ -226,6 +310,8 @@ def upload_content_from_bytes(
         scope_id (str | None): The scope ID. Defaults to None.
         chat_id (str | None): The chat ID. Defaults to None.
         skip_ingestion (bool): Whether to skip ingestion. Defaults to False.
+        ingestion_config (unique_sdk.Content.IngestionConfig | None): The ingestion configuration. Defaults to None.
+        metadata ( dict[str, Any] | None): The metadata for the content. Defaults to None.
 
     Returns:
         Content: The uploaded content.
@@ -241,6 +327,8 @@ def upload_content_from_bytes(
             scope_id=scope_id,
             chat_id=chat_id,
             skip_ingestion=skip_ingestion,
+            ingestion_config=ingestion_config,
+            metadata=metadata,
         )
     except Exception as e:
         logger.error(f"Error while uploading content: {e}")
@@ -256,7 +344,10 @@ def upload_content(
     scope_id: str | None = None,
     chat_id: str | None = None,
     skip_ingestion: bool = False,
-):
+    skip_excel_ingestion: bool = False,
+    ingestion_config: unique_sdk.Content.IngestionConfig | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> Content:
     """
     Uploads content to the knowledge base.
 
@@ -269,6 +360,9 @@ def upload_content(
         scope_id (str | None): The scope ID. Defaults to None.
         chat_id (str | None): The chat ID. Defaults to None.
         skip_ingestion (bool): Whether to skip ingestion. Defaults to False.
+        skip_excel_ingestion (bool): Whether to skip excel ingestion. Defaults to False.
+        ingestion_config (unique_sdk.Content.IngestionConfig | None): The ingestion configuration. Defaults to None.
+        metadata ( dict[str, Any] | None): The metadata for the content. Defaults to None.
 
     Returns:
         Content: The uploaded content.
@@ -284,6 +378,9 @@ def upload_content(
             scope_id=scope_id,
             chat_id=chat_id,
             skip_ingestion=skip_ingestion,
+            skip_excel_ingestion=skip_excel_ingestion,
+            ingestion_config=ingestion_config,
+            metadata=metadata,
         )
     except Exception as e:
         logger.error(f"Error while uploading content: {e}")
@@ -299,7 +396,10 @@ def _trigger_upload_content(
     scope_id: str | None = None,
     chat_id: str | None = None,
     skip_ingestion: bool = False,
-):
+    skip_excel_ingestion: bool = False,
+    ingestion_config: unique_sdk.Content.IngestionConfig | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> Content:
     """
     Uploads content to the knowledge base.
 
@@ -312,6 +412,9 @@ def _trigger_upload_content(
         scope_id (str | None): The scope ID. Defaults to None.
         chat_id (str | None): The chat ID. Defaults to None.
         skip_ingestion (bool): Whether to skip ingestion. Defaults to False.
+        skip_excel_ingestion (bool): Whether to skip excel ingestion. Defaults to False.
+        ingestion_config (unique_sdk.Content.IngestionConfig | None): The ingestion configuration. Defaults to None.
+        metadata (dict[str, Any] | None): The metadata for the content. Defaults to None.
 
     Returns:
         Content: The uploaded content.
@@ -368,16 +471,23 @@ def _trigger_upload_content(
         logger.error(error_msg)
         raise ValueError(error_msg)
 
+    if ingestion_config is None:
+        ingestion_config = {}
+
+    if skip_excel_ingestion:
+        ingestion_config["uniqueIngestionMode"] = "SKIP_EXCEL_INGESTION"
+    elif skip_ingestion:
+        ingestion_config["uniqueIngestionMode"] = "SKIP_INGESTION"
+
     input_dict = {
         "key": content_name,
         "title": content_name,
         "mimeType": mime_type,
         "byteSize": byte_size,
+        "ingestionConfig": ingestion_config,
+        "metadata": metadata,
     }
 
-    if skip_ingestion:
-        input_dict["ingestionConfig"] = {"uniqueIngestionMode": "SKIP_INGESTION"}
-
     if chat_id:
         _upsert_content(
             user_id=user_id,
@@ -395,7 +505,131 @@ def _trigger_upload_content(
             scope_id=scope_id,
         )  # type: ignore
 
-    return Content(**created_content)
+    return Content.model_validate(created_content, by_alias=True, by_name=True)
+
+
+async def _trigger_upload_content_async(
+    user_id: str,
+    company_id: str,
+    content: str | Path | bytes,
+    content_name: str,
+    mime_type: str,
+    scope_id: str | None = None,
+    chat_id: str | None = None,
+    skip_ingestion: bool = False,
+    skip_excel_ingestion: bool = False,
+    ingestion_config: unique_sdk.Content.IngestionConfig | None = None,
+    metadata: dict[str, Any] | None = None,
+):
+    """
+    Asynchronously uploads content to the knowledge base.
+
+    Args:
+        user_id (str): The user ID.
+        company_id (str): The company ID.
+        content (str | Path | bytes): The content to upload. If string or Path, file will be read from disk.
+        content_name (str): The name of the content.
+        mime_type (str): The MIME type of the content.
+        scope_id (str | None): The scope ID. Defaults to None.
+        chat_id (str | None): The chat ID. Defaults to None.
+        skip_ingestion (bool): Whether to skip ingestion. Defaults to False.
+        skip_excel_ingestion (bool): Whether to skip excel ingestion. Defaults to False.
+        ingestion_config (unique_sdk.Content.IngestionConfig | None): The ingestion configuration. Defaults to None.
+        metadata (dict[str, Any] | None): The metadata for the content. Defaults to None.
+
+    Returns:
+        Content: The uploaded content.
+    """
+    # TODO: Remove code duplication with _trigger_upload_content
+
+    if not chat_id and not scope_id:
+        raise ValueError("chat_id or scope_id must be provided")
+
+    byte_size = (
+        os.path.getsize(content) if isinstance(content, (Path, str)) else len(content)
+    )
+    created_content = await _upsert_content_async(
+        user_id=user_id,
+        company_id=company_id,
+        input_data={
+            "key": content_name,
+            "title": content_name,
+            "mimeType": mime_type,
+        },
+        scope_id=scope_id,
+        chat_id=chat_id,
+    )
+
+    write_url = created_content["writeUrl"]
+
+    if not write_url:
+        error_msg = "Write url for uploaded content is missing"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    headers = {
+        "X-Ms-Blob-Content-Type": mime_type,
+        "X-Ms-Blob-Type": "BlockBlob",
+    }
+    # upload to azure blob storage SAS url uploadUrl the pdf file translatedFile make sure it is treated as a application/pdf
+    async with httpx.AsyncClient() as client:
+        if isinstance(content, bytes):
+            response = await client.put(
+                url=write_url,
+                content=content,
+                headers=headers,
+            )
+        else:
+            with open(content, "rb") as file:
+                response = await client.put(
+                    url=write_url,
+                    content=file,
+                    headers=headers,
+                )
+        response.raise_for_status()
+
+    read_url = created_content["readUrl"]
+
+    if not read_url:
+        error_msg = "Read url for uploaded content is missing"
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    if ingestion_config is None:
+        ingestion_config = {}
+
+    if skip_excel_ingestion:
+        ingestion_config["uniqueIngestionMode"] = "SKIP_EXCEL_INGESTION"
+    elif skip_ingestion:
+        ingestion_config["uniqueIngestionMode"] = "SKIP_INGESTION"
+
+    input_dict = {
+        "key": content_name,
+        "title": content_name,
+        "mimeType": mime_type,
+        "byteSize": byte_size,
+        "ingestionConfig": ingestion_config,
+        "metadata": metadata,
+    }
+
+    if chat_id:
+        await _upsert_content_async(
+            user_id=user_id,
+            company_id=company_id,
+            input_data=input_dict,
+            file_url=read_url,
+            chat_id=chat_id,
+        )
+    else:
+        await _upsert_content_async(
+            user_id=user_id,
+            company_id=company_id,
+            input_data=input_dict,
+            file_url=read_url,
+            scope_id=scope_id,
+        )
+
+    return Content.model_validate(created_content, by_alias=True, by_name=True)
 
 
 def request_content_by_id(
@@ -543,3 +777,114 @@ def download_content(
         raise Exception(error_msg)
 
     return content_path
+
+
+def get_content_info(
+    user_id: str,
+    company_id: str,
+    *,
+    metadata_filter: dict[str, Any] | None = None,
+    skip: int | None = None,
+    take: int | None = None,
+    file_path: str | None = None,
+):
+    """Gets the info of a content."""
+
+    get_info_params = unique_sdk.Content.ContentInfoParams(
+        metadataFilter=metadata_filter or None,  # Dict cannot be empty
+    )
+    if skip:
+        get_info_params["skip"] = skip
+    if take:
+        get_info_params["take"] = take
+    if file_path:
+        get_info_params["filePath"] = file_path
+
+    content_info = unique_sdk.Content.get_infos(
+        user_id=user_id, company_id=company_id, **get_info_params
+    )
+    return PaginatedContentInfos.model_validate(
+        content_info, by_alias=True, by_name=True
+    )
+
+
+def get_folder_info(user_id: str, company_id: str, *, scope_id: str) -> FolderInfo:
+    info = unique_sdk.Folder.get_info(
+        user_id=user_id, company_id=company_id, scopeId=scope_id
+    )
+
+    return FolderInfo.model_validate(info, by_alias=True, by_name=True)
+
+
+def update_content(
+    user_id: str,
+    company_id: str,
+    *,
+    content_id: str,
+    metadata: dict[str, Any],
+    file_path: str | None = None,
+    owner_id: str | None = None,
+    parent_folder_path: str | None = None,
+    title: str | None = None,
+) -> ContentInfo:
+    """Updates the metadata of a content."""
+
+    update_params = unique_sdk.Content.UpdateParams(
+        contentId=content_id, metadata=metadata
+    )
+
+    if file_path:
+        update_params["filePath"] = file_path
+    if owner_id:
+        update_params["ownerId"] = owner_id
+    if parent_folder_path:
+        update_params["parentFolderPath"] = parent_folder_path
+    if title:
+        update_params["title"] = title
+
+    content_info = unique_sdk.Content.update(
+        user_id=user_id, company_id=company_id, **update_params
+    )
+    return ContentInfo.model_validate(content_info, by_alias=True, by_name=True)
+
+
+def delete_content(
+    user_id: str,
+    company_id: str,
+    *,
+    content_id: str | None = None,
+    file_path: str | None = None,
+) -> DeleteContentResponse:
+    if content_id:
+        resp = unique_sdk.Content.delete(
+            user_id=user_id, company_id=company_id, contentId=content_id
+        )
+    elif file_path:
+        resp = unique_sdk.Content.delete(
+            user_id=user_id, company_id=company_id, filePath=file_path
+        )
+    else:
+        raise ValueError("content_id or file_path must be provided")
+
+    return DeleteContentResponse.model_validate(resp, by_alias=True, by_name=True)
+
+
+async def delete_content_async(
+    user_id: str,
+    company_id: str,
+    *,
+    content_id: str | None = None,
+    file_path: str | None = None,
+) -> DeleteContentResponse:
+    if content_id:
+        resp = await unique_sdk.Content.delete_async(
+            user_id=user_id, company_id=company_id, contentId=content_id
+        )
+    elif file_path:
+        resp = await unique_sdk.Content.delete_async(
+            user_id=user_id, company_id=company_id, filePath=file_path
+        )
+    else:
+        raise ValueError("content_id or file_path must be provided")
+
+    return DeleteContentResponse.model_validate(resp, by_alias=True, by_name=True)

unique_toolkit/content/schemas.py

@@ -1,7 +1,8 @@
 from datetime import datetime
 from enum import StrEnum
-from typing import Optional
+from typing import Any, Optional
 
+import unique_sdk
 from humps import camelize
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -26,15 +27,41 @@ class ContentMetadata(BaseModel):
 
 class ContentChunk(BaseModel):
     model_config = model_config
-    id: str
-    text: str
-    order: int
-    key: str | None = None
-    chunk_id: str | None = None
-    url: str | None = None
-    title: str | None = None
-    start_page: int | None = None
-    end_page: int | None = None
+    id: str = Field(
+        default="",
+        description="The id of the content this chunk belongs to. The id starts with 'cont_' followed by an alphanumeric string of length 24.",
+        examples=["cont_abcdefgehijklmnopqrstuvwx"],
+    )
+    text: str = Field(default="", description="The text content of the chunk.")
+    order: int = Field(
+        default=0,
+        description="The order of the chunk in the original content. Concatenating the chunks in order will give the original content.",
+    )
+    key: str | None = Field(
+        default=None,
+        description="The key of the chunk. For document chunks this is the the filename",
+    )
+    chunk_id: str | None = Field(
+        default=None,
+        description="The id of the chunk. The id starts with 'chunk_' followed by an alphanumeric string of length 24.",
+        examples=["chunk_abcdefgehijklmnopqrstuv"],
+    )
+    url: str | None = Field(
+        default=None,
+        description="For chunk retrieved from the web this is the url of the chunk.",
+    )
+    title: str | None = Field(
+        default=None,
+        description="The title of the chunk. For document chunks this is the title of the document.",
+    )
+    start_page: int | None = Field(
+        default=None,
+        description="The start page of the chunk. For document chunks this is the start page of the document.",
+    )
+    end_page: int | None = Field(
+        default=None,
+        description="The end page of the chunk. For document chunks this is the end page of the document.",
+    )
 
     object: str | None = None
     metadata: ContentMetadata | None = None
@@ -45,26 +72,66 @@ class ContentChunk(BaseModel):
 
 class Content(BaseModel):
     model_config = model_config
-    id: str
-    key: str
-    title: str | None = None
+    id: str = Field(
+        default="",
+        description="The id of the content. The id starts with 'cont_' followed by an alphanumeric string of length 24.",
+        examples=["cont_abcdefgehijklmnopqrstuvwx"],
+    )
+    key: str = Field(
+        default="",
+        description="The key of the content. For documents this is the the filename",
+    )
+    title: str | None = Field(
+        default=None,
+        description="The title of the content. For documents this is the title of the document.",
+    )
     url: str | None = None
     chunks: list[ContentChunk] = []
     write_url: str | None = None
     read_url: str | None = None
     created_at: datetime | None = None
     updated_at: datetime | None = None
+    expired_at: datetime | None = None
+    metadata: dict[str, Any] | None = None
+    ingestion_config: dict | None = None
+    ingestion_state: str | None = None
 
 
 class ContentReference(BaseModel):
     model_config = model_config
-    id: str
-    message_id: str
+    id: str = Field(
+        default="",
+        description="The id of the content reference. Can be empty on the ChatMessage Object",
+    )
+    message_id: str = Field(
+        default="",
+        description="The id of the message that this reference belongs to. Can be empty on the ChatMessage Object",
+    )
     name: str
     sequence_number: int
     source: str
     source_id: str
     url: str
+    original_index: list[int] = Field(
+        default=[],
+        description="List of indices in the ChatMessage original_content this reference refers to. This is usually the id in the functionCallResponse. List type due to implementation in node-chat",
+    )
+
+    @classmethod
+    def from_sdk_reference(
+        cls, reference: unique_sdk.Message.Reference | unique_sdk.Space.Reference
+    ) -> "ContentReference":
+        kwargs = {
+            "name": reference["name"],
+            "url": reference["url"],
+            "sequence_number": reference["sequenceNumber"],
+            "source": reference["source"],
+            "source_id": reference["sourceId"],
+        }
+        if "originalIndex" in reference:
+            kwargs["original_index"] = reference["originalIndex"]
+
+        return cls.model_validate(kwargs)
 
 
 class ContentSearchType(StrEnum):
@@ -101,3 +168,49 @@ class ContentRerankerConfig(BaseModel):
     model_config = model_config
     deployment_name: str = Field(serialization_alias="deploymentName")
     options: dict | None = None
+
+
+class ContentInfo(BaseModel):
+    model_config = model_config
+    id: str
+    object: str
+    key: str
+    url: str | None = None
+    title: str | None = None
+    metadata: dict[str, Any] | None = None
+    byte_size: int
+    mime_type: str
+    owner_id: str
+    created_at: datetime
+    updated_at: datetime
+    expires_at: datetime | None = None
+    deleted_at: datetime | None = None
+    expired_at: datetime | None = None
+
+
+class PaginatedContentInfos(BaseModel):
+    model_config = model_config
+    object: str
+    content_infos: list[ContentInfo]
+    total_count: int
+
+
+class BaseFolderInfo(BaseModel):
+    model_config = model_config
+    id: str
+    name: str
+    parent_id: str | None
+
+
+class FolderInfo(BaseFolderInfo):
+    model_config = model_config
+    ingestion_config: dict[str, Any]
+    created_at: str | None
+    updated_at: str | None
+    external_id: str | None
+
+
+class DeleteContentResponse(BaseModel):
+    model_config = model_config
+    content_id: str
+    object: str