unique_toolkit 1.71.3__tar.gz → 1.72.0__tar.gz

{unique_toolkit-1.71.3 → unique_toolkit-1.72.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 1.71.3
+Version: 1.72.0
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>

{unique_toolkit-1.71.3 → unique_toolkit-1.72.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_toolkit"
-version = "1.71.3"
+version = "1.72.0"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"

{unique_toolkit-1.71.3 → unique_toolkit-1.72.0}/unique_toolkit/content/functions.py

@@ -876,17 +876,17 @@ def get_content_info(
     skip: int | None = None,
     take: int | None = None,
     file_path: str | None = None,
-):
+) -> PaginatedContentInfos:
     """Gets the info of a content."""
 
     get_info_params = unique_sdk.Content.ContentInfoParams(
         metadataFilter=metadata_filter or None,  # Dict cannot be empty
     )
-    if skip:
+    if skip is not None:
         get_info_params["skip"] = skip
-    if take:
+    if take is not None:
         get_info_params["take"] = take
-    if file_path:
+    if file_path is not None:
         get_info_params["filePath"] = file_path
 
     content_info = unique_sdk.Content.get_infos(
@@ -897,6 +897,35 @@ def get_content_info(
     )
 
 
+async def get_content_info_async(
+    user_id: str,
+    company_id: str,
+    *,
+    metadata_filter: dict[str, Any] | None = None,
+    skip: int | None = None,
+    take: int | None = None,
+    file_path: str | None = None,
+) -> PaginatedContentInfos:
+    """Gets the info of a content."""
+
+    get_info_params = unique_sdk.Content.ContentInfoParams(
+        metadataFilter=metadata_filter or None,  # Dict cannot be empty
+    )
+    if skip is not None:
+        get_info_params["skip"] = skip
+    if take is not None:
+        get_info_params["take"] = take
+    if file_path is not None:
+        get_info_params["filePath"] = file_path
+
+    content_info = unique_sdk.Content.get_infos_async(
+        user_id=user_id, company_id=company_id, **get_info_params
+    )
+    return PaginatedContentInfos.model_validate(
+        await content_info, by_alias=True, by_name=True
+    )
+
+
 def get_folder_info(user_id: str, company_id: str, *, scope_id: str) -> FolderInfo:
     info = unique_sdk.Folder.get_info(
         user_id=user_id, company_id=company_id, scopeId=scope_id
@@ -905,6 +934,16 @@ def get_folder_info(user_id: str, company_id: str, *, scope_id: str) -> FolderIn
     return FolderInfo.model_validate(info, by_alias=True, by_name=True)
 
 
+async def get_folder_info_async(
+    user_id: str, company_id: str, *, scope_id: str
+) -> FolderInfo:
+    info = unique_sdk.Folder.get_info_async(
+        user_id=user_id, company_id=company_id, scopeId=scope_id
+    )
+
+    return FolderInfo.model_validate(await info, by_alias=True, by_name=True)
+
+
 def update_content(
     user_id: str,
     company_id: str,

{unique_toolkit-1.71.3 → unique_toolkit-1.72.0}/unique_toolkit/services/knowledge_base.py

@@ -23,7 +23,9 @@ from unique_toolkit.content.functions import (
     download_content_to_bytes_async,
     download_content_to_file_by_id,
     get_content_info,
+    get_content_info_async,
     get_folder_info,
+    get_folder_info_async,
     search_content_chunks,
     search_content_chunks_async,
     search_contents,
@@ -685,6 +687,80 @@ class KnowledgeBaseService:
             file_path=file_path,
         )
 
+    async def get_paginated_content_infos_async(
+        self,
+        *,
+        metadata_filter: dict[str, Any] | None = None,
+        skip: int | None = None,
+        take: int | None = None,
+        file_path: str | None = None,
+    ) -> PaginatedContentInfos:
+        return await get_content_info_async(
+            user_id=self._user_id,
+            company_id=self._company_id,
+            metadata_filter=metadata_filter,
+            skip=skip,
+            take=take,
+            file_path=file_path,
+        )
+
+    async def get_content_infos_async(
+        self,
+        *,
+        metadata_filter: dict[str, Any] | None = None,
+        step_size: int = 100,
+        max_concurrent_requests: int = 10,
+    ) -> list[ContentInfo]:
+        """
+        Fetches all content infos from the knowledge base using parallel pagination.
+        The API limits responses to 100 items per request, so this method fetches
+        the total count first, then retrieves all pages concurrently (bounded by
+        ``max_concurrent_requests`` to avoid rate limiting or connection exhaustion).
+
+        Args:
+            metadata_filter (dict[str, Any] | None): The metadata filter to use. Defaults to None.
+            step_size (int): Number of items per page. Defaults to 100.
+            max_concurrent_requests (int): Maximum number of concurrent API calls.
+                Defaults to 10.
+
+        Returns:
+            list[ContentInfo]: All content infos visible to the user.
+        """
+
+        info_for_count_of_total_content = await self.get_paginated_content_infos_async(
+            metadata_filter=metadata_filter,
+            take=1,
+        )
+
+        total_count = info_for_count_of_total_content.total_count
+
+        semaphore = asyncio.Semaphore(max_concurrent_requests)
+
+        async def _fetch_page(skip: int) -> PaginatedContentInfos:
+            async with semaphore:
+                return await self.get_paginated_content_infos_async(
+                    metadata_filter=metadata_filter,
+                    skip=skip,
+                    take=step_size,
+                )
+
+        results: list[PaginatedContentInfos | BaseException] = await asyncio.gather(
+            *[_fetch_page(i) for i in range(0, total_count, step_size)],
+            return_exceptions=True,
+        )
+
+        # Log any exceptions that occurred during parallel fetching
+        for result in results:
+            if isinstance(result, BaseException):
+                _LOGGER.error("Error fetching paginated content infos", exc_info=result)
+
+        return [
+            content_info
+            for result in results
+            if not isinstance(result, BaseException)
+            for content_info in result.content_infos
+        ]
+
     def get_file_names_in_folder(self, *, scope_id: str) -> list[str]:
         """
         Get the list of file names in a knowledge base folder
@@ -717,71 +793,132 @@ class KnowledgeBaseService:
             scope_id=scope_id,
         )
 
-    def _resolve_visible_file_tree(self, content_infos: list[ContentInfo]) -> list[str]:
-        # collect all scope ids
-        folder_id_paths: set[str] = set()
-        known_folder_paths: set[str] = set()
-        for content_info in content_infos:
-            if (
-                content_info.metadata
-                and content_info.metadata.get(r"{FullPath}") is not None
-            ):
-                known_folder_paths.add(str(content_info.metadata.get(r"{FullPath}")))
-                continue
+    async def get_folder_info_async(
+        self,
+        *,
+        scope_id: str,
+    ) -> FolderInfo:
+        return await get_folder_info_async(
+            user_id=self._user_id,
+            company_id=self._company_id,
+            scope_id=scope_id,
+        )
 
+    # File Tree Resolution
+    # ------------------------------------------------------------------------------------------------
+
+    @staticmethod
+    def extract_scope_ids(content_infos: list[ContentInfo]) -> set[str]:
+        """Extracts all unique scope IDs from the ``folderIdPath`` metadata field.
+
+        Args:
+            content_infos: The content infos to extract scope IDs from.
+
+        Returns:
+            set[str]: All unique scope IDs found across content infos.
+        """
+        scope_ids: set[str] = set()
+        for content_info in content_infos:
             if (
                 content_info.metadata
-                and content_info.metadata.get("folderIdPath") is not None
+                and (folder_id_path := content_info.metadata.get("folderIdPath"))
+                is not None
+                and isinstance(folder_id_path, str)
             ):
-                folder_id_paths.add(str(content_info.metadata.get("folderIdPath")))
+                scope_ids.update(
+                    sid
+                    for sid in folder_id_path.replace("uniquepathid://", "").split("/")
+                    if sid
+                )
+        return scope_ids
 
-        scope_ids: set[str] = set()
-        for fp in folder_id_paths:
-            scope_ids_list = set(fp.replace("uniquepathid://", "").split("/"))
-            scope_ids.update(scope_ids_list)
+    async def _translate_scope_id_async(self, scope_id: str) -> str | None:
+        """Resolve a single scope ID to its folder name.
 
-        scope_id_to_folder_name: dict[str, str] = {}
-        for scope_id in scope_ids:
-            folder_info = self.get_folder_info(
-                scope_id=scope_id,
+        Returns the folder name, or ``None`` if the lookup fails.
+        """
+        try:
+            folder_info = await self.get_folder_info_async(scope_id=scope_id)
+            return folder_info.name
+        except Exception as e:
+            _LOGGER.warning(
+                f"Could not resolve folder for scope_id {scope_id}", exc_info=e
             )
-            scope_id_to_folder_name[scope_id] = folder_info.name
+            return None
 
-        folder_paths: set[str] = set()
-        for folder_id_path in folder_id_paths:
-            scope_ids_list = folder_id_path.replace("uniquepathid://", "").split("/")
+    async def _translate_scope_ids_async(
+        self,
+        scope_ids: set[str],
+        *,
+        max_concurrent_requests: int = 25,
+    ) -> dict[str, str]:
+        """Translate a set of scope IDs to folder names concurrently.
 
-            if all(scope_id in scope_id_to_folder_name for scope_id in scope_ids_list):
-                folder_path = [
-                    scope_id_to_folder_name[scope_id] for scope_id in scope_ids_list
-                ]
-                folder_paths.add("/".join(folder_path))
+        Scope IDs that cannot be resolved are silently omitted from the result.
 
-        return [
-            p if p.startswith("/") else f"/{p}"
-            for p in folder_paths.union(known_folder_paths)
-        ]
+        Args:
+            scope_ids: The scope IDs to translate.
+            max_concurrent_requests: Maximum number of concurrent API calls.
+                Defaults to 25.
 
-    def resolve_visible_file_tree(
-        self, *, metadata_filter: dict[str, Any] | None = None
-    ) -> list[str]:
+        Returns:
+            dict[str, str]: Mapping from scope ID to folder name.
         """
-        Resolves the visible file tree for the knowledge base for the current user.
+        scope_id_list = list(scope_ids)
+        semaphore = asyncio.Semaphore(max_concurrent_requests)
 
-        Args:
-            metadata_filter (dict[str, Any] | None): The metadata filter to use. Defaults to None.
+        async def _resolve(sid: str) -> str | None:
+            async with semaphore:
+                return await self._translate_scope_id_async(sid)
 
-        Returns:
-            list[str]: The visible file tree.
+        results = await asyncio.gather(*[_resolve(sid) for sid in scope_id_list])
+        return {
+            sid: name for sid, name in zip(scope_id_list, results) if name is not None
+        }
 
+    async def resolve_visible_file_paths_async(
+        self,
+        *,
+        metadata_filter: dict[str, Any] | None = None,
+    ) -> list[tuple[ContentInfo, list[str]]]:
+        """Resolves file paths visible to the current user asynchronously.
 
+        Returns each content item paired with its resolved file path segments.
 
+        Args:
+            metadata_filter: Optional metadata filter to narrow the content scope.
+
+        Returns:
+            list[tuple[ContentInfo, list[str]]]: Each tuple is
+                ``(content_info, [folder1, folder2, ..., filename])``.
         """
-        info = self.get_paginated_content_infos(
-            metadata_filter=metadata_filter,
+        content_infos = await self.get_content_infos_async(
+            metadata_filter=metadata_filter
         )
 
-        return self._resolve_visible_file_tree(content_infos=info.content_infos)
+        scope_ids = self.extract_scope_ids(content_infos)
+        scope_id_to_folder_name = await self._translate_scope_ids_async(scope_ids)
+
+        resolved: list[tuple[ContentInfo, list[str]]] = []
+        for content_info in content_infos:
+            if (
+                content_info.metadata
+                and (folder_id_path := content_info.metadata.get("folderIdPath"))
+                is not None
+                and isinstance(folder_id_path, str)
+            ):
+                file_path = [
+                    scope_id_to_folder_name.get(sid, sid)
+                    for sid in folder_id_path.replace("uniquepathid://", "").split("/")
+                    if sid
+                ]
+            else:
+                file_path = ["_no_folder_path"]
+
+            file_path.append(content_info.key)
+            resolved.append((content_info, file_path))
+
+        return resolved
 
     def _pop_forbidden_metadata_keys(self, metadata: dict[str, Any]) -> dict[str, Any]:
         forbidden_keys = [