admin-api-lib 3.4.0__py3-none-any.whl → 4.1.0__py3-none-any.whl

admin_api_lib/api_endpoints/document_deleter.py

@@ -7,7 +7,12 @@ class DocumentDeleter(ABC):
     """Abstract base class for document deletion endpoint."""
 
     @abstractmethod
-    async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
+    async def adelete_document(
+        self,
+        identification: str,
+        remove_from_key_value_store: bool = True,
+        remove_from_storage: bool = True,
+    ) -> None:
         """
         Delete a document by its identification asynchronously.
 
@@ -17,6 +22,8 @@ class DocumentDeleter(ABC):
             The unique identifier of the document to be deleted.
         remove_from_key_value_store : bool, optional
             If True, the document will also be removed from the key-value store (default is True).
+        remove_from_storage : bool, optional
+            If True, the document will also be removed from the file storage (default is True).
 
         Returns
         -------

admin_api_lib/dependency_container.py

@@ -3,7 +3,7 @@
 from admin_api_lib.impl.api_endpoints.default_file_uploader import DefaultFileUploader
 from dependency_injector.containers import DeclarativeContainer
 from dependency_injector.providers import Configuration, List, Selector, Singleton
-from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import OllamaEmbeddings
 from langfuse import Langfuse
 

admin_api_lib/file_services/file_service.py

@@ -1,77 +1,5 @@
-"""Abstract class for dealing with I/O."""
+"""Re-export core file service interface."""
 
-import abc
-from abc import ABC
-from pathlib import Path
-from typing import BinaryIO
+from rag_core_lib.file_services.file_service import FileService
 
-
-class FileService(ABC):
-    """Abstract class for dealing with I/O."""
-
-    @abc.abstractmethod
-    def download_folder(self, source: str, target: Path) -> None:
-        """Download the remote folder on "source" to the local "target" directory.
-
-        Parameters
-        ----------
-        source: str
-            Path to the remote folder.
-        target: Path
-            Download destination path.
-        """
-
-    @abc.abstractmethod
-    def download_file(self, source: str, target_file: BinaryIO) -> None:
-        """Read a single remote file "source" into the local "target_file" file-like object.
-
-        Example usage
-        =============
-        ```
-        s3_settings: S3Settings = get_s3_settings()
-        s3_service = S3Service(endpoint="endpoint", username="username", password="password", bucket_name="bucket")
-
-        with tempfile.SpooledTemporaryFile(max_size=self._iot_forecast_settings.max_model_size) as temp_file:
-            s3_service.download_file("remote_file", temp_file)
-            # do stuff with temp_file
-        ```
-
-        Parameters
-        ----------
-        source: str
-            Path to the remote folder.
-        target_file: BinaryIO
-            File-like object to save the data to.
-        """
-
-    @abc.abstractmethod
-    def upload_file(self, file_path: str, file_name: str) -> None:
-        """Upload a local file to the Fileservice.
-
-        Parameters
-        ----------
-        file_path : str
-            The path to the local file to be uploaded.
-        file_name : str
-            The target path in the file storage where the file will be stored.
-        """
-
-    @abc.abstractmethod
-    def get_all_sorted_file_names(self) -> list[str]:
-        """Retrieve all file names stored in the file storage.
-
-        Returns
-        -------
-        list[str]
-            A list of file names stored in the file storage.
-        """
-
-    @abc.abstractmethod
-    def delete_file(self, file_name: str) -> None:
-        """Delete a file from the file storage.
-
-        Parameters
-        ----------
-        file_name : str
-            The name of the file to be deleted from the file storage.
-        """
+__all__ = ["FileService"]

admin_api_lib/impl/api_endpoints/default_document_deleter.py

@@ -41,7 +41,21 @@ class DefaultDocumentDeleter(DocumentDeleter):
         self._rag_api = rag_api
         self._key_value_store = key_value_store
 
-    async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
+    @staticmethod
+    def _storage_key_from_identification(identification: str) -> str | None:
+        if identification.startswith("file:"):
+            storage_key = identification[len("file:") :]
+            return storage_key or None
+        if ":" in identification:
+            return None
+        return identification or None
+
+    async def adelete_document(
+        self,
+        identification: str,
+        remove_from_key_value_store: bool = True,
+        remove_from_storage: bool = True,
+    ) -> None:
         """
         Asynchronously delete a document identified by the given identification string.
 
@@ -57,6 +71,8 @@ class DefaultDocumentDeleter(DocumentDeleter):
             The unique identifier of the document to be deleted.
         remove_from_key_value_store : bool, optional
             If True, the document will also be removed from the key-value store (default is True).
+        remove_from_storage : bool, optional
+            If True, the document will also be removed from the file storage (default is True).
 
         Raises
         ------
@@ -67,12 +83,12 @@ class DefaultDocumentDeleter(DocumentDeleter):
         error_messages = ""
         # Delete the document from file service and vector database
         logger.debug("Deleting existing document: %s", identification)
-        try:
-            if remove_from_key_value_store:
-                self._key_value_store.remove(identification)
-            self._file_service.delete_file(identification)
-        except Exception as e:
-            error_messages += f"Error while deleting {identification} from file storage\n {str(e)}\n"
+        if remove_from_key_value_store:
+            self._key_value_store.remove(identification)
+
+        if remove_from_storage:
+            error_messages = self._delete_from_storage(identification, error_messages)
+
         try:
             self._rag_api.remove_information_piece(
                 DeleteRequest(metadata=[KeyValuePair(key="document", value=json.dumps(identification))])
@@ -82,3 +98,14 @@ class DefaultDocumentDeleter(DocumentDeleter):
             error_messages += f"Error while deleting {identification} from vector db\n{str(e)}"
         if error_messages:
             raise HTTPException(404, error_messages)
+
+    def _delete_from_storage(self, identification: str, error_messages: str) -> str:
+        try:
+            storage_key = self._storage_key_from_identification(identification)
+            if storage_key:
+                self._file_service.delete_file(storage_key)
+            else:
+                logger.debug("Skipping file storage deletion for non-file source: %s", identification)
+        except Exception as e:
+            error_messages += f"Error while deleting {identification} from file storage\n {str(e)}\n"
+        return error_messages

admin_api_lib/impl/api_endpoints/default_file_uploader.py

@@ -188,7 +188,11 @@ class DefaultFileUploader(FileUploader):
             # Replace old document
             # deletion is allowed to fail
             with suppress(Exception):
-                await self._document_deleter.adelete_document(source_name, remove_from_key_value_store=False)
+                await self._document_deleter.adelete_document(
+                    source_name,
+                    remove_from_key_value_store=False,
+                    remove_from_storage=False,
+                )
 
             # Run blocking RAG API call in thread pool to avoid blocking event loop
             await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)

admin_api_lib/impl/api_endpoints/default_source_uploader.py

@@ -149,7 +149,11 @@ class DefaultSourceUploader(SourceUploader):
                 )
             )
         except asyncio.TimeoutError:
-            logger.error("Upload of %s timed out after %s seconds", source_name, timeout)
+            logger.error(
+                "Upload of %s timed out after %s seconds (increase SOURCE_UPLOADER_TIMEOUT to allow longer ingestions)",
+                source_name,
+                timeout,
+            )
             self._key_value_store.upsert(source_name, Status.ERROR)
         except Exception:
             logger.exception("Error while uploading %s", source_name)
@@ -193,7 +197,11 @@ class DefaultSourceUploader(SourceUploader):
                 rag_information_pieces.append(self._information_mapper.document2rag_information_piece(doc))
 
             with suppress(Exception):
-                await self._document_deleter.adelete_document(source_name, remove_from_key_value_store=False)
+                await self._document_deleter.adelete_document(
+                    source_name,
+                    remove_from_key_value_store=False,
+                    remove_from_storage=False,
+                )
 
             # Run blocking RAG API call in thread pool to avoid blocking event loop
             await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)

admin_api_lib/impl/chunker/text_chunker.py

@@ -1,7 +1,7 @@
 """Module containing the TextChunker class."""
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 from admin_api_lib.chunker.chunker import Chunker
 

admin_api_lib/impl/file_services/s3_service.py

@@ -1,130 +1,5 @@
-"""Class to handle I/O with S3 storage."""
+"""Re-export core S3 service implementation."""
 
-import logging
-from pathlib import Path
-from typing import BinaryIO
+from rag_core_lib.impl.file_services.s3_service import S3Service
 
-import boto3
-
-from admin_api_lib.file_services.file_service import FileService
-from admin_api_lib.impl.settings.s3_settings import S3Settings
-
-logger = logging.getLogger(__name__)
-
-
-class S3Service(FileService):
-    """Class to handle I/O with S3 storage."""
-
-    def __init__(self, s3_settings: S3Settings):
-        """Class to handle I/O with S3 storage.
-
-        Parameters
-        ----------
-        s3_settings: S3Settings
-            Settings for the s3. Must contain at least the endpoint, access_key_id, secret_access_key and bucket.
-        """
-        self._s3_settings = s3_settings
-        self._s3_client = boto3.client(
-            "s3",
-            endpoint_url=s3_settings.endpoint,
-            aws_access_key_id=s3_settings.access_key_id,
-            aws_secret_access_key=s3_settings.secret_access_key,
-            aws_session_token=None,
-            config=boto3.session.Config(signature_version="s3v4"),
-            verify=False,
-        )
-
-    def download_folder(self, source: str, target: Path) -> None:
-        """Download the remote folder on "source" to the local "target" directory.
-
-        Parameters
-        ----------
-        source: str
-            Path to the remote folder.
-        target: Path
-            Download destination path.
-        """
-        target.mkdir(parents=True, exist_ok=True)
-
-        search_response = self._s3_client.list_objects_v2(
-            Bucket=self._s3_settings.bucket,
-            Prefix=source,
-        )
-        for found_content in search_response.get("Contents", []):
-            file_source = found_content["Key"]
-            target_path = target / file_source[len(source) :]
-            target_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(target_path, "wb") as local_file:
-                self.download_file(file_source, local_file)
-
-    def download_file(self, source: str, target_file: BinaryIO) -> None:
-        """Read a single remote file "source" into the local "target_file" file-like object.
-
-        Example usage
-        =============
-        ```
-        s3_settings: S3Settings = get_s3_settings()
-        s3_service = S3Service(endpoint="endpoint", username="username", password="password", bucket_name="bucket")
-
-        with tempfile.SpooledTemporaryFile(max_size=self._iot_forecast_settings.max_model_size) as temp_file:
-            s3_service.download_file("remote_file", temp_file)
-            # do stuff with temp_file
-        ```
-
-        Parameters
-        ----------
-        source: str
-            Path to the remote folder.
-        target_file: BinaryIO
-            File-like object to save the data to.
-        """
-        self._s3_client.download_fileobj(self._s3_settings.bucket, source, target_file)
-
-    def upload_file(self, file_path: str, file_name: str) -> None:
-        """
-        Upload a local file to the S3 bucket.
-
-        Parameters
-        ----------
-        source : Path
-            The path to the local file to upload.
-        target : str
-            The target path in the S3 bucket where the file will be stored.
-        """
-        self._s3_client.upload_file(
-            Filename=file_path,
-            Bucket=self._s3_settings.bucket,
-            Key=file_name,
-        )
-
-    def get_all_sorted_file_names(self) -> list[str]:
-        """Retrieve all file names stored in the S3 bucket.
-
-        Returns
-        -------
-        list[str]
-            A list of file names stored in the S3 bucket.
-        """
-        file_names = []
-
-        resp = self._s3_client.list_objects_v2(Bucket=self._s3_settings.bucket)
-        if resp.get("Contents"):
-            for obj in resp["Contents"]:
-                file_names.append(obj["Key"])
-        return file_names
-
-    def delete_file(self, file_name: str) -> None:
-        """Delete a file from the S3 bucket.
-
-        Parameters
-        ----------
-        file_name : str
-            The name of the file to be deleted from the S3 bucket.
-        """
-        try:
-            file_name = f"/{file_name}" if not file_name.startswith("/") else file_name
-            self._s3_client.delete_object(Bucket=self._s3_settings.bucket, Key=file_name)
-            logger.info("File %s successfully deleted.", file_name)
-        except Exception:
-            logger.exception("Error deleting file %s", file_name)
-            raise
+__all__ = ["S3Service"]

admin_api_lib/impl/information_enhancer/page_summary_enhancer.py

@@ -1,8 +1,9 @@
 """Module for enhancing the summary of pages by grouping information by page and summarizing each page."""
 
-from asyncio import gather
+import asyncio
 from hashlib import sha256
 from typing import Optional
+from typing import Any
 
 from langchain_core.documents import Document
 from langchain_core.runnables import RunnableConfig
@@ -25,8 +26,36 @@ class PageSummaryEnhancer(SummaryEnhancer):
     """
 
     BASE64_IMAGE_KEY = "base64_image"
+    DOCUMENT_URL_KEY = "document_url"
     DEFAULT_PAGE_NR = 1
 
+    @staticmethod
+    def _parse_max_concurrency(config: Optional[RunnableConfig]) -> int:
+        if not config:
+            return 1
+        raw = config.get("max_concurrency")
+        if raw is None:
+            return 1
+        try:
+            return max(1, int(raw))
+        except (TypeError, ValueError):
+            return 1
+
+    def _group_key(self, piece: Document) -> tuple[Any, ...]:
+        document_url = piece.metadata.get(self.DOCUMENT_URL_KEY)
+        page = piece.metadata.get("page", self.DEFAULT_PAGE_NR)
+
+        # For paged documents (PDF/docling/etc.) keep per-page summaries even if a shared document URL exists.
+        if isinstance(page, int) or (isinstance(page, str) and page != "Unknown Title"):
+            return ("page_number", document_url, page)
+
+        # For sources like sitemaps/confluence, `page` can be a non-unique title (or missing),
+        # so group by the page URL when available to ensure one summary per page.
+        if document_url:
+            return ("document_url", document_url)
+
+        return ("page", page)
+
     async def _asummarize_page(self, page_pieces: list[Document], config: Optional[RunnableConfig]) -> Document:
         full_page_content = " ".join([piece.page_content for piece in page_pieces])
         summary = await self._summarizer.ainvoke(full_page_content, config)
@@ -39,24 +68,46 @@ class PageSummaryEnhancer(SummaryEnhancer):
         return Document(metadata=meta, page_content=summary)
 
     async def _acreate_summary(self, information: list[Document], config: Optional[RunnableConfig]) -> list[Document]:
-        distinct_pages = []
+        grouped = self._group_information(information)
+        max_concurrency = self._parse_max_concurrency(config)
+        return await self._summarize_groups(grouped, config, max_concurrency=max_concurrency)
+
+    def _group_information(self, information: list[Document]) -> list[list[Document]]:
+        ordered_keys: list[tuple[Any, ...]] = []
+        groups: dict[tuple[Any, ...], list[Document]] = {}
         for info in information:
-            if info.metadata.get("page", self.DEFAULT_PAGE_NR) not in distinct_pages:
-                distinct_pages.append(info.metadata.get("page", self.DEFAULT_PAGE_NR))
-
-        grouped = []
-        for page in distinct_pages:
-            group = []
-            for compare_info in information:
-                if compare_info.metadata.get("page", self.DEFAULT_PAGE_NR) == page:
-                    group.append(compare_info)
-            if (
-                self._chunker_settings
-                and len(" ".join([item.page_content for item in group])) < self._chunker_settings.max_size
-            ):
-                continue
-            grouped.append(group)
-
-        summary_tasks = [self._asummarize_page(info_group, config) for info_group in tqdm(grouped)]
-
-        return await gather(*summary_tasks)
+            key = self._group_key(info)
+            if key not in groups:
+                ordered_keys.append(key)
+                groups[key] = []
+            groups[key].append(info)
+        return [groups[key] for key in ordered_keys]
+
+    async def _summarize_groups(
+        self,
+        grouped: list[list[Document]],
+        config: Optional[RunnableConfig],
+        *,
+        max_concurrency: int,
+    ) -> list[Document]:
+        if max_concurrency == 1:
+            summaries: list[Document] = []
+            for info_group in tqdm(grouped):
+                summaries.append(await self._asummarize_page(info_group, config))
+            return summaries
+
+        semaphore = asyncio.Semaphore(max_concurrency)
+        results: list[Document | None] = [None] * len(grouped)
+
+        async def _run(idx: int, info_group: list[Document]) -> tuple[int, Document]:
+            async with semaphore:
+                return idx, await self._asummarize_page(info_group, config)
+
+        tasks = [asyncio.create_task(_run(idx, info_group)) for idx, info_group in enumerate(grouped)]
+        with tqdm(total=len(tasks)) as pbar:
+            for task in asyncio.as_completed(tasks):
+                idx, summary = await task
+                results[idx] = summary
+                pbar.update(1)
+
+        return [summary for summary in results if summary is not None]

admin_api_lib/impl/key_db/file_status_key_value_store.py

@@ -1,6 +1,8 @@
 """Module containing the FileStatusKeyValueStore class."""
 
 import json
+import ssl
+from typing import Any
 
 from redis import Redis
 
@@ -37,9 +39,53 @@ class FileStatusKeyValueStore:
         Parameters
         ----------
         settings : KeyValueSettings
-            The settings object containing the host and port information for the Redis connection.
+            The settings object containing the connection information for the Redis connection.
         """
-        self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True)
+        redis_kwargs: dict[str, Any] = {
+            "host": settings.host,
+            "port": settings.port,
+            "decode_responses": True,
+            **self._build_ssl_kwargs(settings),
+        }
+        if settings.username:
+            redis_kwargs["username"] = settings.username
+        if settings.password:
+            redis_kwargs["password"] = settings.password
+
+        self._redis = Redis(**redis_kwargs)
+
+    @staticmethod
+    def _build_ssl_kwargs(settings: KeyValueSettings) -> dict[str, Any]:
+        """Build Redis SSL settings from configuration, mapping string values to ssl constants."""
+        if not settings.use_ssl:
+            return {}
+
+        cert_reqs_map = {
+            "required": ssl.CERT_REQUIRED,
+            "optional": ssl.CERT_OPTIONAL,
+            "none": ssl.CERT_NONE,
+            "cert_required": ssl.CERT_REQUIRED,
+            "cert_optional": ssl.CERT_OPTIONAL,
+            "cert_none": ssl.CERT_NONE,
+        }
+        ssl_cert_reqs = None
+        if settings.ssl_cert_reqs:
+            ssl_cert_reqs = cert_reqs_map.get(settings.ssl_cert_reqs.lower(), settings.ssl_cert_reqs)
+
+        ssl_kwargs: dict[str, Any] = {
+            "ssl": settings.use_ssl,
+            "ssl_check_hostname": settings.ssl_check_hostname,
+        }
+        if ssl_cert_reqs is not None:
+            ssl_kwargs["ssl_cert_reqs"] = ssl_cert_reqs
+        if settings.ssl_ca_certs:
+            ssl_kwargs["ssl_ca_certs"] = settings.ssl_ca_certs
+        if settings.ssl_certfile:
+            ssl_kwargs["ssl_certfile"] = settings.ssl_certfile
+        if settings.ssl_keyfile:
+            ssl_kwargs["ssl_keyfile"] = settings.ssl_keyfile
+
+        return ssl_kwargs
 
     @staticmethod
     def _to_str(file_name: str, file_status: Status) -> str:

admin_api_lib/impl/settings/key_value_settings.py

@@ -14,6 +14,22 @@ class KeyValueSettings(BaseSettings):
         The hostname of the key value store.
     port : int
         The port number of the key value store.
+    username : str | None
+        Optional username for authenticating with the key value store.
+    password : str | None
+        Optional password for authenticating with the key value store.
+    use_ssl : bool
+        Whether to use SSL/TLS when connecting to the key value store.
+    ssl_cert_reqs : str | None
+        SSL certificate requirement level (e.g., 'required', 'optional', 'none').
+    ssl_ca_certs : str | None
+        Path to a CA bundle file for verifying the server certificate.
+    ssl_certfile : str | None
+        Path to the client SSL certificate file (if mutual TLS is required).
+    ssl_keyfile : str | None
+        Path to the client SSL private key file (if mutual TLS is required).
+    ssl_check_hostname : bool
+        Whether to verify the server hostname against the certificate.
     """
 
     class Config:
@@ -24,3 +40,11 @@ class KeyValueSettings(BaseSettings):
 
     host: str = Field()
     port: int = Field()
+    username: str | None = Field(default=None)
+    password: str | None = Field(default=None)
+    use_ssl: bool = Field(default=False)
+    ssl_cert_reqs: str | None = Field(default=None)
+    ssl_ca_certs: str | None = Field(default=None)
+    ssl_certfile: str | None = Field(default=None)
+    ssl_keyfile: str | None = Field(default=None)
+    ssl_check_hostname: bool = Field(default=True)

admin_api_lib/impl/settings/s3_settings.py

@@ -1,31 +1,5 @@
-"""Contains settings regarding the S3 storage."""
+"""Re-export core S3 settings."""
 
-from pydantic_settings import BaseSettings
+from rag_core_lib.impl.settings.s3_settings import S3Settings
 
-
-class S3Settings(BaseSettings):
-    """
-    Contains settings regarding the S3 storage.
-
-    Attributes
-    ----------
-    secret_access_key : str
-        The secret access key for S3.
-    access_key_id : str
-        The access key ID for S3.
-    endpoint : str
-        The endpoint URL for S3.
-    bucket : str
-        The bucket name in S3.
-    """
-
-    class Config:
-        """Config class for reading Fields from env."""
-
-        env_prefix = "S3_"
-        case_sensitive = False
-
-    secret_access_key: str
-    access_key_id: str
-    endpoint: str
-    bucket: str
+__all__ = ["S3Settings"]

admin_api_lib/impl/summarizer/langchain_summarizer.py

@@ -4,9 +4,9 @@ import asyncio
 import logging
 from typing import Optional
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
 from langchain_core.runnables import Runnable, RunnableConfig, ensure_config
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 from openai import APIConnectionError, APIError, APITimeoutError, RateLimitError
 
 from admin_api_lib.impl.settings.summarizer_settings import SummarizerSettings
@@ -44,6 +44,24 @@ class LangchainSummarizer(Summarizer):
         self._semaphore = semaphore
         self._retry_decorator_settings = create_retry_decorator_settings(summarizer_settings, retry_decorator_settings)
 
+    @staticmethod
+    def _parse_max_concurrency(config: RunnableConfig) -> Optional[int]:
+        """Parse max concurrency from a RunnableConfig.
+
+        Returns
+        -------
+        Optional[int]
+            An integer >= 1 if configured and valid, otherwise None.
+        """
+        max_concurrency = config.get("max_concurrency")
+        if max_concurrency is None:
+            return None
+
+        try:
+            return max(1, int(max_concurrency))
+        except (TypeError, ValueError):
+            return None
+
     async def ainvoke(self, query: SummarizerInput, config: Optional[RunnableConfig] = None) -> SummarizerOutput:
         """
         Asynchronously invokes the summarization process on the given query.
@@ -77,9 +95,8 @@ class LangchainSummarizer(Summarizer):
         langchain_documents = self._chunker.split_documents([document])
         logger.debug("Summarizing %d chunk(s)...", len(langchain_documents))
 
-        # Fan out with concurrency, bounded by your semaphore inside _summarize_chunk
-        tasks = [asyncio.create_task(self._summarize_chunk(doc.page_content, config)) for doc in langchain_documents]
-        outputs = await asyncio.gather(*tasks)
+        max_concurrency = self._parse_max_concurrency(config)
+        outputs = await self._summarize_documents(langchain_documents, config, max_concurrency=max_concurrency)
 
         if len(outputs) == 1:
             return outputs[0]
@@ -93,6 +110,34 @@ class LangchainSummarizer(Summarizer):
         )
         return await self._summarize_chunk(merged, config)
 
+    async def _summarize_documents(
+        self,
+        documents: list[Document],
+        config: RunnableConfig,
+        *,
+        max_concurrency: Optional[int],
+    ) -> list[SummarizerOutput]:
+        """Summarize a set of already-chunked documents.
+
+        Notes
+        -----
+        This optionally limits task fan-out using a per-call semaphore (max_concurrency).
+        The actual LLM call concurrency is always bounded by the instance semaphore held
+        inside `_summarize_chunk`.
+        """
+        if max_concurrency == 1:
+            return [await self._summarize_chunk(doc.page_content, config) for doc in documents]
+
+        limiter: asyncio.Semaphore | None = asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None
+
+        async def _run(doc: Document) -> SummarizerOutput:
+            if limiter is None:
+                return await self._summarize_chunk(doc.page_content, config)
+            async with limiter:
+                return await self._summarize_chunk(doc.page_content, config)
+
+        return await asyncio.gather(*(_run(doc) for doc in documents))
+
     def _create_chain(self) -> Runnable:
         return self._langfuse_manager.get_base_prompt(self.__class__.__name__) | self._langfuse_manager.get_base_llm(
             self.__class__.__name__

{admin_api_lib-3.4.0.dist-info → admin_api_lib-4.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: admin-api-lib
-Version: 3.4.0
+Version: 4.1.0
 Summary: The admin backend is responsible for the document management. This includes deletion, upload and returning the source document.
 License: Apache-2.0
 Author: STACKIT GmbH & Co. KG
@@ -11,19 +11,19 @@ Requires-Python: >=3.13,<4.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: boto3 (>=1.38.10,<2.0.0)
 Requires-Dist: dependency-injector (>=4.46.0,<5.0.0)
-Requires-Dist: fastapi (>=0.118.0,<0.119.0)
-Requires-Dist: langchain-experimental (>=0.3.4,<0.4.0)
-Requires-Dist: langfuse (==3.6.1)
+Requires-Dist: fastapi (>=0.121.2,<0.122.0)
+Requires-Dist: langchain-experimental (>=0.4.0,<0.5.0)
+Requires-Dist: langfuse (>=3.10.1,<4.0.0)
+Requires-Dist: langgraph-checkpoint (>=3.0.0,<4.0.0)
 Requires-Dist: nltk (>=3.9.2,<4.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: python-multipart (>=0.0.20,<0.0.21)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
-Requires-Dist: rag-core-lib (==3.4.0)
+Requires-Dist: rag-core-lib (==4.0.0)
 Requires-Dist: redis (>=6.0.0,<7.0.0)
-Requires-Dist: starlette (>=0.47.2,<0.49.0)
+Requires-Dist: starlette (>=0.49.1)
 Requires-Dist: tenacity (==9.1.2)
 Requires-Dist: tqdm (>=4.67.1,<5.0.0)
 Requires-Dist: uvicorn (>=0.37.0,<0.38.0)
@@ -100,6 +100,8 @@ All settings are powered by `pydantic-settings`, so you can use environment vari
 - `SUMMARIZER_MAXIMUM_INPUT_SIZE`, `SUMMARIZER_MAXIMUM_CONCURRENCY`, `SUMMARIZER_MAX_RETRIES`, etc. – tune summariser limits and retry behaviour.
 - `SOURCE_UPLOADER_TIMEOUT` – adjust how long non-file source ingestions wait before timing out.
 - `USECASE_KEYVALUE_HOST` / `USECASE_KEYVALUE_PORT` – configure the KeyDB/Redis instance that persists document status.
+- `USECASE_KEYVALUE_USERNAME` / `USECASE_KEYVALUE_PASSWORD` – optional credentials for authenticating against KeyDB/Redis.
+- `USECASE_KEYVALUE_USE_SSL`, `USECASE_KEYVALUE_SSL_CERT_REQS`, `USECASE_KEYVALUE_SSL_CA_CERTS`, `USECASE_KEYVALUE_SSL_CERTFILE`, `USECASE_KEYVALUE_SSL_KEYFILE`, `USECASE_KEYVALUE_SSL_CHECK_HOSTNAME` – optional TLS settings for managed Redis deployments (e.g., STACKIT Redis or other SSL-only endpoints).
 
 The Helm chart forwards these values through `adminBackend.envs.*`, keeping deployments declarative. Local development can rely on `.env` as described in the repository root README.
 

{admin_api_lib-3.4.0.dist-info → admin_api_lib-4.1.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 admin_api_lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-admin_api_lib/api_endpoints/document_deleter.py,sha256=1L0mqBBiGV5XXE2-4gpHh2hXuVdtqqz06qLaG3dQx9M,727
+admin_api_lib/api_endpoints/document_deleter.py,sha256=C6PA2OHyLPExBpuMCUU-YYz57aFh_DMkYEjg27lDoV4,941
 admin_api_lib/api_endpoints/document_reference_retriever.py,sha256=eisisp-ZMPn3P1yNkxiqQroz3Rz4Zz8pCrj8JQ9rrro,658
 admin_api_lib/api_endpoints/documents_status_retriever.py,sha256=PxrW4X6mN2z_XJHzTqeolCnRusiiBC4OE8TdI4lUEMg,572
 admin_api_lib/api_endpoints/file_uploader.py,sha256=r7m9G06aSC3mBdVuXCBKTfR5bTmYEjGJSTavhKRuSJk,725
@@ -10,7 +10,7 @@ admin_api_lib/apis/admin_api.py,sha256=RaeVO7A-IW_1kntw19RVUbKAI_BDIQExmgzAAZ5T3
 admin_api_lib/apis/admin_api_base.py,sha256=20M8U8dM91pJa2Wqx_UZTjpU0XCHIffVRac-_KJRMmk,3094
 admin_api_lib/chunker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/chunker/chunker.py,sha256=R2mxwmvz8o3iNzGHaLoMERcsIh82x88ZkKndbRNU-7U,627
-admin_api_lib/dependency_container.py,sha256=Anar3suYlk9isgnqRH9S0Z8MhgvkaYR6PFoadDR3cJ4,10126
+admin_api_lib/dependency_container.py,sha256=E4iupfqq3B60edfOTi-YqkvbSD4ARXmJGLAHXFuZspg,10127
 admin_api_lib/extractor_api_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/extractor_api_client/openapi_client/__init__.py,sha256=ha9QCyCZq3PJKz7pQpTtI0bBypqj6E3Bg1pi7vBIazo,1823
 admin_api_lib/extractor_api_client/openapi_client/api/__init__.py,sha256=6THi7hHMLnN46_cLbVVamrTv-Ab_Wsn4b5YBkhrk65U,140
@@ -33,38 +33,38 @@ admin_api_lib/extractor_api_client/openapi_client/test/test_extraction_request.p
 admin_api_lib/extractor_api_client/openapi_client/test/test_extractor_api.py,sha256=kaTxIEPyQECdkAoT-kzVyYx0jX-P-5U6v73Ndhyg40w,887
 admin_api_lib/extractor_api_client/openapi_client/test/test_information_piece.py,sha256=Aq4h5SB-GY8BbeQR9sFMN1B3Z5UMUO6pRx6XU0b7xqs,1730
 admin_api_lib/extractor_api_client/openapi_client/test/test_key_value_pair.py,sha256=Zd9_HyM-4U2iVtGYO45UI6xkJKVNcj369hFe5zkBhDM,1403
-admin_api_lib/file_services/file_service.py,sha256=nBEga-HEuSlG47BpO4hR4U4cAC_kU7OBRFIuyUlocfU,2286
+admin_api_lib/file_services/file_service.py,sha256=8sJLyWCPRw88ZAY9-4YPcMqYRR6Bgw_pv5A8Dy1Xf5k,137
 admin_api_lib/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/impl/admin_api.py,sha256=k5geH6NorzzKg3_Z0tltLgTeJ0ru5AyFKPQi4nWjdnE,5483
-admin_api_lib/impl/api_endpoints/default_document_deleter.py,sha256=iKqBnk6bFRsZczZUdLBtetHUrZY30xrJWQbC02ZgL2M,3719
+admin_api_lib/impl/api_endpoints/default_document_deleter.py,sha256=9P_UYbnn0wqBOy18jFn44zaY5nykTXfaQ_Zt39bfmF0,4727
 admin_api_lib/impl/api_endpoints/default_document_reference_retriever.py,sha256=H3bQvpMLMjsyUzZMfTziPW7qU3N9D5s6DMKEA4fMITM,2642
 admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py,sha256=ZtLNgmFWGcfU4jNhVPiAKIJT701Z4wVwQAWpPbegxfc,1419
-admin_api_lib/impl/api_endpoints/default_file_uploader.py,sha256=hfBPI1dgY-KUk_7dZU6yU04bbAMxn4hPKOQtxBBg5Ts,9613
-admin_api_lib/impl/api_endpoints/default_source_uploader.py,sha256=NuN_pPsq17IrOYNsxG_0xAM9Vlz5QYsKVhKf-ghYlw4,8583
+admin_api_lib/impl/api_endpoints/default_file_uploader.py,sha256=2tGd3ZQOJMZob_0UreopOliaefvIYq4CTp9soQevoCc,9719
+admin_api_lib/impl/api_endpoints/default_source_uploader.py,sha256=UrKmbyu2l-Jyhbkaiqt66Ks-RMwD_i2vEkOdhgTbwZU,8814
 admin_api_lib/impl/chunker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/impl/chunker/chunker_type.py,sha256=ArEAmQ9OWe3ek7FMb2auFvGk4UXsOr3xzjyB5TXI2i8,247
 admin_api_lib/impl/chunker/semantic_text_chunker.py,sha256=VrbHpY867gQPtMhz6HjqstdBQ-e74kwce_Ma-9LVsJo,10605
-admin_api_lib/impl/chunker/text_chunker.py,sha256=WaSO_9AGfgrUl569HTmpMA-M6kUEOCbVOLudOjpdzlM,1075
+admin_api_lib/impl/chunker/text_chunker.py,sha256=0WC1EsvkqHI16IeopTvrx0Y2XQmXscUFCT8-z6h5nOY,1076
 admin_api_lib/impl/file_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-admin_api_lib/impl/file_services/s3_service.py,sha256=IB68Av69mkkcf4KuXpxnbaJVJuMKHL82zIwuBw94pXg,4428
+admin_api_lib/impl/file_services/s3_service.py,sha256=8NuYwKd3zNdpGWM6UC1R3yUmbG6XOyZrhYbguznzgRY,139
 admin_api_lib/impl/information_enhancer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/impl/information_enhancer/general_enhancer.py,sha256=H4mVxYEGW2owdCcUwbh7z132PTxa3dZssRL0MDJg56Q,1889
-admin_api_lib/impl/information_enhancer/page_summary_enhancer.py,sha256=HfYqnzcmyNDJXxwpZzy8_Ugp24QI9ecgAheQGP1X1_g,2603
+admin_api_lib/impl/information_enhancer/page_summary_enhancer.py,sha256=Ve0KjQMdxESLi2K3-fgZcHf2xMWNeJkfxvcbJCajWJo,4652
 admin_api_lib/impl/information_enhancer/summary_enhancer.py,sha256=CdsE4KGTfui6nBBABwBcEQJ-1SBrPwN7Pk77zPUxGBk,2704
 admin_api_lib/impl/key_db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-admin_api_lib/impl/key_db/file_status_key_value_store.py,sha256=Kktk0Jams9J_TP2V3qVonmso3UOD4GwrAFyURO0YUBA,3697
+admin_api_lib/impl/key_db/file_status_key_value_store.py,sha256=FSmenxyJx7hMWY9rNocAF7zf4zU_iowR3T-JjGyhjNU,5328
 admin_api_lib/impl/mapper/informationpiece2document.py,sha256=dMngfdjBV2JkUtamypwmBvJVWdWcZhXn_XnBmGQLCxg,3924
 admin_api_lib/impl/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/impl/settings/chunker_class_type_settings.py,sha256=w7cgOB2hBattRqIbrQLMBXsPX2UZpmzxnI6n5DrcO2k,541
 admin_api_lib/impl/settings/chunker_settings.py,sha256=TLBPY-PLmCuly8mMgGIlm3gU9FzzyhCgmLdTcfV3vAg,814
 admin_api_lib/impl/settings/document_extractor_settings.py,sha256=gYbdk6M4dwTL9esWfemGGHEJYOdV9BT7c8zGBKLbjlU,520
-admin_api_lib/impl/settings/key_value_settings.py,sha256=d84gYy2bmJ6LhmYEojInm6vEsVRq8MtHM9m0SHPyDho,587
+admin_api_lib/impl/settings/key_value_settings.py,sha256=jokWtwXLwN4_yPaWwmVxJOSKwyR43hzQtQhKqAnIBrw,1789
 admin_api_lib/impl/settings/rag_api_settings.py,sha256=YMxnsiMLjZBQZ2a6C3kZuVgqd42_w4JUC9dTkHnwuaU,484
-admin_api_lib/impl/settings/s3_settings.py,sha256=e5vOOfJRp_22gtG13vf4iXedHXm3E1egLq8t_Ix_AYg,666
+admin_api_lib/impl/settings/s3_settings.py,sha256=K5-zrhueyOKD_CpHQ3U2q16FZ1BZLF5oFnCnUnNoswA,123
 admin_api_lib/impl/settings/source_uploader_settings.py,sha256=ZCVFI3TgSZpVjBVTcJw9QSyTBLvqzYsJ41vpihtI7pY,582
 admin_api_lib/impl/settings/summarizer_settings.py,sha256=NGZ0o25dp7GkfOSAzcoCDSQrpUqiGe28Oi05ho2MAZ8,2895
 admin_api_lib/impl/summarizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-admin_api_lib/impl/summarizer/langchain_summarizer.py,sha256=N_C1iAxmHTTIEausaVwvMfhXi-2EpIslwZ6aJx0wKg8,4700
+admin_api_lib/impl/summarizer/langchain_summarizer.py,sha256=VIsUo4dI3Ebil5lnX4uY6Rdqs8CKlxk9Vvmb9Lx1K5Y,6264
 admin_api_lib/information_enhancer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/information_enhancer/information_enhancer.py,sha256=Bp2fQ3lvOmj192QmSH5NItseyDhWczzCHoeABVx4D3w,1155
 admin_api_lib/main.py,sha256=33fiCU05aKAxz9zaO0YtM2dynLxvJxMJn_4Cf6zk4q8,1582
@@ -101,6 +101,6 @@ admin_api_lib/summarizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 admin_api_lib/summarizer/summarizer.py,sha256=D0rkW0iZSys-68LcO1-PIkE0Faf2Grg-_9wu75Rc1OY,966
 admin_api_lib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 admin_api_lib/utils/utils.py,sha256=eaNQ_NzUEp4hwhCU9EEsUXvbRH_ekVariF7tTsO9Sco,834
-admin_api_lib-3.4.0.dist-info/METADATA,sha256=qQVpMYoe2RcpuEmG3zqhyfl6bn8Oh_VUero7Ixjvdig,7466
-admin_api_lib-3.4.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-admin_api_lib-3.4.0.dist-info/RECORD,,
+admin_api_lib-4.1.0.dist-info/METADATA,sha256=T6a5lgyq6pnbAPxSO4l2r5RHuebfioA62HsNd-x0Br0,7900
+admin_api_lib-4.1.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+admin_api_lib-4.1.0.dist-info/RECORD,,

{admin_api_lib-3.4.0.dist-info → admin_api_lib-4.1.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.2.1
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any