admin-api-lib 3.4.0__tar.gz → 4.1.0__tar.gz

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: admin-api-lib
-Version: 3.4.0
+Version: 4.1.0
 Summary: The admin backend is responsible for the document management. This includes deletion, upload and returning the source document.
 License: Apache-2.0
 Author: STACKIT GmbH & Co. KG
@@ -11,19 +11,19 @@ Requires-Python: >=3.13,<4.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: boto3 (>=1.38.10,<2.0.0)
 Requires-Dist: dependency-injector (>=4.46.0,<5.0.0)
-Requires-Dist: fastapi (>=0.118.0,<0.119.0)
-Requires-Dist: langchain-experimental (>=0.3.4,<0.4.0)
-Requires-Dist: langfuse (==3.6.1)
+Requires-Dist: fastapi (>=0.121.2,<0.122.0)
+Requires-Dist: langchain-experimental (>=0.4.0,<0.5.0)
+Requires-Dist: langfuse (>=3.10.1,<4.0.0)
+Requires-Dist: langgraph-checkpoint (>=3.0.0,<4.0.0)
 Requires-Dist: nltk (>=3.9.2,<4.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: python-multipart (>=0.0.20,<0.0.21)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
-Requires-Dist: rag-core-lib (==3.4.0)
+Requires-Dist: rag-core-lib (==4.0.0)
 Requires-Dist: redis (>=6.0.0,<7.0.0)
-Requires-Dist: starlette (>=0.47.2,<0.49.0)
+Requires-Dist: starlette (>=0.49.1)
 Requires-Dist: tenacity (==9.1.2)
 Requires-Dist: tqdm (>=4.67.1,<5.0.0)
 Requires-Dist: uvicorn (>=0.37.0,<0.38.0)
@@ -100,6 +100,8 @@ All settings are powered by `pydantic-settings`, so you can use environment vari
 - `SUMMARIZER_MAXIMUM_INPUT_SIZE`, `SUMMARIZER_MAXIMUM_CONCURRENCY`, `SUMMARIZER_MAX_RETRIES`, etc. – tune summariser limits and retry behaviour.
 - `SOURCE_UPLOADER_TIMEOUT` – adjust how long non-file source ingestions wait before timing out.
 - `USECASE_KEYVALUE_HOST` / `USECASE_KEYVALUE_PORT` – configure the KeyDB/Redis instance that persists document status.
+- `USECASE_KEYVALUE_USERNAME` / `USECASE_KEYVALUE_PASSWORD` – optional credentials for authenticating against KeyDB/Redis.
+- `USECASE_KEYVALUE_USE_SSL`, `USECASE_KEYVALUE_SSL_CERT_REQS`, `USECASE_KEYVALUE_SSL_CA_CERTS`, `USECASE_KEYVALUE_SSL_CERTFILE`, `USECASE_KEYVALUE_SSL_KEYFILE`, `USECASE_KEYVALUE_SSL_CHECK_HOSTNAME` – optional TLS settings for managed Redis deployments (e.g., STACKIT Redis or other SSL-only endpoints).
 
 The Helm chart forwards these values through `adminBackend.envs.*`, keeping deployments declarative. Local development can rely on `.env` as described in the repository root README.
 

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/README.md

@@ -67,6 +67,8 @@ All settings are powered by `pydantic-settings`, so you can use environment vari
 - `SUMMARIZER_MAXIMUM_INPUT_SIZE`, `SUMMARIZER_MAXIMUM_CONCURRENCY`, `SUMMARIZER_MAX_RETRIES`, etc. – tune summariser limits and retry behaviour.
 - `SOURCE_UPLOADER_TIMEOUT` – adjust how long non-file source ingestions wait before timing out.
 - `USECASE_KEYVALUE_HOST` / `USECASE_KEYVALUE_PORT` – configure the KeyDB/Redis instance that persists document status.
+- `USECASE_KEYVALUE_USERNAME` / `USECASE_KEYVALUE_PASSWORD` – optional credentials for authenticating against KeyDB/Redis.
+- `USECASE_KEYVALUE_USE_SSL`, `USECASE_KEYVALUE_SSL_CERT_REQS`, `USECASE_KEYVALUE_SSL_CA_CERTS`, `USECASE_KEYVALUE_SSL_CERTFILE`, `USECASE_KEYVALUE_SSL_KEYFILE`, `USECASE_KEYVALUE_SSL_CHECK_HOSTNAME` – optional TLS settings for managed Redis deployments (e.g., STACKIT Redis or other SSL-only endpoints).
 
 The Helm chart forwards these values through `adminBackend.envs.*`, keeping deployments declarative. Local development can rely on `.env` as described in the repository root README.
 

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "admin-api-lib"
-version = "v3.4.0"
+version = "4.1.0"
 description = "The admin backend is responsible for the document management. This includes deletion, upload and returning the source document."
 authors = [
     "STACKIT GmbH & Co. KG <data-ai@stackit.cloud>",
@@ -18,6 +18,11 @@ license = "Apache-2.0"
 repository = "https://github.com/stackitcloud/rag-template"
 homepage = "https://pypi.org/project/admin-api-lib"
 
+[[tool.poetry.source]]
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+priority = "explicit"
+
 [tool.flake8]
 exclude= [".eggs", "./libs/*", "./src/admin_api_lib/models/*", "./src/admin_api_lib/rag_backend_client/*", "./src/admin_api_lib/extractor_api_client/*", ".git", ".hg", ".mypy_cache", ".tox", ".venv", ".devcontainer", "venv", "_build", "buck-out", "build", "dist", "**/__init__.py"]
 statistics = true
@@ -74,10 +79,12 @@ known_local_folder = ["admin_api_lib", "rag_core_lib"]
 [tool.pylint]
 max-line-length = 120
 
-[tool.poetry.group.dev.dependencies]
-debugpy = "^1.8.14"
+[tool.poetry.group.test.dependencies]
 pytest = "^8.3.5"
+pytest-asyncio = "^1.0.0"
 coverage = "^7.8.0"
+
+[tool.poetry.group.lint.dependencies]
 flake8 = "^7.2.0"
 flake8-black = "^0.4.0"
 flake8-pyproject = "^1.2.3"
@@ -101,33 +108,31 @@ flake8-tidy-imports = "^4.10.0"
 black = "^25.1.0"
 flake8-logging-format = "^2024.24.12"
 flake8-docstrings = "^1.7.0"
-pytest-asyncio = "^1.0.0"
+
+[tool.poetry.group.dev.dependencies]
+debugpy = "^1.8.14"
 
 [tool.poetry.dependencies]
-rag-core-lib = "3.4.0"
+rag-core-lib = "==4.0.0"
 python = "^3.13"
 uvicorn = "^0.37.0"
-fastapi = "^0.118.0"
+fastapi = "^0.121.2"
 dependency-injector = "^4.46.0"
 python-dateutil = "^2.9.0.post0"
 tenacity = "9.1.2"
 boto3 = "^1.38.10"
 tqdm = "^4.67.1"
-langfuse = "3.6.1"
+langfuse = "^3.10.1"
 redis = "^6.0.0"
 pyyaml = "^6.0.2"
 python-multipart = "^0.0.20"
-starlette = ">=0.47.2,<0.49.0"
-langchain-experimental = "^0.3.4"
+langchain-experimental = "^0.4.0"
 nltk = "^3.9.2"
+starlette = ">=0.49.1"
+langgraph-checkpoint = ">=3.0.0,<4.0.0"
 
 [tool.pytest.ini_options]
 log_cli = true
 log_cli_level = "DEBUG"
 pythonpath = "src"
 testpaths = "src/tests"
-
-[[tool.poetry.source]]
-name = "testpypi"
-url = "https://test.pypi.org/simple/"
-priority = "supplemental"

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/api_endpoints/document_deleter.py

@@ -7,7 +7,12 @@ class DocumentDeleter(ABC):
     """Abstract base class for document deletion endpoint."""
 
     @abstractmethod
-    async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
+    async def adelete_document(
+        self,
+        identification: str,
+        remove_from_key_value_store: bool = True,
+        remove_from_storage: bool = True,
+    ) -> None:
         """
         Delete a document by its identification asynchronously.
 
@@ -17,6 +22,8 @@ class DocumentDeleter(ABC):
             The unique identifier of the document to be deleted.
         remove_from_key_value_store : bool, optional
             If True, the document will also be removed from the key-value store (default is True).
+        remove_from_storage : bool, optional
+            If True, the document will also be removed from the file storage (default is True).
 
         Returns
         -------

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/dependency_container.py

@@ -3,7 +3,7 @@
 from admin_api_lib.impl.api_endpoints.default_file_uploader import DefaultFileUploader
 from dependency_injector.containers import DeclarativeContainer
 from dependency_injector.providers import Configuration, List, Selector, Singleton
-from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import OllamaEmbeddings
 from langfuse import Langfuse
 

admin_api_lib-4.1.0/src/admin_api_lib/file_services/file_service.py

@@ -0,0 +1,5 @@
+"""Re-export core file service interface."""
+
+from rag_core_lib.file_services.file_service import FileService
+
+__all__ = ["FileService"]

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py

@@ -41,7 +41,21 @@ class DefaultDocumentDeleter(DocumentDeleter):
         self._rag_api = rag_api
         self._key_value_store = key_value_store
 
-    async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
+    @staticmethod
+    def _storage_key_from_identification(identification: str) -> str | None:
+        if identification.startswith("file:"):
+            storage_key = identification[len("file:") :]
+            return storage_key or None
+        if ":" in identification:
+            return None
+        return identification or None
+
+    async def adelete_document(
+        self,
+        identification: str,
+        remove_from_key_value_store: bool = True,
+        remove_from_storage: bool = True,
+    ) -> None:
         """
         Asynchronously delete a document identified by the given identification string.
 
@@ -57,6 +71,8 @@ class DefaultDocumentDeleter(DocumentDeleter):
             The unique identifier of the document to be deleted.
         remove_from_key_value_store : bool, optional
             If True, the document will also be removed from the key-value store (default is True).
+        remove_from_storage : bool, optional
+            If True, the document will also be removed from the file storage (default is True).
 
         Raises
         ------
@@ -67,12 +83,12 @@ class DefaultDocumentDeleter(DocumentDeleter):
         error_messages = ""
         # Delete the document from file service and vector database
         logger.debug("Deleting existing document: %s", identification)
-        try:
-            if remove_from_key_value_store:
-                self._key_value_store.remove(identification)
-            self._file_service.delete_file(identification)
-        except Exception as e:
-            error_messages += f"Error while deleting {identification} from file storage\n {str(e)}\n"
+        if remove_from_key_value_store:
+            self._key_value_store.remove(identification)
+
+        if remove_from_storage:
+            error_messages = self._delete_from_storage(identification, error_messages)
+
         try:
             self._rag_api.remove_information_piece(
                 DeleteRequest(metadata=[KeyValuePair(key="document", value=json.dumps(identification))])
@@ -82,3 +98,14 @@ class DefaultDocumentDeleter(DocumentDeleter):
             error_messages += f"Error while deleting {identification} from vector db\n{str(e)}"
         if error_messages:
             raise HTTPException(404, error_messages)
+
+    def _delete_from_storage(self, identification: str, error_messages: str) -> str:
+        try:
+            storage_key = self._storage_key_from_identification(identification)
+            if storage_key:
+                self._file_service.delete_file(storage_key)
+            else:
+                logger.debug("Skipping file storage deletion for non-file source: %s", identification)
+        except Exception as e:
+            error_messages += f"Error while deleting {identification} from file storage\n {str(e)}\n"
+        return error_messages

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py

@@ -188,7 +188,11 @@ class DefaultFileUploader(FileUploader):
             # Replace old document
             # deletion is allowed to fail
             with suppress(Exception):
-                await self._document_deleter.adelete_document(source_name, remove_from_key_value_store=False)
+                await self._document_deleter.adelete_document(
+                    source_name,
+                    remove_from_key_value_store=False,
+                    remove_from_storage=False,
+                )
 
             # Run blocking RAG API call in thread pool to avoid blocking event loop
             await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py

@@ -149,7 +149,11 @@ class DefaultSourceUploader(SourceUploader):
                 )
             )
         except asyncio.TimeoutError:
-            logger.error("Upload of %s timed out after %s seconds", source_name, timeout)
+            logger.error(
+                "Upload of %s timed out after %s seconds (increase SOURCE_UPLOADER_TIMEOUT to allow longer ingestions)",
+                source_name,
+                timeout,
+            )
             self._key_value_store.upsert(source_name, Status.ERROR)
         except Exception:
             logger.exception("Error while uploading %s", source_name)
@@ -193,7 +197,11 @@ class DefaultSourceUploader(SourceUploader):
                 rag_information_pieces.append(self._information_mapper.document2rag_information_piece(doc))
 
             with suppress(Exception):
-                await self._document_deleter.adelete_document(source_name, remove_from_key_value_store=False)
+                await self._document_deleter.adelete_document(
+                    source_name,
+                    remove_from_key_value_store=False,
+                    remove_from_storage=False,
+                )
 
             # Run blocking RAG API call in thread pool to avoid blocking event loop
             await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/impl/chunker/text_chunker.py

@@ -1,7 +1,7 @@
 """Module containing the TextChunker class."""
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 from admin_api_lib.chunker.chunker import Chunker
 

admin_api_lib-4.1.0/src/admin_api_lib/impl/file_services/s3_service.py

@@ -0,0 +1,5 @@
+"""Re-export core S3 service implementation."""
+
+from rag_core_lib.impl.file_services.s3_service import S3Service
+
+__all__ = ["S3Service"]

admin_api_lib-4.1.0/src/admin_api_lib/impl/information_enhancer/page_summary_enhancer.py

@@ -0,0 +1,113 @@
+"""Module for enhancing the summary of pages by grouping information by page and summarizing each page."""
+
+import asyncio
+from hashlib import sha256
+from typing import Optional
+from typing import Any
+
+from langchain_core.documents import Document
+from langchain_core.runnables import RunnableConfig
+from tqdm import tqdm
+
+from admin_api_lib.impl.information_enhancer.summary_enhancer import SummaryEnhancer
+from rag_core_lib.impl.data_types.content_type import ContentType
+
+
+class PageSummaryEnhancer(SummaryEnhancer):
+    """
+    Enhances the summary of pages by grouping information by page and summarizing each page.
+
+    Attributes
+    ----------
+    BASE64_IMAGE_KEY : str
+        Key used to identify base64 encoded images in metadata.
+    DEFAULT_PAGE_NR : int
+        Default page number used when no page metadata is available.
+    """
+
+    BASE64_IMAGE_KEY = "base64_image"
+    DOCUMENT_URL_KEY = "document_url"
+    DEFAULT_PAGE_NR = 1
+
+    @staticmethod
+    def _parse_max_concurrency(config: Optional[RunnableConfig]) -> int:
+        if not config:
+            return 1
+        raw = config.get("max_concurrency")
+        if raw is None:
+            return 1
+        try:
+            return max(1, int(raw))
+        except (TypeError, ValueError):
+            return 1
+
+    def _group_key(self, piece: Document) -> tuple[Any, ...]:
+        document_url = piece.metadata.get(self.DOCUMENT_URL_KEY)
+        page = piece.metadata.get("page", self.DEFAULT_PAGE_NR)
+
+        # For paged documents (PDF/docling/etc.) keep per-page summaries even if a shared document URL exists.
+        if isinstance(page, int) or (isinstance(page, str) and page != "Unknown Title"):
+            return ("page_number", document_url, page)
+
+        # For sources like sitemaps/confluence, `page` can be a non-unique title (or missing),
+        # so group by the page URL when available to ensure one summary per page.
+        if document_url:
+            return ("document_url", document_url)
+
+        return ("page", page)
+
+    async def _asummarize_page(self, page_pieces: list[Document], config: Optional[RunnableConfig]) -> Document:
+        full_page_content = " ".join([piece.page_content for piece in page_pieces])
+        summary = await self._summarizer.ainvoke(full_page_content, config)
+        meta = {key: value for key, value in page_pieces[0].metadata.items() if key != self.BASE64_IMAGE_KEY}
+        meta["id"] = sha256(str.encode(full_page_content)).hexdigest()
+        meta["related"] = meta["related"] + [piece.metadata["id"] for piece in page_pieces]
+        meta["related"] = list(set(meta["related"]))
+        meta["type"] = ContentType.SUMMARY.value
+
+        return Document(metadata=meta, page_content=summary)
+
+    async def _acreate_summary(self, information: list[Document], config: Optional[RunnableConfig]) -> list[Document]:
+        grouped = self._group_information(information)
+        max_concurrency = self._parse_max_concurrency(config)
+        return await self._summarize_groups(grouped, config, max_concurrency=max_concurrency)
+
+    def _group_information(self, information: list[Document]) -> list[list[Document]]:
+        ordered_keys: list[tuple[Any, ...]] = []
+        groups: dict[tuple[Any, ...], list[Document]] = {}
+        for info in information:
+            key = self._group_key(info)
+            if key not in groups:
+                ordered_keys.append(key)
+                groups[key] = []
+            groups[key].append(info)
+        return [groups[key] for key in ordered_keys]
+
+    async def _summarize_groups(
+        self,
+        grouped: list[list[Document]],
+        config: Optional[RunnableConfig],
+        *,
+        max_concurrency: int,
+    ) -> list[Document]:
+        if max_concurrency == 1:
+            summaries: list[Document] = []
+            for info_group in tqdm(grouped):
+                summaries.append(await self._asummarize_page(info_group, config))
+            return summaries
+
+        semaphore = asyncio.Semaphore(max_concurrency)
+        results: list[Document | None] = [None] * len(grouped)
+
+        async def _run(idx: int, info_group: list[Document]) -> tuple[int, Document]:
+            async with semaphore:
+                return idx, await self._asummarize_page(info_group, config)
+
+        tasks = [asyncio.create_task(_run(idx, info_group)) for idx, info_group in enumerate(grouped)]
+        with tqdm(total=len(tasks)) as pbar:
+            for task in asyncio.as_completed(tasks):
+                idx, summary = await task
+                results[idx] = summary
+                pbar.update(1)
+
+        return [summary for summary in results if summary is not None]

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/impl/key_db/file_status_key_value_store.py

@@ -1,6 +1,8 @@
 """Module containing the FileStatusKeyValueStore class."""
 
 import json
+import ssl
+from typing import Any
 
 from redis import Redis
 
@@ -37,9 +39,53 @@ class FileStatusKeyValueStore:
         Parameters
         ----------
         settings : KeyValueSettings
-            The settings object containing the host and port information for the Redis connection.
+            The settings object containing the connection information for the Redis connection.
         """
-        self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True)
+        redis_kwargs: dict[str, Any] = {
+            "host": settings.host,
+            "port": settings.port,
+            "decode_responses": True,
+            **self._build_ssl_kwargs(settings),
+        }
+        if settings.username:
+            redis_kwargs["username"] = settings.username
+        if settings.password:
+            redis_kwargs["password"] = settings.password
+
+        self._redis = Redis(**redis_kwargs)
+
+    @staticmethod
+    def _build_ssl_kwargs(settings: KeyValueSettings) -> dict[str, Any]:
+        """Build Redis SSL settings from configuration, mapping string values to ssl constants."""
+        if not settings.use_ssl:
+            return {}
+
+        cert_reqs_map = {
+            "required": ssl.CERT_REQUIRED,
+            "optional": ssl.CERT_OPTIONAL,
+            "none": ssl.CERT_NONE,
+            "cert_required": ssl.CERT_REQUIRED,
+            "cert_optional": ssl.CERT_OPTIONAL,
+            "cert_none": ssl.CERT_NONE,
+        }
+        ssl_cert_reqs = None
+        if settings.ssl_cert_reqs:
+            ssl_cert_reqs = cert_reqs_map.get(settings.ssl_cert_reqs.lower(), settings.ssl_cert_reqs)
+
+        ssl_kwargs: dict[str, Any] = {
+            "ssl": settings.use_ssl,
+            "ssl_check_hostname": settings.ssl_check_hostname,
+        }
+        if ssl_cert_reqs is not None:
+            ssl_kwargs["ssl_cert_reqs"] = ssl_cert_reqs
+        if settings.ssl_ca_certs:
+            ssl_kwargs["ssl_ca_certs"] = settings.ssl_ca_certs
+        if settings.ssl_certfile:
+            ssl_kwargs["ssl_certfile"] = settings.ssl_certfile
+        if settings.ssl_keyfile:
+            ssl_kwargs["ssl_keyfile"] = settings.ssl_keyfile
+
+        return ssl_kwargs
 
     @staticmethod
     def _to_str(file_name: str, file_status: Status) -> str:

admin_api_lib-4.1.0/src/admin_api_lib/impl/settings/key_value_settings.py

@@ -0,0 +1,50 @@
+"""Contains settings regarding the key values store."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class KeyValueSettings(BaseSettings):
+    """
+    Contains settings regarding the key value store.
+
+    Attributes
+    ----------
+    host : str
+        The hostname of the key value store.
+    port : int
+        The port number of the key value store.
+    username : str | None
+        Optional username for authenticating with the key value store.
+    password : str | None
+        Optional password for authenticating with the key value store.
+    use_ssl : bool
+        Whether to use SSL/TLS when connecting to the key value store.
+    ssl_cert_reqs : str | None
+        SSL certificate requirement level (e.g., 'required', 'optional', 'none').
+    ssl_ca_certs : str | None
+        Path to a CA bundle file for verifying the server certificate.
+    ssl_certfile : str | None
+        Path to the client SSL certificate file (if mutual TLS is required).
+    ssl_keyfile : str | None
+        Path to the client SSL private key file (if mutual TLS is required).
+    ssl_check_hostname : bool
+        Whether to verify the server hostname against the certificate.
+    """
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "USECASE_KEYVALUE_"
+        case_sensitive = False
+
+    host: str = Field()
+    port: int = Field()
+    username: str | None = Field(default=None)
+    password: str | None = Field(default=None)
+    use_ssl: bool = Field(default=False)
+    ssl_cert_reqs: str | None = Field(default=None)
+    ssl_ca_certs: str | None = Field(default=None)
+    ssl_certfile: str | None = Field(default=None)
+    ssl_keyfile: str | None = Field(default=None)
+    ssl_check_hostname: bool = Field(default=True)

admin_api_lib-4.1.0/src/admin_api_lib/impl/settings/s3_settings.py

@@ -0,0 +1,5 @@
+"""Re-export core S3 settings."""
+
+from rag_core_lib.impl.settings.s3_settings import S3Settings
+
+__all__ = ["S3Settings"]

{admin_api_lib-3.4.0 → admin_api_lib-4.1.0}/src/admin_api_lib/impl/summarizer/langchain_summarizer.py

@@ -4,9 +4,9 @@ import asyncio
 import logging
 from typing import Optional
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
 from langchain_core.runnables import Runnable, RunnableConfig, ensure_config
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 from openai import APIConnectionError, APIError, APITimeoutError, RateLimitError
 
 from admin_api_lib.impl.settings.summarizer_settings import SummarizerSettings
@@ -44,6 +44,24 @@ class LangchainSummarizer(Summarizer):
         self._semaphore = semaphore
         self._retry_decorator_settings = create_retry_decorator_settings(summarizer_settings, retry_decorator_settings)
 
+    @staticmethod
+    def _parse_max_concurrency(config: RunnableConfig) -> Optional[int]:
+        """Parse max concurrency from a RunnableConfig.
+
+        Returns
+        -------
+        Optional[int]
+            An integer >= 1 if configured and valid, otherwise None.
+        """
+        max_concurrency = config.get("max_concurrency")
+        if max_concurrency is None:
+            return None
+
+        try:
+            return max(1, int(max_concurrency))
+        except (TypeError, ValueError):
+            return None
+
     async def ainvoke(self, query: SummarizerInput, config: Optional[RunnableConfig] = None) -> SummarizerOutput:
         """
         Asynchronously invokes the summarization process on the given query.
@@ -77,9 +95,8 @@ class LangchainSummarizer(Summarizer):
         langchain_documents = self._chunker.split_documents([document])
         logger.debug("Summarizing %d chunk(s)...", len(langchain_documents))
 
-        # Fan out with concurrency, bounded by your semaphore inside _summarize_chunk
-        tasks = [asyncio.create_task(self._summarize_chunk(doc.page_content, config)) for doc in langchain_documents]
-        outputs = await asyncio.gather(*tasks)
+        max_concurrency = self._parse_max_concurrency(config)
+        outputs = await self._summarize_documents(langchain_documents, config, max_concurrency=max_concurrency)
 
         if len(outputs) == 1:
             return outputs[0]
@@ -93,6 +110,34 @@ class LangchainSummarizer(Summarizer):
         )
         return await self._summarize_chunk(merged, config)
 
+    async def _summarize_documents(
+        self,
+        documents: list[Document],
+        config: RunnableConfig,
+        *,
+        max_concurrency: Optional[int],
+    ) -> list[SummarizerOutput]:
+        """Summarize a set of already-chunked documents.
+
+        Notes
+        -----
+        This optionally limits task fan-out using a per-call semaphore (max_concurrency).
+        The actual LLM call concurrency is always bounded by the instance semaphore held
+        inside `_summarize_chunk`.
+        """
+        if max_concurrency == 1:
+            return [await self._summarize_chunk(doc.page_content, config) for doc in documents]
+
+        limiter: asyncio.Semaphore | None = asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None
+
+        async def _run(doc: Document) -> SummarizerOutput:
+            if limiter is None:
+                return await self._summarize_chunk(doc.page_content, config)
+            async with limiter:
+                return await self._summarize_chunk(doc.page_content, config)
+
+        return await asyncio.gather(*(_run(doc) for doc in documents))
+
     def _create_chain(self) -> Runnable:
         return self._langfuse_manager.get_base_prompt(self.__class__.__name__) | self._langfuse_manager.get_base_llm(
             self.__class__.__name__