admin-api-lib 3.3.0__tar.gz → 4.0.0__tar.gz

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: admin-api-lib
-Version: 3.3.0
+Version: 4.0.0
 Summary: The admin backend is responsible for the document management. This includes deletion, upload and returning the source document.
 License: Apache-2.0
 Author: STACKIT GmbH & Co. KG
@@ -13,16 +13,17 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: boto3 (>=1.38.10,<2.0.0)
 Requires-Dist: dependency-injector (>=4.46.0,<5.0.0)
-Requires-Dist: fastapi (>=0.118.0,<0.119.0)
-Requires-Dist: langchain-experimental (>=0.3.4,<0.4.0)
-Requires-Dist: langfuse (==3.6.1)
+Requires-Dist: fastapi (>=0.121.2,<0.122.0)
+Requires-Dist: langchain-experimental (>=0.4.0,<0.5.0)
+Requires-Dist: langfuse (>=3.10.1,<4.0.0)
+Requires-Dist: langgraph-checkpoint (>=3.0.0,<4.0.0)
 Requires-Dist: nltk (>=3.9.2,<4.0.0)
 Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
 Requires-Dist: python-multipart (>=0.0.20,<0.0.21)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
-Requires-Dist: rag-core-lib (>=3.3.0,<4.0.0)
+Requires-Dist: rag-core-lib (==4.0.0)
 Requires-Dist: redis (>=6.0.0,<7.0.0)
-Requires-Dist: starlette (>=0.47.2,<0.49.0)
+Requires-Dist: starlette (>=0.49.1)
 Requires-Dist: tenacity (==9.1.2)
 Requires-Dist: tqdm (>=4.67.1,<5.0.0)
 Requires-Dist: uvicorn (>=0.37.0,<0.38.0)
@@ -99,6 +100,8 @@ All settings are powered by `pydantic-settings`, so you can use environment vari
 - `SUMMARIZER_MAXIMUM_INPUT_SIZE`, `SUMMARIZER_MAXIMUM_CONCURRENCY`, `SUMMARIZER_MAX_RETRIES`, etc. – tune summariser limits and retry behaviour.
 - `SOURCE_UPLOADER_TIMEOUT` – adjust how long non-file source ingestions wait before timing out.
 - `USECASE_KEYVALUE_HOST` / `USECASE_KEYVALUE_PORT` – configure the KeyDB/Redis instance that persists document status.
+- `USECASE_KEYVALUE_USERNAME` / `USECASE_KEYVALUE_PASSWORD` – optional credentials for authenticating against KeyDB/Redis.
+- `USECASE_KEYVALUE_USE_SSL`, `USECASE_KEYVALUE_SSL_CERT_REQS`, `USECASE_KEYVALUE_SSL_CA_CERTS`, `USECASE_KEYVALUE_SSL_CERTFILE`, `USECASE_KEYVALUE_SSL_KEYFILE`, `USECASE_KEYVALUE_SSL_CHECK_HOSTNAME` – optional TLS settings for managed Redis deployments (e.g., STACKIT Redis or other SSL-only endpoints).
 
 The Helm chart forwards these values through `adminBackend.envs.*`, keeping deployments declarative. Local development can rely on `.env` as described in the repository root README.
 

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/README.md

@@ -67,6 +67,8 @@ All settings are powered by `pydantic-settings`, so you can use environment vari
 - `SUMMARIZER_MAXIMUM_INPUT_SIZE`, `SUMMARIZER_MAXIMUM_CONCURRENCY`, `SUMMARIZER_MAX_RETRIES`, etc. – tune summariser limits and retry behaviour.
 - `SOURCE_UPLOADER_TIMEOUT` – adjust how long non-file source ingestions wait before timing out.
 - `USECASE_KEYVALUE_HOST` / `USECASE_KEYVALUE_PORT` – configure the KeyDB/Redis instance that persists document status.
+- `USECASE_KEYVALUE_USERNAME` / `USECASE_KEYVALUE_PASSWORD` – optional credentials for authenticating against KeyDB/Redis.
+- `USECASE_KEYVALUE_USE_SSL`, `USECASE_KEYVALUE_SSL_CERT_REQS`, `USECASE_KEYVALUE_SSL_CA_CERTS`, `USECASE_KEYVALUE_SSL_CERTFILE`, `USECASE_KEYVALUE_SSL_KEYFILE`, `USECASE_KEYVALUE_SSL_CHECK_HOSTNAME` – optional TLS settings for managed Redis deployments (e.g., STACKIT Redis or other SSL-only endpoints).
 
 The Helm chart forwards these values through `adminBackend.envs.*`, keeping deployments declarative. Local development can rely on `.env` as described in the repository root README.
 

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "admin-api-lib"
-version = "v3.3.0"
+version = "4.0.0"
 description = "The admin backend is responsible for the document management. This includes deletion, upload and returning the source document."
 authors = [
     "STACKIT GmbH & Co. KG <data-ai@stackit.cloud>",
@@ -18,6 +18,11 @@ license = "Apache-2.0"
 repository = "https://github.com/stackitcloud/rag-template"
 homepage = "https://pypi.org/project/admin-api-lib"
 
+[[tool.poetry.source]]
+name = "testpypi"
+url = "https://test.pypi.org/simple/"
+priority = "explicit"
+
 [tool.flake8]
 exclude= [".eggs", "./libs/*", "./src/admin_api_lib/models/*", "./src/admin_api_lib/rag_backend_client/*", "./src/admin_api_lib/extractor_api_client/*", ".git", ".hg", ".mypy_cache", ".tox", ".venv", ".devcontainer", "venv", "_build", "buck-out", "build", "dist", "**/__init__.py"]
 statistics = true
@@ -74,10 +79,12 @@ known_local_folder = ["admin_api_lib", "rag_core_lib"]
 [tool.pylint]
 max-line-length = 120
 
-[tool.poetry.group.dev.dependencies]
-debugpy = "^1.8.14"
+[tool.poetry.group.test.dependencies]
 pytest = "^8.3.5"
+pytest-asyncio = "^1.0.0"
 coverage = "^7.8.0"
+
+[tool.poetry.group.lint.dependencies]
 flake8 = "^7.2.0"
 flake8-black = "^0.4.0"
 flake8-pyproject = "^1.2.3"
@@ -99,36 +106,33 @@ flake8-wot = "^0.2.0"
 flake8-function-order = "^0.0.5"
 flake8-tidy-imports = "^4.10.0"
 black = "^25.1.0"
-# flake8-logging-format = "^2024.24.12"
-# flake8-docstrings = "^1.7.0"
-pytest-asyncio = "^1.0.0"
+flake8-logging-format = "^2024.24.12"
+flake8-docstrings = "^1.7.0"
+
+[tool.poetry.group.dev.dependencies]
+debugpy = "^1.8.14"
 
 [tool.poetry.dependencies]
-rag-core-lib = "^3.3.0"
+rag-core-lib = "==4.0.0"
 python = "^3.13"
 uvicorn = "^0.37.0"
-fastapi = "^0.118.0"
+fastapi = "^0.121.2"
 dependency-injector = "^4.46.0"
 python-dateutil = "^2.9.0.post0"
 tenacity = "9.1.2"
 boto3 = "^1.38.10"
 tqdm = "^4.67.1"
-langfuse = "3.6.1"
+langfuse = "^3.10.1"
 redis = "^6.0.0"
 pyyaml = "^6.0.2"
 python-multipart = "^0.0.20"
-starlette = ">=0.47.2,<0.49.0"
-langchain-experimental = "^0.3.4"
+langchain-experimental = "^0.4.0"
 nltk = "^3.9.2"
+starlette = ">=0.49.1"
+langgraph-checkpoint = ">=3.0.0,<4.0.0"
 
 [tool.pytest.ini_options]
 log_cli = true
 log_cli_level = "DEBUG"
 pythonpath = "src"
 testpaths = "src/tests"
-
-
-[[tool.poetry.source]]
-name = "testpypi"
-url = "https://test.pypi.org/simple/"
-priority = "supplemental"

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/api_endpoints/file_uploader.py

@@ -8,6 +8,7 @@ from admin_api_lib.api_endpoints.uploader_base import UploaderBase
 
 
 class FileUploader(UploaderBase):
+    """File uploader endpoint of the admin API."""
 
     @abstractmethod
     async def upload_file(
@@ -16,7 +17,7 @@ class FileUploader(UploaderBase):
         file: UploadFile,
     ) -> None:
         """
-        Uploads a source file for content extraction.
+        Upload a source file for content extraction.
 
         Parameters
         ----------

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/api_endpoints/source_uploader.py

@@ -21,7 +21,7 @@ class SourceUploader(UploaderBase):
         timeout: Optional[float],
     ) -> None:
         """
-        Uploads the parameters for source content extraction.
+        Upload the parameters for source content extraction.
 
         Parameters
         ----------

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/api_endpoints/uploader_base.py

@@ -7,9 +7,7 @@ class UploaderBase:
     """Base class for uploader API endpoints."""
 
     def __init__(self):
-        """
-        Initialize the UploaderBase.
-        """
+        """Initialize the UploaderBase."""
         self._background_threads = []
 
     def _prune_background_threads(self) -> list[Thread]:

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/apis/admin_api.py

@@ -149,7 +149,7 @@ async def upload_file(
     request: Request,
 ) -> None:
     """
-    Uploads user selected sources.
+    Upload user selected sources.
 
     Parameters
     ----------
@@ -181,7 +181,7 @@ async def upload_source(
     key_value_pair: List[KeyValuePair] = Body(None, description="The key-value pairs for the source"),
 ) -> None:
     """
-    Uploads user selected sources.
+    Upload user selected sources.
 
     Parameters
     ----------

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/dependency_container.py

@@ -3,7 +3,7 @@
 from admin_api_lib.impl.api_endpoints.default_file_uploader import DefaultFileUploader
 from dependency_injector.containers import DeclarativeContainer
 from dependency_injector.providers import Configuration, List, Selector, Singleton
-from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import OllamaEmbeddings
 from langfuse import Langfuse
 

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/impl/api_endpoints/default_document_reference_retriever.py

@@ -2,7 +2,6 @@
 
 import io
 import logging
-import traceback
 
 from fastapi import HTTPException, Response, status
 
@@ -54,10 +53,9 @@ class DefaultDocumentReferenceRetriever(DocumentReferenceRetriever):
                 self._file_service.download_file(identification, document_buffer)
                 logger.debug("DONE retrieving document with id: %s", identification)
                 document_data = document_buffer.getvalue()
-            except Exception as e:
-                logger.error(
-                    "Error retrieving document with id: %s. Error: %s %s", identification, e, traceback.format_exc()
-                )
+            except Exception:
+                # Log full stack trace without embedding the exception object in the message (G200)
+                logger.exception("Error retrieving document with id: %s.", identification)
                 raise ValueError(f"Document with id '{identification}' not found.")
             finally:
                 document_buffer.close()

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py

@@ -1,6 +1,7 @@
+"""Module for the default file uploader implementation."""
+
 import logging
 from pathlib import Path
-import traceback
 import urllib
 import tempfile
 import asyncio
@@ -78,7 +79,7 @@ class DefaultFileUploader(FileUploader):
         file: UploadFile,
     ) -> None:
         """
-        Uploads a source file for content extraction.
+        Upload a source file for content extraction.
 
         Parameters
         ----------
@@ -109,7 +110,7 @@ class DefaultFileUploader(FileUploader):
             raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
         except Exception as e:
             self._key_value_store.upsert(source_name, Status.ERROR)
-            logger.error("Error while uploading %s = %s", source_name, str(e))
+            logger.exception("Error while uploading %s", source_name)
             raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
 
     def _log_task_exception(self, task: asyncio.Task) -> None:
@@ -124,19 +125,16 @@ class DefaultFileUploader(FileUploader):
         if task.done() and not task.cancelled():
             try:
                 task.result()  # This will raise the exception if one occurred
-            except Exception as e:
-                logger.error("Background task failed with exception: %s", str(e))
-                logger.debug("Background task exception traceback: %s", traceback.format_exc())
+            except Exception:
+                logger.exception("Background task failed with exception.")
 
     def _prune_background_tasks(self) -> None:
-        """
-        Remove completed background tasks from the list.
-        """
+        """Remove completed background tasks from the list."""
         self._background_tasks = [task for task in self._background_tasks if not task.done()]
 
     def _check_if_already_in_processing(self, source_name: str) -> None:
         """
-        Checks if the source is already in processing state.
+        Check if the source is already in processing state.
 
         Parameters
         ----------
@@ -196,9 +194,9 @@ class DefaultFileUploader(FileUploader):
             await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)
             self._key_value_store.upsert(source_name, Status.READY)
             logger.info("Source uploaded successfully: %s", source_name)
-        except Exception as e:
+        except Exception:
             self._key_value_store.upsert(source_name, Status.ERROR)
-            logger.error("Error while uploading %s = %s", source_name, str(e))
+            logger.exception("Error while uploading %s", source_name)
 
     def _add_file_url(self, file_name: str, base_url: str, chunked_documents: list[Document]):
         document_url = f"{base_url.rstrip('/')}/document_reference/{urllib.parse.quote_plus(file_name)}"
@@ -229,6 +227,6 @@ class DefaultFileUploader(FileUploader):
 
                 self._file_service.upload_file(Path(temp_file_path), filename)
                 return filename
-        except Exception as e:
-            logger.error("Error during document saving: %s %s", e, traceback.format_exc())
+        except Exception:
+            logger.exception("Error during document saving")
             self._key_value_store.upsert(source_name, Status.ERROR)

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py

@@ -1,3 +1,5 @@
+"""Module for the default source uploader implementation."""
+
 import logging
 import asyncio
 from threading import Thread
@@ -28,6 +30,7 @@ logger = logging.getLogger(__name__)
 
 
 class DefaultSourceUploader(SourceUploader):
+    """The DefaultSourceUploader is responsible for uploading source files for content extraction."""
 
     def __init__(
         self,
@@ -78,7 +81,7 @@ class DefaultSourceUploader(SourceUploader):
         kwargs: list[KeyValuePair],
     ) -> None:
         """
-        Uploads the parameters for source content extraction.
+        Upload the parameters for source content extraction.
 
         Parameters
         ----------
@@ -95,7 +98,6 @@ class DefaultSourceUploader(SourceUploader):
         -------
         None
         """
-
         self._prune_background_threads()
 
         source_name = f"{source_type}:{sanitize_document_name(name)}"
@@ -111,12 +113,12 @@ class DefaultSourceUploader(SourceUploader):
             raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
         except Exception as e:
             self._key_value_store.upsert(source_name, Status.ERROR)
-            logger.error("Error while uploading %s = %s", source_name, str(e))
+            logger.exception("Error while uploading %s", source_name)
             raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))
 
     def _check_if_already_in_processing(self, source_name: str) -> None:
         """
-        Checks if the source is already in processing state.
+        Check if the source is already in processing state.
 
         Parameters
         ----------
@@ -147,10 +149,14 @@ class DefaultSourceUploader(SourceUploader):
                 )
             )
         except asyncio.TimeoutError:
-            logger.error("Upload of %s timed out after %s seconds", source_name, timeout)
+            logger.error(
+                "Upload of %s timed out after %s seconds (increase SOURCE_UPLOADER_TIMEOUT to allow longer ingestions)",
+                source_name,
+                timeout,
+            )
             self._key_value_store.upsert(source_name, Status.ERROR)
         except Exception:
-            logger.error("Error while uploading %s", source_name)
+            logger.exception("Error while uploading %s", source_name)
             self._key_value_store.upsert(source_name, Status.ERROR)
         finally:
             loop.close()
@@ -197,6 +203,6 @@ class DefaultSourceUploader(SourceUploader):
             await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)
             self._key_value_store.upsert(source_name, Status.READY)
             logger.info("Source uploaded successfully: %s", source_name)
-        except Exception as e:
+        except Exception:
             self._key_value_store.upsert(source_name, Status.ERROR)
-            logger.error("Error while uploading %s = %s", source_name, str(e))
+            logger.exception("Error while uploading %s", source_name)

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/impl/chunker/text_chunker.py

@@ -1,7 +1,7 @@
 """Module containing the TextChunker class."""
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 from admin_api_lib.chunker.chunker import Chunker
 

admin_api_lib-4.0.0/src/admin_api_lib/impl/information_enhancer/page_summary_enhancer.py

@@ -0,0 +1,113 @@
+"""Module for enhancing the summary of pages by grouping information by page and summarizing each page."""
+
+import asyncio
+from hashlib import sha256
+from typing import Optional
+from typing import Any
+
+from langchain_core.documents import Document
+from langchain_core.runnables import RunnableConfig
+from tqdm import tqdm
+
+from admin_api_lib.impl.information_enhancer.summary_enhancer import SummaryEnhancer
+from rag_core_lib.impl.data_types.content_type import ContentType
+
+
+class PageSummaryEnhancer(SummaryEnhancer):
+    """
+    Enhances the summary of pages by grouping information by page and summarizing each page.
+
+    Attributes
+    ----------
+    BASE64_IMAGE_KEY : str
+        Key used to identify base64 encoded images in metadata.
+    DEFAULT_PAGE_NR : int
+        Default page number used when no page metadata is available.
+    """
+
+    BASE64_IMAGE_KEY = "base64_image"
+    DOCUMENT_URL_KEY = "document_url"
+    DEFAULT_PAGE_NR = 1
+
+    @staticmethod
+    def _parse_max_concurrency(config: Optional[RunnableConfig]) -> int:
+        if not config:
+            return 1
+        raw = config.get("max_concurrency")
+        if raw is None:
+            return 1
+        try:
+            return max(1, int(raw))
+        except (TypeError, ValueError):
+            return 1
+
+    def _group_key(self, piece: Document) -> tuple[Any, ...]:
+        document_url = piece.metadata.get(self.DOCUMENT_URL_KEY)
+        page = piece.metadata.get("page", self.DEFAULT_PAGE_NR)
+
+        # For paged documents (PDF/docling/etc.) keep per-page summaries even if a shared document URL exists.
+        if isinstance(page, int) or (isinstance(page, str) and page != "Unknown Title"):
+            return ("page_number", document_url, page)
+
+        # For sources like sitemaps/confluence, `page` can be a non-unique title (or missing),
+        # so group by the page URL when available to ensure one summary per page.
+        if document_url:
+            return ("document_url", document_url)
+
+        return ("page", page)
+
+    async def _asummarize_page(self, page_pieces: list[Document], config: Optional[RunnableConfig]) -> Document:
+        full_page_content = " ".join([piece.page_content for piece in page_pieces])
+        summary = await self._summarizer.ainvoke(full_page_content, config)
+        meta = {key: value for key, value in page_pieces[0].metadata.items() if key != self.BASE64_IMAGE_KEY}
+        meta["id"] = sha256(str.encode(full_page_content)).hexdigest()
+        meta["related"] = meta["related"] + [piece.metadata["id"] for piece in page_pieces]
+        meta["related"] = list(set(meta["related"]))
+        meta["type"] = ContentType.SUMMARY.value
+
+        return Document(metadata=meta, page_content=summary)
+
+    async def _acreate_summary(self, information: list[Document], config: Optional[RunnableConfig]) -> list[Document]:
+        grouped = self._group_information(information)
+        max_concurrency = self._parse_max_concurrency(config)
+        return await self._summarize_groups(grouped, config, max_concurrency=max_concurrency)
+
+    def _group_information(self, information: list[Document]) -> list[list[Document]]:
+        ordered_keys: list[tuple[Any, ...]] = []
+        groups: dict[tuple[Any, ...], list[Document]] = {}
+        for info in information:
+            key = self._group_key(info)
+            if key not in groups:
+                ordered_keys.append(key)
+                groups[key] = []
+            groups[key].append(info)
+        return [groups[key] for key in ordered_keys]
+
+    async def _summarize_groups(
+        self,
+        grouped: list[list[Document]],
+        config: Optional[RunnableConfig],
+        *,
+        max_concurrency: int,
+    ) -> list[Document]:
+        if max_concurrency == 1:
+            summaries: list[Document] = []
+            for info_group in tqdm(grouped):
+                summaries.append(await self._asummarize_page(info_group, config))
+            return summaries
+
+        semaphore = asyncio.Semaphore(max_concurrency)
+        results: list[Document | None] = [None] * len(grouped)
+
+        async def _run(idx: int, info_group: list[Document]) -> tuple[int, Document]:
+            async with semaphore:
+                return idx, await self._asummarize_page(info_group, config)
+
+        tasks = [asyncio.create_task(_run(idx, info_group)) for idx, info_group in enumerate(grouped)]
+        with tqdm(total=len(tasks)) as pbar:
+            for task in asyncio.as_completed(tasks):
+                idx, summary = await task
+                results[idx] = summary
+                pbar.update(1)
+
+        return [summary for summary in results if summary is not None]

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/impl/key_db/file_status_key_value_store.py

@@ -1,6 +1,8 @@
 """Module containing the FileStatusKeyValueStore class."""
 
 import json
+import ssl
+from typing import Any
 
 from redis import Redis
 
@@ -37,9 +39,53 @@ class FileStatusKeyValueStore:
         Parameters
         ----------
         settings : KeyValueSettings
-            The settings object containing the host and port information for the Redis connection.
+            The settings object containing the connection information for the Redis connection.
         """
-        self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True)
+        redis_kwargs: dict[str, Any] = {
+            "host": settings.host,
+            "port": settings.port,
+            "decode_responses": True,
+            **self._build_ssl_kwargs(settings),
+        }
+        if settings.username:
+            redis_kwargs["username"] = settings.username
+        if settings.password:
+            redis_kwargs["password"] = settings.password
+
+        self._redis = Redis(**redis_kwargs)
+
+    @staticmethod
+    def _build_ssl_kwargs(settings: KeyValueSettings) -> dict[str, Any]:
+        """Build Redis SSL settings from configuration, mapping string values to ssl constants."""
+        if not settings.use_ssl:
+            return {}
+
+        cert_reqs_map = {
+            "required": ssl.CERT_REQUIRED,
+            "optional": ssl.CERT_OPTIONAL,
+            "none": ssl.CERT_NONE,
+            "cert_required": ssl.CERT_REQUIRED,
+            "cert_optional": ssl.CERT_OPTIONAL,
+            "cert_none": ssl.CERT_NONE,
+        }
+        ssl_cert_reqs = None
+        if settings.ssl_cert_reqs:
+            ssl_cert_reqs = cert_reqs_map.get(settings.ssl_cert_reqs.lower(), settings.ssl_cert_reqs)
+
+        ssl_kwargs: dict[str, Any] = {
+            "ssl": settings.use_ssl,
+            "ssl_check_hostname": settings.ssl_check_hostname,
+        }
+        if ssl_cert_reqs is not None:
+            ssl_kwargs["ssl_cert_reqs"] = ssl_cert_reqs
+        if settings.ssl_ca_certs:
+            ssl_kwargs["ssl_ca_certs"] = settings.ssl_ca_certs
+        if settings.ssl_certfile:
+            ssl_kwargs["ssl_certfile"] = settings.ssl_certfile
+        if settings.ssl_keyfile:
+            ssl_kwargs["ssl_keyfile"] = settings.ssl_keyfile
+
+        return ssl_kwargs
 
     @staticmethod
     def _to_str(file_name: str, file_status: Status) -> str:

admin_api_lib-4.0.0/src/admin_api_lib/impl/settings/key_value_settings.py

@@ -0,0 +1,50 @@
+"""Contains settings regarding the key values store."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class KeyValueSettings(BaseSettings):
+    """
+    Contains settings regarding the key value store.
+
+    Attributes
+    ----------
+    host : str
+        The hostname of the key value store.
+    port : int
+        The port number of the key value store.
+    username : str | None
+        Optional username for authenticating with the key value store.
+    password : str | None
+        Optional password for authenticating with the key value store.
+    use_ssl : bool
+        Whether to use SSL/TLS when connecting to the key value store.
+    ssl_cert_reqs : str | None
+        SSL certificate requirement level (e.g., 'required', 'optional', 'none').
+    ssl_ca_certs : str | None
+        Path to a CA bundle file for verifying the server certificate.
+    ssl_certfile : str | None
+        Path to the client SSL certificate file (if mutual TLS is required).
+    ssl_keyfile : str | None
+        Path to the client SSL private key file (if mutual TLS is required).
+    ssl_check_hostname : bool
+        Whether to verify the server hostname against the certificate.
+    """
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "USECASE_KEYVALUE_"
+        case_sensitive = False
+
+    host: str = Field()
+    port: int = Field()
+    username: str | None = Field(default=None)
+    password: str | None = Field(default=None)
+    use_ssl: bool = Field(default=False)
+    ssl_cert_reqs: str | None = Field(default=None)
+    ssl_ca_certs: str | None = Field(default=None)
+    ssl_certfile: str | None = Field(default=None)
+    ssl_keyfile: str | None = Field(default=None)
+    ssl_check_hostname: bool = Field(default=True)

{admin_api_lib-3.3.0 → admin_api_lib-4.0.0}/src/admin_api_lib/impl/summarizer/langchain_summarizer.py

@@ -4,9 +4,9 @@ import asyncio
 import logging
 from typing import Optional
 
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
 from langchain_core.runnables import Runnable, RunnableConfig, ensure_config
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 from openai import APIConnectionError, APIError, APITimeoutError, RateLimitError
 
 from admin_api_lib.impl.settings.summarizer_settings import SummarizerSettings
@@ -44,6 +44,24 @@ class LangchainSummarizer(Summarizer):
         self._semaphore = semaphore
         self._retry_decorator_settings = create_retry_decorator_settings(summarizer_settings, retry_decorator_settings)
 
+    @staticmethod
+    def _parse_max_concurrency(config: RunnableConfig) -> Optional[int]:
+        """Parse max concurrency from a RunnableConfig.
+
+        Returns
+        -------
+        Optional[int]
+            An integer >= 1 if configured and valid, otherwise None.
+        """
+        max_concurrency = config.get("max_concurrency")
+        if max_concurrency is None:
+            return None
+
+        try:
+            return max(1, int(max_concurrency))
+        except (TypeError, ValueError):
+            return None
+
     async def ainvoke(self, query: SummarizerInput, config: Optional[RunnableConfig] = None) -> SummarizerOutput:
         """
         Asynchronously invokes the summarization process on the given query.
@@ -77,9 +95,8 @@ class LangchainSummarizer(Summarizer):
         langchain_documents = self._chunker.split_documents([document])
         logger.debug("Summarizing %d chunk(s)...", len(langchain_documents))
 
-        # Fan out with concurrency, bounded by your semaphore inside _summarize_chunk
-        tasks = [asyncio.create_task(self._summarize_chunk(doc.page_content, config)) for doc in langchain_documents]
-        outputs = await asyncio.gather(*tasks)
+        max_concurrency = self._parse_max_concurrency(config)
+        outputs = await self._summarize_documents(langchain_documents, config, max_concurrency=max_concurrency)
 
         if len(outputs) == 1:
             return outputs[0]
@@ -93,6 +110,34 @@ class LangchainSummarizer(Summarizer):
         )
         return await self._summarize_chunk(merged, config)
 
+    async def _summarize_documents(
+        self,
+        documents: list[Document],
+        config: RunnableConfig,
+        *,
+        max_concurrency: Optional[int],
+    ) -> list[SummarizerOutput]:
+        """Summarize a set of already-chunked documents.
+
+        Notes
+        -----
+        This optionally limits task fan-out using a per-call semaphore (max_concurrency).
+        The actual LLM call concurrency is always bounded by the instance semaphore held
+        inside `_summarize_chunk`.
+        """
+        if max_concurrency == 1:
+            return [await self._summarize_chunk(doc.page_content, config) for doc in documents]
+
+        limiter: asyncio.Semaphore | None = asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None
+
+        async def _run(doc: Document) -> SummarizerOutput:
+            if limiter is None:
+                return await self._summarize_chunk(doc.page_content, config)
+            async with limiter:
+                return await self._summarize_chunk(doc.page_content, config)
+
+        return await asyncio.gather(*(_run(doc) for doc in documents))
+
     def _create_chain(self) -> Runnable:
         return self._langfuse_manager.get_base_prompt(self.__class__.__name__) | self._langfuse_manager.get_base_llm(
             self.__class__.__name__

admin_api_lib-3.3.0/src/admin_api_lib/impl/information_enhancer/page_summary_enhancer.py

@@ -1,62 +0,0 @@
-"""Module for enhancing the summary of pages by grouping information by page and summarizing each page."""
-
-from asyncio import gather
-from hashlib import sha256
-from typing import Optional
-
-from langchain_core.documents import Document
-from langchain_core.runnables import RunnableConfig
-from tqdm import tqdm
-
-from admin_api_lib.impl.information_enhancer.summary_enhancer import SummaryEnhancer
-from rag_core_lib.impl.data_types.content_type import ContentType
-
-
-class PageSummaryEnhancer(SummaryEnhancer):
-    """
-    Enhances the summary of pages by grouping information by page and summarizing each page.
-
-    Attributes
-    ----------
-    BASE64_IMAGE_KEY : str
-        Key used to identify base64 encoded images in metadata.
-    DEFAULT_PAGE_NR : int
-        Default page number used when no page metadata is available.
-    """
-
-    BASE64_IMAGE_KEY = "base64_image"
-    DEFAULT_PAGE_NR = 1
-
-    async def _asummarize_page(self, page_pieces: list[Document], config: Optional[RunnableConfig]) -> Document:
-        full_page_content = " ".join([piece.page_content for piece in page_pieces])
-        summary = await self._summarizer.ainvoke(full_page_content, config)
-        meta = {key: value for key, value in page_pieces[0].metadata.items() if key != self.BASE64_IMAGE_KEY}
-        meta["id"] = sha256(str.encode(full_page_content)).hexdigest()
-        meta["related"] = meta["related"] + [piece.metadata["id"] for piece in page_pieces]
-        meta["related"] = list(set(meta["related"]))
-        meta["type"] = ContentType.SUMMARY.value
-
-        return Document(metadata=meta, page_content=summary)
-
-    async def _acreate_summary(self, information: list[Document], config: Optional[RunnableConfig]) -> list[Document]:
-        distinct_pages = []
-        for info in information:
-            if info.metadata.get("page", self.DEFAULT_PAGE_NR) not in distinct_pages:
-                distinct_pages.append(info.metadata.get("page", self.DEFAULT_PAGE_NR))
-
-        grouped = []
-        for page in distinct_pages:
-            group = []
-            for compare_info in information:
-                if compare_info.metadata.get("page", self.DEFAULT_PAGE_NR) == page:
-                    group.append(compare_info)
-            if (
-                self._chunker_settings
-                and len(" ".join([item.page_content for item in group])) < self._chunker_settings.max_size
-            ):
-                continue
-            grouped.append(group)
-
-        summary_tasks = [self._asummarize_page(info_group, config) for info_group in tqdm(grouped)]
-
-        return await gather(*summary_tasks)

admin_api_lib-3.3.0/src/admin_api_lib/impl/settings/key_value_settings.py

@@ -1,26 +0,0 @@
-"""Contains settings regarding the key values store."""
-
-from pydantic import Field
-from pydantic_settings import BaseSettings
-
-
-class KeyValueSettings(BaseSettings):
-    """
-    Contains settings regarding the key value store.
-
-    Attributes
-    ----------
-    host : str
-        The hostname of the key value store.
-    port : int
-        The port number of the key value store.
-    """
-
-    class Config:
-        """Config class for reading Fields from env."""
-
-        env_prefix = "USECASE_KEYVALUE_"
-        case_sensitive = False
-
-    host: str = Field()
-    port: int = Field()