admin-api-lib 3.2.0__py3-none-any.whl

admin_api_lib/impl/key_db/file_status_key_value_store.py

@@ -0,0 +1,111 @@
+"""Module containing the FileStatusKeyValueStore class."""
+
+import json
+
+from redis import Redis
+
+from admin_api_lib.impl.settings.key_value_settings import KeyValueSettings
+from admin_api_lib.models.status import Status
+
+
+class FileStatusKeyValueStore:
+    """
+    A key-value store for managing file statuses using Redis.
+
+    This class provides methods to upsert, remove, and retrieve file status information
+    from a Redis store. Each file status is stored as a JSON string containing the file name
+    and its associated status.
+
+    Attributes
+    ----------
+    STORAGE_KEY : str
+        The key under which all file statuses are stored in Redis.
+    INNER_FILENAME_KEY : str
+        The key used for the file name in the JSON string.
+    INNER_STATUS_KEY : str
+        The key used for the file status in the JSON string.
+    """
+
+    STORAGE_KEY = "stackit-rag-template-files"
+    INNER_FILENAME_KEY = "filename"
+    INNER_STATUS_KEY = "status"
+
+    def __init__(self, settings: KeyValueSettings):
+        """
+        Initialize the FileStatusKeyValueStore with the given settings.
+
+        Parameters
+        ----------
+        settings : KeyValueSettings
+            The settings object containing the host and port information for the Redis connection.
+        """
+        self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True)
+
+    @staticmethod
+    def _to_str(file_name: str, file_status: Status) -> str:
+        return json.dumps(
+            {
+                FileStatusKeyValueStore.INNER_FILENAME_KEY: file_name,
+                FileStatusKeyValueStore.INNER_STATUS_KEY: file_status,
+            }
+        )
+
+    @staticmethod
+    def _from_str(redis_content: str) -> tuple[str, Status]:
+        content_dict = json.loads(redis_content)
+        return (
+            content_dict[FileStatusKeyValueStore.INNER_FILENAME_KEY],
+            content_dict[FileStatusKeyValueStore.INNER_STATUS_KEY],
+        )
+
+    def upsert(self, file_name: str, file_status: Status) -> None:
+        """
+        Upserts the status of a file in the key-value store.
+
+        This method first removes any existing entry for the given file name and then adds the new status.
+
+        Parameters
+        ----------
+        file_name : str
+            The name of the file whose status is to be upserted.
+        file_status : Status
+            The status to be associated with the file.
+
+        Returns
+        -------
+        None
+        """
+        self.remove(file_name)
+        self._redis.sadd(self.STORAGE_KEY, FileStatusKeyValueStore._to_str(file_name, file_status))
+
+    def remove(self, file_name: str) -> None:
+        """
+        Remove the specified file name from the key-value store.
+
+        Parameters
+        ----------
+        file_name : str
+            The name of the file to be removed from the key-value store.
+
+        Returns
+        -------
+        None
+        """
+        all_documents = self.get_all()
+        correct_file_name = [x for x in all_documents if x[0] == file_name]
+        for file_name_related in correct_file_name:
+            self._redis.srem(
+                self.STORAGE_KEY, FileStatusKeyValueStore._to_str(file_name_related[0], file_name_related[1])
+            )
+
+    def get_all(self) -> list[tuple[str, Status]]:
+        """
+        Retrieve all file status information from the Redis store.
+
+        Returns
+        -------
+        list[tuple[str, Status]]
+            A list of tuples where each tuple contains a string and a Status object.
+        """
+        all_file_informations = list(self._redis.smembers(self.STORAGE_KEY))
+        return [FileStatusKeyValueStore._from_str(x) for x in all_file_informations]

admin_api_lib/impl/mapper/informationpiece2document.py

@@ -0,0 +1,108 @@
+"""Module for mapping between InformationPiece and LangchainDocument."""
+
+import json
+
+from langchain_core.documents import Document as LangchainDocument
+
+from admin_api_lib.extractor_api_client.openapi_client.models.content_type import (
+    ContentType as ExtractorInformaType,
+)
+from admin_api_lib.extractor_api_client.openapi_client.models.information_piece import (
+    InformationPiece as ExtractorInformationPiece,
+)
+from admin_api_lib.rag_backend_client.openapi_client.models.information_piece import (
+    InformationPiece as RagInformationPiece,
+)
+from admin_api_lib.rag_backend_client.openapi_client.models.key_value_pair import (
+    KeyValuePair as RagKeyValue,
+)
+from rag_core_lib.impl.data_types.content_type import ContentType as RagInformationType
+
+
+class InformationPiece2Document:
+    """The InformationPiece2Document class.
+
+    A utility class for converting between ExtractorInformationPiece and LangchainDocument,
+    and between LangchainDocument and RagInformationPiece.
+
+    Attributes
+    ----------
+    LOOKUP_TABLE : dict
+        A dictionary mapping ExtractorInformaType to RagInformationType.
+    METADATA_TYPE_KEY : str
+        The key used to store the type of information piece in metadata.
+    """
+
+    LOOKUP_TABLE = {
+        ExtractorInformaType.IMAGE: RagInformationType.IMAGE,
+        ExtractorInformaType.TABLE: RagInformationType.TABLE,
+        ExtractorInformaType.TEXT: RagInformationType.TEXT,
+    }
+    METADATA_TYPE_KEY = "type"
+
+    @staticmethod
+    def extractor_information_piece2document(info: ExtractorInformationPiece) -> LangchainDocument:
+        """
+        Convert an ExtractorInformationPiece instance to a LangchainDocument instance.
+
+        Parameters
+        ----------
+        info : ExtractorInformationPiece
+            The information piece to be converted, containing metadata page content, type.
+
+        Returns
+        -------
+        LangchainDocument
+            The converted LangchainDocument with the page content, metadata and type.
+
+        Notes
+        -----
+        The metadata of the resulting LangchainDocument includes all key-value pairs from the
+        input metadata, with an additional entry for the type of the information piece.
+        """
+        metadata = {x.key: x.value for x in info.metadata}
+        metadata[InformationPiece2Document.METADATA_TYPE_KEY] = InformationPiece2Document.infotype2infotype(
+            info.type
+        ).value
+
+        return LangchainDocument(page_content=info.page_content, metadata=metadata)
+
+    @staticmethod
+    def document2rag_information_piece(document: LangchainDocument) -> RagInformationPiece:
+        """
+        Convert a LangchainDocument to a RagInformationPiece.
+
+        Parameters
+        ----------
+        document : LangchainDocument
+            The document to be converted, containing metadata, page content and type.
+
+        Returns
+        -------
+        RagInformationPiece
+            The converted information piece with type, metadata, and page content.
+        """
+        metadata = [RagKeyValue(key=str(key), value=json.dumps(value)) for key, value in document.metadata.items()]
+        content_type = RagInformationType(document.metadata[InformationPiece2Document.METADATA_TYPE_KEY].upper())
+        return RagInformationPiece(
+            type=content_type,
+            metadata=metadata,
+            page_content=document.page_content,
+        )
+
+    @staticmethod
+    def infotype2infotype(info_type: ExtractorInformaType) -> RagInformationType:
+        """
+        Convert from ExtractorInformaType to RagInformationType.
+
+        Parameters
+        ----------
+        info_type : ExtractorInformaType
+            The external information type to be converted.
+
+        Returns
+        -------
+        RagInformationType
+            The corresponding internal information type.
+        """
+        return InformationPiece2Document.LOOKUP_TABLE[info_type]

admin_api_lib/impl/settings/chunker_class_type_settings.py

@@ -0,0 +1,18 @@
+"""Settings for selecting the embedder implementation."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+from admin_api_lib.impl.chunker.chunker_type import ChunkerType
+
+
+class ChunkerClassTypeSettings(BaseSettings):
+    """Settings controlling which chunker implementation is used."""
+
+    class Config:
+        """Configure environment integration for the settings."""
+
+        env_prefix = "CHUNKER_CLASS_TYPE_"
+        case_sensitive = False
+
+    chunker_type: ChunkerType = Field(default=ChunkerType.RECURSIVE)

admin_api_lib/impl/settings/chunker_settings.py

@@ -0,0 +1,29 @@
+"""Contains settings regarding the chunker."""
+
+from typing import Literal
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ChunkerSettings(BaseSettings):
+    """Contains settings regarding the chunker configuration."""
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "CHUNKER_"
+        case_sensitive = False
+
+    max_size: int = Field(default=1000, gt=0)
+    overlap: int = Field(default=100, ge=0)
+
+    breakpoint_threshold_type: Literal[
+        "percentile",
+        "standard_deviation",
+        "interquartile",
+        "gradient",
+    ] = Field(default="percentile")
+    breakpoint_threshold_amount: float = Field(default=95.0, ge=0.0)
+    buffer_size: int = Field(default=1, ge=0)
+    min_size: int = Field(default=200, gt=0)

admin_api_lib/impl/settings/document_extractor_settings.py

@@ -0,0 +1,21 @@
+"""Contains settings regarding the chunker."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class DocumentExtractorSettings(BaseSettings):
+    """Contains settings regarding the document extractor microservice.
+
+    Attributes
+    ----------
+    host (str): The url to the api.
+    """
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "DOCUMENT_EXTRACTOR_"
+        case_sensitive = False
+
+    host: str = Field(default="http://extractor:8080")

admin_api_lib/impl/settings/key_value_settings.py

@@ -0,0 +1,26 @@
+"""Contains settings regarding the key values store."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class KeyValueSettings(BaseSettings):
+    """
+    Contains settings regarding the key value store.
+
+    Attributes
+    ----------
+    host : str
+        The hostname of the key value store.
+    port : int
+        The port number of the key value store.
+    """
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "USECASE_KEYVALUE_"
+        case_sensitive = False
+
+    host: str = Field()
+    port: int = Field()

admin_api_lib/impl/settings/rag_api_settings.py

@@ -0,0 +1,21 @@
+"""Contains settings regarding the chunker."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class RAGAPISettings(BaseSettings):
+    """Contains settings regarding the rag api microservice.
+
+    Attributes
+    ----------
+    host (str): The url to the api.
+    """
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "RAG_API"
+        case_sensitive = False
+
+    host: str = Field(default="http://backend:8080")

admin_api_lib/impl/settings/s3_settings.py

@@ -0,0 +1,31 @@
+"""Contains settings regarding the S3 storage."""
+
+from pydantic_settings import BaseSettings
+
+
+class S3Settings(BaseSettings):
+    """
+    Contains settings regarding the S3 storage.
+
+    Attributes
+    ----------
+    secret_access_key : str
+        The secret access key for S3.
+    access_key_id : str
+        The access key ID for S3.
+    endpoint : str
+        The endpoint URL for S3.
+    bucket : str
+        The bucket name in S3.
+    """
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "S3_"
+        case_sensitive = False
+
+    secret_access_key: str
+    access_key_id: str
+    endpoint: str
+    bucket: str

admin_api_lib/impl/settings/source_uploader_settings.py

@@ -0,0 +1,23 @@
+"""Contains settings regarding the SourceUploader."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class SourceUploaderSettings(BaseSettings):
+    """
+    Contains settings regarding the SourceUploader.
+
+    Attributes
+    ----------
+    timeout : float
+       The timeout for the SourceUploader.
+    """
+
+    class Config:
+        """Config class for reading Fields from env."""
+
+        env_prefix = "SOURCE_UPLOADER_"
+        case_sensitive = False
+
+    timeout: float = Field(default=3600.0, description="Timeout for the SourceUploader in seconds.")

admin_api_lib/impl/settings/summarizer_settings.py

@@ -0,0 +1,86 @@
+"""Contains settings for summarizer."""
+
+from typing import Optional
+from pydantic import Field, PositiveInt, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class SummarizerSettings(BaseSettings):
+    """
+    Contains settings regarding the summarizer.
+
+    Attributes
+    ----------
+    maximum_input_size : int
+        The maximum size of the input that the summarizer can handle. Default is 8000.
+    maximum_concurrency : int
+        The maximum number of concurrent summarization processes. Default is 10.
+    max_retries: Optional[PositiveInt]
+        Total retries, not counting the initial attempt.
+    retry_base_delay: Optional[float]
+        Base delay in seconds for the first retry.
+    retry_max_delay: Optional[float]
+        Maximum delay cap in seconds for any single wait.
+    backoff_factor: Optional[float]
+        Exponential backoff factor (>= 1).
+    attempt_cap: Optional[int]
+        Cap for exponent growth (backoff_factor ** attempt_cap).
+    jitter_min: Optional[float]
+        Minimum jitter in seconds.
+    jitter_max: Optional[float]
+        Maximum jitter in seconds.
+    """
+
+    model_config = SettingsConfigDict(env_prefix="SUMMARIZER_", case_sensitive=False)
+
+    maximum_input_size: int = Field(default=8000)
+    maximum_concurrency: int = Field(default=10)
+    max_retries: Optional[PositiveInt] = Field(
+        default=None,
+        title="Max Retries",
+        description="Total retries, not counting the initial attempt.",
+    )
+    retry_base_delay: Optional[float] = Field(
+        default=None,
+        ge=0,
+        title="Retry Base Delay",
+        description="Base delay in seconds for the first retry.",
+    )
+    retry_max_delay: Optional[float] = Field(
+        default=None,
+        gt=0,
+        title="Retry Max Delay",
+        description="Maximum delay cap in seconds for any single wait.",
+    )
+    backoff_factor: Optional[float] = Field(
+        default=None,
+        ge=1.0,
+        title="Backoff Factor",
+        description="Exponential backoff factor (>= 1).",
+    )
+    attempt_cap: Optional[int] = Field(
+        default=None,
+        ge=0,
+        title="Attempt Cap",
+        description="Cap for exponent growth (backoff_factor ** attempt_cap).",
+    )
+    jitter_min: Optional[float] = Field(
+        default=None,
+        ge=0.0,
+        title="Jitter Min (s)",
+        description="Minimum jitter in seconds.",
+    )
+    jitter_max: Optional[float] = Field(
+        default=None,
+        ge=0.0,
+        title="Jitter Max (s)",
+        description="Maximum jitter in seconds.",
+    )
+
+    @model_validator(mode="after")
+    def _check_relations(self) -> "SummarizerSettings":
+        if not self.jitter_min or not self.jitter_max:
+            return self
+        if self.jitter_max < self.jitter_min:
+            raise ValueError("jitter_max must be >= jitter_min")
+        return self

admin_api_lib/impl/summarizer/langchain_summarizer.py

@@ -0,0 +1,117 @@
+"""Module for the LangchainSummarizer class."""
+
+import asyncio
+import logging
+from typing import Optional
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+from langchain_core.runnables import Runnable, RunnableConfig, ensure_config
+from openai import APIConnectionError, APIError, APITimeoutError, RateLimitError
+
+from admin_api_lib.impl.settings.summarizer_settings import SummarizerSettings
+from admin_api_lib.summarizer.summarizer import (
+    Summarizer,
+    SummarizerInput,
+    SummarizerOutput,
+)
+from rag_core_lib.impl.langfuse_manager.langfuse_manager import LangfuseManager
+from rag_core_lib.impl.settings.retry_decorator_settings import RetryDecoratorSettings
+from rag_core_lib.impl.utils.async_threadsafe_semaphore import AsyncThreadsafeSemaphore
+from rag_core_lib.impl.utils.retry_decorator import create_retry_decorator_settings, retry_with_backoff
+
+logger = logging.getLogger(__name__)
+
+
+class LangchainSummarizer(Summarizer):
+    """Is responsible for summarizing input data.
+
+    LangchainSummarizer is responsible for summarizing input data using the LangfuseManager,
+    RecursiveCharacterTextSplitter, and AsyncThreadsafeSemaphore. It handles chunking of the input
+    document and retries the summarization process if an error occurs.
+    """
+
+    def __init__(
+        self,
+        langfuse_manager: LangfuseManager,
+        chunker: RecursiveCharacterTextSplitter,
+        semaphore: AsyncThreadsafeSemaphore,
+        summarizer_settings: SummarizerSettings,
+        retry_decorator_settings: RetryDecoratorSettings,
+    ):
+        self._chunker = chunker
+        self._langfuse_manager = langfuse_manager
+        self._semaphore = semaphore
+        self._retry_decorator_settings = create_retry_decorator_settings(summarizer_settings, retry_decorator_settings)
+
+    async def ainvoke(self, query: SummarizerInput, config: Optional[RunnableConfig] = None) -> SummarizerOutput:
+        """
+        Asynchronously invokes the summarization process on the given query.
+
+        Parameters
+        ----------
+        query : SummarizerInput
+            The input data to be summarized.
+        config : Optional[RunnableConfig], optional
+            Configuration options for the summarization process, by default None.
+
+        Returns
+        -------
+        SummarizerOutput
+            The summarized output.
+
+        Raises
+        ------
+        Exception
+            If the summary creation fails after the allowed number of tries.
+
+        Notes
+        -----
+        This method handles chunking of the input document and retries the summarization
+        process if an error occurs, up to the number of tries specified in the config.
+        """
+        assert query, "Query is empty: %s" % query  # noqa S101
+        config = ensure_config(config)
+
+        document = Document(page_content=query)
+        langchain_documents = self._chunker.split_documents([document])
+        logger.debug("Summarizing %d chunk(s)...", len(langchain_documents))
+
+        # Fan out with concurrency, bounded by your semaphore inside _summarize_chunk
+        tasks = [asyncio.create_task(self._summarize_chunk(doc.page_content, config)) for doc in langchain_documents]
+        outputs = await asyncio.gather(*tasks)
+
+        if len(outputs) == 1:
+            return outputs[0]
+
+        merged = " ".join(outputs)
+
+        logger.debug(
+            "Reduced number of chars from %d to %d",
+            len("".join([x.page_content for x in langchain_documents])),
+            len(merged),
+        )
+        return await self._summarize_chunk(merged, config)
+
+    def _create_chain(self) -> Runnable:
+        return self._langfuse_manager.get_base_prompt(self.__class__.__name__) | self._langfuse_manager.get_base_llm(
+            self.__class__.__name__
+        )
+
+    def _retry_with_backoff_wrapper(self):
+        return retry_with_backoff(
+            settings=self._retry_decorator_settings,
+            exceptions=(APIError, RateLimitError, APITimeoutError, APIConnectionError),
+            rate_limit_exceptions=(RateLimitError,),
+            logger=logger,
+        )
+
+    async def _summarize_chunk(self, text: str, config: Optional[RunnableConfig]) -> SummarizerOutput:
+        @self._retry_with_backoff_wrapper()
+        async def _call(text: str, config: Optional[RunnableConfig]) -> SummarizerOutput:
+            response = await self._create_chain().ainvoke({"text": text}, config)
+            return response.content if hasattr(response, "content") else str(response)
+
+        # Hold the semaphore for the entire retry lifecycle
+        async with self._semaphore:
+            return await _call(text, config)

admin_api_lib/information_enhancer/information_enhancer.py

@@ -0,0 +1,34 @@
+"""Module for the InformationEnhancer abstract base class."""
+
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from langchain_core.documents import Document
+from langchain_core.runnables import RunnableConfig
+
+from rag_core_lib.runnables.async_runnable import AsyncRunnable
+
+RetrieverInput = list[Document]
+RetrieverOutput = list[Document]
+
+
+class InformationEnhancer(AsyncRunnable[RetrieverInput, RetrieverOutput], ABC):
+    """The base class for an information enhancer."""
+
+    @abstractmethod
+    async def ainvoke(self, information: RetrieverInput, config: Optional[RunnableConfig] = None) -> RetrieverOutput:
+        """
+        Asynchronously invokes the information enhancer with the given input and configuration.
+
+        Parameters
+        ----------
+        information : RetrieverInput
+            The input information to be processed by the information enhancer.
+        config : Optional[RunnableConfig]
+            The configuration settings for the information enhancer, by default None.
+
+        Returns
+        -------
+        RetrieverOutput
+            The output after processing the input information.
+        """