admin-api-lib 3.2.0__py3-none-any.whl

admin_api_lib/impl/chunker/semantic_text_chunker.py

@@ -0,0 +1,252 @@
+"""Semantic text chunker backed by LangChain's semantic splitter.
+
+Adds optional max/min chunk size enforcement via RecursiveCharacterTextSplitter
+when both values are provided and ``max_chunk_size > min_chunk_size``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+import re
+from inspect import signature
+from typing import Any, Type
+import logging
+
+from langchain_core.documents import Document
+from langchain_core.embeddings import Embeddings
+from langchain_experimental.text_splitter import SemanticChunker as LangchainSemanticChunker
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from nltk.tokenize import PunktSentenceTokenizer
+
+from admin_api_lib.chunker.chunker import Chunker
+
+
+logger = logging.getLogger(__name__)
+
+
+class SemanticTextChunker(Chunker):
+    """Semantic text chunker backed by LangChain's semantic splitter with optional max/min chunk size enforcement."""
+
+    def __init__(
+        self,
+        embeddings: Embeddings,
+        *,
+        breakpoint_threshold_type: str | None = None,
+        breakpoint_threshold_amount: float | None = None,
+        buffer_size: int | None = None,
+        min_chunk_size: int | None = None,
+        max_chunk_size: int | None = None,
+        overlap: int | None = None,
+        chunker_cls: Type[LangchainSemanticChunker] = LangchainSemanticChunker,
+        recursive_text_splitter: RecursiveCharacterTextSplitter | None = None,
+    ) -> None:
+        """Initialise the semantic chunker.
+
+        Parameters
+        ----------
+        embeddings : Embeddings
+            The embeddings backend that powers semantic similarity detection.
+        breakpoint_threshold_type : str | None, optional
+            Strategy used to derive semantic breakpoints.
+        breakpoint_threshold_amount : float | None, optional
+            Threshold to apply for the selected breakpoint strategy.
+        buffer_size : int | None, optional
+            Number of neighbouring sentences to include for context.
+        min_chunk_size : int | None, optional
+            Minimum chunk size enforced by the chunker.
+        max_chunk_size : int | None, optional
+            Maximum chunk size enforced by the chunker.
+        overlap : int | None, optional
+            Number of overlapping characters between chunks.
+        chunker_cls : type[LangchainSemanticChunker], optional
+            Concrete semantic chunker implementation to instantiate. Defaults to
+            :class:`langchain_text_splitters.SemanticChunker`.
+        recursive_text_splitter : RecursiveCharacterTextSplitter | None, optional
+            Optional pre-configured recursive text splitter to use for max/min chunk size enforcement.
+        """
+        self._min_chunk_size = min_chunk_size
+        self._max_chunk_size = max_chunk_size
+        self._overlap = overlap
+
+        init_params = _supported_parameters(chunker_cls)
+        candidate_kwargs: dict[str, Any] = {
+            "breakpoint_threshold_type": breakpoint_threshold_type,
+            "breakpoint_threshold_amount": breakpoint_threshold_amount,
+            "buffer_size": buffer_size,
+            "min_chunk_size": min_chunk_size,
+        }
+        filtered_kwargs = {
+            key: value for key, value in candidate_kwargs.items() if value is not None and key in init_params
+        }
+
+        self._semantic_chunker = chunker_cls(
+            embeddings=embeddings,
+            **filtered_kwargs,
+        )
+
+        # Configure a recursive splitter for max/min enforcement when requested.
+        # If none provided, instantiate a sensible default using max_chunk_size and overlap.
+        self._recursive_splitter: RecursiveCharacterTextSplitter | None = None
+        if self._min_chunk_size and self._max_chunk_size and self._max_chunk_size > self._min_chunk_size:
+            if recursive_text_splitter is not None:
+                self._recursive_splitter = recursive_text_splitter
+            else:
+                self._recursive_splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=int(self._max_chunk_size),
+                    chunk_overlap=int(self._overlap or 0),
+                )
+
+    def chunk(self, documents: Iterable[Document]) -> list[Document]:
+        """Split documents into chunks.
+
+        The documents are first processed by the semantic chunker,
+        and then any oversized chunks are re-split using the recursive text splitter,
+        if `self._recursive_splitter` and `self._min_chunk_size`/`self._max_chunk_size` are configured.
+
+        Parameters
+        ----------
+        documents : Iterable[Document]
+            Documents to be chunked.
+
+        Returns
+        -------
+        list[Document]
+            Chunked documents.
+        """
+        documents_list = list(documents)
+        if not documents_list:
+            return []
+
+        sem_chunks = self._semantic_chunker.split_documents(documents_list)
+
+        # If no max/min enforcement requested, return directly
+        if not self._recursive_splitter:
+            return sem_chunks
+
+        # Enforce max size by re-splitting only oversized chunks, then ensure minimum size
+        final_chunks: list[Document] = []
+        for chunk in sem_chunks:
+            text = chunk.page_content or ""
+            if len(text) <= self._max_chunk_size:  # type: ignore[arg-type]
+                final_chunks.append(chunk)
+                continue
+
+            # Split this oversized chunk using the recursive splitter, preserving metadata
+            sub_chunks = self._recursive_splitter.split_documents([chunk])  # type: ignore[union-attr]
+
+            # Ensure minimum size by balancing the last small chunk with its predecessor
+            balanced = self._rebalance_min_size(sub_chunks)
+            final_chunks.extend(balanced)
+
+        return final_chunks
+
+    def _rebalance_min_size(self, chunks: list[Document]) -> list[Document]:
+        """Rebalance chunks so that the trailing chunk meets ``min_chunk_size`` when possible.
+
+        Strategy
+        --------
+        - If the last chunk is smaller than ``min_chunk_size`` and there is a previous chunk,
+          combine both and re-split into one or two chunks such that each is within
+          [min_chunk_size, max_chunk_size]. This guarantees no tiny tail when feasible.
+        - Otherwise, return the chunks unchanged.
+        """
+        if not chunks or len(chunks) == 1:
+            return chunks
+
+        last = chunks[-1]
+        prev = chunks[-2]
+
+        overlap = int(self._overlap or 0)
+        prev_text = prev.page_content
+        last_text = last.page_content
+        tail = last_text[overlap:] if overlap > 0 else last_text
+        combined_text = prev_text + "\n" + tail
+        combined_len = len(combined_text)
+
+        # Case 1: Combined fits entirely under max -> single merged chunk
+        if combined_len <= self._max_chunk_size:
+            merged = Document(page_content=combined_text, metadata=prev.metadata)
+            return chunks[:-2] + [merged]
+
+        # Case 2: Split combined into two parts within [min, max] using sentence boundaries if possible
+        # Compute candidate breakpoints at sentence ends
+        boundaries = self._sentence_boundaries(combined_text)
+        # Ideal target for the first part: stay within [min,max] and leave >= min for the tail
+        target_first = combined_len - self._min_chunk_size
+        target_first = max(self._min_chunk_size, min(target_first, self._max_chunk_size))
+
+        # Filter boundaries that satisfy constraints for both parts
+        valid = self._filter_boundaries(boundaries, combined_len)
+
+        cut_at = None
+        if valid:
+            # choose boundary closest to target_first
+            cut_at = min(valid, key=lambda i: abs(i - target_first))
+        else:
+            # As a conservative fallback, try any boundary <= max that leaves a non-empty tail
+            candidates = [i for i in boundaries if i <= self._max_chunk_size and combined_len - i > 0]
+            if candidates:
+                cut_at = max(candidates)
+
+        if cut_at is None:
+            # Could not find a safe sentence boundary; keep original chunks
+            return chunks
+
+        first_text = combined_text[:cut_at]
+        second_text = combined_text[cut_at:]
+        first = Document(page_content=first_text, metadata=prev.metadata)
+        second = Document(page_content=second_text, metadata=prev.metadata)
+        return chunks[:-2] + [first, second]
+
+    def _filter_boundaries(self, boundaries: list[int], combined_len: int) -> list[int]:
+        """Filter boundaries to find valid split points."""
+        valid = []
+        for idx in boundaries:
+            size1 = idx
+            size2 = combined_len - idx
+            if size1 < self._min_chunk_size or size1 > self._max_chunk_size:
+                continue
+            if size2 < self._min_chunk_size:  # leave at least min for the tail
+                continue
+            valid.append(idx)
+        return valid
+
+    def _sentence_boundaries(self, text: str) -> list[int]:
+        """Return indices in ``text`` that are good sentence breakpoints.
+
+        Tries NLTK's sentence tokenizer if available, otherwise uses a regex-based
+        heuristic to detect sentence ends. Indices are character offsets suitable
+        for slicing ``text[:idx]`` and ``text[idx:]``.
+        """
+        try:
+            tokenizer = PunktSentenceTokenizer()
+            spans = list(tokenizer.span_tokenize(text))
+            return [end for (_, end) in spans]
+        except Exception:
+            logger.info("NLTK Punkt tokenizer unavailable, falling back to regex-based sentence boundary detection.")
+        # Regex heuristic: sentence end punctuation followed by whitespace/newline
+        boundaries: list[int] = []
+        for m in re.finditer(r"(?<=[\.!?])[\"'”)]*\s+", text):
+            boundaries.append(m.end())
+        # Ensure we don't return 0 or len(text) as boundaries
+        return [i for i in boundaries if 0 < i < len(text)]
+
+
+def _supported_parameters(chunker_cls: type) -> set[str]:
+    """Return constructor parameters supported by ``chunker_cls``.
+
+    Parameters
+    ----------
+    chunker_cls : type
+        Semantic chunker class whose constructor signature should be inspected.
+
+    Returns
+    -------
+    set[str]
+        Set of keyword-parameter names accepted by the constructor.
+    """
+    try:
+        params = signature(chunker_cls.__init__).parameters
+    except (TypeError, ValueError):  # pragma: no cover - defensive, should not occur
+        return set()
+    return {name for name in params if name != "self"}

admin_api_lib/impl/chunker/text_chunker.py

@@ -0,0 +1,33 @@
+"""Module containing the TextChunker class."""
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+
+from admin_api_lib.chunker.chunker import Chunker
+
+
+class TextChunker(Chunker):
+    """A class that chunks text documents into smaller chunks."""
+
+    def __init__(self, splitter: RecursiveCharacterTextSplitter):
+        # NOTE:  `CharacterTextSplitter` does not take chunk_size into consideration
+        #       See: https://github.com/langchain-ai/langchain/issues/10410#issuecomment-1712595675
+        #       for that reason, we use the recursive splitter
+        self._splitter = splitter
+
+    def chunk(self, documents: list[Document]) -> list[Document]:
+        """
+        Chunk the given documents into smaller chunks.
+
+        Parameters
+        ----------
+        documents : list[Document]
+            The documents to be chunked.
+
+        Returns
+        -------
+        list[Document]
+            The list of chunked documents.
+
+        """
+        return self._splitter.split_documents(documents)

admin_api_lib/impl/file_services/s3_service.py

@@ -0,0 +1,130 @@
+"""Class to handle I/O with S3 storage."""
+
+import logging
+from pathlib import Path
+from typing import BinaryIO
+
+import boto3
+
+from admin_api_lib.file_services.file_service import FileService
+from admin_api_lib.impl.settings.s3_settings import S3Settings
+
+logger = logging.getLogger(__name__)
+
+
+class S3Service(FileService):
+    """Class to handle I/O with S3 storage."""
+
+    def __init__(self, s3_settings: S3Settings):
+        """Class to handle I/O with S3 storage.
+
+        Parameters
+        ----------
+        s3_settings: S3Settings
+            Settings for the s3. Must contain at least the endpoint, access_key_id, secret_access_key and bucket.
+        """
+        self._s3_settings = s3_settings
+        self._s3_client = boto3.client(
+            "s3",
+            endpoint_url=s3_settings.endpoint,
+            aws_access_key_id=s3_settings.access_key_id,
+            aws_secret_access_key=s3_settings.secret_access_key,
+            aws_session_token=None,
+            config=boto3.session.Config(signature_version="s3v4"),
+            verify=False,
+        )
+
+    def download_folder(self, source: str, target: Path) -> None:
+        """Download the remote folder on "source" to the local "target" directory.
+
+        Parameters
+        ----------
+        source: str
+            Path to the remote folder.
+        target: Path
+            Download destination path.
+        """
+        target.mkdir(parents=True, exist_ok=True)
+
+        search_response = self._s3_client.list_objects_v2(
+            Bucket=self._s3_settings.bucket,
+            Prefix=source,
+        )
+        for found_content in search_response.get("Contents", []):
+            file_source = found_content["Key"]
+            target_path = target / file_source[len(source) :]
+            target_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(target_path, "wb") as local_file:
+                self.download_file(file_source, local_file)
+
+    def download_file(self, source: str, target_file: BinaryIO) -> None:
+        """Read a single remote file "source" into the local "target_file" file-like object.
+
+        Example usage
+        =============
+        ```
+        s3_settings: S3Settings = get_s3_settings()
+        s3_service = S3Service(endpoint="endpoint", username="username", password="password", bucket_name="bucket")
+
+        with tempfile.SpooledTemporaryFile(max_size=self._iot_forecast_settings.max_model_size) as temp_file:
+            s3_service.download_file("remote_file", temp_file)
+            # do stuff with temp_file
+        ```
+
+        Parameters
+        ----------
+        source: str
+            Path to the remote folder.
+        target_file: BinaryIO
+            File-like object to save the data to.
+        """
+        self._s3_client.download_fileobj(self._s3_settings.bucket, source, target_file)
+
+    def upload_file(self, file_path: str, file_name: str) -> None:
+        """
+        Upload a local file to the S3 bucket.
+
+        Parameters
+        ----------
+        source : Path
+            The path to the local file to upload.
+        target : str
+            The target path in the S3 bucket where the file will be stored.
+        """
+        self._s3_client.upload_file(
+            Filename=file_path,
+            Bucket=self._s3_settings.bucket,
+            Key=file_name,
+        )
+
+    def get_all_sorted_file_names(self) -> list[str]:
+        """Retrieve all file names stored in the S3 bucket.
+
+        Returns
+        -------
+        list[str]
+            A list of file names stored in the S3 bucket.
+        """
+        file_names = []
+
+        resp = self._s3_client.list_objects_v2(Bucket=self._s3_settings.bucket)
+        if resp.get("Contents"):
+            for obj in resp["Contents"]:
+                file_names.append(obj["Key"])
+        return file_names
+
+    def delete_file(self, file_name: str) -> None:
+        """Delete a file from the S3 bucket.
+
+        Parameters
+        ----------
+        file_name : str
+            The name of the file to be deleted from the S3 bucket.
+        """
+        try:
+            file_name = f"/{file_name}" if not file_name.startswith("/") else file_name
+            self._s3_client.delete_object(Bucket=self._s3_settings.bucket, Key=file_name)
+            logger.info("File %s successfully deleted.", file_name)
+        except Exception:
+            logger.exception("Error deleting file %s", file_name)
+            raise

admin_api_lib/impl/information_enhancer/general_enhancer.py

@@ -0,0 +1,52 @@
+"""Module containing the GeneralEnhancer class."""
+
+from asyncio import gather
+from typing import Optional
+
+from langchain_core.runnables import RunnableConfig, ensure_config
+
+from admin_api_lib.information_enhancer.information_enhancer import (
+    InformationEnhancer,
+    RetrieverInput,
+    RetrieverOutput,
+)
+
+
+class GeneralEnhancer(InformationEnhancer):
+    """The GeneralEnhancer aggregates multiple InformationEnhancer instances.
+
+    InformationEnhancers are applied asynchronously to the input information.
+    """
+
+    def __init__(self, enhancers: list[InformationEnhancer]):
+        """Initialize the GeneralEnhancer with a list of InformationEnhancer instances.
+
+        Parameters
+        ----------
+        enhancers : list of InformationEnhancer
+            A list of InformationEnhancer instances to be used by the GeneralEnhancer.
+        """
+        super().__init__()
+        self._enhancers = enhancers
+
+    async def ainvoke(self, information: RetrieverInput, config: Optional[RunnableConfig] = None) -> RetrieverOutput:
+        """Asynchronously invokes the each information enhancer with the given input and configuration.
+
+        Parameters
+        ----------
+        information : RetrieverInput
+            The input information to be processed by the general information enhancer.
+        config : Optional[RunnableConfig], optional
+            The configuration settings for the general information enhancer, by default None.
+
+        Returns
+        -------
+        RetrieverOutput
+            The output after processing the input information.
+        """
+        config = ensure_config(config)
+        summarize_tasks = [enhancer.ainvoke(information, config) for enhancer in self._enhancers]
+        summary_results = await gather(*summarize_tasks)
+        for summaries in summary_results:
+            information += summaries
+        return information

admin_api_lib/impl/information_enhancer/page_summary_enhancer.py

@@ -0,0 +1,62 @@
+"""Module for enhancing the summary of pages by grouping information by page and summarizing each page."""
+
+from asyncio import gather
+from hashlib import sha256
+from typing import Optional
+
+from langchain_core.documents import Document
+from langchain_core.runnables import RunnableConfig
+from tqdm import tqdm
+
+from admin_api_lib.impl.information_enhancer.summary_enhancer import SummaryEnhancer
+from rag_core_lib.impl.data_types.content_type import ContentType
+
+
+class PageSummaryEnhancer(SummaryEnhancer):
+    """
+    Enhances the summary of pages by grouping information by page and summarizing each page.
+
+    Attributes
+    ----------
+    BASE64_IMAGE_KEY : str
+        Key used to identify base64 encoded images in metadata.
+    DEFAULT_PAGE_NR : int
+        Default page number used when no page metadata is available.
+    """
+
+    BASE64_IMAGE_KEY = "base64_image"
+    DEFAULT_PAGE_NR = 1
+
+    async def _asummarize_page(self, page_pieces: list[Document], config: Optional[RunnableConfig]) -> Document:
+        full_page_content = " ".join([piece.page_content for piece in page_pieces])
+        summary = await self._summarizer.ainvoke(full_page_content, config)
+        meta = {key: value for key, value in page_pieces[0].metadata.items() if key != self.BASE64_IMAGE_KEY}
+        meta["id"] = sha256(str.encode(full_page_content)).hexdigest()
+        meta["related"] = meta["related"] + [piece.metadata["id"] for piece in page_pieces]
+        meta["related"] = list(set(meta["related"]))
+        meta["type"] = ContentType.SUMMARY.value
+
+        return Document(metadata=meta, page_content=summary)
+
+    async def _acreate_summary(self, information: list[Document], config: Optional[RunnableConfig]) -> list[Document]:
+        distinct_pages = []
+        for info in information:
+            if info.metadata.get("page", self.DEFAULT_PAGE_NR) not in distinct_pages:
+                distinct_pages.append(info.metadata.get("page", self.DEFAULT_PAGE_NR))
+
+        grouped = []
+        for page in distinct_pages:
+            group = []
+            for compare_info in information:
+                if compare_info.metadata.get("page", self.DEFAULT_PAGE_NR) == page:
+                    group.append(compare_info)
+            if (
+                self._chunker_settings
+                and len(" ".join([item.page_content for item in group])) < self._chunker_settings.max_size
+            ):
+                continue
+            grouped.append(group)
+
+        summary_tasks = [self._asummarize_page(info_group, config) for info_group in tqdm(grouped)]
+
+        return await gather(*summary_tasks)

admin_api_lib/impl/information_enhancer/summary_enhancer.py

@@ -0,0 +1,74 @@
+"""Module with SummaryEnhancer class that enhances information by generating summaries using a provided Summarizer."""
+
+from abc import abstractmethod
+from typing import Optional
+
+from admin_api_lib.impl.settings.chunker_settings import ChunkerSettings
+from langchain_core.documents import Document
+from langchain_core.runnables import RunnableConfig, ensure_config
+
+from admin_api_lib.information_enhancer.information_enhancer import (
+    InformationEnhancer,
+    RetrieverInput,
+    RetrieverOutput,
+)
+from admin_api_lib.summarizer.summarizer import Summarizer
+from rag_core_lib.impl.data_types.content_type import ContentType
+
+
+class SummaryEnhancer(InformationEnhancer):
+    """The SummaryEnhancer enhances information by generating summaries using a provided Summarizer instance.
+
+    Attributes
+    ----------
+    INFORMATION_METADATA_TYPE : str
+        A constant string representing the type of information metadata.
+    """
+
+    INFORMATION_METADATA_TYPE = "type"
+
+    def __init__(self, summarizer: Summarizer, chunker_settings: ChunkerSettings = None):
+        """
+        Initialize the SummaryEnhancer with a given Summarizer instance.
+
+        Parameters
+        ----------
+        summarizer : Summarizer
+            An instance of the Summarizer class used to generate summaries.
+        """
+        super().__init__()
+        self._summarizer = summarizer
+        self._chunker_settings = chunker_settings
+
+    @staticmethod
+    def _is_relevant(information: Document) -> bool:
+        match information.metadata.get(SummaryEnhancer.INFORMATION_METADATA_TYPE, ContentType.SUMMARY):  # noqa:R503
+            case ContentType.SUMMARY | ContentType.IMAGE:
+                return False
+            case _:
+                return True
+
+    async def ainvoke(self, information: RetrieverInput, config: Optional[RunnableConfig] = None) -> RetrieverOutput:
+        """
+        Asynchronously invokes the summary enhancer on the provided information.
+
+        Parameters
+        ----------
+        information : RetrieverInput
+            The input information to be processed and summarized.
+        config : Optional[RunnableConfig], optional
+            Configuration for the runnable, by default None.
+
+        Returns
+        -------
+        RetrieverOutput
+            The summarized output of the provided information.
+        """
+        config = ensure_config(config)
+        pieces_to_summarize = [info for info in information if self._is_relevant(info)]
+        return await self._acreate_summary(pieces_to_summarize, config)
+
+    @abstractmethod
+    async def _acreate_summary(
+        self, information: list[Document], config: Optional[RunnableConfig]
+    ) -> list[Document]: ...