admin-api-lib 3.2.0__tar.gz

admin_api_lib-3.2.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.3
+Name: admin-api-lib
+Version: 3.2.0
+Summary: The admin backend is responsible for the document management. This includes deletion, upload and returning the source document.
+Author: STACKIT Data and AI Consulting
+Author-email: data-ai-consulting@stackit.cloud
+Requires-Python: >=3.13,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: boto3 (>=1.38.10,<2.0.0)
+Requires-Dist: dependency-injector (>=4.46.0,<5.0.0)
+Requires-Dist: fastapi (>=0.118.0,<0.119.0)
+Requires-Dist: langchain-experimental (>=0.3.4,<0.4.0)
+Requires-Dist: langfuse (==3.6.1)
+Requires-Dist: nltk (>=3.9.2,<4.0.0)
+Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
+Requires-Dist: python-multipart (>=0.0.20,<0.0.21)
+Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
+Requires-Dist: rag-core-lib (==3.2.0)
+Requires-Dist: redis (>=6.0.0,<7.0.0)
+Requires-Dist: starlette (>=0.47.2,<0.49.0)
+Requires-Dist: tenacity (==9.1.2)
+Requires-Dist: tqdm (>=4.67.1,<5.0.0)
+Requires-Dist: uvicorn (>=0.37.0,<0.38.0)

admin_api_lib-3.2.0/pyproject.toml

@@ -0,0 +1,119 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "admin-api-lib"
+version = "v3.2.0"
+description = "The admin backend is responsible for the document management. This includes deletion, upload and returning the source document."
+authors = ["STACKIT Data and AI Consulting <data-ai-consulting@stackit.cloud>"]
+packages = [{ include = "admin_api_lib", from = "src" }]
+
+[tool.flake8]
+exclude= [".eggs", "./libs/*", "./src/admin_api_lib/models/*", "./src/admin_api_lib/rag_backend_client/*", "./src/admin_api_lib/extractor_api_client/*", ".git", ".hg", ".mypy_cache", ".tox", ".venv", ".devcontainer", "venv", "_build", "buck-out", "build", "dist", "**/__init__.py"]
+statistics = true
+show-source = false
+max-complexity = 8
+max-annotations-complexity = 3
+docstring-convention = 'numpy'
+max-line-length = 120
+ignore = ["E203", "W503", "E704"]
+inline-quotes = '"'
+docstring-quotes = '"""'
+multiline-quotes = '"""'
+dictionaries = ["en_US", "python", "technical", "pandas"]
+ban-relative-imports = true
+per-file-ignores = """
+  ./src/admin_api_lib/prompt_templates/summarize_prompt.py: E501,
+  ./src/admin_api_lib/apis/admin_api.py: B008,WOT001,
+  ./src/admin_api_lib/impl/admin_api.py: B008,
+  ./src/admin_api_lib/dependency_container.py: CCE002,CCE001,WOT001,
+  ./src/admin_api_lib/apis/admin_api_base.py: WOT001,
+  ./tests/*: S101,S106,D100,D103,PT011,N802,E501,
+  ./src/admin_api_lib/impl/settings/confluence_settings.py: C901,N805,
+  ./src/admin_api_lib/impl/utils/comma_separated_bool_list.py: R505,
+  ./src/admin_api_lib/impl/utils/comma_separated_str_list.py: R505,
+"""
+
+[tool.black]
+line-length = 120
+exclude = """
+/(
+      .eggs
+    | .git
+    | .hg
+    | .mypy_cache
+    | .nox
+    | .pants.d
+    | .tox
+    | .venv
+    | _build
+    | buck-out
+    | build
+    | dist
+    | node_modules
+    | venv
+)/
+"""
+
+[tool.isort]
+profile = "black"
+skip = ['.eggs', '.git', '.hg', '.mypy_cache', '.nox', '.pants.d', '.tox', '.venv', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'venv']
+skip_gitignore = true
+known_local_folder = ["admin_api_lib", "rag_core_lib"]
+
+[tool.pylint]
+max-line-length = 120
+
+[tool.poetry.group.dev.dependencies]
+debugpy = "^1.8.14"
+pytest = "^8.3.5"
+coverage = "^7.8.0"
+flake8 = "^7.2.0"
+flake8-black = "^0.4.0"
+flake8-pyproject = "^1.2.3"
+flake8-quotes = "^3.4.0"
+flake8-return = "^1.2.0"
+flake8-annotations-complexity = "^0.1.0"
+flake8-bandit = "^4.1.1"
+flake8-bugbear = "^24.12.12"
+flake8-builtins = "^2.5.0"
+flake8-comprehensions = "^3.15.0"
+flake8-eradicate = "^1.5.0"
+flake8-expression-complexity = "^0.0.11"
+flake8-pytest-style = "^2.1.0"
+pep8-naming = "^0.15.1"
+flake8-eol = "^0.0.8"
+flake8-exceptions = "^0.0.1a0"
+flake8-simplify = "^0.22.0"
+flake8-wot = "^0.2.0"
+flake8-function-order = "^0.0.5"
+flake8-tidy-imports = "^4.10.0"
+black = "^25.1.0"
+# flake8-logging-format = "^2024.24.12"
+# flake8-docstrings = "^1.7.0"
+pytest-asyncio = "^1.0.0"
+
+[tool.poetry.dependencies]
+rag-core-lib = "3.2.0"
+python = "^3.13"
+uvicorn = "^0.37.0"
+fastapi = "^0.118.0"
+dependency-injector = "^4.46.0"
+python-dateutil = "^2.9.0.post0"
+tenacity = "9.1.2"
+boto3 = "^1.38.10"
+tqdm = "^4.67.1"
+langfuse = "3.6.1"
+redis = "^6.0.0"
+pyyaml = "^6.0.2"
+python-multipart = "^0.0.20"
+starlette = ">=0.47.2,<0.49.0"
+langchain-experimental = "^0.3.4"
+nltk = "^3.9.2"
+
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "DEBUG"
+pythonpath = "src"
+testpaths = "src/tests"

admin_api_lib-3.2.0/src/admin_api_lib/api_endpoints/document_deleter.py

@@ -0,0 +1,24 @@
+"""Module for the document deletion endpoint."""
+
+from abc import ABC, abstractmethod
+
+
+class DocumentDeleter(ABC):
+    """Abstract base class for document deletion endpoint."""
+
+    @abstractmethod
+    async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
+        """
+        Delete a document by its identification asynchronously.
+
+        Parameters
+        ----------
+        identification : str
+            The unique identifier of the document to be deleted.
+        remove_from_key_value_store : bool, optional
+            If True, the document will also be removed from the key-value store (default is True).
+
+        Returns
+        -------
+        None
+        """

admin_api_lib-3.2.0/src/admin_api_lib/api_endpoints/document_reference_retriever.py

@@ -0,0 +1,25 @@
+"""Module for the DocumentReferenceRetriever abstract base class."""
+
+from abc import ABC, abstractmethod
+
+from fastapi import Response
+
+
+class DocumentReferenceRetriever(ABC):
+    """Abstract base class for retrieving document references."""
+
+    @abstractmethod
+    async def adocument_reference_id_get(self, identification: str) -> Response:
+        """
+        Get document reference by ID asynchronously.
+
+        Parameters
+        ----------
+        identification : str
+            The ID of the document reference to retrieve.
+
+        Returns
+        -------
+        Response
+            The response containing the document reference.
+        """

admin_api_lib-3.2.0/src/admin_api_lib/api_endpoints/documents_status_retriever.py

@@ -0,0 +1,20 @@
+"""Module abstract base class for the DocumentsStatusRetriever."""
+
+from abc import ABC, abstractmethod
+
+from admin_api_lib.models.document_status import DocumentStatus
+
+
+class DocumentsStatusRetriever(ABC):
+    """Abstract base class for retrieving all documents."""
+
+    @abstractmethod
+    async def aget_all_documents_status(self) -> list[DocumentStatus]:
+        """
+        Get all documents and their statuses asynchronously.
+
+        Returns
+        -------
+        list[DocumentStatus]
+            A list containing document names and their statuses.
+        """

admin_api_lib-3.2.0/src/admin_api_lib/api_endpoints/file_uploader.py

@@ -0,0 +1,31 @@
+"""Module for the upload file endpoint."""
+
+from abc import abstractmethod
+
+from fastapi import UploadFile
+
+from admin_api_lib.api_endpoints.uploader_base import UploaderBase
+
+
+class FileUploader(UploaderBase):
+
+    @abstractmethod
+    async def upload_file(
+        self,
+        base_url: str,
+        file: UploadFile,
+    ) -> None:
+        """
+        Uploads a source file for content extraction.
+
+        Parameters
+        ----------
+        base_url : str
+            The base url of the service. Is used to determine the download link of the file.
+        file : UploadFile
+            The file to process.
+
+        Returns
+        -------
+        None
+        """

admin_api_lib-3.2.0/src/admin_api_lib/api_endpoints/source_uploader.py

@@ -0,0 +1,40 @@
+"""Module for the upload source endpoint."""
+
+from abc import abstractmethod
+from typing import Optional
+
+from pydantic import StrictStr
+
+from admin_api_lib.api_endpoints.uploader_base import UploaderBase
+from admin_api_lib.models.key_value_pair import KeyValuePair
+
+
+class SourceUploader(UploaderBase):
+    """Abstract base class for source uploader API endpoints."""
+
+    @abstractmethod
+    async def upload_source(
+        self,
+        source_type: StrictStr,
+        name: StrictStr,
+        kwargs: list[KeyValuePair],
+        timeout: Optional[float],
+    ) -> None:
+        """
+        Uploads the parameters for source content extraction.
+
+        Parameters
+        ----------
+        source_type : str
+            The type of the source. Is used by the extractor service to determine the correct extraction method.
+        name : str
+            Display name of the source.
+        kwargs : list[KeyValuePair]
+            List of KeyValuePair with parameters used for the extraction.
+        timeout : float, optional
+            Timeout for the operation.
+
+        Returns
+        -------
+        None
+        """

admin_api_lib-3.2.0/src/admin_api_lib/api_endpoints/uploader_base.py

@@ -0,0 +1,30 @@
+"""Module for the base class of uploader API endpoints."""
+
+from threading import Thread
+
+
+class UploaderBase:
+    """Base class for uploader API endpoints."""
+
+    def __init__(self):
+        """
+        Initialize the UploaderBase.
+        """
+        self._background_threads = []
+
+    def _prune_background_threads(self) -> list[Thread]:
+        """
+        Prune background threads that are no longer running.
+
+        Returns
+        -------
+        list[Thread]
+            A list of background threads that are still alive.
+        """
+        tmp_background_threads = []
+        for thread in self._background_threads:
+            if not thread.is_alive():
+                thread.join()
+            else:
+                tmp_background_threads.append(thread)
+        self._background_threads = tmp_background_threads

admin_api_lib-3.2.0/src/admin_api_lib/apis/admin_api.py

@@ -0,0 +1,197 @@
+"""Module for the Admin API."""
+
+# coding: utf-8
+
+from typing import Dict, List  # noqa: F401
+import importlib
+import pkgutil
+from typing_extensions import Annotated
+
+import admin_api_lib.impl
+
+from fastapi import (  # noqa: F401
+    APIRouter,
+    Body,
+    Cookie,
+    Depends,
+    Form,
+    UploadFile,
+    Request,
+    Header,
+    HTTPException,
+    Path,
+    Query,
+    Response,
+    Security,
+    status,
+)
+from pydantic import Field, StrictStr
+
+
+from admin_api_lib.apis.admin_api_base import BaseAdminApi
+from admin_api_lib.models.document_status import DocumentStatus
+from admin_api_lib.models.http_validation_error import HTTPValidationError
+from admin_api_lib.models.key_value_pair import KeyValuePair
+from admin_api_lib.models.extra_models import TokenModel  # noqa: F401
+
+router = APIRouter()
+
+ns_pkg = admin_api_lib.impl
+for _, name, _ in pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + "."):
+    importlib.import_module(name)
+
+
+@router.delete(
+    "/delete_document/{identification}",
+    responses={
+        200: {"description": "Deleted"},
+        500: {"description": "Internal server error"},
+        422: {"model": HTTPValidationError, "description": "Validation Error"},
+    },
+    tags=["admin"],
+    summary="Delete Document",
+    response_model_by_alias=True,
+)
+async def delete_document(
+    identification: StrictStr = Path(..., description=""),
+) -> None:
+    """
+    Asynchronously deletes a document based on the provided identification.
+
+    Parameters
+    ----------
+    identification : str
+        The unique identifier of the document to be deleted.
+
+    Returns
+    -------
+    None
+    """
+    if not BaseAdminApi.subclasses:
+        raise HTTPException(status_code=500, detail="Not implemented")
+    return await BaseAdminApi.subclasses[0]().delete_document(identification)
+
+
+@router.get(
+    "/document_reference/{identification}",
+    responses={
+        200: {"model": UploadFile, "description": "Returns the pdf in binary form."},
+        400: {"model": str, "description": "Bad request"},
+        404: {"model": str, "description": "Document not found."},
+        500: {"model": str, "description": "Internal server error"},
+        422: {"model": HTTPValidationError, "description": "Validation Error"},
+    },
+    tags=["admin"],
+    summary="Document Reference Id Get",
+    response_model_by_alias=True,
+)
+async def document_reference(
+    identification: Annotated[StrictStr, Field(description="Identifier of the document.")] = Path(
+        ..., description="Identifier of the document."
+    ),
+) -> Response:
+    """
+    Asynchronously retrieve a document reference by its identification.
+
+    Parameters
+    ----------
+    identification : str
+        The unique identifier for the document reference.
+
+    Returns
+    -------
+    Response
+        The response object containing the document reference details.
+    """
+    if not BaseAdminApi.subclasses:
+        raise HTTPException(status_code=500, detail="Not implemented")
+    return await BaseAdminApi.subclasses[0]().document_reference(identification)
+
+
+@router.get(
+    "/all_documents_status",
+    responses={
+        200: {"model": List[DocumentStatus], "description": "List of document links"},
+        500: {"description": "Internal server error"},
+    },
+    tags=["admin"],
+    summary="Get All Documents Status",
+    response_model_by_alias=True,
+)
+async def get_all_documents_status() -> List[DocumentStatus]:
+    """
+    Asynchronously retrieves the status of all documents.
+
+    Returns
+    -------
+    list[DocumentStatus]
+        A list containing the status of all documents.
+    """
+    if not BaseAdminApi.subclasses:
+        raise HTTPException(status_code=500, detail="Not implemented")
+    return await BaseAdminApi.subclasses[0]().get_all_documents_status()
+
+
+@router.post(
+    "/upload_file",
+    responses={
+        200: {"description": "ok"},
+        400: {"description": "Bad request"},
+        422: {"description": "Unprocessable Content"},
+        500: {"description": "Internal server error"},
+    },
+    tags=["admin"],
+    summary="Upload File",
+    response_model_by_alias=True,
+)
+async def upload_file(
+    file: UploadFile,
+    request: Request,
+) -> None:
+    """
+    Uploads user selected sources.
+
+    Parameters
+    ----------
+    file : UploadFile
+        The file to be uploaded.
+    request : Request
+        The HTTP request object containing metadata about the upload request.
+    """
+    if not BaseAdminApi.subclasses:
+        raise HTTPException(status_code=500, detail="Not implemented")
+    return await BaseAdminApi.subclasses[0]().upload_file(file, request)
+
+
+@router.post(
+    "/upload_source",
+    responses={
+        200: {"description": "ok"},
+        400: {"description": "Bad request"},
+        422: {"description": "Unprocessable Content"},
+        500: {"description": "Internal server error"},
+    },
+    tags=["admin"],
+    summary="Upload Source",
+    response_model_by_alias=True,
+)
+async def upload_source(
+    source_type: StrictStr = Query(None, description="The type of the source"),
+    name: StrictStr = Query(None, description="The name of the source", alias="name"),
+    key_value_pair: List[KeyValuePair] = Body(None, description="The key-value pairs for the source"),
+) -> None:
+    """
+    Uploads user selected sources.
+
+    Parameters
+    ----------
+    source_type : str
+        The type of the source. Is used by the extractor service to determine the correct extractor to use.
+    name : str
+        Display name of the source.
+    key_value_pair : List[KeyValuePair]
+        List of KeyValuePair with parameters used for the extraction.
+    """
+    if not BaseAdminApi.subclasses:
+        raise HTTPException(status_code=500, detail="Not implemented")
+    return await BaseAdminApi.subclasses[0]().upload_source(source_type, name, key_value_pair)

admin_api_lib-3.2.0/src/admin_api_lib/apis/admin_api_base.py

@@ -0,0 +1,120 @@
+"""Module for the base AdminApi interface."""
+
+# coding: utf-8
+# flake8: noqa: D105
+
+from typing import ClassVar, Dict, List, Tuple  # noqa: F401
+from typing_extensions import Annotated
+
+from pydantic import Field, StrictStr
+from fastapi import Request, Response, UploadFile
+
+from admin_api_lib.models.document_status import DocumentStatus
+from admin_api_lib.models.key_value_pair import KeyValuePair
+
+
+class BaseAdminApi:
+    """
+    The base AdminApi interface.
+
+    Attributes
+    ----------
+    subclasses : ClassVar[Tuple]
+        A tuple that holds all subclasses of BaseAdminApi.
+    """
+
+    subclasses: ClassVar[Tuple] = ()
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        BaseAdminApi.subclasses = BaseAdminApi.subclasses + (cls,)
+
+    async def delete_document(
+        self,
+        identification: StrictStr,
+    ) -> None:
+        """
+        Asynchronously deletes a document based on the provided identification.
+
+        Parameters
+        ----------
+        identification : str
+            The unique identifier of the document to be deleted.
+
+        Returns
+        -------
+        None
+        """
+
+    async def document_reference(
+        self,
+        identification: Annotated[StrictStr, Field(description="Identifier of the document.")],
+    ) -> Response:
+        """
+        Asynchronously retrieve a document reference by its identification.
+
+        Parameters
+        ----------
+        identification : str
+            The unique identifier for the document reference.
+
+        Returns
+        -------
+        Response
+            The response object containing the document reference details.
+        """
+
+    async def get_all_documents_status(
+        self,
+    ) -> list[DocumentStatus]:
+        """
+        Asynchronously retrieves the status of all documents.
+
+        Returns
+        -------
+        list[DocumentStatus]
+            A list containing the status of all documents.
+        """
+
+    async def upload_source(
+        self,
+        source_type: StrictStr,
+        name: StrictStr,
+        key_value_pair: List[KeyValuePair],
+    ) -> None:
+        """
+        Asynchronously uploads user selected source.
+
+        Parameters
+        ----------
+        source_type : str
+            The type of the source. Is used by the extractor service to determine the correct extractor to use.
+        name : str
+            Display name of the source.
+        key_value_pair : list[KeyValuePair]
+            List of KeyValuePair with parameters used for the extraction.
+
+        Returns
+        -------
+        None
+        """
+
+    async def upload_file(
+        self,
+        file: UploadFile,
+        request: Request,
+    ) -> None:
+        """
+        Asynchronously uploads user-selected documents.
+
+        Parameters
+        ----------
+        file : UploadFile
+            The file object containing the source documents to be uploaded.
+        request : Request
+            The request object containing metadata about the upload request.
+
+        Returns
+        -------
+        None
+        """

admin_api_lib-3.2.0/src/admin_api_lib/chunker/chunker.py

@@ -0,0 +1,25 @@
+"""Module for the Chunker abstract base class."""
+
+from abc import ABC, abstractmethod
+
+from langchain_core.documents import Document
+
+
+class Chunker(ABC):
+    """Abstract base class for chunking documents into smaller parts."""
+
+    @abstractmethod
+    def chunk(self, documents: Document) -> list[Document]:
+        """
+        Chunk the given documents into smaller parts.
+
+        Parameters
+        ----------
+        documents : Document
+            The documents to be chunked.
+
+        Returns
+        -------
+        list of Document
+            A list of smaller parts obtained by chunking the documents.
+        """