admin-api-lib 3.2.0__py3-none-any.whl

admin_api_lib/extractor_api_client/openapi_client/test/test_extraction_parameters.py

@@ -0,0 +1,59 @@
+# coding: utf-8
+
+"""
+extractor-api-lib
+
+No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+The version of the OpenAPI document: 1.0.0
+Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from admin_api_lib.extractor_api_client.openapi_client.models.extraction_parameters import ExtractionParameters
+
+
+class TestExtractionParameters(unittest.TestCase):
+    """ExtractionParameters unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> ExtractionParameters:
+        """Test ExtractionParameters
+        include_optional is a boolean, when False only required
+        params are included, when True both required and
+        optional params are included"""
+        # uncomment below to create an instance of `ExtractionParameters`
+        """
+        model = ExtractionParameters()
+        if include_optional:
+            return ExtractionParameters(
+                document_name = '',
+                confluence_kwargs = [
+                    {"value":"value","key":"key"}
+                    ],
+                type = ''
+            )
+        else:
+            return ExtractionParameters(
+                document_name = '',
+                type = '',
+        )
+        """
+
+    def testExtractionParameters(self):
+        """Test ExtractionParameters"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

admin_api_lib/extractor_api_client/openapi_client/test/test_extraction_request.py

@@ -0,0 +1,56 @@
+# coding: utf-8
+
+"""
+extractor-api-lib
+
+No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+The version of the OpenAPI document: 1.0.0
+Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from admin_api_lib.extractor_api_client.openapi_client.models.extraction_request import ExtractionRequest
+
+
+class TestExtractionRequest(unittest.TestCase):
+    """ExtractionRequest unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> ExtractionRequest:
+        """Test ExtractionRequest
+        include_optional is a boolean, when False only required
+        params are included, when True both required and
+        optional params are included"""
+        # uncomment below to create an instance of `ExtractionRequest`
+        """
+        model = ExtractionRequest()
+        if include_optional:
+            return ExtractionRequest(
+                path_on_s3 = '',
+                document_name = ''
+            )
+        else:
+            return ExtractionRequest(
+                path_on_s3 = '',
+                document_name = '',
+        )
+        """
+
+    def testExtractionRequest(self):
+        """Test ExtractionRequest"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

admin_api_lib/extractor_api_client/openapi_client/test/test_extractor_api.py

@@ -0,0 +1,39 @@
+# coding: utf-8
+
+"""
+extractor-api-lib
+
+No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+The version of the OpenAPI document: 1.0.0
+Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from admin_api_lib.extractor_api_client.openapi_client.api.extractor_api import ExtractorApi
+
+
+class TestExtractorApi(unittest.TestCase):
+    """ExtractorApi unit test stubs"""
+
+    def setUp(self) -> None:
+        self.api = ExtractorApi()
+
+    def tearDown(self) -> None:
+        pass
+
+    def test_extract_from_file_post(self) -> None:
+        """Test case for extract_from_file_post"""
+        pass
+
+    def test_extract_from_source(self) -> None:
+        """Test case for extract_from_source"""
+        pass
+
+
+if __name__ == "__main__":
+    unittest.main()

admin_api_lib/extractor_api_client/openapi_client/test/test_information_piece.py

@@ -0,0 +1,62 @@
+# coding: utf-8
+
+"""
+extractor-api-lib
+
+No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+The version of the OpenAPI document: 1.0.0
+Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from admin_api_lib.extractor_api_client.openapi_client.models.information_piece import InformationPiece
+
+
+class TestInformationPiece(unittest.TestCase):
+    """InformationPiece unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> InformationPiece:
+        """Test InformationPiece
+        include_optional is a boolean, when False only required
+        params are included, when True both required and
+        optional params are included"""
+        # uncomment below to create an instance of `InformationPiece`
+        """
+        model = InformationPiece()
+        if include_optional:
+            return InformationPiece(
+                metadata = [
+                    {"value":"value","key":"key"}
+                    ],
+                page_content = '',
+                type = 'IMAGE'
+            )
+        else:
+            return InformationPiece(
+                metadata = [
+                    {"value":"value","key":"key"}
+                    ],
+                page_content = '',
+                type = 'IMAGE',
+        )
+        """
+
+    def testInformationPiece(self):
+        """Test InformationPiece"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

admin_api_lib/extractor_api_client/openapi_client/test/test_key_value_pair.py

@@ -0,0 +1,54 @@
+# coding: utf-8
+
+"""
+extractor-api-lib
+
+No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
+
+The version of the OpenAPI document: 1.0.0
+Generated by OpenAPI Generator (https://openapi-generator.tech)
+
+Do not edit the class manually.
+"""  # noqa: E501
+
+
+import unittest
+
+from admin_api_lib.extractor_api_client.openapi_client.models.key_value_pair import KeyValuePair
+
+
+class TestKeyValuePair(unittest.TestCase):
+    """KeyValuePair unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def make_instance(self, include_optional) -> KeyValuePair:
+        """Test KeyValuePair
+        include_optional is a boolean, when False only required
+        params are included, when True both required and
+        optional params are included"""
+        # uncomment below to create an instance of `KeyValuePair`
+        """
+        model = KeyValuePair()
+        if include_optional:
+            return KeyValuePair(
+                key = None,
+                value = None
+            )
+        else:
+            return KeyValuePair(
+        )
+        """
+
+    def testKeyValuePair(self):
+        """Test KeyValuePair"""
+        # inst_req_only = self.make_instance(include_optional=False)
+        # inst_req_and_optional = self.make_instance(include_optional=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

admin_api_lib/file_services/file_service.py

@@ -0,0 +1,77 @@
+"""Abstract class for dealing with I/O."""
+
+import abc
+from abc import ABC
+from pathlib import Path
+from typing import BinaryIO
+
+
+class FileService(ABC):
+    """Abstract class for dealing with I/O."""
+
+    @abc.abstractmethod
+    def download_folder(self, source: str, target: Path) -> None:
+        """Download the remote folder on "source" to the local "target" directory.
+
+        Parameters
+        ----------
+        source: str
+            Path to the remote folder.
+        target: Path
+            Download destination path.
+        """
+
+    @abc.abstractmethod
+    def download_file(self, source: str, target_file: BinaryIO) -> None:
+        """Read a single remote file "source" into the local "target_file" file-like object.
+
+        Example usage
+        =============
+        ```
+        s3_settings: S3Settings = get_s3_settings()
+        s3_service = S3Service(endpoint="endpoint", username="username", password="password", bucket_name="bucket")
+
+        with tempfile.SpooledTemporaryFile(max_size=self._iot_forecast_settings.max_model_size) as temp_file:
+            s3_service.download_file("remote_file", temp_file)
+            # do stuff with temp_file
+        ```
+
+        Parameters
+        ----------
+        source: str
+            Path to the remote folder.
+        target_file: BinaryIO
+            File-like object to save the data to.
+        """
+
+    @abc.abstractmethod
+    def upload_file(self, file_path: str, file_name: str) -> None:
+        """Upload a local file to the Fileservice.
+
+        Parameters
+        ----------
+        file_path : str
+            The path to the local file to be uploaded.
+        file_name : str
+            The target path in the file storage where the file will be stored.
+        """
+
+    @abc.abstractmethod
+    def get_all_sorted_file_names(self) -> list[str]:
+        """Retrieve all file names stored in the file storage.
+
+        Returns
+        -------
+        list[str]
+            A list of file names stored in the file storage.
+        """
+
+    @abc.abstractmethod
+    def delete_file(self, file_name: str) -> None:
+        """Delete a file from the file storage.
+
+        Parameters
+        ----------
+        file_name : str
+            The name of the file to be deleted from the file storage.
+        """

admin_api_lib/impl/admin_api.py

@@ -0,0 +1,167 @@
+"""Module containing the implementation of the Admin API."""
+
+import logging
+
+
+from pydantic import StrictStr
+from dependency_injector.wiring import Provide, inject
+from fastapi import Depends, Request, Response, UploadFile
+
+from admin_api_lib.api_endpoints.file_uploader import FileUploader
+from admin_api_lib.api_endpoints.source_uploader import SourceUploader
+from admin_api_lib.models.key_value_pair import KeyValuePair
+from admin_api_lib.api_endpoints.document_deleter import DocumentDeleter
+from admin_api_lib.api_endpoints.document_reference_retriever import (
+    DocumentReferenceRetriever,
+)
+from admin_api_lib.api_endpoints.documents_status_retriever import (
+    DocumentsStatusRetriever,
+)
+from admin_api_lib.apis.admin_api_base import BaseAdminApi
+from admin_api_lib.dependency_container import DependencyContainer
+from admin_api_lib.models.document_status import DocumentStatus
+
+logger = logging.getLogger(__name__)
+
+
+class AdminApi(BaseAdminApi):
+    """The Class for the Admin API.
+
+    AdminApi class provides various asynchronous methods to interact with documents, including deleting,
+    retrieving status, loading from Confluence, retrieving by reference ID, and uploading documents.
+    """
+
+    def __init__(self):
+        """
+        Initialize the AdminAPI class.
+
+        This constructor calls the parent class's initializer and sets up
+        an empty list to hold background threads.
+        """
+        super().__init__()
+        self._background_threads = []
+
+    @inject
+    async def delete_document(
+        self,
+        identification: str,
+        document_deleter: DocumentDeleter = Depends(Provide[DependencyContainer.document_deleter]),
+    ) -> None:
+        """
+        Delete a document asynchronously.
+
+        Parameters
+        ----------
+        identification : str
+            The unique identifier of the document to be deleted.
+        document_deleter : DocumentDeleter
+            The document deleter instance, injected by dependency injection
+            (default is Depends(Provide[DependencyContainer.document_deleter])).
+
+        Returns
+        -------
+        None
+        """
+        await document_deleter.adelete_document(identification)
+
+    @inject
+    async def get_all_documents_status(
+        self,
+        document_status_retriever: DocumentsStatusRetriever = Depends(
+            Provide[DependencyContainer.documents_status_retriever]
+        ),
+    ) -> list[DocumentStatus]:
+        """
+        Asynchronously retrieve the status of all documents.
+
+        Parameters
+        ----------
+        document_status_retriever : DocumentsStatusRetriever
+            An instance of DocumentsStatusRetriever
+            (default is Depends(Provide[DependencyContainer.documents_status_retriever])).
+
+        Returns
+        -------
+        list[DocumentStatus]
+            A list containing the status of all documents.
+        """
+        return await document_status_retriever.aget_all_documents_status()
+
+    @inject
+    async def upload_source(
+        self,
+        source_type: StrictStr,
+        name: StrictStr,
+        kwargs: list[KeyValuePair],
+        source_uploader: SourceUploader = Depends(Provide[DependencyContainer.source_uploader]),
+    ) -> None:
+        """
+        Asynchronously uploads user-selected source documents.
+
+        Parameters
+        ----------
+        source_type : StrictStr
+            The type of the source document to be uploaded.
+        name : StrictStr
+            The name of the source document to be uploaded.
+        kwargs : list[KeyValuePair]
+           Additional parameters required for the extractor.
+        source_uploader : SourceUploader
+            An instance of SourceUploader to handle the upload process.
+
+        Returns
+        -------
+        None
+        """
+        await source_uploader.upload_source(source_type, name, kwargs)
+
+    @inject
+    async def upload_file(
+        self,
+        file: UploadFile,
+        request: Request,
+        file_uploader: FileUploader = Depends(Provide[DependencyContainer.file_uploader]),
+    ) -> None:
+        """
+        Asynchronously uploads a file to the server.
+
+        Parameters
+        ----------
+        file : UploadFile
+            The file object to be uploaded.
+        request : Request
+            The HTTP request object containing metadata about the upload request.
+        file_uploader : FileUploader, optional
+            An instance of FileUploader to handle the upload process.
+
+        Returns
+        -------
+        None
+        """
+        await file_uploader.upload_file(str(request.base_url), file)
+
+    @inject
+    async def document_reference(
+        self,
+        identification: str,
+        document_reference_retriever: DocumentReferenceRetriever = Depends(
+            Provide[DependencyContainer.document_reference_retriever]
+        ),
+    ) -> Response:
+        """
+        Retrieve the document with the given identification.
+
+        Parameters
+        ----------
+        identification : str
+            The identification of the document.
+        document_reference_retriever : DocumentReferenceRetriever, optional
+            The service to retrieve the document reference.
+            Defaults to Depends(Provide[DependencyContainer.document_reference_retriever]).
+
+        Returns
+        -------
+        Response
+            The document in binary form.
+        """
+        return await document_reference_retriever.adocument_reference_id_get(identification)

admin_api_lib/impl/api_endpoints/default_document_deleter.py

@@ -0,0 +1,84 @@
+"""Module for the DefaultDocumentDeleter class."""
+
+import json
+import logging
+
+from fastapi import HTTPException
+
+from admin_api_lib.api_endpoints.document_deleter import DocumentDeleter
+from admin_api_lib.file_services.file_service import FileService
+from admin_api_lib.impl.key_db.file_status_key_value_store import (
+    FileStatusKeyValueStore,
+)
+from admin_api_lib.rag_backend_client.openapi_client.api.rag_api import RagApi
+from admin_api_lib.rag_backend_client.openapi_client.models.delete_request import (
+    DeleteRequest,
+)
+from admin_api_lib.rag_backend_client.openapi_client.models.key_value_pair import (
+    KeyValuePair,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DefaultDocumentDeleter(DocumentDeleter):
+    """A class used to delete documents from file storage and vector database."""
+
+    def __init__(self, file_service: FileService, rag_api: RagApi, key_value_store: FileStatusKeyValueStore):
+        """
+        Initialize the DefaultDocumentDeleter.
+
+        Parameters
+        ----------
+        file_service : FileService
+            The service responsible for file operations with s3 storage.
+        rag_api : RagApi
+            The API client for interacting with the RAG backend system.
+        key_value_store : FileStatusKeyValueStore
+            The key-value store to store file names and the corresponding file statuses.
+        """
+        self._file_service = file_service
+        self._rag_api = rag_api
+        self._key_value_store = key_value_store
+
+    async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
+        """
+        Asynchronously delete a document identified by the given identification string.
+
+        This method attempts to delete the document from both the S3 storage and the vector database.
+        If any errors occur during the deletion process, an HTTPException is raised with the error messages.
+        If the source document is from a service like Confluence, no document on the S3 storage exists, and nothing
+        can be deleted from the S3 storage. However, this does not prevent the deletion of the document from the
+        vector database. If the document does not exist on the S3 storage, the deletion process will continue.
+
+        Parameters
+        ----------
+        identification : str
+            The unique identifier of the document to be deleted.
+        remove_from_key_value_store : bool, optional
+            If True, the document will also be removed from the key-value store (default is True).
+
+        Raises
+        ------
+        HTTPException
+            If any errors occur during the deletion process, an HTTPException is raised with a 404 status code
+            and the error messages.
+        """
+        error_messages = ""
+        # Delete the document from file service and vector database
+        logger.debug("Deleting existing document: %s", identification)
+        try:
+            if remove_from_key_value_store:
+                self._key_value_store.remove(identification)
+            self._file_service.delete_file(identification)
+        except Exception as e:
+            error_messages += f"Error while deleting {identification} from file storage\n {str(e)}\n"
+        try:
+            self._rag_api.remove_information_piece(
+                DeleteRequest(metadata=[KeyValuePair(key="document", value=json.dumps(identification))])
+            )
+            logger.info("Deleted information pieces belonging to %s from rag.", identification)
+        except Exception as e:
+            error_messages += f"Error while deleting {identification} from vector db\n{str(e)}"
+        if error_messages:
+            raise HTTPException(404, error_messages)

admin_api_lib/impl/api_endpoints/default_document_reference_retriever.py

@@ -0,0 +1,72 @@
+"""Module for the DefaultDocumentReferenceRetriever class."""
+
+import io
+import logging
+import traceback
+
+from fastapi import HTTPException, Response, status
+
+from admin_api_lib.api_endpoints.document_reference_retriever import (
+    DocumentReferenceRetriever,
+)
+from admin_api_lib.file_services.file_service import FileService
+
+logger = logging.getLogger(__name__)
+
+
+class DefaultDocumentReferenceRetriever(DocumentReferenceRetriever):
+    """A class to retrieve document references using a file service."""
+
+    def __init__(self, file_service: FileService):
+        """
+        Initialize the DefaultDocumentReferenceRetriever.
+
+        Parameters
+        ----------
+        file_service : FileService
+            An instance of FileService used to handle file operations.
+        """
+        self._file_service = file_service
+
+    async def adocument_reference_id_get(self, identification: str) -> Response:
+        """
+        Retrieve the document with the given identification asynchronously.
+
+        Parameters
+        ----------
+        identification : str
+            The identification string of the document.
+
+        Returns
+        -------
+        Response
+            The document in binary form wrapped in a FastAPI Response object.
+
+        Raises
+        ------
+        HTTPException
+            If the document with the given identification is not found or any other value error occurs.
+        """
+        try:
+            logger.debug("START retrieving document with id: %s", identification)
+            document_buffer = io.BytesIO()
+            try:
+                self._file_service.download_file(identification, document_buffer)
+                logger.debug("DONE retrieving document with id: %s", identification)
+                document_data = document_buffer.getvalue()
+            except Exception as e:
+                logger.error(
+                    "Error retrieving document with id: %s. Error: %s %s", identification, e, traceback.format_exc()
+                )
+                raise ValueError(f"Document with id '{identification}' not found.")
+            finally:
+                document_buffer.close()
+        except ValueError as e:
+            raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
+
+        media_type = "application/pdf" if identification.endswith(".pdf") else "application/octet-stream"
+        headers = {
+            "Content-Disposition": f'inline; filename="{identification.encode("utf-8").decode()}"',
+            "Content-Type": media_type,
+        }
+        return Response(document_data, status_code=200, headers=headers, media_type=media_type)