admin-api-lib 4.0.0__py3-none-any.whl → 4.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,12 @@ class DocumentDeleter(ABC):
7
7
  """Abstract base class for document deletion endpoint."""
8
8
 
9
9
  @abstractmethod
10
- async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
10
+ async def adelete_document(
11
+ self,
12
+ identification: str,
13
+ remove_from_key_value_store: bool = True,
14
+ remove_from_storage: bool = True,
15
+ ) -> None:
11
16
  """
12
17
  Delete a document by its identification asynchronously.
13
18
 
@@ -17,6 +22,8 @@ class DocumentDeleter(ABC):
17
22
  The unique identifier of the document to be deleted.
18
23
  remove_from_key_value_store : bool, optional
19
24
  If True, the document will also be removed from the key-value store (default is True).
25
+ remove_from_storage : bool, optional
26
+ If True, the document will also be removed from the file storage (default is True).
20
27
 
21
28
  Returns
22
29
  -------
@@ -1,77 +1,5 @@
1
- """Abstract class for dealing with I/O."""
1
+ """Re-export core file service interface."""
2
2
 
3
- import abc
4
- from abc import ABC
5
- from pathlib import Path
6
- from typing import BinaryIO
3
+ from rag_core_lib.file_services.file_service import FileService
7
4
 
8
-
9
- class FileService(ABC):
10
- """Abstract class for dealing with I/O."""
11
-
12
- @abc.abstractmethod
13
- def download_folder(self, source: str, target: Path) -> None:
14
- """Download the remote folder on "source" to the local "target" directory.
15
-
16
- Parameters
17
- ----------
18
- source: str
19
- Path to the remote folder.
20
- target: Path
21
- Download destination path.
22
- """
23
-
24
- @abc.abstractmethod
25
- def download_file(self, source: str, target_file: BinaryIO) -> None:
26
- """Read a single remote file "source" into the local "target_file" file-like object.
27
-
28
- Example usage
29
- =============
30
- ```
31
- s3_settings: S3Settings = get_s3_settings()
32
- s3_service = S3Service(endpoint="endpoint", username="username", password="password", bucket_name="bucket")
33
-
34
- with tempfile.SpooledTemporaryFile(max_size=self._iot_forecast_settings.max_model_size) as temp_file:
35
- s3_service.download_file("remote_file", temp_file)
36
- # do stuff with temp_file
37
- ```
38
-
39
- Parameters
40
- ----------
41
- source: str
42
- Path to the remote folder.
43
- target_file: BinaryIO
44
- File-like object to save the data to.
45
- """
46
-
47
- @abc.abstractmethod
48
- def upload_file(self, file_path: str, file_name: str) -> None:
49
- """Upload a local file to the Fileservice.
50
-
51
- Parameters
52
- ----------
53
- file_path : str
54
- The path to the local file to be uploaded.
55
- file_name : str
56
- The target path in the file storage where the file will be stored.
57
- """
58
-
59
- @abc.abstractmethod
60
- def get_all_sorted_file_names(self) -> list[str]:
61
- """Retrieve all file names stored in the file storage.
62
-
63
- Returns
64
- -------
65
- list[str]
66
- A list of file names stored in the file storage.
67
- """
68
-
69
- @abc.abstractmethod
70
- def delete_file(self, file_name: str) -> None:
71
- """Delete a file from the file storage.
72
-
73
- Parameters
74
- ----------
75
- file_name : str
76
- The name of the file to be deleted from the file storage.
77
- """
5
+ __all__ = ["FileService"]
@@ -41,7 +41,21 @@ class DefaultDocumentDeleter(DocumentDeleter):
41
41
  self._rag_api = rag_api
42
42
  self._key_value_store = key_value_store
43
43
 
44
- async def adelete_document(self, identification: str, remove_from_key_value_store: bool = True) -> None:
44
+ @staticmethod
45
+ def _storage_key_from_identification(identification: str) -> str | None:
46
+ if identification.startswith("file:"):
47
+ storage_key = identification[len("file:") :]
48
+ return storage_key or None
49
+ if ":" in identification:
50
+ return None
51
+ return identification or None
52
+
53
+ async def adelete_document(
54
+ self,
55
+ identification: str,
56
+ remove_from_key_value_store: bool = True,
57
+ remove_from_storage: bool = True,
58
+ ) -> None:
45
59
  """
46
60
  Asynchronously delete a document identified by the given identification string.
47
61
 
@@ -57,6 +71,8 @@ class DefaultDocumentDeleter(DocumentDeleter):
57
71
  The unique identifier of the document to be deleted.
58
72
  remove_from_key_value_store : bool, optional
59
73
  If True, the document will also be removed from the key-value store (default is True).
74
+ remove_from_storage : bool, optional
75
+ If True, the document will also be removed from the file storage (default is True).
60
76
 
61
77
  Raises
62
78
  ------
@@ -67,12 +83,12 @@ class DefaultDocumentDeleter(DocumentDeleter):
67
83
  error_messages = ""
68
84
  # Delete the document from file service and vector database
69
85
  logger.debug("Deleting existing document: %s", identification)
70
- try:
71
- if remove_from_key_value_store:
72
- self._key_value_store.remove(identification)
73
- self._file_service.delete_file(identification)
74
- except Exception as e:
75
- error_messages += f"Error while deleting {identification} from file storage\n {str(e)}\n"
86
+ if remove_from_key_value_store:
87
+ self._key_value_store.remove(identification)
88
+
89
+ if remove_from_storage:
90
+ error_messages = self._delete_from_storage(identification, error_messages)
91
+
76
92
  try:
77
93
  self._rag_api.remove_information_piece(
78
94
  DeleteRequest(metadata=[KeyValuePair(key="document", value=json.dumps(identification))])
@@ -82,3 +98,14 @@ class DefaultDocumentDeleter(DocumentDeleter):
82
98
  error_messages += f"Error while deleting {identification} from vector db\n{str(e)}"
83
99
  if error_messages:
84
100
  raise HTTPException(404, error_messages)
101
+
102
+ def _delete_from_storage(self, identification: str, error_messages: str) -> str:
103
+ try:
104
+ storage_key = self._storage_key_from_identification(identification)
105
+ if storage_key:
106
+ self._file_service.delete_file(storage_key)
107
+ else:
108
+ logger.debug("Skipping file storage deletion for non-file source: %s", identification)
109
+ except Exception as e:
110
+ error_messages += f"Error while deleting {identification} from file storage\n {str(e)}\n"
111
+ return error_messages
@@ -188,7 +188,11 @@ class DefaultFileUploader(FileUploader):
188
188
  # Replace old document
189
189
  # deletion is allowed to fail
190
190
  with suppress(Exception):
191
- await self._document_deleter.adelete_document(source_name, remove_from_key_value_store=False)
191
+ await self._document_deleter.adelete_document(
192
+ source_name,
193
+ remove_from_key_value_store=False,
194
+ remove_from_storage=False,
195
+ )
192
196
 
193
197
  # Run blocking RAG API call in thread pool to avoid blocking event loop
194
198
  await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)
@@ -197,7 +197,11 @@ class DefaultSourceUploader(SourceUploader):
197
197
  rag_information_pieces.append(self._information_mapper.document2rag_information_piece(doc))
198
198
 
199
199
  with suppress(Exception):
200
- await self._document_deleter.adelete_document(source_name, remove_from_key_value_store=False)
200
+ await self._document_deleter.adelete_document(
201
+ source_name,
202
+ remove_from_key_value_store=False,
203
+ remove_from_storage=False,
204
+ )
201
205
 
202
206
  # Run blocking RAG API call in thread pool to avoid blocking event loop
203
207
  await asyncio.to_thread(self._rag_api.upload_information_piece, rag_information_pieces)
@@ -1,130 +1,5 @@
1
- """Class to handle I/O with S3 storage."""
1
+ """Re-export core S3 service implementation."""
2
2
 
3
- import logging
4
- from pathlib import Path
5
- from typing import BinaryIO
3
+ from rag_core_lib.impl.file_services.s3_service import S3Service
6
4
 
7
- import boto3
8
-
9
- from admin_api_lib.file_services.file_service import FileService
10
- from admin_api_lib.impl.settings.s3_settings import S3Settings
11
-
12
- logger = logging.getLogger(__name__)
13
-
14
-
15
- class S3Service(FileService):
16
- """Class to handle I/O with S3 storage."""
17
-
18
- def __init__(self, s3_settings: S3Settings):
19
- """Class to handle I/O with S3 storage.
20
-
21
- Parameters
22
- ----------
23
- s3_settings: S3Settings
24
- Settings for the s3. Must contain at least the endpoint, access_key_id, secret_access_key and bucket.
25
- """
26
- self._s3_settings = s3_settings
27
- self._s3_client = boto3.client(
28
- "s3",
29
- endpoint_url=s3_settings.endpoint,
30
- aws_access_key_id=s3_settings.access_key_id,
31
- aws_secret_access_key=s3_settings.secret_access_key,
32
- aws_session_token=None,
33
- config=boto3.session.Config(signature_version="s3v4"),
34
- verify=False,
35
- )
36
-
37
- def download_folder(self, source: str, target: Path) -> None:
38
- """Download the remote folder on "source" to the local "target" directory.
39
-
40
- Parameters
41
- ----------
42
- source: str
43
- Path to the remote folder.
44
- target: Path
45
- Download destination path.
46
- """
47
- target.mkdir(parents=True, exist_ok=True)
48
-
49
- search_response = self._s3_client.list_objects_v2(
50
- Bucket=self._s3_settings.bucket,
51
- Prefix=source,
52
- )
53
- for found_content in search_response.get("Contents", []):
54
- file_source = found_content["Key"]
55
- target_path = target / file_source[len(source) :]
56
- target_path.parent.mkdir(parents=True, exist_ok=True)
57
- with open(target_path, "wb") as local_file:
58
- self.download_file(file_source, local_file)
59
-
60
- def download_file(self, source: str, target_file: BinaryIO) -> None:
61
- """Read a single remote file "source" into the local "target_file" file-like object.
62
-
63
- Example usage
64
- =============
65
- ```
66
- s3_settings: S3Settings = get_s3_settings()
67
- s3_service = S3Service(endpoint="endpoint", username="username", password="password", bucket_name="bucket")
68
-
69
- with tempfile.SpooledTemporaryFile(max_size=self._iot_forecast_settings.max_model_size) as temp_file:
70
- s3_service.download_file("remote_file", temp_file)
71
- # do stuff with temp_file
72
- ```
73
-
74
- Parameters
75
- ----------
76
- source: str
77
- Path to the remote folder.
78
- target_file: BinaryIO
79
- File-like object to save the data to.
80
- """
81
- self._s3_client.download_fileobj(self._s3_settings.bucket, source, target_file)
82
-
83
- def upload_file(self, file_path: str, file_name: str) -> None:
84
- """
85
- Upload a local file to the S3 bucket.
86
-
87
- Parameters
88
- ----------
89
- source : Path
90
- The path to the local file to upload.
91
- target : str
92
- The target path in the S3 bucket where the file will be stored.
93
- """
94
- self._s3_client.upload_file(
95
- Filename=file_path,
96
- Bucket=self._s3_settings.bucket,
97
- Key=file_name,
98
- )
99
-
100
- def get_all_sorted_file_names(self) -> list[str]:
101
- """Retrieve all file names stored in the S3 bucket.
102
-
103
- Returns
104
- -------
105
- list[str]
106
- A list of file names stored in the S3 bucket.
107
- """
108
- file_names = []
109
-
110
- resp = self._s3_client.list_objects_v2(Bucket=self._s3_settings.bucket)
111
- if resp.get("Contents"):
112
- for obj in resp["Contents"]:
113
- file_names.append(obj["Key"])
114
- return file_names
115
-
116
- def delete_file(self, file_name: str) -> None:
117
- """Delete a file from the S3 bucket.
118
-
119
- Parameters
120
- ----------
121
- file_name : str
122
- The name of the file to be deleted from the S3 bucket.
123
- """
124
- try:
125
- file_name = f"/{file_name}" if not file_name.startswith("/") else file_name
126
- self._s3_client.delete_object(Bucket=self._s3_settings.bucket, Key=file_name)
127
- logger.info("File %s successfully deleted.", file_name)
128
- except Exception:
129
- logger.exception("Error deleting file %s", file_name)
130
- raise
5
+ __all__ = ["S3Service"]
@@ -1,31 +1,5 @@
1
- """Contains settings regarding the S3 storage."""
1
+ """Re-export core S3 settings."""
2
2
 
3
- from pydantic_settings import BaseSettings
3
+ from rag_core_lib.impl.settings.s3_settings import S3Settings
4
4
 
5
-
6
- class S3Settings(BaseSettings):
7
- """
8
- Contains settings regarding the S3 storage.
9
-
10
- Attributes
11
- ----------
12
- secret_access_key : str
13
- The secret access key for S3.
14
- access_key_id : str
15
- The access key ID for S3.
16
- endpoint : str
17
- The endpoint URL for S3.
18
- bucket : str
19
- The bucket name in S3.
20
- """
21
-
22
- class Config:
23
- """Config class for reading Fields from env."""
24
-
25
- env_prefix = "S3_"
26
- case_sensitive = False
27
-
28
- secret_access_key: str
29
- access_key_id: str
30
- endpoint: str
31
- bucket: str
5
+ __all__ = ["S3Settings"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: admin-api-lib
3
- Version: 4.0.0
3
+ Version: 4.1.0
4
4
  Summary: The admin backend is responsible for the document management. This includes deletion, upload and returning the source document.
5
5
  License: Apache-2.0
6
6
  Author: STACKIT GmbH & Co. KG
@@ -1,5 +1,5 @@
1
1
  admin_api_lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- admin_api_lib/api_endpoints/document_deleter.py,sha256=1L0mqBBiGV5XXE2-4gpHh2hXuVdtqqz06qLaG3dQx9M,727
2
+ admin_api_lib/api_endpoints/document_deleter.py,sha256=C6PA2OHyLPExBpuMCUU-YYz57aFh_DMkYEjg27lDoV4,941
3
3
  admin_api_lib/api_endpoints/document_reference_retriever.py,sha256=eisisp-ZMPn3P1yNkxiqQroz3Rz4Zz8pCrj8JQ9rrro,658
4
4
  admin_api_lib/api_endpoints/documents_status_retriever.py,sha256=PxrW4X6mN2z_XJHzTqeolCnRusiiBC4OE8TdI4lUEMg,572
5
5
  admin_api_lib/api_endpoints/file_uploader.py,sha256=r7m9G06aSC3mBdVuXCBKTfR5bTmYEjGJSTavhKRuSJk,725
@@ -33,20 +33,20 @@ admin_api_lib/extractor_api_client/openapi_client/test/test_extraction_request.p
33
33
  admin_api_lib/extractor_api_client/openapi_client/test/test_extractor_api.py,sha256=kaTxIEPyQECdkAoT-kzVyYx0jX-P-5U6v73Ndhyg40w,887
34
34
  admin_api_lib/extractor_api_client/openapi_client/test/test_information_piece.py,sha256=Aq4h5SB-GY8BbeQR9sFMN1B3Z5UMUO6pRx6XU0b7xqs,1730
35
35
  admin_api_lib/extractor_api_client/openapi_client/test/test_key_value_pair.py,sha256=Zd9_HyM-4U2iVtGYO45UI6xkJKVNcj369hFe5zkBhDM,1403
36
- admin_api_lib/file_services/file_service.py,sha256=nBEga-HEuSlG47BpO4hR4U4cAC_kU7OBRFIuyUlocfU,2286
36
+ admin_api_lib/file_services/file_service.py,sha256=8sJLyWCPRw88ZAY9-4YPcMqYRR6Bgw_pv5A8Dy1Xf5k,137
37
37
  admin_api_lib/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
38
38
  admin_api_lib/impl/admin_api.py,sha256=k5geH6NorzzKg3_Z0tltLgTeJ0ru5AyFKPQi4nWjdnE,5483
39
- admin_api_lib/impl/api_endpoints/default_document_deleter.py,sha256=iKqBnk6bFRsZczZUdLBtetHUrZY30xrJWQbC02ZgL2M,3719
39
+ admin_api_lib/impl/api_endpoints/default_document_deleter.py,sha256=9P_UYbnn0wqBOy18jFn44zaY5nykTXfaQ_Zt39bfmF0,4727
40
40
  admin_api_lib/impl/api_endpoints/default_document_reference_retriever.py,sha256=H3bQvpMLMjsyUzZMfTziPW7qU3N9D5s6DMKEA4fMITM,2642
41
41
  admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py,sha256=ZtLNgmFWGcfU4jNhVPiAKIJT701Z4wVwQAWpPbegxfc,1419
42
- admin_api_lib/impl/api_endpoints/default_file_uploader.py,sha256=hfBPI1dgY-KUk_7dZU6yU04bbAMxn4hPKOQtxBBg5Ts,9613
43
- admin_api_lib/impl/api_endpoints/default_source_uploader.py,sha256=-K78-jkKSCrQDhBOMc3Y_qEXVRdM0IpRY_Gf8ad1-W4,8708
42
+ admin_api_lib/impl/api_endpoints/default_file_uploader.py,sha256=2tGd3ZQOJMZob_0UreopOliaefvIYq4CTp9soQevoCc,9719
43
+ admin_api_lib/impl/api_endpoints/default_source_uploader.py,sha256=UrKmbyu2l-Jyhbkaiqt66Ks-RMwD_i2vEkOdhgTbwZU,8814
44
44
  admin_api_lib/impl/chunker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
45
  admin_api_lib/impl/chunker/chunker_type.py,sha256=ArEAmQ9OWe3ek7FMb2auFvGk4UXsOr3xzjyB5TXI2i8,247
46
46
  admin_api_lib/impl/chunker/semantic_text_chunker.py,sha256=VrbHpY867gQPtMhz6HjqstdBQ-e74kwce_Ma-9LVsJo,10605
47
47
  admin_api_lib/impl/chunker/text_chunker.py,sha256=0WC1EsvkqHI16IeopTvrx0Y2XQmXscUFCT8-z6h5nOY,1076
48
48
  admin_api_lib/impl/file_services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
49
- admin_api_lib/impl/file_services/s3_service.py,sha256=IB68Av69mkkcf4KuXpxnbaJVJuMKHL82zIwuBw94pXg,4428
49
+ admin_api_lib/impl/file_services/s3_service.py,sha256=8NuYwKd3zNdpGWM6UC1R3yUmbG6XOyZrhYbguznzgRY,139
50
50
  admin_api_lib/impl/information_enhancer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
51
  admin_api_lib/impl/information_enhancer/general_enhancer.py,sha256=H4mVxYEGW2owdCcUwbh7z132PTxa3dZssRL0MDJg56Q,1889
52
52
  admin_api_lib/impl/information_enhancer/page_summary_enhancer.py,sha256=Ve0KjQMdxESLi2K3-fgZcHf2xMWNeJkfxvcbJCajWJo,4652
@@ -60,7 +60,7 @@ admin_api_lib/impl/settings/chunker_settings.py,sha256=TLBPY-PLmCuly8mMgGIlm3gU9
60
60
  admin_api_lib/impl/settings/document_extractor_settings.py,sha256=gYbdk6M4dwTL9esWfemGGHEJYOdV9BT7c8zGBKLbjlU,520
61
61
  admin_api_lib/impl/settings/key_value_settings.py,sha256=jokWtwXLwN4_yPaWwmVxJOSKwyR43hzQtQhKqAnIBrw,1789
62
62
  admin_api_lib/impl/settings/rag_api_settings.py,sha256=YMxnsiMLjZBQZ2a6C3kZuVgqd42_w4JUC9dTkHnwuaU,484
63
- admin_api_lib/impl/settings/s3_settings.py,sha256=e5vOOfJRp_22gtG13vf4iXedHXm3E1egLq8t_Ix_AYg,666
63
+ admin_api_lib/impl/settings/s3_settings.py,sha256=K5-zrhueyOKD_CpHQ3U2q16FZ1BZLF5oFnCnUnNoswA,123
64
64
  admin_api_lib/impl/settings/source_uploader_settings.py,sha256=ZCVFI3TgSZpVjBVTcJw9QSyTBLvqzYsJ41vpihtI7pY,582
65
65
  admin_api_lib/impl/settings/summarizer_settings.py,sha256=NGZ0o25dp7GkfOSAzcoCDSQrpUqiGe28Oi05ho2MAZ8,2895
66
66
  admin_api_lib/impl/summarizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -101,6 +101,6 @@ admin_api_lib/summarizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
101
101
  admin_api_lib/summarizer/summarizer.py,sha256=D0rkW0iZSys-68LcO1-PIkE0Faf2Grg-_9wu75Rc1OY,966
102
102
  admin_api_lib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
103
103
  admin_api_lib/utils/utils.py,sha256=eaNQ_NzUEp4hwhCU9EEsUXvbRH_ekVariF7tTsO9Sco,834
104
- admin_api_lib-4.0.0.dist-info/METADATA,sha256=EcuMQqTXIsCgk8U-lesJoznxJR3cTKFEbiegdyIns80,7900
105
- admin_api_lib-4.0.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
106
- admin_api_lib-4.0.0.dist-info/RECORD,,
104
+ admin_api_lib-4.1.0.dist-info/METADATA,sha256=T6a5lgyq6pnbAPxSO4l2r5RHuebfioA62HsNd-x0Br0,7900
105
+ admin_api_lib-4.1.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
106
+ admin_api_lib-4.1.0.dist-info/RECORD,,