th2rag 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- th2rag/__init__.py +0 -0
- th2rag/auth/__init__.py +0 -0
- th2rag/auth/exceptions.py +2 -0
- th2rag/auth/router.py +43 -0
- th2rag/auth/schemas.py +15 -0
- th2rag/auth/service.py +115 -0
- th2rag/clients/__init__.py +0 -0
- th2rag/clients/storage/config.py +13 -0
- th2rag/clients/storage/local.py +48 -0
- th2rag/clients/storage/s3.py +93 -0
- th2rag/config.py +267 -0
- th2rag/conversations/__init__.py +0 -0
- th2rag/conversations/dependencies.py +30 -0
- th2rag/conversations/exceptions.py +6 -0
- th2rag/conversations/router.py +47 -0
- th2rag/conversations/schemas.py +43 -0
- th2rag/conversations/service.py +51 -0
- th2rag/database.py +72 -0
- th2rag/dataset/__init__.py +0 -0
- th2rag/dataset/dependencies.py +25 -0
- th2rag/dataset/exceptions.py +2 -0
- th2rag/dataset/router.py +53 -0
- th2rag/dataset/schemas.py +23 -0
- th2rag/dataset/service.py +78 -0
- th2rag/dependencies.py +143 -0
- th2rag/knowledge/__init__.py +0 -0
- th2rag/knowledge/dependencies.py +25 -0
- th2rag/knowledge/exceptions.py +2 -0
- th2rag/knowledge/mage_client.py +505 -0
- th2rag/knowledge/router.py +714 -0
- th2rag/knowledge/schemas.py +108 -0
- th2rag/knowledge/service.py +392 -0
- th2rag/logger.py +21 -0
- th2rag/main.py +51 -0
- th2rag/messages/__init__.py +0 -0
- th2rag/messages/exceptions.py +2 -0
- th2rag/messages/router.py +63 -0
- th2rag/messages/schemas.py +68 -0
- th2rag/messages/service.py +210 -0
- th2rag/models.py +215 -0
- th2rag/pagination.py +68 -0
- th2rag/rag/__init__.py +0 -0
- th2rag/rag/chunkers/__init__.py +0 -0
- th2rag/rag/chunkers/hybrid_chunker.py +23 -0
- th2rag/rag/constants.py +15 -0
- th2rag/rag/converters/__init__.py +0 -0
- th2rag/rag/converters/documents_converter.py +123 -0
- th2rag/rag/converters/json_converter.py +521 -0
- th2rag/rag/converters/legacy_converter.py +124 -0
- th2rag/rag/converters/pdf_converter.py +384 -0
- th2rag/rag/converters/spreadsheet_converter.py +159 -0
- th2rag/rag/converters/txt_converter.py +98 -0
- th2rag/rag/embeddings/__init__.py +0 -0
- th2rag/rag/embeddings/base.py +21 -0
- th2rag/rag/embeddings/exceptions.py +2 -0
- th2rag/rag/embeddings/sentence_transformer.py +97 -0
- th2rag/rag/generation/base.py +22 -0
- th2rag/rag/generation/exceptions.py +2 -0
- th2rag/rag/generation/litellm_generator.py +143 -0
- th2rag/rag/generation/mistral_generator.py +269 -0
- th2rag/rag/generation/test_litellm.py +33 -0
- th2rag/rag/retrieval/__init__.py +0 -0
- th2rag/rag/retrieval/base.py +11 -0
- th2rag/rag/retrieval/exceptions.py +10 -0
- th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
- th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
- th2rag/rag/retrieval/lancedb/schemas.py +19 -0
- th2rag/rag/services/__init__.py +0 -0
- th2rag/rag/services/rag_service.py +99 -0
- th2rag/rag/services/rag_service_builder.py +42 -0
- th2rag/rag/tasks/process_pdf.py +586 -0
- th2rag/users/dependencies.py +44 -0
- th2rag/users/exceptions.py +6 -0
- th2rag/users/router.py +92 -0
- th2rag/users/schemas.py +61 -0
- th2rag/users/service.py +140 -0
- th2rag/utils/__init__.py +0 -0
- th2rag/utils/helpers.py +40 -0
- th2rag/utils/logger.py +15 -0
- th2rag/utils/security.py +76 -0
- th2rag/utils/welcome_router.py +98 -0
- th2rag/webhooks/__init__.py +0 -0
- th2rag/webhooks/sender.py +126 -0
- th2rag-0.0.2.dist-info/METADATA +1002 -0
- th2rag-0.0.2.dist-info/RECORD +87 -0
- th2rag-0.0.2.dist-info/WHEEL +4 -0
- th2rag-0.0.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
5
|
+
|
|
6
|
+
from th2rag.models import Status
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class KnowledgeBase(BaseModel):
|
|
10
|
+
name: str
|
|
11
|
+
description: str
|
|
12
|
+
prompt: str | None = None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Knowledge(KnowledgeBase):
|
|
16
|
+
knowledge_id: int
|
|
17
|
+
user_id: int
|
|
18
|
+
knowledge_path: str
|
|
19
|
+
status: Status
|
|
20
|
+
callback_url: str | None = None
|
|
21
|
+
error_message: str | None = None
|
|
22
|
+
created_at: datetime
|
|
23
|
+
updated_at: datetime
|
|
24
|
+
|
|
25
|
+
model_config = ConfigDict(from_attributes=True)
|
|
26
|
+
|
|
27
|
+
class KnowledgeWithPaths(Knowledge):
|
|
28
|
+
"""Extended Knowledge schema that includes all file paths"""
|
|
29
|
+
file_paths: list[str] = Field(description="List of all S3 file paths")
|
|
30
|
+
file_names: list[str] = Field(description="List of original filenames")
|
|
31
|
+
|
|
32
|
+
def get_file_path(self, index: int) -> str:
|
|
33
|
+
"""
|
|
34
|
+
Get file path by index.
|
|
35
|
+
"""
|
|
36
|
+
if 0 <= index < len(self.file_paths):
|
|
37
|
+
return self.file_paths[index]
|
|
38
|
+
raise IndexError(f"File index {index} out of range (0-{len(self.file_paths)-1})")
|
|
39
|
+
|
|
40
|
+
def get_file_name(self, index: int) -> str:
|
|
41
|
+
"""
|
|
42
|
+
Get filename by index.
|
|
43
|
+
"""
|
|
44
|
+
if 0 <= index < len(self.file_names):
|
|
45
|
+
return self.file_names[index]
|
|
46
|
+
raise IndexError(f"File index {index} out of range (0-{len(self.file_names)-1})")
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def file_count(self) -> int:
|
|
50
|
+
"""Get the total number of files"""
|
|
51
|
+
return len(self.file_paths)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class KnowledgeCreate(KnowledgeBase):
|
|
55
|
+
pass
|
|
56
|
+
|
|
57
|
+
class FileProcessingInfo(BaseModel):
|
|
58
|
+
"""Information about individual file processing"""
|
|
59
|
+
filename: str
|
|
60
|
+
source: str
|
|
61
|
+
chunk_count: int | None = None
|
|
62
|
+
status: Literal["completed", "failed"] | None = "completed"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ProcessingInfo(BaseModel):
|
|
66
|
+
"""Information about PDF processing status"""
|
|
67
|
+
mode: Literal["sync", "async"] = Field(
|
|
68
|
+
description="Processing mode: 'sync' (waited for completion) or 'async' (background)"
|
|
69
|
+
)
|
|
70
|
+
status: Literal["processing", "completed", "failed", "partial"] = Field(
|
|
71
|
+
description="Current processing status"
|
|
72
|
+
)
|
|
73
|
+
total_files: int = Field(
|
|
74
|
+
description="Total number of files being processed"
|
|
75
|
+
)
|
|
76
|
+
successful_files: int | None = Field(
|
|
77
|
+
default=None,
|
|
78
|
+
description="Number of files successfully processed"
|
|
79
|
+
)
|
|
80
|
+
failed_files: int | None = Field(
|
|
81
|
+
default=None,
|
|
82
|
+
description="Number of files that failed processing"
|
|
83
|
+
)
|
|
84
|
+
total_chunks: int | None = Field(
|
|
85
|
+
default=None,
|
|
86
|
+
description="Total number of chunks created across all files"
|
|
87
|
+
)
|
|
88
|
+
files: list[FileProcessingInfo] | None = Field(
|
|
89
|
+
default=None,
|
|
90
|
+
description="Individual file processing information"
|
|
91
|
+
)
|
|
92
|
+
method: Literal["mage", "background_task"] | None = Field(
|
|
93
|
+
default=None,
|
|
94
|
+
description="Processing method for async mode"
|
|
95
|
+
)
|
|
96
|
+
run_id: str | None = Field(
|
|
97
|
+
default=None,
|
|
98
|
+
description="Mage run ID (only for async with Mage)"
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class KnowledgeCreateResponse(BaseModel):
|
|
103
|
+
"""Response when creating a new knowledge document"""
|
|
104
|
+
result: KnowledgeWithPaths = Field( description="The created knowledge document with file paths")
|
|
105
|
+
processing: ProcessingInfo = Field(description="Processing information")
|
|
106
|
+
message: str = Field(description="Human-readable message about the processing status")
|
|
107
|
+
|
|
108
|
+
model_config = ConfigDict(from_attributes=True)
|
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
from sqlalchemy import select
|
|
2
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
3
|
+
|
|
4
|
+
from th2rag.clients.storage.s3 import delete_file_from_s3, upload_file_to_s3
|
|
5
|
+
from th2rag.knowledge import exceptions, schemas
|
|
6
|
+
from th2rag.models import Knowledge, Status
|
|
7
|
+
from th2rag.pagination import PageParams, PageResponse, paginate
|
|
8
|
+
from th2rag.rag.retrieval.lancedb.lancedb_storage import LanceDBStorage
|
|
9
|
+
from th2rag.users import schemas as user_schemas
|
|
10
|
+
from th2rag.utils.logger import setup_logger
|
|
11
|
+
from th2rag.config import settings
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
import os
|
|
14
|
+
|
|
15
|
+
logger = setup_logger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
async def get_all_knowledges(
|
|
19
|
+
page_params: PageParams, current_user: user_schemas.User, db: AsyncSession
|
|
20
|
+
) -> PageResponse[schemas.Knowledge]:
|
|
21
|
+
stmt = select(Knowledge)
|
|
22
|
+
if current_user.role != "ADMIN":
|
|
23
|
+
stmt = stmt.where(Knowledge.user_id == current_user.user_id)
|
|
24
|
+
|
|
25
|
+
# First, check and auto-fail any timed out PENDING documents
|
|
26
|
+
# Get all PENDING documents for this user
|
|
27
|
+
pending_stmt = stmt.where(Knowledge.status == Status.PENDING)
|
|
28
|
+
pending_result = await db.execute(pending_stmt)
|
|
29
|
+
pending_docs = pending_result.scalars().all()
|
|
30
|
+
|
|
31
|
+
# Check each PENDING document for timeout
|
|
32
|
+
failed_count = 0
|
|
33
|
+
timeout_threshold = settings.knowledge_pending_timeout_minutes
|
|
34
|
+
|
|
35
|
+
for knowledge in pending_docs:
|
|
36
|
+
if knowledge.created_at:
|
|
37
|
+
# Handle both timezone-aware and timezone-naive datetimes
|
|
38
|
+
created_at = knowledge.created_at
|
|
39
|
+
if created_at.tzinfo is None:
|
|
40
|
+
created_at = created_at.replace(tzinfo=timezone.utc)
|
|
41
|
+
|
|
42
|
+
age = datetime.now(timezone.utc) - created_at
|
|
43
|
+
age_minutes = age.total_seconds() / 60
|
|
44
|
+
|
|
45
|
+
if age_minutes > timeout_threshold:
|
|
46
|
+
logger.warning(
|
|
47
|
+
f"Auto-failing knowledge {knowledge.knowledge_id} - "
|
|
48
|
+
f"exceeded timeout ({age_minutes:.1f} > {timeout_threshold} minutes)"
|
|
49
|
+
)
|
|
50
|
+
knowledge.status = Status.FAILED
|
|
51
|
+
failed_count += 1
|
|
52
|
+
|
|
53
|
+
# Commit all status updates
|
|
54
|
+
if failed_count > 0:
|
|
55
|
+
await db.commit()
|
|
56
|
+
logger.info(f"Auto-failed {failed_count} knowledge document(s) in list view")
|
|
57
|
+
|
|
58
|
+
# Now return paginated results, sorted by knowledge_id descending (newest first)
|
|
59
|
+
stmt = stmt.order_by(Knowledge.knowledge_id.desc())
|
|
60
|
+
return await paginate(db, stmt, page_params, schemas.Knowledge)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def get_knowledge_by_id(knowledge_id: int, db: AsyncSession) -> schemas.Knowledge:
|
|
64
|
+
stmt = select(Knowledge).where(Knowledge.knowledge_id == knowledge_id)
|
|
65
|
+
result = await db.execute(stmt)
|
|
66
|
+
knowledge = result.scalar_one_or_none()
|
|
67
|
+
|
|
68
|
+
if not knowledge:
|
|
69
|
+
raise exceptions.KnowledgeNotFoundException("Knowledge not found")
|
|
70
|
+
|
|
71
|
+
# Auto-fail logic: Check if PENDING and timed out
|
|
72
|
+
if knowledge.status == Status.PENDING and knowledge.created_at:
|
|
73
|
+
# Handle both timezone-aware and timezone-naive datetimes
|
|
74
|
+
created_at = knowledge.created_at
|
|
75
|
+
if created_at.tzinfo is None:
|
|
76
|
+
# If created_at is naive, treat it as UTC
|
|
77
|
+
created_at = created_at.replace(tzinfo=timezone.utc)
|
|
78
|
+
|
|
79
|
+
age = datetime.now(timezone.utc) - created_at
|
|
80
|
+
age_minutes = age.total_seconds() / 60
|
|
81
|
+
timeout_threshold = settings.knowledge_pending_timeout_minutes
|
|
82
|
+
|
|
83
|
+
if age_minutes > timeout_threshold:
|
|
84
|
+
logger.warning(
|
|
85
|
+
f"Auto-failing knowledge {knowledge_id} - "
|
|
86
|
+
f"exceeded timeout ({age_minutes:.1f} > {timeout_threshold} minutes)"
|
|
87
|
+
)
|
|
88
|
+
knowledge.status = Status.FAILED
|
|
89
|
+
await db.commit()
|
|
90
|
+
await db.refresh(knowledge)
|
|
91
|
+
|
|
92
|
+
return schemas.Knowledge.model_validate(knowledge)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
# Source unique de vérité pour les extensions acceptées à l'upload.
|
|
96
|
+
# TOUTE extension ici DOIT être routable par convert_any_doc
|
|
97
|
+
# (garde-fou permanent dans test/knowledge/test_lot4_whitelist.py).
|
|
98
|
+
# Formats legacy (.doc/.ppt/.xls) nécessitent LibreOffice ;
|
|
99
|
+
# sans lui, l'upload est rejeté avec un message explicite.
|
|
100
|
+
ALLOWED_EXTENSIONS: frozenset[str] = frozenset({
|
|
101
|
+
# Office natifs Docling
|
|
102
|
+
".pdf",
|
|
103
|
+
".docx",
|
|
104
|
+
".pptx",
|
|
105
|
+
# Office legacy (via LibreOffice headless si installé)
|
|
106
|
+
".doc",
|
|
107
|
+
".ppt",
|
|
108
|
+
".xls",
|
|
109
|
+
# Spreadsheets
|
|
110
|
+
".csv",
|
|
111
|
+
".xlsx",
|
|
112
|
+
# Données structurées
|
|
113
|
+
".json",
|
|
114
|
+
# Texte brut
|
|
115
|
+
".txt",
|
|
116
|
+
# Web / markup (Lot 2)
|
|
117
|
+
".html",
|
|
118
|
+
".htm",
|
|
119
|
+
".md",
|
|
120
|
+
".markdown",
|
|
121
|
+
".adoc",
|
|
122
|
+
".asciidoc",
|
|
123
|
+
# Images OCR (Lot 2)
|
|
124
|
+
".png",
|
|
125
|
+
".jpg",
|
|
126
|
+
".jpeg",
|
|
127
|
+
".tiff",
|
|
128
|
+
".tif",
|
|
129
|
+
".bmp",
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
_ALLOWED_EXTENSIONS_STR: str = ", ".join(sorted(ALLOWED_EXTENSIONS))
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def validate_upload_extension(ext: str) -> None:
|
|
136
|
+
"""
|
|
137
|
+
Valide qu'une extension de fichier est supportée pour l'upload.
|
|
138
|
+
Lève ValueError avec message clair si non supportée.
|
|
139
|
+
À appeler AVANT création de la knowledge pour éviter un FAILED tardif.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
ext: Extension avec le point, ex: '.pdf', '.xyz'
|
|
143
|
+
|
|
144
|
+
Raises:
|
|
145
|
+
ValueError: Si l'extension n'est pas dans ALLOWED_EXTENSIONS.
|
|
146
|
+
"""
|
|
147
|
+
if ext.lower() not in ALLOWED_EXTENSIONS:
|
|
148
|
+
raise ValueError(
|
|
149
|
+
f"Format {ext} non supporté. "
|
|
150
|
+
f"Formats acceptés : {_ALLOWED_EXTENSIONS_STR}"
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
async def create_knowledge_with_multiple_files(
|
|
155
|
+
name: str,
|
|
156
|
+
description: str,
|
|
157
|
+
prompt: str,
|
|
158
|
+
files_data: list[dict],
|
|
159
|
+
user_id: int,
|
|
160
|
+
db: AsyncSession,
|
|
161
|
+
callback_url: str | None = None,
|
|
162
|
+
) -> schemas.KnowledgeWithPaths:
|
|
163
|
+
"""
|
|
164
|
+
Create knowledge with one or more files.
|
|
165
|
+
|
|
166
|
+
This function handles both single and multiple file uploads uniformly.
|
|
167
|
+
Files are uploaded to S3 and stored in the database.
|
|
168
|
+
|
|
169
|
+
Args:
|
|
170
|
+
name: Knowledge name
|
|
171
|
+
description: Knowledge description
|
|
172
|
+
prompt: Chatbot prompt
|
|
173
|
+
files_data: List of file dictionaries with 'filename' and 'bytes'
|
|
174
|
+
user_id: User ID
|
|
175
|
+
db: Database session
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
KnowledgeWithPaths object with all file information
|
|
179
|
+
"""
|
|
180
|
+
if not files_data:
|
|
181
|
+
raise ValueError("At least one file is required")
|
|
182
|
+
|
|
183
|
+
# Validate that no files are empty (zero bytes)
|
|
184
|
+
for file_data in files_data:
|
|
185
|
+
file_size = len(file_data["bytes"])
|
|
186
|
+
if file_size == 0:
|
|
187
|
+
raise ValueError(
|
|
188
|
+
f"File '{file_data['filename']}' is empty (0 bytes) and cannot be uploaded"
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
# Create knowledge record first to get the ID
|
|
192
|
+
new_knowledge = Knowledge(
|
|
193
|
+
name=name,
|
|
194
|
+
description=description,
|
|
195
|
+
knowledge_path="",
|
|
196
|
+
knowledge_paths=[],
|
|
197
|
+
filenames=[file_data["filename"] for file_data in files_data],
|
|
198
|
+
prompt=prompt,
|
|
199
|
+
user_id=user_id,
|
|
200
|
+
status=Status.PENDING,
|
|
201
|
+
callback_url=callback_url,
|
|
202
|
+
)
|
|
203
|
+
db.add(new_knowledge)
|
|
204
|
+
await db.commit()
|
|
205
|
+
await db.refresh(new_knowledge)
|
|
206
|
+
|
|
207
|
+
# Upload all files to S3 and collect their URLs
|
|
208
|
+
s3_urls = []
|
|
209
|
+
filenames = []
|
|
210
|
+
logger.info("[Indexing] Started Upload files to storage: s3")
|
|
211
|
+
|
|
212
|
+
try:
|
|
213
|
+
for file_data in files_data:
|
|
214
|
+
s3_url = upload_file_to_s3(
|
|
215
|
+
file_data["bytes"], file_data["filename"], new_knowledge.knowledge_id
|
|
216
|
+
)
|
|
217
|
+
s3_urls.append(s3_url)
|
|
218
|
+
filenames.append(file_data["filename"])
|
|
219
|
+
logger.info(f"Uploaded '{file_data['filename']}' to S3: {s3_url}")
|
|
220
|
+
logger.info("[Indexing] finished Upload files to storage: s3")
|
|
221
|
+
|
|
222
|
+
# Update knowledge with S3 URLs
|
|
223
|
+
new_knowledge.knowledge_path = s3_urls[0]
|
|
224
|
+
new_knowledge.knowledge_paths = s3_urls
|
|
225
|
+
await db.commit()
|
|
226
|
+
await db.refresh(new_knowledge)
|
|
227
|
+
|
|
228
|
+
except Exception as e:
|
|
229
|
+
# If upload fails, delete the knowledge record
|
|
230
|
+
logger.error(
|
|
231
|
+
f"Failed to upload files for knowledge {new_knowledge.knowledge_id}: {e}"
|
|
232
|
+
)
|
|
233
|
+
await db.delete(new_knowledge)
|
|
234
|
+
await db.commit()
|
|
235
|
+
raise
|
|
236
|
+
|
|
237
|
+
# Return enhanced schema with paths
|
|
238
|
+
logger.info(
|
|
239
|
+
f"Created knowledge {new_knowledge.knowledge_id} with {len(s3_urls)} file(s)"
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
return schemas.KnowledgeWithPaths(
|
|
243
|
+
**schemas.Knowledge.model_validate(new_knowledge).model_dump(),
|
|
244
|
+
file_paths=s3_urls,
|
|
245
|
+
file_names=filenames,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
async def update_knowledge_status(
|
|
250
|
+
knowledge_id: int,
|
|
251
|
+
new_status: Status,
|
|
252
|
+
db: AsyncSession,
|
|
253
|
+
error_message: str | None = None,
|
|
254
|
+
) -> None:
|
|
255
|
+
"""Update the processing status of a knowledge document.
|
|
256
|
+
|
|
257
|
+
Args:
|
|
258
|
+
knowledge_id: ID of the knowledge document.
|
|
259
|
+
new_status: Target status.
|
|
260
|
+
db: Database session.
|
|
261
|
+
error_message: Optional reason for FAILED status or partial-success summary.
|
|
262
|
+
If provided, it is written to the error_message column (truncated to 2000 chars).
|
|
263
|
+
If None, the existing value in error_message is left untouched.
|
|
264
|
+
"""
|
|
265
|
+
stmt = select(Knowledge).where(Knowledge.knowledge_id == knowledge_id)
|
|
266
|
+
result = await db.execute(stmt)
|
|
267
|
+
knowledge = result.scalar_one_or_none()
|
|
268
|
+
|
|
269
|
+
if not knowledge:
|
|
270
|
+
raise exceptions.KnowledgeNotFoundException("Knowledge not found")
|
|
271
|
+
|
|
272
|
+
knowledge.status = new_status
|
|
273
|
+
if error_message is not None:
|
|
274
|
+
knowledge.error_message = error_message[:2000]
|
|
275
|
+
await db.commit()
|
|
276
|
+
logger.info(f"Updated knowledge {knowledge_id} status to {new_status.value}")
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
async def check_and_fail_if_timeout(knowledge_id: int, db: AsyncSession) -> bool:
|
|
280
|
+
"""
|
|
281
|
+
Check if a knowledge document has exceeded the timeout threshold.
|
|
282
|
+
If it has and is still PENDING, update it to FAILED.
|
|
283
|
+
|
|
284
|
+
Args:
|
|
285
|
+
knowledge_id: ID of the knowledge document
|
|
286
|
+
db: Database session
|
|
287
|
+
|
|
288
|
+
Returns:
|
|
289
|
+
True if the document was auto-failed, False otherwise
|
|
290
|
+
"""
|
|
291
|
+
stmt = select(Knowledge).where(Knowledge.knowledge_id == knowledge_id)
|
|
292
|
+
result = await db.execute(stmt)
|
|
293
|
+
knowledge = result.scalar_one_or_none()
|
|
294
|
+
|
|
295
|
+
if not knowledge:
|
|
296
|
+
raise exceptions.KnowledgeNotFoundException("Knowledge not found")
|
|
297
|
+
|
|
298
|
+
# Only check if status is PENDING
|
|
299
|
+
if knowledge.status != Status.PENDING:
|
|
300
|
+
return False
|
|
301
|
+
|
|
302
|
+
# Check if created_at exists and calculate age
|
|
303
|
+
if not knowledge.created_at:
|
|
304
|
+
logger.warning(f"Knowledge {knowledge_id} has no created_at timestamp")
|
|
305
|
+
return False
|
|
306
|
+
|
|
307
|
+
# Calculate age in minutes
|
|
308
|
+
age = datetime.now(timezone.utc) - knowledge.created_at
|
|
309
|
+
age_minutes = age.total_seconds() / 60
|
|
310
|
+
|
|
311
|
+
timeout_threshold = settings.knowledge_pending_timeout_minutes
|
|
312
|
+
|
|
313
|
+
# If exceeded timeout, mark as FAILED
|
|
314
|
+
if age_minutes > timeout_threshold:
|
|
315
|
+
logger.warning(
|
|
316
|
+
f"Auto-failing knowledge {knowledge_id} - "
|
|
317
|
+
f"exceeded timeout ({age_minutes:.1f} > {timeout_threshold} minutes)"
|
|
318
|
+
)
|
|
319
|
+
knowledge.status = Status.FAILED
|
|
320
|
+
await db.commit()
|
|
321
|
+
return True
|
|
322
|
+
|
|
323
|
+
return False
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
async def delete_knowledge(knowledge_id: int, db: AsyncSession) -> None:
|
|
327
|
+
"""
|
|
328
|
+
Delete a knowledge document and all associated resources.
|
|
329
|
+
|
|
330
|
+
This includes:
|
|
331
|
+
- All S3 files (primary and additional)
|
|
332
|
+
- All vector database chunks
|
|
333
|
+
- The database record
|
|
334
|
+
"""
|
|
335
|
+
stmt = select(Knowledge).where(Knowledge.knowledge_id == knowledge_id)
|
|
336
|
+
result = await db.execute(stmt)
|
|
337
|
+
knowledge = result.scalar_one_or_none()
|
|
338
|
+
|
|
339
|
+
if not knowledge:
|
|
340
|
+
raise exceptions.KnowledgeNotFoundException("Knowledge not found")
|
|
341
|
+
|
|
342
|
+
# Delete all associated files from S3
|
|
343
|
+
deleted_files = 0
|
|
344
|
+
failed_deletions = []
|
|
345
|
+
|
|
346
|
+
try:
|
|
347
|
+
# Get all file paths (primary + additional)
|
|
348
|
+
all_paths = set()
|
|
349
|
+
all_paths.add(knowledge.knowledge_path)
|
|
350
|
+
|
|
351
|
+
# JSONB fields are already lists, no need to parse JSON
|
|
352
|
+
if hasattr(knowledge, "knowledge_paths") and knowledge.knowledge_paths:
|
|
353
|
+
if isinstance(knowledge.knowledge_paths, list):
|
|
354
|
+
all_paths.update(knowledge.knowledge_paths)
|
|
355
|
+
else:
|
|
356
|
+
logger.warning(
|
|
357
|
+
f"knowledge_paths is not a list: {type(knowledge.knowledge_paths)}"
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
# Delete each file from S3
|
|
361
|
+
for path in all_paths:
|
|
362
|
+
try:
|
|
363
|
+
delete_file_from_s3(path)
|
|
364
|
+
deleted_files += 1
|
|
365
|
+
logger.info(f"Deleted S3 file: {path}")
|
|
366
|
+
except Exception as e:
|
|
367
|
+
failed_deletions.append(path)
|
|
368
|
+
logger.warning(f"Could not delete S3 object {path}: {e}")
|
|
369
|
+
|
|
370
|
+
logger.info(
|
|
371
|
+
f"Deleted {deleted_files} file(s) from S3 for knowledge {knowledge_id}"
|
|
372
|
+
)
|
|
373
|
+
if failed_deletions:
|
|
374
|
+
logger.warning(
|
|
375
|
+
f"Failed to delete {len(failed_deletions)} file(s): {failed_deletions}"
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
except Exception as e:
|
|
379
|
+
logger.error(f"Error during S3 cleanup for knowledge {knowledge_id}: {e}")
|
|
380
|
+
|
|
381
|
+
# Delete from database
|
|
382
|
+
await db.delete(knowledge)
|
|
383
|
+
await db.commit()
|
|
384
|
+
logger.info(f"Deleted knowledge {knowledge_id} from database")
|
|
385
|
+
|
|
386
|
+
# Delete from vector database
|
|
387
|
+
try:
|
|
388
|
+
lancedb_storage = LanceDBStorage()
|
|
389
|
+
lancedb_storage.delete_chunks(doc_id=knowledge_id)
|
|
390
|
+
logger.info(f"Deleted chunks for knowledge {knowledge_id} from vector database")
|
|
391
|
+
except Exception as e:
|
|
392
|
+
logger.error(f"Error deleting chunks from vector database: {e}")
|
th2rag/logger.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
# create a logger instance
|
|
5
|
+
logger = logging.getLogger()
|
|
6
|
+
|
|
7
|
+
# create a formatter
|
|
8
|
+
formatter = logging.Formatter(fmt="%(asctime)s - %(levelname)s - %(message)s")
|
|
9
|
+
|
|
10
|
+
# create handlers
|
|
11
|
+
stream_handler = logging.StreamHandler(sys.stdout)
|
|
12
|
+
file_handler = logging.FileHandler("app.log")
|
|
13
|
+
|
|
14
|
+
# set formatter for the handlers
|
|
15
|
+
stream_handler.setFormatter(formatter)
|
|
16
|
+
file_handler.setFormatter(formatter)
|
|
17
|
+
|
|
18
|
+
# add handlers to the logger
|
|
19
|
+
logger.handlers = [stream_handler, file_handler]
|
|
20
|
+
|
|
21
|
+
logger.setLevel(logging.INFO)
|
th2rag/main.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from logging import getLogger
|
|
3
|
+
|
|
4
|
+
from fastapi import FastAPI
|
|
5
|
+
from fastapi.responses import FileResponse
|
|
6
|
+
from fastapi.staticfiles import StaticFiles
|
|
7
|
+
|
|
8
|
+
from th2rag.auth.router import router as auth_router
|
|
9
|
+
from th2rag.config import settings
|
|
10
|
+
from th2rag.conversations.router import router as conversations_router
|
|
11
|
+
from th2rag.dataset.router import router as datasets_router
|
|
12
|
+
from th2rag.knowledge.router import router as knowledge_router
|
|
13
|
+
from th2rag.text_to_sql.router import router as text_to_sql_router
|
|
14
|
+
from th2rag.users.router import router as users_router
|
|
15
|
+
from th2rag.utils.welcome_router import router as welcome_router
|
|
16
|
+
|
|
17
|
+
print(f"BYPASS_AUTH setting: {settings.bypass_auth}")
|
|
18
|
+
|
|
19
|
+
logger = getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
os.getenv("SHINYPROXY_PUBLIC_PATH")
|
|
23
|
+
route_path = settings.root_path
|
|
24
|
+
route_path = None
|
|
25
|
+
|
|
26
|
+
if os.getenv("SHINYPROXY_PUBLIC_PATH"):
|
|
27
|
+
route_path = "/app_direct/th2rag_api/"
|
|
28
|
+
|
|
29
|
+
logger.info("Route Path is {route_path}")
|
|
30
|
+
|
|
31
|
+
app = FastAPI(root_path = route_path)
|
|
32
|
+
# Mount static files directory (optional, for other static assets)
|
|
33
|
+
app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
34
|
+
|
|
35
|
+
# Add favicon endpoint
|
|
36
|
+
@app.get('/favicon.ico', include_in_schema=False)
|
|
37
|
+
async def favicon():
|
|
38
|
+
return FileResponse('static/favicon.ico')
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
app.include_router(auth_router)
|
|
42
|
+
app.include_router(welcome_router)
|
|
43
|
+
app.include_router(users_router)
|
|
44
|
+
app.include_router(conversations_router)
|
|
45
|
+
app.include_router(knowledge_router)
|
|
46
|
+
app.include_router(auth_router)
|
|
47
|
+
app.include_router(datasets_router)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
app.include_router(text_to_sql_router)
|
|
File without changes
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
from typing import Annotated
|
|
2
|
+
|
|
3
|
+
from fastapi import APIRouter, Depends, HTTPException, Path, status
|
|
4
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
5
|
+
|
|
6
|
+
from th2rag.conversations.dependencies import require_conversation_owner_or_admin
|
|
7
|
+
from th2rag.conversations.schemas import Conversation
|
|
8
|
+
from th2rag.database import get_db
|
|
9
|
+
from th2rag.dependencies import get_rag_service
|
|
10
|
+
from th2rag.messages import exceptions, schemas, service
|
|
11
|
+
from th2rag.pagination import PageParams, PageResponse
|
|
12
|
+
from th2rag.rag.services.rag_service import RAGService
|
|
13
|
+
|
|
14
|
+
router = APIRouter(tags=["messages"])
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@router.get("", response_model=PageResponse[schemas.Message])
|
|
18
|
+
async def get_all_messages(
|
|
19
|
+
conversation: Conversation = Depends(require_conversation_owner_or_admin),
|
|
20
|
+
page_params: PageParams = Depends(),
|
|
21
|
+
db: AsyncSession = Depends(get_db),
|
|
22
|
+
):
|
|
23
|
+
return await service.get_all_messages(conversation.conversation_id, page_params, db)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@router.get("/{message_id}", response_model=schemas.Message)
|
|
27
|
+
async def get_message_by_id(
|
|
28
|
+
message_id: Annotated[int, Path(description="The ID of the message")],
|
|
29
|
+
conversation: Conversation = Depends(require_conversation_owner_or_admin),
|
|
30
|
+
db: AsyncSession = Depends(get_db),
|
|
31
|
+
):
|
|
32
|
+
try:
|
|
33
|
+
return await service.get_message_by_id(conversation.conversation_id, message_id, db)
|
|
34
|
+
except exceptions.MessageNotFoundException as e:
|
|
35
|
+
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@router.post("", response_model=schemas.Message)
|
|
39
|
+
async def create_message(
|
|
40
|
+
message_in: schemas.MessageCreate,
|
|
41
|
+
conversation: Conversation = Depends(require_conversation_owner_or_admin),
|
|
42
|
+
db: AsyncSession = Depends(get_db),
|
|
43
|
+
rag_service: RAGService = Depends(get_rag_service),
|
|
44
|
+
):
|
|
45
|
+
# Don't inject rag_service as dependency - let service handle it conditionally
|
|
46
|
+
return await service.create_and_respond_message(
|
|
47
|
+
conversation.conversation_id, message_in=message_in, db=db,rag_service=rag_service
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@router.post("/{message_id}/feedback", response_model=schemas.Message)
|
|
52
|
+
async def add_feedback_to_message(
|
|
53
|
+
feedback_in: schemas.FeedbackCreate,
|
|
54
|
+
message_id: Annotated[int, Path(description="The ID of the message")],
|
|
55
|
+
conversation: Conversation = Depends(require_conversation_owner_or_admin),
|
|
56
|
+
db: AsyncSession = Depends(get_db),
|
|
57
|
+
):
|
|
58
|
+
try:
|
|
59
|
+
return await service.add_feedback_to_message(
|
|
60
|
+
conversation.conversation_id, message_id, feedback_in, db
|
|
61
|
+
)
|
|
62
|
+
except exceptions.MessageNotFoundException as e:
|
|
63
|
+
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|