th2rag 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. th2rag/__init__.py +0 -0
  2. th2rag/auth/__init__.py +0 -0
  3. th2rag/auth/exceptions.py +2 -0
  4. th2rag/auth/router.py +43 -0
  5. th2rag/auth/schemas.py +15 -0
  6. th2rag/auth/service.py +115 -0
  7. th2rag/clients/__init__.py +0 -0
  8. th2rag/clients/storage/config.py +13 -0
  9. th2rag/clients/storage/local.py +48 -0
  10. th2rag/clients/storage/s3.py +93 -0
  11. th2rag/config.py +267 -0
  12. th2rag/conversations/__init__.py +0 -0
  13. th2rag/conversations/dependencies.py +30 -0
  14. th2rag/conversations/exceptions.py +6 -0
  15. th2rag/conversations/router.py +47 -0
  16. th2rag/conversations/schemas.py +43 -0
  17. th2rag/conversations/service.py +51 -0
  18. th2rag/database.py +72 -0
  19. th2rag/dataset/__init__.py +0 -0
  20. th2rag/dataset/dependencies.py +25 -0
  21. th2rag/dataset/exceptions.py +2 -0
  22. th2rag/dataset/router.py +53 -0
  23. th2rag/dataset/schemas.py +23 -0
  24. th2rag/dataset/service.py +78 -0
  25. th2rag/dependencies.py +143 -0
  26. th2rag/knowledge/__init__.py +0 -0
  27. th2rag/knowledge/dependencies.py +25 -0
  28. th2rag/knowledge/exceptions.py +2 -0
  29. th2rag/knowledge/mage_client.py +505 -0
  30. th2rag/knowledge/router.py +714 -0
  31. th2rag/knowledge/schemas.py +108 -0
  32. th2rag/knowledge/service.py +392 -0
  33. th2rag/logger.py +21 -0
  34. th2rag/main.py +51 -0
  35. th2rag/messages/__init__.py +0 -0
  36. th2rag/messages/exceptions.py +2 -0
  37. th2rag/messages/router.py +63 -0
  38. th2rag/messages/schemas.py +68 -0
  39. th2rag/messages/service.py +210 -0
  40. th2rag/models.py +215 -0
  41. th2rag/pagination.py +68 -0
  42. th2rag/rag/__init__.py +0 -0
  43. th2rag/rag/chunkers/__init__.py +0 -0
  44. th2rag/rag/chunkers/hybrid_chunker.py +23 -0
  45. th2rag/rag/constants.py +15 -0
  46. th2rag/rag/converters/__init__.py +0 -0
  47. th2rag/rag/converters/documents_converter.py +123 -0
  48. th2rag/rag/converters/json_converter.py +521 -0
  49. th2rag/rag/converters/legacy_converter.py +124 -0
  50. th2rag/rag/converters/pdf_converter.py +384 -0
  51. th2rag/rag/converters/spreadsheet_converter.py +159 -0
  52. th2rag/rag/converters/txt_converter.py +98 -0
  53. th2rag/rag/embeddings/__init__.py +0 -0
  54. th2rag/rag/embeddings/base.py +21 -0
  55. th2rag/rag/embeddings/exceptions.py +2 -0
  56. th2rag/rag/embeddings/sentence_transformer.py +97 -0
  57. th2rag/rag/generation/base.py +22 -0
  58. th2rag/rag/generation/exceptions.py +2 -0
  59. th2rag/rag/generation/litellm_generator.py +143 -0
  60. th2rag/rag/generation/mistral_generator.py +269 -0
  61. th2rag/rag/generation/test_litellm.py +33 -0
  62. th2rag/rag/retrieval/__init__.py +0 -0
  63. th2rag/rag/retrieval/base.py +11 -0
  64. th2rag/rag/retrieval/exceptions.py +10 -0
  65. th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
  66. th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
  67. th2rag/rag/retrieval/lancedb/schemas.py +19 -0
  68. th2rag/rag/services/__init__.py +0 -0
  69. th2rag/rag/services/rag_service.py +99 -0
  70. th2rag/rag/services/rag_service_builder.py +42 -0
  71. th2rag/rag/tasks/process_pdf.py +586 -0
  72. th2rag/users/dependencies.py +44 -0
  73. th2rag/users/exceptions.py +6 -0
  74. th2rag/users/router.py +92 -0
  75. th2rag/users/schemas.py +61 -0
  76. th2rag/users/service.py +140 -0
  77. th2rag/utils/__init__.py +0 -0
  78. th2rag/utils/helpers.py +40 -0
  79. th2rag/utils/logger.py +15 -0
  80. th2rag/utils/security.py +76 -0
  81. th2rag/utils/welcome_router.py +98 -0
  82. th2rag/webhooks/__init__.py +0 -0
  83. th2rag/webhooks/sender.py +126 -0
  84. th2rag-0.0.2.dist-info/METADATA +1002 -0
  85. th2rag-0.0.2.dist-info/RECORD +87 -0
  86. th2rag-0.0.2.dist-info/WHEEL +4 -0
  87. th2rag-0.0.2.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,714 @@
1
+ import asyncio
2
+ import ipaddress
3
+ import os
4
+ from urllib.parse import urlparse
5
+
6
+ from fastapi import (
7
+ APIRouter,
8
+ BackgroundTasks,
9
+ Depends,
10
+ File,
11
+ Form,
12
+ HTTPException,
13
+ UploadFile,
14
+ status,
15
+ )
16
+ from fastapi.responses import HTMLResponse
17
+ from sqlalchemy.ext.asyncio import AsyncSession
18
+
19
+ from th2rag.config import settings
20
+ from th2rag.database import get_db
21
+ from th2rag.dependencies import get_current_user
22
+ from th2rag.knowledge import dependencies, schemas, service
23
+ from th2rag.knowledge.mage_client import process_knowledge_upload_batch
24
+ from th2rag.pagination import PageParams, PageResponse
25
+ from th2rag.rag.tasks.process_pdf import process_pdfs_background, process_pdfs_sync
26
+ from th2rag.users import schemas as user_schemas
27
+ from th2rag.utils.logger import setup_logger
28
+
29
+ router = APIRouter(prefix="/knowledge", tags=["knowledge"])
30
+ logger = setup_logger(__name__)
31
+
32
+
33
+ def _validate_callback_url(url: str | None) -> None:
34
+ """Validate callback_url to prevent SSRF attacks."""
35
+ if url is None:
36
+ return
37
+ parsed = urlparse(url)
38
+ if parsed.scheme not in ("http", "https"):
39
+ raise HTTPException(status_code=400, detail="callback_url must use http or https")
40
+ hostname = parsed.hostname or ""
41
+ if hostname in ("localhost", "127.0.0.1", "0.0.0.0", "::1"):
42
+ raise HTTPException(status_code=400, detail="callback_url cannot point to localhost")
43
+ try:
44
+ ip = ipaddress.ip_address(hostname)
45
+ if ip.is_private or ip.is_loopback or ip.is_link_local:
46
+ raise HTTPException(status_code=400, detail="callback_url cannot point to a private network")
47
+ except ValueError:
48
+ pass # hostname is a DNS name, not an IP — OK
49
+
50
+
51
+ @router.get("/", response_model=PageResponse[schemas.Knowledge])
52
+ async def get_all_knowledges(
53
+ page_params: PageParams = Depends(),
54
+ db: AsyncSession = Depends(get_db),
55
+ current_user: user_schemas.User = Depends(get_current_user),
56
+ ):
57
+ """
58
+ Get all knowledge documents for the current user.
59
+ Admin users can see all documents.
60
+ """
61
+ return await service.get_all_knowledges(page_params, current_user, db)
62
+
63
+
64
+ @router.get("/upload-ui", response_class=HTMLResponse)
65
+ async def upload_ui():
66
+ """
67
+ Serve a custom HTML interface for multi-file upload with drag & drop.
68
+ This provides a better user experience than Swagger UI for selecting multiple files.
69
+
70
+ Note: Swagger UI has limitations rendering multiple file inputs properly.
71
+ Use this UI or test via cURL/Postman for better multi-file selection.
72
+
73
+ This endpoint does not require authentication to serve the HTML page,
74
+ but the upload endpoint itself requires authentication.
75
+ """
76
+ return """
77
+ <!DOCTYPE html>
78
+ <html lang="en">
79
+ <head>
80
+ <meta charset="UTF-8">
81
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
82
+ <title>Upload Knowledge - Multi-File Selector</title>
83
+ <style>
84
+ * { margin: 0; padding: 0; box-sizing: border-box; }
85
+ body {
86
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
87
+ background: linear-gradient(135deg, #013DFF 0%, #764ba2 100%);
88
+ min-height: 100vh;
89
+ display: flex;
90
+ align-items: center;
91
+ justify-content: center;
92
+ padding: 20px;
93
+ }
94
+ .container {
95
+ background: white;
96
+ border-radius: 16px;
97
+ box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
98
+ max-width: 800px;
99
+ width: 100%;
100
+ padding: 40px;
101
+ }
102
+ h1 { color: #333; margin-bottom: 10px; font-size: 28px; }
103
+ .subtitle { color: #666; margin-bottom: 30px; font-size: 14px; }
104
+ .form-group { margin-bottom: 24px; }
105
+ label {
106
+ display: block;
107
+ color: #333;
108
+ font-weight: 600;
109
+ margin-bottom: 8px;
110
+ font-size: 14px;
111
+ }
112
+ input[type="text"], textarea {
113
+ width: 100%;
114
+ padding: 12px;
115
+ border: 2px solid #e0e0e0;
116
+ border-radius: 8px;
117
+ font-size: 14px;
118
+ transition: border-color 0.3s;
119
+ }
120
+ input[type="text"]:focus, textarea:focus {
121
+ outline: none;
122
+ border-color: #013DFF
123
+ ;
124
+ }
125
+ textarea { resize: vertical; min-height: 80px; }
126
+ .file-drop-area {
127
+ border: 3px dashed #013DFF
128
+ ;
129
+ border-radius: 12px;
130
+ padding: 40px;
131
+ text-align: center;
132
+ cursor: pointer;
133
+ transition: all 0.3s;
134
+ background: #f8f9ff;
135
+ }
136
+ .file-drop-area:hover {
137
+ border-color: #764ba2;
138
+ background: #f0f2ff;
139
+ }
140
+ .file-drop-area.dragover {
141
+ background: #e8ebff;
142
+ border-color: #764ba2;
143
+ transform: scale(1.02);
144
+ }
145
+ .file-icon { font-size: 48px; margin-bottom: 16px; }
146
+ .file-label {
147
+ font-size: 16px;
148
+ color: #013DFF
149
+ ;
150
+ font-weight: 600;
151
+ margin-bottom: 8px;
152
+ }
153
+ .file-hint { font-size: 13px; color: #999; }
154
+ #fileInput { display: none; }
155
+ .file-list { margin-top: 20px; }
156
+ .file-item {
157
+ display: flex;
158
+ align-items: center;
159
+ justify-content: space-between;
160
+ padding: 12px;
161
+ background: #f8f9fa;
162
+ border-radius: 8px;
163
+ margin-bottom: 8px;
164
+ }
165
+ .file-info { display: flex; align-items: center; gap: 12px; }
166
+ .file-name { font-weight: 500; color: #333; }
167
+ .file-size { color: #999; font-size: 13px; }
168
+ .remove-btn {
169
+ background: #ff4444;
170
+ color: white;
171
+ border: none;
172
+ padding: 6px 12px;
173
+ border-radius: 6px;
174
+ cursor: pointer;
175
+ font-size: 13px;
176
+ transition: background 0.3s;
177
+ }
178
+ .remove-btn:hover { background: #cc0000; }
179
+ .submit-btn {
180
+ width: 100%;
181
+ padding: 16px;
182
+ background: linear-gradient(135deg, #013DFF
183
+ 0%, #764ba2 100%);
184
+ color: white;
185
+ border: none;
186
+ border-radius: 8px;
187
+ font-size: 16px;
188
+ font-weight: 600;
189
+ cursor: pointer;
190
+ transition: transform 0.2s;
191
+ margin-top: 24px;
192
+ }
193
+ .submit-btn:hover:not(:disabled) { transform: translateY(-2px); }
194
+ .submit-btn:disabled { opacity: 0.5; cursor: not-allowed; }
195
+ .status-message {
196
+ margin-top: 20px;
197
+ padding: 16px;
198
+ border-radius: 8px;
199
+ display: none;
200
+ }
201
+ .status-message.success {
202
+ background: #d4edda;
203
+ color: #155724;
204
+ border: 1px solid #c3e6cb;
205
+ }
206
+ .status-message.error {
207
+ background: #f8d7da;
208
+ color: #721c24;
209
+ border: 1px solid #f5c6cb;
210
+ }
211
+ .status-message.info {
212
+ background: #d1ecf1;
213
+ color: #0c5460;
214
+ border: 1px solid #bee5eb;
215
+ }
216
+ .progress-bar {
217
+ width: 100%;
218
+ height: 4px;
219
+ background: #e0e0e0;
220
+ border-radius: 2px;
221
+ overflow: hidden;
222
+ margin-top: 16px;
223
+ display: none;
224
+ }
225
+ .progress-fill {
226
+ height: 100%;
227
+ background: linear-gradient(90deg, #013DFF
228
+ 0%, #764ba2 100%);
229
+ width: 0%;
230
+ transition: width 0.3s;
231
+ }
232
+ </style>
233
+ </head>
234
+ <body>
235
+ <div class="container">
236
+ <h1>📚 Create Knowledge Base</h1>
237
+ <p class="subtitle">Upload multiple documents (PDF, DOCX, PPTX, CSV, XLSX) at once...</p>
238
+
239
+ <form id="uploadForm">
240
+ <div class="form-group">
241
+ <label for="name">Knowledge Name *</label>
242
+ <input type="text" id="name" name="name" required placeholder="e.g., Company Policies">
243
+ </div>
244
+
245
+ <div class="form-group">
246
+ <label for="description">Description *</label>
247
+ <textarea id="description" name="description" required placeholder="Describe what this knowledge base contains..."></textarea>
248
+ </div>
249
+
250
+ <div class="form-group">
251
+ <label for="prompt">ChatBot Prompt *</label>
252
+ <textarea id="prompt" name="prompt" required placeholder="Enter the system prompt for the chatbot..."></textarea>
253
+ </div>
254
+
255
+ <div class="form-group">
256
+ <label>Supported formats: PDF, DOCX, PPTX, DOC, PPT, XLS, CSV, XLSX, JSON, TXT, HTML, MD, ADOC, PNG, JPG, TIFF, BMP (select multiple)</label>
257
+ <div class="file-drop-area" id="dropArea">
258
+ <div class="file-icon">📄</div>
259
+ <div class="file-label">Click to select files or drag & drop</div>
260
+ <div class="file-hint">Select multiple PDFs or JSON files using Ctrl+Click (Cmd+Click on Mac)</div>
261
+ </div>
262
+ <input type="file" id="fileInput" multiple accept=".pdf,.docx,.pptx,.doc,.ppt,.xls,.csv,.xlsx,.json,.txt,.html,.htm,.md,.markdown,.adoc,.asciidoc,.png,.jpg,.jpeg,.tiff,.tif,.bmp">
263
+ <div class="file-list" id="fileList"></div>
264
+ </div>
265
+
266
+ <button type="submit" class="submit-btn" id="submitBtn">
267
+ Upload & Create Knowledge Base
268
+ </button>
269
+
270
+ <div class="progress-bar" id="progressBar">
271
+ <div class="progress-fill" id="progressFill"></div>
272
+ </div>
273
+
274
+ <div class="status-message" id="statusMessage"></div>
275
+ </form>
276
+ </div>
277
+
278
+ <script>
279
+ const dropArea = document.getElementById('dropArea');
280
+ const fileInput = document.getElementById('fileInput');
281
+ const fileList = document.getElementById('fileList');
282
+ const form = document.getElementById('uploadForm');
283
+ const submitBtn = document.getElementById('submitBtn');
284
+ const statusMessage = document.getElementById('statusMessage');
285
+ const progressBar = document.getElementById('progressBar');
286
+ const progressFill = document.getElementById('progressFill');
287
+ let selectedFiles = [];
288
+
289
+ dropArea.addEventListener('click', () => fileInput.click());
290
+ fileInput.addEventListener('change', (e) => handleFiles(e.target.files));
291
+
292
+ ['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => {
293
+ dropArea.addEventListener(eventName, preventDefaults, false);
294
+ });
295
+
296
+ function preventDefaults(e) { e.preventDefault(); e.stopPropagation(); }
297
+
298
+ ['dragenter', 'dragover'].forEach(eventName => {
299
+ dropArea.addEventListener(eventName, () => dropArea.classList.add('dragover'));
300
+ });
301
+
302
+ ['dragleave', 'drop'].forEach(eventName => {
303
+ dropArea.addEventListener(eventName, () => dropArea.classList.remove('dragover'));
304
+ });
305
+
306
+ dropArea.addEventListener('drop', (e) => {
307
+ const files = e.dataTransfer.files;
308
+ handleFiles(files);
309
+ });
310
+
311
+ const ALLOWED_EXTS = new Set([
312
+ '.pdf', '.docx', '.pptx', '.doc', '.ppt', '.xls',
313
+ '.csv', '.xlsx', '.json', '.txt',
314
+ '.html', '.htm', '.md', '.markdown', '.adoc', '.asciidoc',
315
+ '.png', '.jpg', '.jpeg', '.tiff', '.tif', '.bmp'
316
+ ]);
317
+
318
+ function handleFiles(files) {
319
+ const validFiles = Array.from(files).filter(file => {
320
+ const ext = '.' + file.name.toLowerCase().split('.').pop();
321
+ return ALLOWED_EXTS.has(ext);
322
+ });
323
+ if (validFiles.length === 0) {
324
+ const exts = Array.from(ALLOWED_EXTS).join(', ');
325
+ showStatus('Please select a supported file. Supported formats: ' + exts, 'error');
326
+ return;
327
+ }
328
+ selectedFiles = [...selectedFiles, ...validFiles];
329
+ displayFiles();
330
+ hideStatus();
331
+ }
332
+
333
+ function displayFiles() {
334
+ fileList.innerHTML = '';
335
+ selectedFiles.forEach((file, index) => {
336
+ const fileItem = document.createElement('div');
337
+ fileItem.className = 'file-item';
338
+ const sizeInMB = (file.size / (1024 * 1024)).toFixed(2);
339
+ fileItem.innerHTML = `
340
+ <div class="file-info">
341
+ <span class="file-name">${file.name}</span>
342
+ <span class="file-size">(${sizeInMB} MB)</span>
343
+ </div>
344
+ <button type="button" class="remove-btn" onclick="removeFile(${index})">Remove</button>
345
+ `;
346
+ fileList.appendChild(fileItem);
347
+ });
348
+ }
349
+
350
+ function removeFile(index) {
351
+ selectedFiles.splice(index, 1);
352
+ displayFiles();
353
+ }
354
+
355
+ function showStatus(message, type) {
356
+ statusMessage.textContent = message;
357
+ statusMessage.className = `status-message ${type}`;
358
+ statusMessage.style.display = 'block';
359
+ }
360
+
361
+ function hideStatus() { statusMessage.style.display = 'none'; }
362
+
363
+ function showProgress(percent) {
364
+ progressBar.style.display = 'block';
365
+ progressFill.style.width = percent + '%';
366
+ }
367
+
368
+ function hideProgress() {
369
+ progressBar.style.display = 'none';
370
+ progressFill.style.width = '0%';
371
+ }
372
+
373
+ form.addEventListener('submit', async (e) => {
374
+ e.preventDefault();
375
+
376
+ if (selectedFiles.length === 0) {
377
+ showStatus('Please select at least one PDF or JSON file', 'error');
378
+ return;
379
+ }
380
+
381
+ const formData = new FormData();
382
+ formData.append('name', document.getElementById('name').value);
383
+ formData.append('description', document.getElementById('description').value);
384
+ formData.append('prompt', document.getElementById('prompt').value);
385
+ selectedFiles.forEach(file => formData.append('files', file));
386
+
387
+ submitBtn.disabled = true;
388
+ showStatus('Uploading files...', 'info');
389
+ showProgress(30);
390
+
391
+ try {
392
+ const response = await fetch('/knowledge/', {
393
+ method: 'POST',
394
+ body: formData,
395
+ credentials: 'include', // Include cookies for authentication
396
+ });
397
+
398
+ showProgress(100);
399
+
400
+ if (response.ok) {
401
+ const result = await response.json();
402
+ showStatus(
403
+ `✅ Success! Created knowledge base with ID ${result.result.knowledge_id}. ${result.message}`,
404
+ 'success'
405
+ );
406
+ setTimeout(() => {
407
+ form.reset();
408
+ selectedFiles = [];
409
+ displayFiles();
410
+ hideProgress();
411
+ }, 3000);
412
+ } else {
413
+ const error = await response.json();
414
+ showStatus(`❌ Error: ${error.detail || 'Upload failed'}`, 'error');
415
+ hideProgress();
416
+ }
417
+ } catch (error) {
418
+ showStatus(`❌ Network error: ${error.message}`, 'error');
419
+ hideProgress();
420
+ } finally {
421
+ submitBtn.disabled = false;
422
+ }
423
+ });
424
+
425
+ window.removeFile = removeFile;
426
+ </script>
427
+ </body>
428
+ </html>
429
+ """
430
+
431
+
432
+ @router.get("/{knowledge_id}", response_model=schemas.Knowledge)
433
+ async def get_knowledge_by_id(
434
+ knowledge: schemas.Knowledge = Depends(dependencies.require_knowledge_owner_or_admin),
435
+ ):
436
+ """
437
+ Get a specific knowledge document by ID.
438
+ Only the owner or admin can access it.
439
+ """
440
+ return knowledge
441
+
442
+
443
+ @router.post("/", status_code=status.HTTP_201_CREATED)
444
+ async def create_knowledge(
445
+ background_tasks: BackgroundTasks,
446
+ name: str = Form(..., description="The name of the knowledge"),
447
+ description: str = Form(..., description="The description of the knowledge"),
448
+ prompt: str = Form(..., description="The prompt of the ChatBot"),
449
+ files: list[UploadFile] = File(..., description="PDF files to upload (use /knowledge/upload-ui for better multi-select experience)"),
450
+ callback_url: str | None = Form(None, description="Optional webhook URL to notify when indexation completes"),
451
+ db: AsyncSession = Depends(get_db),
452
+ current_user: user_schemas.User = Depends(get_current_user),
453
+ ):
454
+ """
455
+ Upload PDF files to create a knowledge base.
456
+ """
457
+ _validate_callback_url(callback_url)
458
+
459
+ if not files:
460
+ raise HTTPException(
461
+ status_code=status.HTTP_400_BAD_REQUEST,
462
+ detail="At least one file must be provided."
463
+ )
464
+
465
+ # Read and validate all files
466
+ file_data_list = []
467
+ total_size_bytes = 0
468
+
469
+ for file in files:
470
+ ext = os.path.splitext(file.filename or "")[1].lower()
471
+ try:
472
+ service.validate_upload_extension(ext)
473
+ except ValueError as exc:
474
+ raise HTTPException(
475
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
476
+ detail=str(exc),
477
+ ) from exc
478
+ try:
479
+ file_bytes = await file.read()
480
+ file_size_bytes = len(file_bytes)
481
+
482
+ # Reject empty files (zero bytes)
483
+ if file_size_bytes == 0:
484
+ raise HTTPException(
485
+ status_code=status.HTTP_400_BAD_REQUEST,
486
+ detail=f"File '{file.filename}' is empty and cannot be uploaded."
487
+ )
488
+
489
+ total_size_bytes += file_size_bytes
490
+
491
+ file_data_list.append({
492
+ "filename": file.filename,
493
+ "bytes": file_bytes,
494
+ "size_bytes": file_size_bytes,
495
+ "size_mb": file_size_bytes / (1024 * 1024)
496
+ })
497
+ logger.info(f"Read file: {file.filename} ({file_data_list[-1]['size_mb']:.2f} MB)")
498
+
499
+ except HTTPException:
500
+ # Re-raise HTTPException as-is
501
+ raise
502
+ except Exception as e:
503
+ logger.error(f"Error reading file '{file.filename}': {e}")
504
+ raise HTTPException(
505
+ status_code=status.HTTP_400_BAD_REQUEST,
506
+ detail=f"Error reading file '{file.filename}'."
507
+ )
508
+
509
+ total_size_mb = total_size_bytes / (1024 * 1024)
510
+ logger.info(
511
+ f"Total upload size: {total_size_mb:.2f} MB across {len(files)} file(s)"
512
+ )
513
+
514
+ # Check total size against max limit
515
+ if total_size_mb > settings.pdf_max_file_size_mb:
516
+ raise HTTPException(
517
+ status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
518
+ detail=f"Total file size too large. Maximum is {settings.pdf_max_file_size_mb} MB. "
519
+ f"Your total is {total_size_mb:.2f} MB."
520
+ )
521
+
522
+ # Create ONE knowledge record with all files
523
+ try:
524
+ knowledge = await service.create_knowledge_with_multiple_files(
525
+ name=name,
526
+ description=description,
527
+ prompt=prompt,
528
+ files_data=file_data_list,
529
+ user_id=current_user.user_id,
530
+ db=db,
531
+ callback_url=callback_url,
532
+ )
533
+ logger.info(
534
+ f"Created knowledge record: ID={knowledge.knowledge_id} "
535
+ f"with {len(file_data_list)} file(s)"
536
+ )
537
+
538
+ except Exception as e:
539
+ logger.error(f"Failed to create knowledge record: {e}")
540
+ raise HTTPException(
541
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
542
+ detail=f"Failed to create knowledge record: {str(e)}"
543
+ )
544
+
545
+ # Collect all sources for processing
546
+ sources = knowledge.file_paths # Use file_paths directly
547
+ filenames = knowledge.file_names # Use file_names directly
548
+
549
+ # Determine processing mode based on TOTAL size
550
+ processing_mode = settings.get_pdf_processing_mode(total_size_mb)
551
+ processing_mode = "async"
552
+
553
+ logger.info(
554
+ f"Processing mode: {processing_mode} for knowledge {knowledge.knowledge_id} "
555
+ f"with {len(sources)} file(s) "
556
+ f"(total size: {total_size_mb:.2f} MB, threshold: {settings.pdf_auto_async_threshold_mb} MB)"
557
+ )
558
+
559
+ if processing_mode == "sync":
560
+ # Process all files synchronously - user waits for completion
561
+ try:
562
+ logger.info(
563
+ f"Starting synchronous processing for knowledge {knowledge.knowledge_id} "
564
+ f"with {len(sources)} file(s)"
565
+ )
566
+
567
+ # Process all files and wait for results
568
+ result = await process_pdfs_sync(
569
+ doc_id=knowledge.knowledge_id,
570
+ sources=sources,
571
+ db=db,
572
+ callback_url=callback_url,
573
+ )
574
+
575
+ # Build response with file details
576
+ processed_files = []
577
+ for idx, file_result in enumerate(result.get("files", [])):
578
+ processed_files.append({
579
+ "filename": filenames[idx] if idx < len(filenames) else file_result.get("source"),
580
+ "source": file_result.get("source"),
581
+ "chunk_count": file_result.get("chunk_count"),
582
+ "status": file_result.get("status")
583
+ })
584
+
585
+ logger.info(
586
+ f"Synchronous processing completed for knowledge {knowledge.knowledge_id}: "
587
+ f"{result['total_chunks']} total chunks from {result['successful_files']} file(s)"
588
+ )
589
+
590
+ return {
591
+ "result": knowledge,
592
+ "processing": {
593
+ "mode": "sync",
594
+ "status": "completed",
595
+ "total_files": len(sources),
596
+ "successful_files": result.get("successful_files", 0),
597
+ "failed_files": result.get("failed_files", 0),
598
+ "total_chunks": result.get("total_chunks", 0),
599
+ "files": processed_files
600
+ },
601
+ "message": (
602
+ f"Successfully processed {result['successful_files']}/{len(sources)} file(s) "
603
+ f"with {result['total_chunks']} total chunks."
604
+ )
605
+ }
606
+
607
+ except asyncio.TimeoutError:
608
+ logger.warning(
609
+ f"Synchronous processing timed out for knowledge {knowledge.knowledge_id}"
610
+ )
611
+ raise HTTPException(
612
+ status_code=status.HTTP_408_REQUEST_TIMEOUT,
613
+ detail=f"PDF processing exceeded timeout of {settings.pdf_processing_timeout}s."
614
+ )
615
+ except Exception as e:
616
+ logger.error(f"Synchronous processing failed: {e}")
617
+ raise HTTPException(
618
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
619
+ detail=f"PDF processing failed: {str(e)}"
620
+ )
621
+
622
+ else:
623
+ # Process all files asynchronously - immediate response
624
+ logger.info(
625
+ f"Starting asynchronous processing for knowledge {knowledge.knowledge_id} "
626
+ f"with {len(sources)} file(s)"
627
+ )
628
+
629
+ # Prepare file info for response
630
+ file_sources = [
631
+ {
632
+ "filename": filenames[idx],
633
+ "source": sources[idx],
634
+ "size_mb": file_data_list[idx]["size_mb"]
635
+ }
636
+ for idx in range(len(sources))
637
+ ]
638
+
639
+ try:
640
+ # Try Mage AI pipeline first
641
+ mage_result = process_knowledge_upload_batch(
642
+ doc_id=str(knowledge.knowledge_id),
643
+ sources=sources
644
+ )
645
+
646
+ if mage_result:
647
+ run_id = mage_result.get('run_id')
648
+ logger.info(
649
+ f"Mage AI pipeline triggered for knowledge {knowledge.knowledge_id}. "
650
+ f"Run ID: {run_id}"
651
+ )
652
+ processing_info = {
653
+ "method": "mage",
654
+ "run_id": run_id,
655
+ "files": file_sources
656
+ }
657
+ else:
658
+ logger.warning(
659
+ f"Mage trigger failed for knowledge {knowledge.knowledge_id}, "
660
+ f"using background task"
661
+ )
662
+ # Fallback to background task
663
+ background_tasks.add_task(
664
+ process_pdfs_background,
665
+ doc_id=knowledge.knowledge_id,
666
+ sources=sources,
667
+ callback_url=callback_url,
668
+ )
669
+ processing_info = {
670
+ "method": "background_task",
671
+ "files": file_sources
672
+ }
673
+
674
+ except Exception as mage_error:
675
+ logger.error(
676
+ f"Error triggering Mage AI for knowledge {knowledge.knowledge_id}: {mage_error}"
677
+ )
678
+ # Fallback to background task
679
+ background_tasks.add_task(
680
+ process_pdfs_background,
681
+ doc_id=knowledge.knowledge_id,
682
+ sources=sources,
683
+ callback_url=callback_url,
684
+ )
685
+ processing_info = {
686
+ "method": "background_task",
687
+ "files": file_sources
688
+ }
689
+
690
+ return {
691
+ "result": knowledge,
692
+ "processing": {
693
+ "mode": "async",
694
+ "status": "processing",
695
+ "total_files": len(sources),
696
+ **processing_info
697
+ },
698
+ "message": (
699
+ f"Your {len(sources)} file(s) {'is' if len(sources) == 1 else 'are'} being processed "
700
+ f"in the background under knowledge ID {knowledge.knowledge_id}."
701
+ )
702
+ }
703
+
704
+
705
+ @router.delete("/{knowledge_id}", status_code=status.HTTP_204_NO_CONTENT)
706
+ async def delete_knowledge(
707
+ knowledge: schemas.Knowledge = Depends(dependencies.require_knowledge_owner_or_admin),
708
+ db: AsyncSession = Depends(get_db),
709
+ ):
710
+ """
711
+ Delete a knowledge document and all its associated data.
712
+ This includes all S3 files and vector database entries.
713
+ """
714
+ await service.delete_knowledge(knowledge.knowledge_id, db)