th2rag 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. th2rag/__init__.py +0 -0
  2. th2rag/auth/__init__.py +0 -0
  3. th2rag/auth/exceptions.py +2 -0
  4. th2rag/auth/router.py +43 -0
  5. th2rag/auth/schemas.py +15 -0
  6. th2rag/auth/service.py +115 -0
  7. th2rag/clients/__init__.py +0 -0
  8. th2rag/clients/storage/config.py +13 -0
  9. th2rag/clients/storage/local.py +48 -0
  10. th2rag/clients/storage/s3.py +93 -0
  11. th2rag/config.py +267 -0
  12. th2rag/conversations/__init__.py +0 -0
  13. th2rag/conversations/dependencies.py +30 -0
  14. th2rag/conversations/exceptions.py +6 -0
  15. th2rag/conversations/router.py +47 -0
  16. th2rag/conversations/schemas.py +43 -0
  17. th2rag/conversations/service.py +51 -0
  18. th2rag/database.py +72 -0
  19. th2rag/dataset/__init__.py +0 -0
  20. th2rag/dataset/dependencies.py +25 -0
  21. th2rag/dataset/exceptions.py +2 -0
  22. th2rag/dataset/router.py +53 -0
  23. th2rag/dataset/schemas.py +23 -0
  24. th2rag/dataset/service.py +78 -0
  25. th2rag/dependencies.py +143 -0
  26. th2rag/knowledge/__init__.py +0 -0
  27. th2rag/knowledge/dependencies.py +25 -0
  28. th2rag/knowledge/exceptions.py +2 -0
  29. th2rag/knowledge/mage_client.py +505 -0
  30. th2rag/knowledge/router.py +714 -0
  31. th2rag/knowledge/schemas.py +108 -0
  32. th2rag/knowledge/service.py +392 -0
  33. th2rag/logger.py +21 -0
  34. th2rag/main.py +51 -0
  35. th2rag/messages/__init__.py +0 -0
  36. th2rag/messages/exceptions.py +2 -0
  37. th2rag/messages/router.py +63 -0
  38. th2rag/messages/schemas.py +68 -0
  39. th2rag/messages/service.py +210 -0
  40. th2rag/models.py +215 -0
  41. th2rag/pagination.py +68 -0
  42. th2rag/rag/__init__.py +0 -0
  43. th2rag/rag/chunkers/__init__.py +0 -0
  44. th2rag/rag/chunkers/hybrid_chunker.py +23 -0
  45. th2rag/rag/constants.py +15 -0
  46. th2rag/rag/converters/__init__.py +0 -0
  47. th2rag/rag/converters/documents_converter.py +123 -0
  48. th2rag/rag/converters/json_converter.py +521 -0
  49. th2rag/rag/converters/legacy_converter.py +124 -0
  50. th2rag/rag/converters/pdf_converter.py +384 -0
  51. th2rag/rag/converters/spreadsheet_converter.py +159 -0
  52. th2rag/rag/converters/txt_converter.py +98 -0
  53. th2rag/rag/embeddings/__init__.py +0 -0
  54. th2rag/rag/embeddings/base.py +21 -0
  55. th2rag/rag/embeddings/exceptions.py +2 -0
  56. th2rag/rag/embeddings/sentence_transformer.py +97 -0
  57. th2rag/rag/generation/base.py +22 -0
  58. th2rag/rag/generation/exceptions.py +2 -0
  59. th2rag/rag/generation/litellm_generator.py +143 -0
  60. th2rag/rag/generation/mistral_generator.py +269 -0
  61. th2rag/rag/generation/test_litellm.py +33 -0
  62. th2rag/rag/retrieval/__init__.py +0 -0
  63. th2rag/rag/retrieval/base.py +11 -0
  64. th2rag/rag/retrieval/exceptions.py +10 -0
  65. th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
  66. th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
  67. th2rag/rag/retrieval/lancedb/schemas.py +19 -0
  68. th2rag/rag/services/__init__.py +0 -0
  69. th2rag/rag/services/rag_service.py +99 -0
  70. th2rag/rag/services/rag_service_builder.py +42 -0
  71. th2rag/rag/tasks/process_pdf.py +586 -0
  72. th2rag/users/dependencies.py +44 -0
  73. th2rag/users/exceptions.py +6 -0
  74. th2rag/users/router.py +92 -0
  75. th2rag/users/schemas.py +61 -0
  76. th2rag/users/service.py +140 -0
  77. th2rag/utils/__init__.py +0 -0
  78. th2rag/utils/helpers.py +40 -0
  79. th2rag/utils/logger.py +15 -0
  80. th2rag/utils/security.py +76 -0
  81. th2rag/utils/welcome_router.py +98 -0
  82. th2rag/webhooks/__init__.py +0 -0
  83. th2rag/webhooks/sender.py +126 -0
  84. th2rag-0.0.2.dist-info/METADATA +1002 -0
  85. th2rag-0.0.2.dist-info/RECORD +87 -0
  86. th2rag-0.0.2.dist-info/WHEEL +4 -0
  87. th2rag-0.0.2.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,505 @@
1
+ import json
2
+ import os
3
+ from typing import Any
4
+
5
+ import requests
6
+ from dotenv import load_dotenv
7
+
8
+ load_dotenv()
9
+
10
+
11
+ class MageAPIClient:
12
+ _instance = None
13
+
14
+ def __new__(cls):
15
+ if cls._instance is None:
16
+ cls._instance = super(MageAPIClient, cls).__new__(cls)
17
+ cls._instance._initialized = False
18
+ return cls._instance
19
+
20
+ def __init__(self):
21
+ if self._initialized:
22
+ return
23
+
24
+ self.base_url = os.getenv('BASE_URL', 'http://localhost:6789')
25
+ self.oauth_token = os.getenv('OAUTH_TOKEN')
26
+ self.api_key = os.getenv('API_KEY')
27
+ self.project_name = os.getenv('PROJECT_NAME', 'default_repo')
28
+ self._initialized = True
29
+
30
+ def _get_headers(self) -> dict[str, str]:
31
+ headers = {
32
+ 'Content-Type': 'application/json',
33
+ 'X-API-KEY': self.api_key,
34
+ }
35
+ if self.oauth_token:
36
+ headers['Cookie'] = f'oauth_token={self.oauth_token}'
37
+ return headers
38
+
39
+ def pipeline_exists(self, pipeline_uuid: str) -> bool:
40
+ """Check if a pipeline exists."""
41
+ try:
42
+ url = f'{self.base_url}/api/pipelines/{pipeline_uuid}'
43
+ response = requests.get(url, headers=self._get_headers(), timeout=10)
44
+
45
+ if response.status_code == 200:
46
+ data = response.json()
47
+ if "error" in data:
48
+ return False
49
+ return True
50
+ return False
51
+ except Exception as e:
52
+ print(f"Error checking pipeline existence: {e}")
53
+ return False
54
+
55
+ def create_pipeline(self, pipeline_uuid: str, pipeline_type: str = "python") -> dict[str, Any] | None:
56
+ """Create a new pipeline."""
57
+ payload = {
58
+ "pipeline": {
59
+ "name": pipeline_uuid,
60
+ "type": pipeline_type
61
+ }
62
+ }
63
+
64
+ try:
65
+ url = f'{self.base_url}/api/pipelines'
66
+ response = requests.post(url, headers=self._get_headers(), json=payload, timeout=10)
67
+
68
+ if response.status_code == 200:
69
+ data = response.json()
70
+ if "error" in data:
71
+ print("❌ Failed to create pipeline.")
72
+ print(f"Response: {json.dumps(data, indent=2)}")
73
+ return None
74
+ print(f"✅ Pipeline '{pipeline_uuid}' created successfully!")
75
+ return data
76
+ else:
77
+ print(f"❌ Failed to create pipeline. Status: {response.status_code}")
78
+ print(f"Response: {response.text}")
79
+ return None
80
+ except Exception as e:
81
+ print(f"❌ Error creating pipeline: {e}")
82
+ return None
83
+
84
+ def block_exists(self, pipeline_uuid: str, block_uuid: str) -> bool:
85
+ """Check if a block exists in a pipeline."""
86
+ try:
87
+ url = f'{self.base_url}/api/pipelines/{pipeline_uuid}/blocks/{block_uuid}'
88
+ response = requests.get(url, headers=self._get_headers(), timeout=10)
89
+
90
+ if response.status_code == 200:
91
+ data = response.json()
92
+ if "error" in data:
93
+ return False
94
+ return "block" in data and data["block"] is not None
95
+ return False
96
+ except Exception as e:
97
+ print(f"Error checking block existence: {e}")
98
+ return False
99
+
100
+ def create_block(self, pipeline_uuid: str, block_name: str, block_content: str,
101
+ block_type: str = "data_loader") -> dict[str, Any] | None:
102
+ """Create a new block in a pipeline."""
103
+ payload = {
104
+ "block": {
105
+ "name": block_name,
106
+ "type": block_type,
107
+ "language": "python",
108
+ "content": block_content,
109
+ "priority": 0,
110
+ "configuration": {
111
+ "data_source": None
112
+ }
113
+ },
114
+ "api_key": self.api_key
115
+ }
116
+
117
+ try:
118
+ url = f'{self.base_url}/api/pipelines/{pipeline_uuid}/blocks'
119
+ response = requests.post(url, headers=self._get_headers(), json=payload, timeout=15)
120
+
121
+ if response.status_code in [200, 201]:
122
+ data = response.json()
123
+ if "error" in data:
124
+ print("❌ Failed to create block.")
125
+ print(f"Response: {json.dumps(data, indent=2)}")
126
+ return None
127
+ print(f"✅ Block '{block_name}' created successfully!")
128
+ return data
129
+ else:
130
+ print(f"❌ Failed to create block. Status: {response.status_code}")
131
+ print(f"Response: {response.text}")
132
+ return None
133
+ except Exception as e:
134
+ print(f"❌ Error creating block: {e}")
135
+ return None
136
+
137
+ def get_pipeline_schedules(self, pipeline_uuid: str) -> list:
138
+ """Get all pipeline schedules/triggers."""
139
+ url = f"{self.base_url}/api/pipelines/{pipeline_uuid}/pipeline_schedules"
140
+ try:
141
+ response = requests.get(
142
+ url,
143
+ headers=self._get_headers(),
144
+ timeout=15
145
+ )
146
+ response.raise_for_status()
147
+ return response.json().get("pipeline_schedules", [])
148
+ except Exception as e:
149
+ print(f"Error fetching schedules: {e}")
150
+ return []
151
+
152
+ def create_api_trigger(self, pipeline_uuid: str, trigger_name: str) -> dict[str, Any] | None:
153
+ """Create an API trigger for a pipeline (only once)."""
154
+ url = f"{self.base_url}/api/pipelines/{pipeline_uuid}/pipeline_schedules?project={self.project_name}"
155
+
156
+ payload = {
157
+ "pipeline_schedule": {
158
+ "name": trigger_name,
159
+ "schedule_type": "api",
160
+ "status": "active"
161
+ }
162
+ }
163
+
164
+ try:
165
+ response = requests.post(url, headers=self._get_headers(), json=payload, timeout=15)
166
+ data = response.json()
167
+
168
+ if "error" in data:
169
+ print("❌ Mage returned an error during trigger creation")
170
+ print(json.dumps(data, indent=2))
171
+ return None
172
+
173
+ schedule = data.get("pipeline_schedule")
174
+ if schedule and schedule.get("id") and schedule.get("token"):
175
+ print(f"✅ API Trigger '{trigger_name}' created successfully!")
176
+ return schedule
177
+ else:
178
+ print("❌ Trigger creation failed")
179
+ print(json.dumps(data, indent=2))
180
+ return None
181
+ except Exception as e:
182
+ print(f"❌ Error creating API trigger: {e}")
183
+ return None
184
+
185
+ def trigger_pipeline(self, schedule_id: int, trigger_token: str,
186
+ run_variables: dict[str, Any] | None = None) -> dict[str, Any] | None:
187
+ """Trigger a pipeline execution (creates a new run)."""
188
+ url = f"{self.base_url}/api/pipeline_schedules/{schedule_id}/pipeline_runs/{trigger_token}"
189
+
190
+ payload = {}
191
+ if run_variables:
192
+ payload["pipeline_run"] = {
193
+ "variables": run_variables
194
+ }
195
+
196
+ try:
197
+ response = requests.post(url, headers=self._get_headers(), json=payload, timeout=15)
198
+ response.raise_for_status()
199
+
200
+ data = response.json()
201
+ if "error" in data:
202
+ print("❌ Mage returned an error during execution")
203
+ print(json.dumps(data, indent=2))
204
+ return None
205
+
206
+ print("✅ Pipeline run created successfully!")
207
+ return data
208
+ except Exception as e:
209
+ print(f"❌ Error triggering pipeline: {e}")
210
+ return None
211
+
212
+
213
+ class PipelineOrchestrator:
214
+
215
+ # Configuration from environment
216
+ PIPELINE_UUID = os.getenv('MAGE_PIPELINE_UUID', 'process_pdf_batch')
217
+ TRIGGER_NAME = os.getenv('MAGE_TRIGGER_NAME', 'batch_runner_trigger')
218
+ BLOCK_NAME = "process_pdfs_batch_loader"
219
+
220
+ def __init__(self):
221
+ self.client = MageAPIClient()
222
+ self.schedule_id = None
223
+ self.trigger_token = None
224
+ self._initialized = False
225
+
226
+ def _get_block_content(self) -> str:
227
+ """Returns the block content for batch PDF processing with shared model loading."""
228
+ return '''if 'data_loader' not in globals():
229
+ from mage_ai.data_preparation.decorators import data_loader
230
+
231
+ @data_loader
232
+ def process_pdfs_batch_loader(*args, **kwargs):
233
+ """
234
+ Process multiple PDFs for a single knowledge document.
235
+ Models are loaded once and reused for all files.
236
+ """
237
+ from th2rag.rag.tasks.process_pdf import process_pdfs_background
238
+
239
+ # Get parameters
240
+ sources = kwargs.get("sources") # List of S3 URLs
241
+ doc_id = kwargs.get("doc_id")
242
+
243
+ print(f"doc_id = {doc_id}")
244
+ print(f"sources = {sources}")
245
+ print(f"Processing {len(sources) if sources else 0} file(s)")
246
+
247
+ if not sources or not doc_id:
248
+ raise ValueError(f"Missing required parameters. doc_id: {doc_id}, sources: {sources}")
249
+
250
+ if not isinstance(sources, list):
251
+ raise ValueError(f"sources must be a list, got {type(sources)}")
252
+
253
+ # Call the batch background processor
254
+ # This function loads models ONCE and processes all files
255
+ result = process_pdfs_background(doc_id=int(doc_id), sources=sources)
256
+
257
+ return {
258
+ "status": "completed",
259
+ "doc_id": doc_id,
260
+ "total_files": len(sources),
261
+ "result": result
262
+ }
263
+ '''
264
+
265
+ def _ensure_pipeline_exists(self) -> bool:
266
+ """Ensure pipeline exists, create if it doesn't."""
267
+ print(f"\n[1/4] Checking if pipeline '{self.PIPELINE_UUID}' exists...")
268
+
269
+ if not self.client.pipeline_exists(self.PIPELINE_UUID):
270
+ print(f" Pipeline not found. Creating '{self.PIPELINE_UUID}'...")
271
+ result = self.client.create_pipeline(self.PIPELINE_UUID)
272
+ if not result:
273
+ print("❌ Failed to create pipeline. Aborting.")
274
+ return False
275
+ else:
276
+ print(f" ✓ Pipeline '{self.PIPELINE_UUID}' already exists.")
277
+
278
+ return True
279
+
280
+ def _ensure_block_exists(self) -> bool:
281
+ """Ensure block exists, create if it doesn't."""
282
+ print(f"\n[2/4] Checking if block '{self.BLOCK_NAME}' exists...")
283
+
284
+ if not self.client.block_exists(self.PIPELINE_UUID, self.BLOCK_NAME):
285
+ print(f" Block not found. Creating '{self.BLOCK_NAME}'...")
286
+ block_content = self._get_block_content()
287
+ result = self.client.create_block(
288
+ self.PIPELINE_UUID,
289
+ self.BLOCK_NAME,
290
+ block_content
291
+ )
292
+ if not result:
293
+ print("❌ Failed to create block. Aborting.")
294
+ return False
295
+ else:
296
+ print(f" ✓ Block '{self.BLOCK_NAME}' already exists.")
297
+
298
+ return True
299
+
300
+ def _load_or_create_trigger(self) -> bool:
301
+ """Load existing trigger or create if it doesn't exist (ONCE ONLY)."""
302
+ print(f"\n[3/4] Checking for API trigger '{self.TRIGGER_NAME}'...")
303
+
304
+ # Check if trigger already exists
305
+ schedules = self.client.get_pipeline_schedules(self.PIPELINE_UUID)
306
+ existing_trigger = next(
307
+ (s for s in schedules if s.get("name") == self.TRIGGER_NAME),
308
+ None
309
+ )
310
+
311
+ if existing_trigger:
312
+ self.schedule_id = existing_trigger.get("id")
313
+ self.trigger_token = existing_trigger.get("token")
314
+ print(f" ✓ Trigger '{self.TRIGGER_NAME}' already exists.")
315
+ print(f" Schedule ID: {self.schedule_id}")
316
+ print(f" Trigger Token: {self.trigger_token}")
317
+ return True
318
+
319
+ # Create trigger if it doesn't exist
320
+ print(f" Trigger not found. Creating '{self.TRIGGER_NAME}'...")
321
+ trigger_info = self.client.create_api_trigger(self.PIPELINE_UUID, self.TRIGGER_NAME)
322
+ if not trigger_info:
323
+ print("❌ Failed to create API trigger. Aborting.")
324
+ return False
325
+
326
+ self.schedule_id = trigger_info.get("id")
327
+ self.trigger_token = trigger_info.get("token")
328
+
329
+ print(f" Schedule ID: {self.schedule_id}")
330
+ print(f" Trigger Token: {self.trigger_token}")
331
+
332
+ return True
333
+
334
+ def initialize(self) -> bool:
335
+ """Initialize pipeline infrastructure (call once at startup)."""
336
+ if self._initialized:
337
+ return True
338
+
339
+ print("=" * 70)
340
+ print("INITIALIZING MAGE PIPELINE INFRASTRUCTURE")
341
+ print("=" * 70)
342
+
343
+ # Step 1: Ensure pipeline exists
344
+ if not self._ensure_pipeline_exists():
345
+ return False
346
+
347
+ # Step 2: Ensure block exists
348
+ if not self._ensure_block_exists():
349
+ return False
350
+
351
+ # Step 3: Load or create trigger (ONCE)
352
+ if not self._load_or_create_trigger():
353
+ return False
354
+
355
+ self._initialized = True
356
+ print("\n✅ Pipeline infrastructure initialized successfully!")
357
+ print("=" * 70)
358
+ return True
359
+
360
+ def execute_batch_upload(self, doc_id: str, sources: list[str]) -> dict[str, Any] | None:
361
+ """
362
+ Execute pipeline for batch PDF processing.
363
+ This creates a SINGLE RUN that processes ALL files with shared model loading.
364
+
365
+ Args:
366
+ doc_id: Document ID
367
+ sources: List of S3 URLs to process
368
+
369
+ Returns:
370
+ Execution result or None if failed
371
+ """
372
+ # Ensure infrastructure is initialized
373
+ if not self._initialized:
374
+ if not self.initialize():
375
+ return None
376
+
377
+ print("\n" + "=" * 70)
378
+ print(f"CREATING BATCH PIPELINE RUN - {len(sources)} FILES")
379
+ print("=" * 70)
380
+
381
+ # Create a single run with all sources
382
+ print("\n[4/4] Creating pipeline run for batch processing...")
383
+ print(f" doc_id: {doc_id}")
384
+ print(f" sources: {len(sources)} file(s)")
385
+ for idx, source in enumerate(sources, 1):
386
+ print(f" [{idx}] {source}")
387
+
388
+ run_variables = {
389
+ "doc_id": doc_id,
390
+ "sources": sources # Pass entire list as a single variable
391
+ }
392
+
393
+ execution_result = self.client.trigger_pipeline(
394
+ self.schedule_id,
395
+ self.trigger_token,
396
+ run_variables
397
+ )
398
+
399
+ if execution_result:
400
+ print("\n" + "=" * 70)
401
+ print("BATCH PIPELINE RUN STARTED")
402
+ print("=" * 70)
403
+ print(f"Schedule ID : {self.schedule_id}")
404
+ print(f"Trigger Token : {self.trigger_token}")
405
+ print(f"Doc ID : {doc_id}")
406
+ print(f"Total Files : {len(sources)}")
407
+
408
+ run_info = execution_result.get('pipeline_run', {})
409
+ run_id = run_info.get('id')
410
+ status = run_info.get('status')
411
+
412
+ if run_id:
413
+ print(f"Run ID : {run_id}")
414
+ print(f"Status : {status}")
415
+ print(f"\nView in Mage: {self.client.base_url}/pipelines/{self.PIPELINE_UUID}/runs/{run_id}")
416
+
417
+ print("=" * 70)
418
+
419
+ return {
420
+ "run_id": run_id,
421
+ "status": status,
422
+ "doc_id": doc_id,
423
+ "total_files": len(sources),
424
+ "sources": sources
425
+ }
426
+ else:
427
+ print("❌ Failed to create pipeline run.")
428
+ return None
429
+
430
+
431
+ # Global orchestrator instance (singleton pattern)
432
+ _orchestrator_instance = None
433
+
434
+ def get_orchestrator() -> PipelineOrchestrator:
435
+ """Get or create the global orchestrator instance."""
436
+ global _orchestrator_instance
437
+ if _orchestrator_instance is None:
438
+ _orchestrator_instance = PipelineOrchestrator()
439
+ _orchestrator_instance.initialize()
440
+ return _orchestrator_instance
441
+
442
+
443
+ def process_knowledge_upload_batch(doc_id: str, sources: list[str]) -> dict[str, Any] | None:
444
+ """
445
+ Process a knowledge upload with multiple files in a single batch.
446
+ Models are loaded once and reused for all files.
447
+
448
+ Args:
449
+ doc_id: Document ID
450
+ sources: List of S3 URLs to process
451
+
452
+ Returns:
453
+ Execution result or None if failed
454
+ """
455
+ orchestrator = get_orchestrator()
456
+ return orchestrator.execute_batch_upload(doc_id, sources)
457
+
458
+
459
+ def main():
460
+ """Main entry point - simulates batch knowledge uploads."""
461
+ print("\n" + "=" * 70)
462
+ print("MAGE PDF BATCH PROCESSOR - KNOWLEDGE UPLOAD HANDLER")
463
+ print("=" * 70)
464
+
465
+ # Initialize once
466
+ orchestrator = get_orchestrator()
467
+
468
+ # Simulate batch uploads
469
+ while True:
470
+ print("\n" + "-" * 70)
471
+ doc_id = input("\nEnter Document ID (or 'q' to quit): ").strip()
472
+
473
+ if doc_id.lower() == 'q':
474
+ break
475
+
476
+ num_files = input("How many files? ").strip()
477
+
478
+ try:
479
+ num_files = int(num_files)
480
+ except ValueError:
481
+ print("\n❌ Error: Number of files must be an integer.")
482
+ continue
483
+
484
+ sources = []
485
+ for i in range(num_files):
486
+ source = input(f"Enter Source URL #{i+1}: ").strip()
487
+ if source:
488
+ sources.append(source)
489
+
490
+ if not all([doc_id, sources]):
491
+ print("\n❌ Error: Document ID and at least one Source are required.")
492
+ continue
493
+
494
+ # Process batch upload - creates ONE RUN for all files
495
+ result = process_knowledge_upload_batch(doc_id, sources)
496
+
497
+ if result:
498
+ print(f"\n✅ Batch upload processed! Run ID: {result['run_id']}")
499
+ print(f" Processing {result['total_files']} file(s)")
500
+ else:
501
+ print("\n❌ Upload failed. Check the errors above.")
502
+
503
+
504
+ if __name__ == "__main__":
505
+ main()