th2rag 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- th2rag/__init__.py +0 -0
- th2rag/auth/__init__.py +0 -0
- th2rag/auth/exceptions.py +2 -0
- th2rag/auth/router.py +43 -0
- th2rag/auth/schemas.py +15 -0
- th2rag/auth/service.py +115 -0
- th2rag/clients/__init__.py +0 -0
- th2rag/clients/storage/config.py +13 -0
- th2rag/clients/storage/local.py +48 -0
- th2rag/clients/storage/s3.py +93 -0
- th2rag/config.py +267 -0
- th2rag/conversations/__init__.py +0 -0
- th2rag/conversations/dependencies.py +30 -0
- th2rag/conversations/exceptions.py +6 -0
- th2rag/conversations/router.py +47 -0
- th2rag/conversations/schemas.py +43 -0
- th2rag/conversations/service.py +51 -0
- th2rag/database.py +72 -0
- th2rag/dataset/__init__.py +0 -0
- th2rag/dataset/dependencies.py +25 -0
- th2rag/dataset/exceptions.py +2 -0
- th2rag/dataset/router.py +53 -0
- th2rag/dataset/schemas.py +23 -0
- th2rag/dataset/service.py +78 -0
- th2rag/dependencies.py +143 -0
- th2rag/knowledge/__init__.py +0 -0
- th2rag/knowledge/dependencies.py +25 -0
- th2rag/knowledge/exceptions.py +2 -0
- th2rag/knowledge/mage_client.py +505 -0
- th2rag/knowledge/router.py +714 -0
- th2rag/knowledge/schemas.py +108 -0
- th2rag/knowledge/service.py +392 -0
- th2rag/logger.py +21 -0
- th2rag/main.py +51 -0
- th2rag/messages/__init__.py +0 -0
- th2rag/messages/exceptions.py +2 -0
- th2rag/messages/router.py +63 -0
- th2rag/messages/schemas.py +68 -0
- th2rag/messages/service.py +210 -0
- th2rag/models.py +215 -0
- th2rag/pagination.py +68 -0
- th2rag/rag/__init__.py +0 -0
- th2rag/rag/chunkers/__init__.py +0 -0
- th2rag/rag/chunkers/hybrid_chunker.py +23 -0
- th2rag/rag/constants.py +15 -0
- th2rag/rag/converters/__init__.py +0 -0
- th2rag/rag/converters/documents_converter.py +123 -0
- th2rag/rag/converters/json_converter.py +521 -0
- th2rag/rag/converters/legacy_converter.py +124 -0
- th2rag/rag/converters/pdf_converter.py +384 -0
- th2rag/rag/converters/spreadsheet_converter.py +159 -0
- th2rag/rag/converters/txt_converter.py +98 -0
- th2rag/rag/embeddings/__init__.py +0 -0
- th2rag/rag/embeddings/base.py +21 -0
- th2rag/rag/embeddings/exceptions.py +2 -0
- th2rag/rag/embeddings/sentence_transformer.py +97 -0
- th2rag/rag/generation/base.py +22 -0
- th2rag/rag/generation/exceptions.py +2 -0
- th2rag/rag/generation/litellm_generator.py +143 -0
- th2rag/rag/generation/mistral_generator.py +269 -0
- th2rag/rag/generation/test_litellm.py +33 -0
- th2rag/rag/retrieval/__init__.py +0 -0
- th2rag/rag/retrieval/base.py +11 -0
- th2rag/rag/retrieval/exceptions.py +10 -0
- th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
- th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
- th2rag/rag/retrieval/lancedb/schemas.py +19 -0
- th2rag/rag/services/__init__.py +0 -0
- th2rag/rag/services/rag_service.py +99 -0
- th2rag/rag/services/rag_service_builder.py +42 -0
- th2rag/rag/tasks/process_pdf.py +586 -0
- th2rag/users/dependencies.py +44 -0
- th2rag/users/exceptions.py +6 -0
- th2rag/users/router.py +92 -0
- th2rag/users/schemas.py +61 -0
- th2rag/users/service.py +140 -0
- th2rag/utils/__init__.py +0 -0
- th2rag/utils/helpers.py +40 -0
- th2rag/utils/logger.py +15 -0
- th2rag/utils/security.py +76 -0
- th2rag/utils/welcome_router.py +98 -0
- th2rag/webhooks/__init__.py +0 -0
- th2rag/webhooks/sender.py +126 -0
- th2rag-0.0.2.dist-info/METADATA +1002 -0
- th2rag-0.0.2.dist-info/RECORD +87 -0
- th2rag-0.0.2.dist-info/WHEEL +4 -0
- th2rag-0.0.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
from dotenv import load_dotenv
|
|
7
|
+
|
|
8
|
+
load_dotenv()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MageAPIClient:
|
|
12
|
+
_instance = None
|
|
13
|
+
|
|
14
|
+
def __new__(cls):
|
|
15
|
+
if cls._instance is None:
|
|
16
|
+
cls._instance = super(MageAPIClient, cls).__new__(cls)
|
|
17
|
+
cls._instance._initialized = False
|
|
18
|
+
return cls._instance
|
|
19
|
+
|
|
20
|
+
def __init__(self):
|
|
21
|
+
if self._initialized:
|
|
22
|
+
return
|
|
23
|
+
|
|
24
|
+
self.base_url = os.getenv('BASE_URL', 'http://localhost:6789')
|
|
25
|
+
self.oauth_token = os.getenv('OAUTH_TOKEN')
|
|
26
|
+
self.api_key = os.getenv('API_KEY')
|
|
27
|
+
self.project_name = os.getenv('PROJECT_NAME', 'default_repo')
|
|
28
|
+
self._initialized = True
|
|
29
|
+
|
|
30
|
+
def _get_headers(self) -> dict[str, str]:
|
|
31
|
+
headers = {
|
|
32
|
+
'Content-Type': 'application/json',
|
|
33
|
+
'X-API-KEY': self.api_key,
|
|
34
|
+
}
|
|
35
|
+
if self.oauth_token:
|
|
36
|
+
headers['Cookie'] = f'oauth_token={self.oauth_token}'
|
|
37
|
+
return headers
|
|
38
|
+
|
|
39
|
+
def pipeline_exists(self, pipeline_uuid: str) -> bool:
|
|
40
|
+
"""Check if a pipeline exists."""
|
|
41
|
+
try:
|
|
42
|
+
url = f'{self.base_url}/api/pipelines/{pipeline_uuid}'
|
|
43
|
+
response = requests.get(url, headers=self._get_headers(), timeout=10)
|
|
44
|
+
|
|
45
|
+
if response.status_code == 200:
|
|
46
|
+
data = response.json()
|
|
47
|
+
if "error" in data:
|
|
48
|
+
return False
|
|
49
|
+
return True
|
|
50
|
+
return False
|
|
51
|
+
except Exception as e:
|
|
52
|
+
print(f"Error checking pipeline existence: {e}")
|
|
53
|
+
return False
|
|
54
|
+
|
|
55
|
+
def create_pipeline(self, pipeline_uuid: str, pipeline_type: str = "python") -> dict[str, Any] | None:
|
|
56
|
+
"""Create a new pipeline."""
|
|
57
|
+
payload = {
|
|
58
|
+
"pipeline": {
|
|
59
|
+
"name": pipeline_uuid,
|
|
60
|
+
"type": pipeline_type
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
url = f'{self.base_url}/api/pipelines'
|
|
66
|
+
response = requests.post(url, headers=self._get_headers(), json=payload, timeout=10)
|
|
67
|
+
|
|
68
|
+
if response.status_code == 200:
|
|
69
|
+
data = response.json()
|
|
70
|
+
if "error" in data:
|
|
71
|
+
print("❌ Failed to create pipeline.")
|
|
72
|
+
print(f"Response: {json.dumps(data, indent=2)}")
|
|
73
|
+
return None
|
|
74
|
+
print(f"✅ Pipeline '{pipeline_uuid}' created successfully!")
|
|
75
|
+
return data
|
|
76
|
+
else:
|
|
77
|
+
print(f"❌ Failed to create pipeline. Status: {response.status_code}")
|
|
78
|
+
print(f"Response: {response.text}")
|
|
79
|
+
return None
|
|
80
|
+
except Exception as e:
|
|
81
|
+
print(f"❌ Error creating pipeline: {e}")
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
def block_exists(self, pipeline_uuid: str, block_uuid: str) -> bool:
|
|
85
|
+
"""Check if a block exists in a pipeline."""
|
|
86
|
+
try:
|
|
87
|
+
url = f'{self.base_url}/api/pipelines/{pipeline_uuid}/blocks/{block_uuid}'
|
|
88
|
+
response = requests.get(url, headers=self._get_headers(), timeout=10)
|
|
89
|
+
|
|
90
|
+
if response.status_code == 200:
|
|
91
|
+
data = response.json()
|
|
92
|
+
if "error" in data:
|
|
93
|
+
return False
|
|
94
|
+
return "block" in data and data["block"] is not None
|
|
95
|
+
return False
|
|
96
|
+
except Exception as e:
|
|
97
|
+
print(f"Error checking block existence: {e}")
|
|
98
|
+
return False
|
|
99
|
+
|
|
100
|
+
def create_block(self, pipeline_uuid: str, block_name: str, block_content: str,
|
|
101
|
+
block_type: str = "data_loader") -> dict[str, Any] | None:
|
|
102
|
+
"""Create a new block in a pipeline."""
|
|
103
|
+
payload = {
|
|
104
|
+
"block": {
|
|
105
|
+
"name": block_name,
|
|
106
|
+
"type": block_type,
|
|
107
|
+
"language": "python",
|
|
108
|
+
"content": block_content,
|
|
109
|
+
"priority": 0,
|
|
110
|
+
"configuration": {
|
|
111
|
+
"data_source": None
|
|
112
|
+
}
|
|
113
|
+
},
|
|
114
|
+
"api_key": self.api_key
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
url = f'{self.base_url}/api/pipelines/{pipeline_uuid}/blocks'
|
|
119
|
+
response = requests.post(url, headers=self._get_headers(), json=payload, timeout=15)
|
|
120
|
+
|
|
121
|
+
if response.status_code in [200, 201]:
|
|
122
|
+
data = response.json()
|
|
123
|
+
if "error" in data:
|
|
124
|
+
print("❌ Failed to create block.")
|
|
125
|
+
print(f"Response: {json.dumps(data, indent=2)}")
|
|
126
|
+
return None
|
|
127
|
+
print(f"✅ Block '{block_name}' created successfully!")
|
|
128
|
+
return data
|
|
129
|
+
else:
|
|
130
|
+
print(f"❌ Failed to create block. Status: {response.status_code}")
|
|
131
|
+
print(f"Response: {response.text}")
|
|
132
|
+
return None
|
|
133
|
+
except Exception as e:
|
|
134
|
+
print(f"❌ Error creating block: {e}")
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
def get_pipeline_schedules(self, pipeline_uuid: str) -> list:
|
|
138
|
+
"""Get all pipeline schedules/triggers."""
|
|
139
|
+
url = f"{self.base_url}/api/pipelines/{pipeline_uuid}/pipeline_schedules"
|
|
140
|
+
try:
|
|
141
|
+
response = requests.get(
|
|
142
|
+
url,
|
|
143
|
+
headers=self._get_headers(),
|
|
144
|
+
timeout=15
|
|
145
|
+
)
|
|
146
|
+
response.raise_for_status()
|
|
147
|
+
return response.json().get("pipeline_schedules", [])
|
|
148
|
+
except Exception as e:
|
|
149
|
+
print(f"Error fetching schedules: {e}")
|
|
150
|
+
return []
|
|
151
|
+
|
|
152
|
+
def create_api_trigger(self, pipeline_uuid: str, trigger_name: str) -> dict[str, Any] | None:
|
|
153
|
+
"""Create an API trigger for a pipeline (only once)."""
|
|
154
|
+
url = f"{self.base_url}/api/pipelines/{pipeline_uuid}/pipeline_schedules?project={self.project_name}"
|
|
155
|
+
|
|
156
|
+
payload = {
|
|
157
|
+
"pipeline_schedule": {
|
|
158
|
+
"name": trigger_name,
|
|
159
|
+
"schedule_type": "api",
|
|
160
|
+
"status": "active"
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
try:
|
|
165
|
+
response = requests.post(url, headers=self._get_headers(), json=payload, timeout=15)
|
|
166
|
+
data = response.json()
|
|
167
|
+
|
|
168
|
+
if "error" in data:
|
|
169
|
+
print("❌ Mage returned an error during trigger creation")
|
|
170
|
+
print(json.dumps(data, indent=2))
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
schedule = data.get("pipeline_schedule")
|
|
174
|
+
if schedule and schedule.get("id") and schedule.get("token"):
|
|
175
|
+
print(f"✅ API Trigger '{trigger_name}' created successfully!")
|
|
176
|
+
return schedule
|
|
177
|
+
else:
|
|
178
|
+
print("❌ Trigger creation failed")
|
|
179
|
+
print(json.dumps(data, indent=2))
|
|
180
|
+
return None
|
|
181
|
+
except Exception as e:
|
|
182
|
+
print(f"❌ Error creating API trigger: {e}")
|
|
183
|
+
return None
|
|
184
|
+
|
|
185
|
+
def trigger_pipeline(self, schedule_id: int, trigger_token: str,
|
|
186
|
+
run_variables: dict[str, Any] | None = None) -> dict[str, Any] | None:
|
|
187
|
+
"""Trigger a pipeline execution (creates a new run)."""
|
|
188
|
+
url = f"{self.base_url}/api/pipeline_schedules/{schedule_id}/pipeline_runs/{trigger_token}"
|
|
189
|
+
|
|
190
|
+
payload = {}
|
|
191
|
+
if run_variables:
|
|
192
|
+
payload["pipeline_run"] = {
|
|
193
|
+
"variables": run_variables
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
try:
|
|
197
|
+
response = requests.post(url, headers=self._get_headers(), json=payload, timeout=15)
|
|
198
|
+
response.raise_for_status()
|
|
199
|
+
|
|
200
|
+
data = response.json()
|
|
201
|
+
if "error" in data:
|
|
202
|
+
print("❌ Mage returned an error during execution")
|
|
203
|
+
print(json.dumps(data, indent=2))
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
print("✅ Pipeline run created successfully!")
|
|
207
|
+
return data
|
|
208
|
+
except Exception as e:
|
|
209
|
+
print(f"❌ Error triggering pipeline: {e}")
|
|
210
|
+
return None
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class PipelineOrchestrator:
|
|
214
|
+
|
|
215
|
+
# Configuration from environment
|
|
216
|
+
PIPELINE_UUID = os.getenv('MAGE_PIPELINE_UUID', 'process_pdf_batch')
|
|
217
|
+
TRIGGER_NAME = os.getenv('MAGE_TRIGGER_NAME', 'batch_runner_trigger')
|
|
218
|
+
BLOCK_NAME = "process_pdfs_batch_loader"
|
|
219
|
+
|
|
220
|
+
def __init__(self):
|
|
221
|
+
self.client = MageAPIClient()
|
|
222
|
+
self.schedule_id = None
|
|
223
|
+
self.trigger_token = None
|
|
224
|
+
self._initialized = False
|
|
225
|
+
|
|
226
|
+
def _get_block_content(self) -> str:
|
|
227
|
+
"""Returns the block content for batch PDF processing with shared model loading."""
|
|
228
|
+
return '''if 'data_loader' not in globals():
|
|
229
|
+
from mage_ai.data_preparation.decorators import data_loader
|
|
230
|
+
|
|
231
|
+
@data_loader
|
|
232
|
+
def process_pdfs_batch_loader(*args, **kwargs):
|
|
233
|
+
"""
|
|
234
|
+
Process multiple PDFs for a single knowledge document.
|
|
235
|
+
Models are loaded once and reused for all files.
|
|
236
|
+
"""
|
|
237
|
+
from th2rag.rag.tasks.process_pdf import process_pdfs_background
|
|
238
|
+
|
|
239
|
+
# Get parameters
|
|
240
|
+
sources = kwargs.get("sources") # List of S3 URLs
|
|
241
|
+
doc_id = kwargs.get("doc_id")
|
|
242
|
+
|
|
243
|
+
print(f"doc_id = {doc_id}")
|
|
244
|
+
print(f"sources = {sources}")
|
|
245
|
+
print(f"Processing {len(sources) if sources else 0} file(s)")
|
|
246
|
+
|
|
247
|
+
if not sources or not doc_id:
|
|
248
|
+
raise ValueError(f"Missing required parameters. doc_id: {doc_id}, sources: {sources}")
|
|
249
|
+
|
|
250
|
+
if not isinstance(sources, list):
|
|
251
|
+
raise ValueError(f"sources must be a list, got {type(sources)}")
|
|
252
|
+
|
|
253
|
+
# Call the batch background processor
|
|
254
|
+
# This function loads models ONCE and processes all files
|
|
255
|
+
result = process_pdfs_background(doc_id=int(doc_id), sources=sources)
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
"status": "completed",
|
|
259
|
+
"doc_id": doc_id,
|
|
260
|
+
"total_files": len(sources),
|
|
261
|
+
"result": result
|
|
262
|
+
}
|
|
263
|
+
'''
|
|
264
|
+
|
|
265
|
+
def _ensure_pipeline_exists(self) -> bool:
|
|
266
|
+
"""Ensure pipeline exists, create if it doesn't."""
|
|
267
|
+
print(f"\n[1/4] Checking if pipeline '{self.PIPELINE_UUID}' exists...")
|
|
268
|
+
|
|
269
|
+
if not self.client.pipeline_exists(self.PIPELINE_UUID):
|
|
270
|
+
print(f" Pipeline not found. Creating '{self.PIPELINE_UUID}'...")
|
|
271
|
+
result = self.client.create_pipeline(self.PIPELINE_UUID)
|
|
272
|
+
if not result:
|
|
273
|
+
print("❌ Failed to create pipeline. Aborting.")
|
|
274
|
+
return False
|
|
275
|
+
else:
|
|
276
|
+
print(f" ✓ Pipeline '{self.PIPELINE_UUID}' already exists.")
|
|
277
|
+
|
|
278
|
+
return True
|
|
279
|
+
|
|
280
|
+
def _ensure_block_exists(self) -> bool:
|
|
281
|
+
"""Ensure block exists, create if it doesn't."""
|
|
282
|
+
print(f"\n[2/4] Checking if block '{self.BLOCK_NAME}' exists...")
|
|
283
|
+
|
|
284
|
+
if not self.client.block_exists(self.PIPELINE_UUID, self.BLOCK_NAME):
|
|
285
|
+
print(f" Block not found. Creating '{self.BLOCK_NAME}'...")
|
|
286
|
+
block_content = self._get_block_content()
|
|
287
|
+
result = self.client.create_block(
|
|
288
|
+
self.PIPELINE_UUID,
|
|
289
|
+
self.BLOCK_NAME,
|
|
290
|
+
block_content
|
|
291
|
+
)
|
|
292
|
+
if not result:
|
|
293
|
+
print("❌ Failed to create block. Aborting.")
|
|
294
|
+
return False
|
|
295
|
+
else:
|
|
296
|
+
print(f" ✓ Block '{self.BLOCK_NAME}' already exists.")
|
|
297
|
+
|
|
298
|
+
return True
|
|
299
|
+
|
|
300
|
+
def _load_or_create_trigger(self) -> bool:
|
|
301
|
+
"""Load existing trigger or create if it doesn't exist (ONCE ONLY)."""
|
|
302
|
+
print(f"\n[3/4] Checking for API trigger '{self.TRIGGER_NAME}'...")
|
|
303
|
+
|
|
304
|
+
# Check if trigger already exists
|
|
305
|
+
schedules = self.client.get_pipeline_schedules(self.PIPELINE_UUID)
|
|
306
|
+
existing_trigger = next(
|
|
307
|
+
(s for s in schedules if s.get("name") == self.TRIGGER_NAME),
|
|
308
|
+
None
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
if existing_trigger:
|
|
312
|
+
self.schedule_id = existing_trigger.get("id")
|
|
313
|
+
self.trigger_token = existing_trigger.get("token")
|
|
314
|
+
print(f" ✓ Trigger '{self.TRIGGER_NAME}' already exists.")
|
|
315
|
+
print(f" Schedule ID: {self.schedule_id}")
|
|
316
|
+
print(f" Trigger Token: {self.trigger_token}")
|
|
317
|
+
return True
|
|
318
|
+
|
|
319
|
+
# Create trigger if it doesn't exist
|
|
320
|
+
print(f" Trigger not found. Creating '{self.TRIGGER_NAME}'...")
|
|
321
|
+
trigger_info = self.client.create_api_trigger(self.PIPELINE_UUID, self.TRIGGER_NAME)
|
|
322
|
+
if not trigger_info:
|
|
323
|
+
print("❌ Failed to create API trigger. Aborting.")
|
|
324
|
+
return False
|
|
325
|
+
|
|
326
|
+
self.schedule_id = trigger_info.get("id")
|
|
327
|
+
self.trigger_token = trigger_info.get("token")
|
|
328
|
+
|
|
329
|
+
print(f" Schedule ID: {self.schedule_id}")
|
|
330
|
+
print(f" Trigger Token: {self.trigger_token}")
|
|
331
|
+
|
|
332
|
+
return True
|
|
333
|
+
|
|
334
|
+
def initialize(self) -> bool:
|
|
335
|
+
"""Initialize pipeline infrastructure (call once at startup)."""
|
|
336
|
+
if self._initialized:
|
|
337
|
+
return True
|
|
338
|
+
|
|
339
|
+
print("=" * 70)
|
|
340
|
+
print("INITIALIZING MAGE PIPELINE INFRASTRUCTURE")
|
|
341
|
+
print("=" * 70)
|
|
342
|
+
|
|
343
|
+
# Step 1: Ensure pipeline exists
|
|
344
|
+
if not self._ensure_pipeline_exists():
|
|
345
|
+
return False
|
|
346
|
+
|
|
347
|
+
# Step 2: Ensure block exists
|
|
348
|
+
if not self._ensure_block_exists():
|
|
349
|
+
return False
|
|
350
|
+
|
|
351
|
+
# Step 3: Load or create trigger (ONCE)
|
|
352
|
+
if not self._load_or_create_trigger():
|
|
353
|
+
return False
|
|
354
|
+
|
|
355
|
+
self._initialized = True
|
|
356
|
+
print("\n✅ Pipeline infrastructure initialized successfully!")
|
|
357
|
+
print("=" * 70)
|
|
358
|
+
return True
|
|
359
|
+
|
|
360
|
+
def execute_batch_upload(self, doc_id: str, sources: list[str]) -> dict[str, Any] | None:
|
|
361
|
+
"""
|
|
362
|
+
Execute pipeline for batch PDF processing.
|
|
363
|
+
This creates a SINGLE RUN that processes ALL files with shared model loading.
|
|
364
|
+
|
|
365
|
+
Args:
|
|
366
|
+
doc_id: Document ID
|
|
367
|
+
sources: List of S3 URLs to process
|
|
368
|
+
|
|
369
|
+
Returns:
|
|
370
|
+
Execution result or None if failed
|
|
371
|
+
"""
|
|
372
|
+
# Ensure infrastructure is initialized
|
|
373
|
+
if not self._initialized:
|
|
374
|
+
if not self.initialize():
|
|
375
|
+
return None
|
|
376
|
+
|
|
377
|
+
print("\n" + "=" * 70)
|
|
378
|
+
print(f"CREATING BATCH PIPELINE RUN - {len(sources)} FILES")
|
|
379
|
+
print("=" * 70)
|
|
380
|
+
|
|
381
|
+
# Create a single run with all sources
|
|
382
|
+
print("\n[4/4] Creating pipeline run for batch processing...")
|
|
383
|
+
print(f" doc_id: {doc_id}")
|
|
384
|
+
print(f" sources: {len(sources)} file(s)")
|
|
385
|
+
for idx, source in enumerate(sources, 1):
|
|
386
|
+
print(f" [{idx}] {source}")
|
|
387
|
+
|
|
388
|
+
run_variables = {
|
|
389
|
+
"doc_id": doc_id,
|
|
390
|
+
"sources": sources # Pass entire list as a single variable
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
execution_result = self.client.trigger_pipeline(
|
|
394
|
+
self.schedule_id,
|
|
395
|
+
self.trigger_token,
|
|
396
|
+
run_variables
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
if execution_result:
|
|
400
|
+
print("\n" + "=" * 70)
|
|
401
|
+
print("BATCH PIPELINE RUN STARTED")
|
|
402
|
+
print("=" * 70)
|
|
403
|
+
print(f"Schedule ID : {self.schedule_id}")
|
|
404
|
+
print(f"Trigger Token : {self.trigger_token}")
|
|
405
|
+
print(f"Doc ID : {doc_id}")
|
|
406
|
+
print(f"Total Files : {len(sources)}")
|
|
407
|
+
|
|
408
|
+
run_info = execution_result.get('pipeline_run', {})
|
|
409
|
+
run_id = run_info.get('id')
|
|
410
|
+
status = run_info.get('status')
|
|
411
|
+
|
|
412
|
+
if run_id:
|
|
413
|
+
print(f"Run ID : {run_id}")
|
|
414
|
+
print(f"Status : {status}")
|
|
415
|
+
print(f"\nView in Mage: {self.client.base_url}/pipelines/{self.PIPELINE_UUID}/runs/{run_id}")
|
|
416
|
+
|
|
417
|
+
print("=" * 70)
|
|
418
|
+
|
|
419
|
+
return {
|
|
420
|
+
"run_id": run_id,
|
|
421
|
+
"status": status,
|
|
422
|
+
"doc_id": doc_id,
|
|
423
|
+
"total_files": len(sources),
|
|
424
|
+
"sources": sources
|
|
425
|
+
}
|
|
426
|
+
else:
|
|
427
|
+
print("❌ Failed to create pipeline run.")
|
|
428
|
+
return None
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
# Global orchestrator instance (singleton pattern)
|
|
432
|
+
_orchestrator_instance = None
|
|
433
|
+
|
|
434
|
+
def get_orchestrator() -> PipelineOrchestrator:
|
|
435
|
+
"""Get or create the global orchestrator instance."""
|
|
436
|
+
global _orchestrator_instance
|
|
437
|
+
if _orchestrator_instance is None:
|
|
438
|
+
_orchestrator_instance = PipelineOrchestrator()
|
|
439
|
+
_orchestrator_instance.initialize()
|
|
440
|
+
return _orchestrator_instance
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def process_knowledge_upload_batch(doc_id: str, sources: list[str]) -> dict[str, Any] | None:
|
|
444
|
+
"""
|
|
445
|
+
Process a knowledge upload with multiple files in a single batch.
|
|
446
|
+
Models are loaded once and reused for all files.
|
|
447
|
+
|
|
448
|
+
Args:
|
|
449
|
+
doc_id: Document ID
|
|
450
|
+
sources: List of S3 URLs to process
|
|
451
|
+
|
|
452
|
+
Returns:
|
|
453
|
+
Execution result or None if failed
|
|
454
|
+
"""
|
|
455
|
+
orchestrator = get_orchestrator()
|
|
456
|
+
return orchestrator.execute_batch_upload(doc_id, sources)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def main():
|
|
460
|
+
"""Main entry point - simulates batch knowledge uploads."""
|
|
461
|
+
print("\n" + "=" * 70)
|
|
462
|
+
print("MAGE PDF BATCH PROCESSOR - KNOWLEDGE UPLOAD HANDLER")
|
|
463
|
+
print("=" * 70)
|
|
464
|
+
|
|
465
|
+
# Initialize once
|
|
466
|
+
orchestrator = get_orchestrator()
|
|
467
|
+
|
|
468
|
+
# Simulate batch uploads
|
|
469
|
+
while True:
|
|
470
|
+
print("\n" + "-" * 70)
|
|
471
|
+
doc_id = input("\nEnter Document ID (or 'q' to quit): ").strip()
|
|
472
|
+
|
|
473
|
+
if doc_id.lower() == 'q':
|
|
474
|
+
break
|
|
475
|
+
|
|
476
|
+
num_files = input("How many files? ").strip()
|
|
477
|
+
|
|
478
|
+
try:
|
|
479
|
+
num_files = int(num_files)
|
|
480
|
+
except ValueError:
|
|
481
|
+
print("\n❌ Error: Number of files must be an integer.")
|
|
482
|
+
continue
|
|
483
|
+
|
|
484
|
+
sources = []
|
|
485
|
+
for i in range(num_files):
|
|
486
|
+
source = input(f"Enter Source URL #{i+1}: ").strip()
|
|
487
|
+
if source:
|
|
488
|
+
sources.append(source)
|
|
489
|
+
|
|
490
|
+
if not all([doc_id, sources]):
|
|
491
|
+
print("\n❌ Error: Document ID and at least one Source are required.")
|
|
492
|
+
continue
|
|
493
|
+
|
|
494
|
+
# Process batch upload - creates ONE RUN for all files
|
|
495
|
+
result = process_knowledge_upload_batch(doc_id, sources)
|
|
496
|
+
|
|
497
|
+
if result:
|
|
498
|
+
print(f"\n✅ Batch upload processed! Run ID: {result['run_id']}")
|
|
499
|
+
print(f" Processing {result['total_files']} file(s)")
|
|
500
|
+
else:
|
|
501
|
+
print("\n❌ Upload failed. Check the errors above.")
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
if __name__ == "__main__":
|
|
505
|
+
main()
|