th2rag 0.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. th2rag-0.0.2/PKG-INFO +1002 -0
  2. th2rag-0.0.2/README.md +953 -0
  3. th2rag-0.0.2/pyproject.toml +214 -0
  4. th2rag-0.0.2/pyproject.toml.orig +218 -0
  5. th2rag-0.0.2/src/th2rag/__init__.py +0 -0
  6. th2rag-0.0.2/src/th2rag/auth/__init__.py +0 -0
  7. th2rag-0.0.2/src/th2rag/auth/exceptions.py +2 -0
  8. th2rag-0.0.2/src/th2rag/auth/router.py +43 -0
  9. th2rag-0.0.2/src/th2rag/auth/schemas.py +15 -0
  10. th2rag-0.0.2/src/th2rag/auth/service.py +115 -0
  11. th2rag-0.0.2/src/th2rag/clients/__init__.py +0 -0
  12. th2rag-0.0.2/src/th2rag/clients/storage/config.py +13 -0
  13. th2rag-0.0.2/src/th2rag/clients/storage/local.py +48 -0
  14. th2rag-0.0.2/src/th2rag/clients/storage/s3.py +93 -0
  15. th2rag-0.0.2/src/th2rag/config.py +267 -0
  16. th2rag-0.0.2/src/th2rag/conversations/__init__.py +0 -0
  17. th2rag-0.0.2/src/th2rag/conversations/dependencies.py +30 -0
  18. th2rag-0.0.2/src/th2rag/conversations/exceptions.py +6 -0
  19. th2rag-0.0.2/src/th2rag/conversations/router.py +47 -0
  20. th2rag-0.0.2/src/th2rag/conversations/schemas.py +43 -0
  21. th2rag-0.0.2/src/th2rag/conversations/service.py +51 -0
  22. th2rag-0.0.2/src/th2rag/database.py +72 -0
  23. th2rag-0.0.2/src/th2rag/dataset/__init__.py +0 -0
  24. th2rag-0.0.2/src/th2rag/dataset/dependencies.py +25 -0
  25. th2rag-0.0.2/src/th2rag/dataset/exceptions.py +2 -0
  26. th2rag-0.0.2/src/th2rag/dataset/router.py +53 -0
  27. th2rag-0.0.2/src/th2rag/dataset/schemas.py +23 -0
  28. th2rag-0.0.2/src/th2rag/dataset/service.py +78 -0
  29. th2rag-0.0.2/src/th2rag/dependencies.py +143 -0
  30. th2rag-0.0.2/src/th2rag/knowledge/__init__.py +0 -0
  31. th2rag-0.0.2/src/th2rag/knowledge/dependencies.py +25 -0
  32. th2rag-0.0.2/src/th2rag/knowledge/exceptions.py +2 -0
  33. th2rag-0.0.2/src/th2rag/knowledge/mage_client.py +505 -0
  34. th2rag-0.0.2/src/th2rag/knowledge/router.py +714 -0
  35. th2rag-0.0.2/src/th2rag/knowledge/schemas.py +108 -0
  36. th2rag-0.0.2/src/th2rag/knowledge/service.py +392 -0
  37. th2rag-0.0.2/src/th2rag/logger.py +21 -0
  38. th2rag-0.0.2/src/th2rag/main.py +51 -0
  39. th2rag-0.0.2/src/th2rag/messages/__init__.py +0 -0
  40. th2rag-0.0.2/src/th2rag/messages/exceptions.py +2 -0
  41. th2rag-0.0.2/src/th2rag/messages/router.py +63 -0
  42. th2rag-0.0.2/src/th2rag/messages/schemas.py +68 -0
  43. th2rag-0.0.2/src/th2rag/messages/service.py +210 -0
  44. th2rag-0.0.2/src/th2rag/models.py +215 -0
  45. th2rag-0.0.2/src/th2rag/pagination.py +68 -0
  46. th2rag-0.0.2/src/th2rag/rag/__init__.py +0 -0
  47. th2rag-0.0.2/src/th2rag/rag/chunkers/__init__.py +0 -0
  48. th2rag-0.0.2/src/th2rag/rag/chunkers/hybrid_chunker.py +23 -0
  49. th2rag-0.0.2/src/th2rag/rag/constants.py +15 -0
  50. th2rag-0.0.2/src/th2rag/rag/converters/__init__.py +0 -0
  51. th2rag-0.0.2/src/th2rag/rag/converters/documents_converter.py +123 -0
  52. th2rag-0.0.2/src/th2rag/rag/converters/json_converter.py +521 -0
  53. th2rag-0.0.2/src/th2rag/rag/converters/legacy_converter.py +124 -0
  54. th2rag-0.0.2/src/th2rag/rag/converters/pdf_converter.py +384 -0
  55. th2rag-0.0.2/src/th2rag/rag/converters/spreadsheet_converter.py +159 -0
  56. th2rag-0.0.2/src/th2rag/rag/converters/txt_converter.py +98 -0
  57. th2rag-0.0.2/src/th2rag/rag/embeddings/__init__.py +0 -0
  58. th2rag-0.0.2/src/th2rag/rag/embeddings/base.py +21 -0
  59. th2rag-0.0.2/src/th2rag/rag/embeddings/exceptions.py +2 -0
  60. th2rag-0.0.2/src/th2rag/rag/embeddings/sentence_transformer.py +97 -0
  61. th2rag-0.0.2/src/th2rag/rag/generation/base.py +22 -0
  62. th2rag-0.0.2/src/th2rag/rag/generation/exceptions.py +2 -0
  63. th2rag-0.0.2/src/th2rag/rag/generation/litellm_generator.py +143 -0
  64. th2rag-0.0.2/src/th2rag/rag/generation/mistral_generator.py +269 -0
  65. th2rag-0.0.2/src/th2rag/rag/generation/test_litellm.py +33 -0
  66. th2rag-0.0.2/src/th2rag/rag/retrieval/__init__.py +0 -0
  67. th2rag-0.0.2/src/th2rag/rag/retrieval/base.py +11 -0
  68. th2rag-0.0.2/src/th2rag/rag/retrieval/exceptions.py +10 -0
  69. th2rag-0.0.2/src/th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
  70. th2rag-0.0.2/src/th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
  71. th2rag-0.0.2/src/th2rag/rag/retrieval/lancedb/schemas.py +19 -0
  72. th2rag-0.0.2/src/th2rag/rag/services/__init__.py +0 -0
  73. th2rag-0.0.2/src/th2rag/rag/services/rag_service.py +99 -0
  74. th2rag-0.0.2/src/th2rag/rag/services/rag_service_builder.py +42 -0
  75. th2rag-0.0.2/src/th2rag/rag/tasks/process_pdf.py +586 -0
  76. th2rag-0.0.2/src/th2rag/users/dependencies.py +44 -0
  77. th2rag-0.0.2/src/th2rag/users/exceptions.py +6 -0
  78. th2rag-0.0.2/src/th2rag/users/router.py +92 -0
  79. th2rag-0.0.2/src/th2rag/users/schemas.py +61 -0
  80. th2rag-0.0.2/src/th2rag/users/service.py +140 -0
  81. th2rag-0.0.2/src/th2rag/utils/__init__.py +0 -0
  82. th2rag-0.0.2/src/th2rag/utils/helpers.py +40 -0
  83. th2rag-0.0.2/src/th2rag/utils/logger.py +15 -0
  84. th2rag-0.0.2/src/th2rag/utils/security.py +76 -0
  85. th2rag-0.0.2/src/th2rag/utils/welcome_router.py +98 -0
  86. th2rag-0.0.2/src/th2rag/webhooks/__init__.py +0 -0
  87. th2rag-0.0.2/src/th2rag/webhooks/sender.py +126 -0
th2rag-0.0.2/PKG-INFO ADDED
@@ -0,0 +1,1002 @@
1
+ Metadata-Version: 2.3
2
+ Name: th2rag
3
+ Version: 0.0.2
4
+ Summary: Add your description here
5
+ Author: Imene Boukhelkhal, David Gnaglo, Farid Azouaou
6
+ Author-email: Imene Boukhelkhal <imene.boukhelkhal@thaink2.com>, David Gnaglo <david.gnaglo@thaink2.com>, Farid Azouaou <farid.azouaou@thaink2.com>
7
+ Requires-Dist: python-jose[cryptography]
8
+ Requires-Dist: fastapi>=0.115.0
9
+ Requires-Dist: uvicorn[standard]>=0.34.0
10
+ Requires-Dist: httpx>=0.28.0
11
+ Requires-Dist: python-multipart>=0.0.20
12
+ Requires-Dist: sqlalchemy>=1.4.40,<2.0.0
13
+ Requires-Dist: alembic>=1.15.0
14
+ Requires-Dist: psycopg2-binary>=2.9.10
15
+ Requires-Dist: asyncpg>=0.30.0
16
+ Requires-Dist: aiosqlite>=0.21.0
17
+ Requires-Dist: pyjwt>=2.10.0
18
+ Requires-Dist: passlib[bcrypt]>=1.7.4
19
+ Requires-Dist: cryptography>=44.0.0
20
+ Requires-Dist: python-dotenv>=1.0.0
21
+ Requires-Dist: pydantic>=2.10.0
22
+ Requires-Dist: pydantic-settings>=2.8.0
23
+ Requires-Dist: email-validator>=2.2.0
24
+ Requires-Dist: boto3>=1.37.0
25
+ Requires-Dist: lancedb==0.19
26
+ Requires-Dist: pyarrow==21.0.0
27
+ Requires-Dist: sentence-transformers>=3.4.0
28
+ Requires-Dist: transformers>=4.49.0
29
+ Requires-Dist: docling>=2.28.0
30
+ Requires-Dist: docling-core>=2.23.0
31
+ Requires-Dist: docling-ibm-models>=3.4.0
32
+ Requires-Dist: docling-parse>=4.0.0
33
+ Requires-Dist: pypdfium2>=4.30.0
34
+ Requires-Dist: pillow>=11.1.0
35
+ Requires-Dist: pandas>=2.2.0
36
+ Requires-Dist: python-docx>=1.1.0
37
+ Requires-Dist: python-pptx>=1.0.0
38
+ Requires-Dist: openpyxl>=3.1.0
39
+ Requires-Dist: xlsxwriter>=3.2.0
40
+ Requires-Dist: requests>=2.32.0
41
+ Requires-Dist: rich>=13.9.0
42
+ Requires-Dist: bcrypt==4.3.0
43
+ Requires-Dist: greenlet>=3.3.0
44
+ Requires-Dist: plotly>=6.5.1
45
+ Requires-Dist: litellm>=1.81.6
46
+ Requires-Dist: pypdf2>=3.0.1
47
+ Requires-Python: >=3.10
48
+ Description-Content-Type: text/markdown
49
+
50
+ # 🧠 RAG Backend API
51
+
52
+ A production-ready Retrieval-Augmented Generation (RAG) system built with FastAPI, featuring document processing, vector search, and conversational AI capabilities.
53
+
54
+ ## 📋 Table of Contents
55
+
56
+ - [Features](#-features)
57
+ - [Tech Stack](#-tech-stack)
58
+ - [Prerequisites](#-prerequisites)
59
+ - [Installation](#-installation)
60
+ - [Configuration](#-configuration)
61
+ - [Running the Application](#-running-the-application)
62
+ - [Testing](#-testing)
63
+ - [Development Workflow](#-development-workflow)
64
+ - [Project Structure](#-project-structure)
65
+ - [API Documentation](#-api-documentation)
66
+ - [ContrXuting](#-contrXuting)
67
+
68
+ ---
69
+
70
+ ## ✨ Features
71
+
72
+ - 🔐 JWT-based authentication with refresh tokens
73
+ - 📄 PDF document processing and vectorization
74
+ - 🗃️ Vector similarity search using LanceDB
75
+ - 💬 Conversational AI with context retrieval
76
+ - 📊 Dataset management and metadata tracking
77
+ - ☁️ S3-compatXle storage integration
78
+ - 🐳 Full Docker support with health checks
79
+ - 🧪 Comprehensive test suite
80
+
81
+ ---
82
+
83
+ ## 🛠️ Tech Stack
84
+
85
+ - **Language**: Python 3.12+
86
+ - **Framework**: FastAPI
87
+ - **Package Manager**: UV (modern Python package manager)
88
+ - **ORM**: SQLAlchemy 2.0
89
+ - **Database**: PostgreSQL
90
+ - **Vector Store**: LanceDB
91
+ - **Embeddings**: Sentence Transformers (gtr-t5-large)
92
+ - **LLM**: Mistral AI
93
+ - **Document Processing**: Docling
94
+ - **Storage**: S3-compatXle (Scaleway, AWS, etc.)
95
+ - **Containerization**: Docker + Docker Compose
96
+
97
+ ---
98
+
99
+ ## 📦 Prerequisites
100
+
101
+ ### System Requirements
102
+
103
+ #### Windows Users 🪟
104
+
105
+ ```powershell
106
+ # 1. Install Chocolatey (Package Manager) - Run as Administrator
107
+ Set-ExecutionPolicy Bypass -Scope Process -Force
108
+ [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
109
+ iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
110
+
111
+ # 2. Install Python 3.12
112
+ choco install python312 -y
113
+
114
+ # 3. Install PostgreSQL (for local development)
115
+ choco install postgresql15 -y
116
+
117
+ # 4. Install Visual C++ Build Tools (required for some Python packages)
118
+ choco install visualstudio2022buildtools -y
119
+
120
+ # 5. Restart your terminal
121
+ ```
122
+
123
+ #### macOS Users 🍎
124
+
125
+ ```bash
126
+ # 1. Install Homebrew (if not installed)
127
+ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
128
+
129
+ # 2. Install Python 3.12
130
+ brew install python@3.12
131
+
132
+ # 3. Install PostgreSQL (for local development)
133
+ brew install postgresql@15
134
+
135
+ # 4. Install OpenCV (required for document processing)
136
+ brew install opencv
137
+
138
+ # 5. Restart your terminal
139
+ ```
140
+
141
+ ### Install UV Package Manager
142
+
143
+ #### Windows 🪟
144
+
145
+ ```powershell
146
+ # Install UV using PowerShell (Run as Administrator)
147
+ powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
148
+
149
+ # Close and reopen your terminal, then verify
150
+ uv --version
151
+ ```
152
+
153
+ #### macOS/Linux 🍎🐧
154
+
155
+ ```bash
156
+ # Install UV
157
+ curl -LsSf https://astral.sh/uv/install.sh | sh
158
+
159
+ # Reload shell configuration
160
+ source ~/.zshrc # or ~/.bashrc for bash users
161
+
162
+ # Verify installation
163
+ uv --version
164
+ ```
165
+
166
+ ### Install Git (If Not Already Installed)
167
+
168
+ #### Windows 🪟
169
+
170
+ ```powershell
171
+ # Install Git
172
+ choco install git -y
173
+
174
+ # Configure Git (replace with your info)
175
+ git config --global user.name "Your Name"
176
+ git config --global user.email "your.email@company.com"
177
+ ```
178
+
179
+ #### macOS 🍎
180
+
181
+ ```bash
182
+ # Install Git
183
+ brew install git
184
+
185
+ # Configure Git (replace with your info)
186
+ git config --global user.name "Your Name"
187
+ git config --global user.email "your.email@company.com"
188
+ ```
189
+
190
+ ### Install Docker (Optional but Recommended)
191
+
192
+ - **Windows**: [Download Docker Desktop for Windows](https://www.docker.com/products/docker-desktop/)
193
+ - **macOS**: [Download Docker Desktop for Mac](https://www.docker.com/products/docker-desktop/)
194
+
195
+ ---
196
+
197
+ ## 🚀 Installation
198
+
199
+ ### 1. Clone the Repository
200
+
201
+ ```bash
202
+ # Clone the repository
203
+ git clone <repository-url>
204
+ cd rag-backend-api
205
+
206
+ # Switch to development branch (if applicable)
207
+ git checkout develop
208
+ ```
209
+
210
+ ### 2. Create Virtual Environment
211
+
212
+ #### Windows 🪟
213
+
214
+ ```powershell
215
+ # Create UV virtual environment with Python 3.12
216
+ uv venv --python 3.12
217
+
218
+ # Activate virtual environment
219
+ .venv\Scripts\activate
220
+
221
+ # Verify Python version
222
+ python --version # Should show Python 3.12.x
223
+ ```
224
+
225
+ #### macOS/Linux 🍎🐧
226
+
227
+ ```bash
228
+ # Create UV virtual environment with Python 3.12
229
+ uv venv --python 3.12
230
+
231
+ # Activate virtual environment
232
+ source .venv/bin/activate
233
+
234
+ # Verify Python version
235
+ python --version # Should show Python 3.12.x
236
+ ```
237
+
238
+ ### 3. Install Dependencies
239
+
240
+ ```bash
241
+ # Install all dependencies (including dev dependencies)
242
+ uv sync
243
+
244
+ # For production only (without dev dependencies)
245
+ uv sync --no-dev
246
+ ```
247
+
248
+ #### Platform-Specific Troubleshooting
249
+
250
+ **Windows 🪟**
251
+
252
+ ```powershell
253
+ # If psycopg2-binary fails
254
+ # Make sure PostgreSQL is installed, then:
255
+ $env:PATH += ";C:\Program Files\PostgreSQL\15\bin"
256
+ uv pip install psycopg2-binary
257
+
258
+ # If Visual C++ errors occur
259
+ # Install Visual Studio Build Tools from:
260
+ # https://visualstudio.microsoft.com/visual-cpp-build-tools/
261
+ ```
262
+
263
+ **macOS ARM (M1/M2/M3) 🍎**
264
+
265
+ ```bash
266
+ # If PyTorch installation fails
267
+ uv pip install torch==2.6.0 torchvision==0.21.0
268
+
269
+ # If psycopg2 fails
270
+ brew install postgresql@15
271
+ export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
272
+ uv pip install psycopg2-binary
273
+ ```
274
+
275
+ ---
276
+
277
+ ## ⚙️ Configuration
278
+
279
+ ### 1. Create Environment File
280
+
281
+ #### Windows 🪟
282
+
283
+ ```powershell
284
+ # Copy example configuration
285
+ Copy-Item .env.example .env
286
+
287
+ # Edit with notepad
288
+ notepad .env
289
+ ```
290
+
291
+ #### macOS/Linux 🍎🐧
292
+
293
+ ```bash
294
+ # Copy example configuration
295
+ cp .env.example .env
296
+
297
+ # Edit with your preferred editor
298
+ nano .env # or vim, code, etc.
299
+ ```
300
+
301
+ ### 2. Configure Environment Variables
302
+
303
+ Update the `.env` file with your actual values:
304
+
305
+ ```env
306
+ # Application Mode
307
+ WORKING_MODE=development
308
+
309
+ # Database Configuration
310
+ DB_HOST=localhost
311
+ DB_PORT=5432
312
+ DB_NAME=rag_db
313
+ DB_USER=your_db_user
314
+ DB_PASSWORD=your_secure_password
315
+ DB_SCHEMA=public
316
+
317
+ # S3 Storage Configuration
318
+ S3_REGION=eu-west-3
319
+ S3_ACCESS_KEY=your_access_key
320
+ S3_ACCESS_KEY_SECRET=your_secret_key
321
+ S3_ENDPOINT=https://s3.eu-west-3.amazonaws.com
322
+ S3_BUCKET_NAME=your_bucket_name
323
+
324
+ # Mistral AI API
325
+ MISTRAL_API_KEY=your_mistral_api_key
326
+ MISTRAL_API_URL=https://api.mistral.ai/v1
327
+
328
+ # JWT Authentication
329
+ SECRET_KEY=your-super-secret-key-minimum-32-characters
330
+ ACCESS_TOKEN_EXPIRE_MINUTES=60
331
+ ALGORITHM=HS256
332
+
333
+ # Password Encryption
334
+ ENCRYPT_KEY=your-encryption-key
335
+
336
+ # Development Settings (optional)
337
+ ECHO_SQL=false
338
+ ```
339
+
340
+ > ⚠️ **Security Warning**: Never commit `.env` files to version control!
341
+
342
+ ### 3. Initialize Database
343
+
344
+ ```bash
345
+ # Run database migrations
346
+ alembic upgrade head
347
+ ```
348
+
349
+ ---
350
+
351
+ ## 🔐 Database Access (Important)
352
+
353
+ The PostgreSQL database is **IP-whitelisted** for security reasons.
354
+
355
+ ➡️ **Before running the application**, you must ask the project owner/admin to add your public IP address to the database whitelist.
356
+
357
+ ⚠️ If your IP is not whitelisted, the database connection will fail.
358
+
359
+ ---
360
+
361
+ ### 📡 How to Get Your Public IP Address
362
+
363
+ #### macOS / Linux 🍎🐧
364
+ ```bash
365
+ curl ifconfig.me
366
+ ```
367
+ or
368
+ ```bash
369
+ curl https://api.ipify.org
370
+ ```
371
+
372
+ #### Windows 🪟 (PowerShell)
373
+ ```powershell
374
+ curl ifconfig.me
375
+ ```
376
+ or
377
+ ```powershell
378
+ (Invoke-WebRequest -uri "https://api.ipify.org").Content
379
+ ```
380
+
381
+ ---
382
+
383
+ ### 📌 Next Steps
384
+ Send the resulting IP address to the administrator so it can be approved and added to the whitelist.
385
+
386
+ > ⚠️ **Note:** If you are on a dynamic network (VPN, mobile hotspot, etc.), your IP may change and need to be re-approved.
387
+
388
+ ## 🎯 Running the Application
389
+
390
+ ### Option 1: Local Development (Recommended) ⚡
391
+
392
+ #### Windows 🪟
393
+
394
+ ```powershell
395
+ # Activate virtual environment
396
+ .venv\Scripts\activate
397
+
398
+ # Start the API server with auto-reload
399
+ uv run uvicorn th2rag.main:app --reload --host 0.0.0.0 --port 8000
400
+ ```
401
+
402
+ #### macOS/Linux 🍎🐧
403
+
404
+ ```bash
405
+ # Activate virtual environment
406
+ source .venv/bin/activate
407
+
408
+ # Start the API server with auto-reload
409
+ uv run uvicorn th2rag.main:app --reload --host 0.0.0.0 --port 8000
410
+ ```
411
+
412
+ **Access Points:**
413
+ - 📚 Swagger UI: http://localhost:8000/docs
414
+ - 📖 ReDoc: http://localhost:8000/redoc
415
+ - ❤️ Health Check: http://localhost:8000/health
416
+
417
+ ### Option 2: Docker (Production-like) 🐳
418
+
419
+ ```bash
420
+ # Start all services
421
+ docker-compose up -d
422
+
423
+ # View logs
424
+ docker-compose logs -f api
425
+
426
+ # Stop services
427
+ docker-compose down
428
+ ```
429
+
430
+ #### Updating Environment Variables in Docker
431
+
432
+ ```bash
433
+ # Important: Restart does NOT reload .env!
434
+ # You must stop and recreate containers:
435
+
436
+ docker-compose down
437
+ docker-compose up -d
438
+ docker-compose logs -f api
439
+ ```
440
+
441
+ ---
442
+
443
+ ## 🧪 Testing
444
+
445
+ ### Run Tests Locally
446
+
447
+ #### Windows 🪟
448
+
449
+ ```powershell
450
+ # Activate virtual environment
451
+ .venv\Scripts\activate
452
+
453
+ # Run all tests
454
+ uv run pytest
455
+
456
+ # Run with verbose output
457
+ uv run pytest -v
458
+
459
+ # Run specific test file
460
+ uv run pytest test\test_auth.py
461
+
462
+ # Run with coverage
463
+ uv run pytest --cov=src --cov-report=html
464
+ uv run pytest --cov=src --cov-report=term-missing
465
+
466
+ # Open coverage report
467
+ start htmlcov\index.html
468
+ ```
469
+
470
+ #### macOS/Linux 🍎🐧
471
+
472
+ ```bash
473
+ # Activate virtual environment
474
+ source .venv/bin/activate
475
+
476
+ # Run all tests
477
+ uv run pytest
478
+
479
+ # Run with verbose output
480
+ uv run pytest -v
481
+
482
+ # Run specific test file
483
+ uv run pytest test/test_auth.py
484
+
485
+ # Run with coverage
486
+ uv run pytest --cov=src --cov-report=html
487
+ uv run pytest --cov=src --cov-report=term-missing
488
+
489
+ # Open coverage report
490
+ open htmlcov/index.html
491
+ ```
492
+
493
+ ### Run Tests in Docker
494
+
495
+ ```bash
496
+ # Ensure containers are running
497
+ docker-compose up -d
498
+
499
+ # Run tests inside container
500
+ docker-compose exec api pytest -v
501
+ ```
502
+
503
+ ---
504
+
505
+ ## 🔄 Development Workflow
506
+
507
+ ### Branch Strategy
508
+
509
+ This project follows a Git workflow with feature branches:
510
+
511
+ ```bash
512
+ # 1. Create a new feature branch from main/develop
513
+ git checkout -b feature/your-feature-name
514
+
515
+ # 2. Make your changes and commit regularly
516
+ git add .
517
+ git commit -m "feat: description of your changes"
518
+
519
+ # 3. Push to remote
520
+ git push origin feature/your-feature-name
521
+
522
+ # 4. Create a Pull Request on GitHub
523
+ ```
524
+
525
+ ### Pre-commit Setup (Code Quality Checks)
526
+
527
+ Pre-commit hooks automatically check your code before each commit.
528
+
529
+ #### 1. Install Pre-commit
530
+
531
+ **Windows 🪟**
532
+ ```powershell
533
+ uv pip install pre-commit
534
+ ```
535
+
536
+ **macOS/Linux 🍎🐧**
537
+ ```bash
538
+ uv pip install pre-commit
539
+ ```
540
+
541
+ #### 2. Install Git Hooks
542
+
543
+ ```bash
544
+ # Install the pre-commit hooks
545
+ pre-commit install
546
+
547
+ # (Optional) Run against all files to test
548
+ pre-commit run --all-files
549
+ ```
550
+
551
+ #### 3. Create Pre-commit Configuration
552
+
553
+ Create `.pre-commit-config.yaml` in your project root:
554
+
555
+ ```yaml
556
+ repos:
557
+ - repo: https://github.com/pre-commit/pre-commit-hooks
558
+ rev: v4.5.0
559
+ hooks:
560
+ - id: trailing-whitespace
561
+ - id: end-of-file-fixer
562
+ - id: check-yaml
563
+ - id: check-added-large-files
564
+ args: ['--maxkb=1000']
565
+ - id: check-json
566
+ - id: check-merge-conflict
567
+ - id: detect-private-key
568
+
569
+ - repo: https://github.com/psf/black
570
+ rev: 23.12.1
571
+ hooks:
572
+ - id: black
573
+ language_version: python3.12
574
+
575
+ - repo: https://github.com/astral-sh/ruff-pre-commit
576
+ rev: v0.1.9
577
+ hooks:
578
+ - id: ruff
579
+ args: [--fix, --exit-non-zero-on-fix]
580
+
581
+ - repo: https://github.com/pre-commit/mirrors-mypy
582
+ rev: v1.8.0
583
+ hooks:
584
+ - id: mypy
585
+ additional_dependencies: [types-all]
586
+ args: [--ignore-missing-imports]
587
+ ```
588
+
589
+ ### Commit Message Convention
590
+
591
+ Follow conventional commits for clear history:
592
+
593
+ ```bash
594
+ # Format: <type>(<scope>): <subject>
595
+
596
+ git commit -m "feat(auth): add refresh token endpoint"
597
+ git commit -m "fix(database): resolve connection pool issue"
598
+ git commit -m "docs(readme): update installation instructions"
599
+ git commit -m "test(auth): add JWT validation tests"
600
+ git commit -m "chore(deps): update dependencies"
601
+ ```
602
+
603
+ **Commit Types:**
604
+ - `feat`: New feature
605
+ - `fix`: Bug fix
606
+ - `docs`: Documentation changes
607
+ - `style`: Code style changes (formatting)
608
+ - `refactor`: Code refactoring
609
+ - `test`: Adding or updating tests
610
+ - `chore`: Maintenance tasks
611
+
612
+ ### Pushing Your X Branch
613
+
614
+ #### Step-by-Step Guide
615
+
616
+ **1. Check your current status**
617
+
618
+ ```bash
619
+ # See what files have changed
620
+ git status
621
+
622
+ # See the actual changes
623
+ git diff
624
+ ```
625
+
626
+ **2. Stage your changes**
627
+
628
+ ```bash
629
+ # Stage specific files
630
+ git add pyproject.toml
631
+ git add .pre-commit-config.yaml
632
+ git add docker-compose.yml
633
+ git add README.md
634
+ git add .env.example
635
+
636
+ # OR stage all changes
637
+ git add .
638
+ ```
639
+
640
+ **3. Commit your changes**
641
+
642
+ ```bash
643
+ # Commit with a descriptive message
644
+ git commit -m "feat(migration): migrate to UV package manager and update documentation"
645
+ ```
646
+
647
+ **4. Create and push your branch**
648
+
649
+ **Windows 🪟**
650
+ ```powershell
651
+ # Create and switch to X branch
652
+ git checkout -b X
653
+
654
+ # Push to remote repository
655
+ git push -u origin X
656
+ ```
657
+
658
+ **macOS/Linux 🍎🐧**
659
+ ```bash
660
+ # Create and switch to X branch
661
+ git checkout -b X
662
+
663
+ # Push to remote repository
664
+ git push -u origin X
665
+ ```
666
+
667
+ **5. Create a Pull Request**
668
+
669
+ - Go to your GitHub repository
670
+ - Click "Compare & pull request" button
671
+ - Select base branch (usually `main` or `develop`)
672
+ - Add description of your changes
673
+ - Assign reviewers from your team
674
+ - Submit the pull request
675
+
676
+ ### Daily Workflow Example
677
+
678
+ ```bash
679
+ # 1. Start of day - update your branch
680
+ git checkout X
681
+ git pull origin X
682
+
683
+ # 2. Make changes to files
684
+ # ... edit code ...
685
+
686
+ # 3. Check what changed
687
+ git status
688
+ git diff
689
+
690
+ # 4. Run tests before committing
691
+ uv run pytest
692
+
693
+ # 5. Pre-commit will automatically run when you commit
694
+ git add .
695
+ git commit -m "feat(rag): improve document chunking algorithm"
696
+
697
+ # 6. Push changes
698
+ git push origin X
699
+
700
+ # 7. If you need to sync with main branch
701
+ git checkout main
702
+ git pull origin main
703
+ git checkout X
704
+ git merge main
705
+ # Resolve any conflicts if they occur
706
+ git push origin X
707
+ ```
708
+
709
+ ### Handling Pre-commit Failures
710
+
711
+ If pre-commit hooks fail:
712
+
713
+ ```bash
714
+ # Pre-commit will show what failed
715
+ # Fix the issues, then:
716
+
717
+ git add .
718
+ git commit -m "your message"
719
+
720
+ # If automatic fixes were applied, stage them:
721
+ git add .
722
+ git commit --amend --no-edit
723
+ ```
724
+
725
+ ---
726
+
727
+ ## 📁 Project Structure
728
+
729
+ ```
730
+ .
731
+ ├── .github/ # GitHub workflows and templates
732
+ ├── alembic/ # Database migrations
733
+ │ ├── versions/ # Migration scripts
734
+ │ └── env.py # Alembic environment
735
+ ├── data/ # LanceDB vector storage
736
+ ├── src/ # Main application source
737
+ │ ├── auth/ # JWT authentication
738
+ │ ├── clients/ # S3 client integration
739
+ │ ├── conversations/ # Conversation management
740
+ │ ├── dataset/ # Dataset handling
741
+ │ ├── knowledge/ # Document management
742
+ │ ├── messages/ # Message operations
743
+ │ ├── rag/ # RAG pipeline
744
+ │ │ ├── chunkers/ # Document chunking
745
+ │ │ ├── converters/ # Format conversion
746
+ │ │ ├── embeddings/ # Vector embeddings
747
+ │ │ ├── generation/ # LLM generation
748
+ │ │ ├── retrieval/ # Vector search
749
+ │ │ ├── services/ # RAG orchestration
750
+ │ │ └── tasks/ # Background jobs
751
+ │ ├── users/ # User management
752
+ │ ├── utils/ # Utilities
753
+ │ └── main.py # Application entry
754
+ ├── test/ # Test suite
755
+ ├── .env.example # Environment template
756
+ ├── .gitignore # Git ignore rules
757
+ ├── .pre-commit-config.yaml # Pre-commit hooks
758
+ ├── alembic.ini # Alembic config
759
+ ├── docker-compose.yml # Docker services
760
+ ├── Dockerfile # Container definition
761
+ ├── pyproject.toml # Project dependencies
762
+ └── README.md # This file
763
+ ```
764
+
765
+ ---
766
+
767
+ ## 📚 API Documentation
768
+
769
+ ### Authentication
770
+ - `POST /auth/login` - Login and get tokens
771
+ - `POST /auth/refresh-token` - Refresh access token
772
+
773
+ ### Users
774
+ - `GET /users` - List users (admin)
775
+ - `POST /users` - Create user
776
+ - `GET /users/me` - Get current user
777
+ - `GET /users/{id}` - Get user by ID
778
+ - `DELETE /users/{id}` - Delete user
779
+
780
+ ### Conversations
781
+ - `GET /conversations` - List conversations
782
+ - `POST /conversations` - Create conversation
783
+ - `GET /conversations/{id}` - Get conversation details
784
+ - `DELETE /conversations/{id}` - Delete conversation
785
+
786
+ ### Messages
787
+ - `GET /conversations/{id}/messages` - List messages
788
+ - `POST /conversations/{id}/messages` - Send message and get AI response
789
+ - `POST /conversations/{id}/messages/{message_id}/feedback` - Add feedback
790
+
791
+ ### Knowledge Documents
792
+ - `GET /knowledge` - List documents
793
+ - `POST /knowledge` - Upload PDF document
794
+ - `GET /knowledge/{id}` - Get document details
795
+ - `DELETE /knowledge/{id}` - Delete document and vectors
796
+
797
+ ### Datasets
798
+ - `GET /datasets` - List datasets
799
+ - `POST /datasets` - Create dataset
800
+ - `PUT /datasets/{id}` - Update dataset
801
+ - `DELETE /datasets/{id}` - Delete dataset
802
+
803
+ For detailed API schemas, visit: http://localhost:8000/docs
804
+
805
+ ---
806
+
807
+ ## 🧠 RAG Engine
808
+
809
+ The `rag/` module is responsible for the core Retrieval-Augmented Generation (RAG) pipeline. It orchestrates document conversion, chunking, embedding, retrieval, and final answer generation.
810
+
811
+ ### 📄 1. Document Conversion
812
+
813
+ * **`converters/pdf_converter.py`**: Converts PDF files (local or S3) to an internal `Document` format using `docling`.
814
+
815
+ ```python
816
+ convert_pdf(source: str) -> dict
817
+ ```
818
+
819
+ ### 🛫 2. Chunking
820
+
821
+ * **`chunkers/hybrid_chunker.py`**: Wraps `docling`'s `HybridChunker` to split documents into semantic chunks.
822
+
823
+ ```python
824
+ HybridTextChunker.chunk_document(dl_doc) -> list
825
+ ```
826
+
827
+ ### 🖐️ 3. Embeddings
828
+
829
+ * **`embeddings/sentence_transformer.py`**: Uses `sentence-transformers/gtr-t5-large` to convert text chunks into dense vectors.
830
+
831
+ ```python
832
+ SentenceTransformerEmbedding.encode(text: str) -> List[float]
833
+ ```
834
+
835
+ ### 🗆️ 4. Vector Store (LanceDB)
836
+
837
+ * **`retrieval/lancedb/lancedb_storage.py`**: Manages chunk storage/retrieval in LanceDB.
838
+ * **`retrieval/lancedb/schemas.py`**: Pydantic schema definitions for LanceDB.
839
+
840
+ ```python
841
+ LanceDBStorage.add_chunks(chunks)
842
+ LanceDBStorage.query(query_vector, doc_id, limit)
843
+ ```
844
+
845
+ ### 🔍 5. Retrieval
846
+
847
+ * **`retrieval/lancedb/lancedb_retriever.py`**: Retrieves similar document chunks using LanceDB + embeddings.
848
+
849
+ ```python
850
+ LanceDBRetriever.retrieve(query: str, doc_id: int, limit: int) -> List[Dict]
851
+ ```
852
+
853
+ ### 📏 6. Generation
854
+
855
+ * **`generation/mistral_generator.py`**: Calls Mistral API to generate responses using retrieved context.
856
+
857
+ ```python
858
+ MistralGenerator.generate(question, retrieved_context, prompt, history) -> str
859
+ ```
860
+
861
+ ### ⚖️ 7. RAG Service
862
+
863
+ * **`services/rag_service.py`**: Orchestrates the full RAG pipeline.
864
+ * **`services/rag_service_builder.py`**: Builds the RAG service with configurable components.
865
+
866
+ ```python
867
+ RAGService.answer_question(question, doc_id, limit, prompt, history)
868
+ ```
869
+
870
+ ### ⚙️ 8. Background Processing
871
+
872
+ * **`tasks/process_pdf.py`**: Asynchronously handles PDF-to-vector pipeline including download, convert, chunk, embed, and store.
873
+
874
+ ```python
875
+ process_pdf_background(doc_id, source)
876
+ ```
877
+
878
+ ---
879
+
880
+ ## 🤝 ContrXuting
881
+
882
+ ### Code Review Checklist
883
+
884
+ Before submitting a PR, ensure:
885
+
886
+ - ✅ All tests pass (`uv run pytest`)
887
+ - ✅ Pre-commit hooks pass
888
+ - ✅ Code is documented with docstrings
889
+ - ✅ New features have tests
890
+ - ✅ README updated if needed
891
+ - ✅ Commit messages follow convention
892
+ - ✅ No sensitive data in commits
893
+
894
+ ### Adding New Features
895
+
896
+ **1. Database Changes**
897
+ ```bash
898
+ # Edit src/models.py
899
+ # Generate migration
900
+ alembic revision --autogenerate -m "add new feature"
901
+ # Review and apply
902
+ alembic upgrade head
903
+ ```
904
+
905
+ **2. New Embedding Model**
906
+ ```python
907
+ # src/rag/embeddings/custom_embedding.py
908
+ from th2rag.rag.embeddings.base import BaseEmbedding
909
+
910
+ class CustomEmbedding(BaseEmbedding):
911
+ def encode(self, text: str) -> List[float]:
912
+ # Implementation
913
+ pass
914
+ ```
915
+
916
+ **3. New LLM Generator**
917
+ ```python
918
+ # src/rag/generation/custom_generator.py
919
+ from th2rag.rag.generation.base import BaseGenerator
920
+
921
+ class CustomGenerator(BaseGenerator):
922
+ def generate(self, question: str, context: str) -> str:
923
+ # Implementation
924
+ pass
925
+ ```
926
+
927
+ ---
928
+
929
+ ## 🆘 Troubleshooting
930
+
931
+ ### Windows-Specific Issues 🪟
932
+
933
+ **Port already in use:**
934
+ ```powershell
935
+ # Find process
936
+ netstat -ano | findstr :8000
937
+ # Kill process (replace PID)
938
+ taskkill /PID <PID> /F
939
+ ```
940
+
941
+ **Python not found:**
942
+ ```powershell
943
+ # Add Python to PATH
944
+ $env:Path += ";C:\Python312;C:\Python312\Scripts"
945
+ ```
946
+
947
+ **Permission errors:**
948
+ ```powershell
949
+ # Run PowerShell as Administrator
950
+ # Or adjust execution policy
951
+ Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
952
+ ```
953
+
954
+ ### macOS-Specific Issues 🍎
955
+
956
+ **Port already in use:**
957
+ ```bash
958
+ # Find and kill process
959
+ lsof -ti:8000 | xargs kill -9
960
+ ```
961
+
962
+ **SSL certificate errors:**
963
+ ```bash
964
+ # Update certificates
965
+ pip install --upgrade certifi
966
+ ```
967
+
968
+ ### Common Issues (All Platforms)
969
+
970
+ **Database connection failed:**
971
+ - Verify PostgreSQL is running
972
+ - Check credentials in `.env`
973
+ - Ensure database exists
974
+
975
+ **Docker container won't start:**
976
+ ```bash
977
+ docker-compose logs api
978
+ docker-compose down -v
979
+ docker-compose up -d --build
980
+ ```
981
+
982
+ **Pre-commit hooks fail:**
983
+ ```bash
984
+ # Update hooks
985
+ pre-commit autoupdate
986
+ pre-commit run --all-files
987
+ ```
988
+
989
+ ---
990
+
991
+ ## 📞 Support
992
+
993
+ - 📧 Email: your-team@company.com
994
+ - 💬 Slack: #rag-backend-support
995
+ - 🐛 Issues: [GitHub Issues](link-to-issues)
996
+ - 📖 Wiki: [Project Wiki](link-to-wiki)
997
+
998
+ ---
999
+
1000
+ ## 📄 License
1001
+
1002
+ [Specify your license here]