th2rag 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- th2rag-0.0.2/PKG-INFO +1002 -0
- th2rag-0.0.2/README.md +953 -0
- th2rag-0.0.2/pyproject.toml +214 -0
- th2rag-0.0.2/pyproject.toml.orig +218 -0
- th2rag-0.0.2/src/th2rag/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/auth/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/auth/exceptions.py +2 -0
- th2rag-0.0.2/src/th2rag/auth/router.py +43 -0
- th2rag-0.0.2/src/th2rag/auth/schemas.py +15 -0
- th2rag-0.0.2/src/th2rag/auth/service.py +115 -0
- th2rag-0.0.2/src/th2rag/clients/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/clients/storage/config.py +13 -0
- th2rag-0.0.2/src/th2rag/clients/storage/local.py +48 -0
- th2rag-0.0.2/src/th2rag/clients/storage/s3.py +93 -0
- th2rag-0.0.2/src/th2rag/config.py +267 -0
- th2rag-0.0.2/src/th2rag/conversations/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/conversations/dependencies.py +30 -0
- th2rag-0.0.2/src/th2rag/conversations/exceptions.py +6 -0
- th2rag-0.0.2/src/th2rag/conversations/router.py +47 -0
- th2rag-0.0.2/src/th2rag/conversations/schemas.py +43 -0
- th2rag-0.0.2/src/th2rag/conversations/service.py +51 -0
- th2rag-0.0.2/src/th2rag/database.py +72 -0
- th2rag-0.0.2/src/th2rag/dataset/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/dataset/dependencies.py +25 -0
- th2rag-0.0.2/src/th2rag/dataset/exceptions.py +2 -0
- th2rag-0.0.2/src/th2rag/dataset/router.py +53 -0
- th2rag-0.0.2/src/th2rag/dataset/schemas.py +23 -0
- th2rag-0.0.2/src/th2rag/dataset/service.py +78 -0
- th2rag-0.0.2/src/th2rag/dependencies.py +143 -0
- th2rag-0.0.2/src/th2rag/knowledge/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/knowledge/dependencies.py +25 -0
- th2rag-0.0.2/src/th2rag/knowledge/exceptions.py +2 -0
- th2rag-0.0.2/src/th2rag/knowledge/mage_client.py +505 -0
- th2rag-0.0.2/src/th2rag/knowledge/router.py +714 -0
- th2rag-0.0.2/src/th2rag/knowledge/schemas.py +108 -0
- th2rag-0.0.2/src/th2rag/knowledge/service.py +392 -0
- th2rag-0.0.2/src/th2rag/logger.py +21 -0
- th2rag-0.0.2/src/th2rag/main.py +51 -0
- th2rag-0.0.2/src/th2rag/messages/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/messages/exceptions.py +2 -0
- th2rag-0.0.2/src/th2rag/messages/router.py +63 -0
- th2rag-0.0.2/src/th2rag/messages/schemas.py +68 -0
- th2rag-0.0.2/src/th2rag/messages/service.py +210 -0
- th2rag-0.0.2/src/th2rag/models.py +215 -0
- th2rag-0.0.2/src/th2rag/pagination.py +68 -0
- th2rag-0.0.2/src/th2rag/rag/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/rag/chunkers/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/rag/chunkers/hybrid_chunker.py +23 -0
- th2rag-0.0.2/src/th2rag/rag/constants.py +15 -0
- th2rag-0.0.2/src/th2rag/rag/converters/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/rag/converters/documents_converter.py +123 -0
- th2rag-0.0.2/src/th2rag/rag/converters/json_converter.py +521 -0
- th2rag-0.0.2/src/th2rag/rag/converters/legacy_converter.py +124 -0
- th2rag-0.0.2/src/th2rag/rag/converters/pdf_converter.py +384 -0
- th2rag-0.0.2/src/th2rag/rag/converters/spreadsheet_converter.py +159 -0
- th2rag-0.0.2/src/th2rag/rag/converters/txt_converter.py +98 -0
- th2rag-0.0.2/src/th2rag/rag/embeddings/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/rag/embeddings/base.py +21 -0
- th2rag-0.0.2/src/th2rag/rag/embeddings/exceptions.py +2 -0
- th2rag-0.0.2/src/th2rag/rag/embeddings/sentence_transformer.py +97 -0
- th2rag-0.0.2/src/th2rag/rag/generation/base.py +22 -0
- th2rag-0.0.2/src/th2rag/rag/generation/exceptions.py +2 -0
- th2rag-0.0.2/src/th2rag/rag/generation/litellm_generator.py +143 -0
- th2rag-0.0.2/src/th2rag/rag/generation/mistral_generator.py +269 -0
- th2rag-0.0.2/src/th2rag/rag/generation/test_litellm.py +33 -0
- th2rag-0.0.2/src/th2rag/rag/retrieval/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/rag/retrieval/base.py +11 -0
- th2rag-0.0.2/src/th2rag/rag/retrieval/exceptions.py +10 -0
- th2rag-0.0.2/src/th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
- th2rag-0.0.2/src/th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
- th2rag-0.0.2/src/th2rag/rag/retrieval/lancedb/schemas.py +19 -0
- th2rag-0.0.2/src/th2rag/rag/services/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/rag/services/rag_service.py +99 -0
- th2rag-0.0.2/src/th2rag/rag/services/rag_service_builder.py +42 -0
- th2rag-0.0.2/src/th2rag/rag/tasks/process_pdf.py +586 -0
- th2rag-0.0.2/src/th2rag/users/dependencies.py +44 -0
- th2rag-0.0.2/src/th2rag/users/exceptions.py +6 -0
- th2rag-0.0.2/src/th2rag/users/router.py +92 -0
- th2rag-0.0.2/src/th2rag/users/schemas.py +61 -0
- th2rag-0.0.2/src/th2rag/users/service.py +140 -0
- th2rag-0.0.2/src/th2rag/utils/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/utils/helpers.py +40 -0
- th2rag-0.0.2/src/th2rag/utils/logger.py +15 -0
- th2rag-0.0.2/src/th2rag/utils/security.py +76 -0
- th2rag-0.0.2/src/th2rag/utils/welcome_router.py +98 -0
- th2rag-0.0.2/src/th2rag/webhooks/__init__.py +0 -0
- th2rag-0.0.2/src/th2rag/webhooks/sender.py +126 -0
th2rag-0.0.2/PKG-INFO
ADDED
|
@@ -0,0 +1,1002 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: th2rag
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Author: Imene Boukhelkhal, David Gnaglo, Farid Azouaou
|
|
6
|
+
Author-email: Imene Boukhelkhal <imene.boukhelkhal@thaink2.com>, David Gnaglo <david.gnaglo@thaink2.com>, Farid Azouaou <farid.azouaou@thaink2.com>
|
|
7
|
+
Requires-Dist: python-jose[cryptography]
|
|
8
|
+
Requires-Dist: fastapi>=0.115.0
|
|
9
|
+
Requires-Dist: uvicorn[standard]>=0.34.0
|
|
10
|
+
Requires-Dist: httpx>=0.28.0
|
|
11
|
+
Requires-Dist: python-multipart>=0.0.20
|
|
12
|
+
Requires-Dist: sqlalchemy>=1.4.40,<2.0.0
|
|
13
|
+
Requires-Dist: alembic>=1.15.0
|
|
14
|
+
Requires-Dist: psycopg2-binary>=2.9.10
|
|
15
|
+
Requires-Dist: asyncpg>=0.30.0
|
|
16
|
+
Requires-Dist: aiosqlite>=0.21.0
|
|
17
|
+
Requires-Dist: pyjwt>=2.10.0
|
|
18
|
+
Requires-Dist: passlib[bcrypt]>=1.7.4
|
|
19
|
+
Requires-Dist: cryptography>=44.0.0
|
|
20
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
21
|
+
Requires-Dist: pydantic>=2.10.0
|
|
22
|
+
Requires-Dist: pydantic-settings>=2.8.0
|
|
23
|
+
Requires-Dist: email-validator>=2.2.0
|
|
24
|
+
Requires-Dist: boto3>=1.37.0
|
|
25
|
+
Requires-Dist: lancedb==0.19
|
|
26
|
+
Requires-Dist: pyarrow==21.0.0
|
|
27
|
+
Requires-Dist: sentence-transformers>=3.4.0
|
|
28
|
+
Requires-Dist: transformers>=4.49.0
|
|
29
|
+
Requires-Dist: docling>=2.28.0
|
|
30
|
+
Requires-Dist: docling-core>=2.23.0
|
|
31
|
+
Requires-Dist: docling-ibm-models>=3.4.0
|
|
32
|
+
Requires-Dist: docling-parse>=4.0.0
|
|
33
|
+
Requires-Dist: pypdfium2>=4.30.0
|
|
34
|
+
Requires-Dist: pillow>=11.1.0
|
|
35
|
+
Requires-Dist: pandas>=2.2.0
|
|
36
|
+
Requires-Dist: python-docx>=1.1.0
|
|
37
|
+
Requires-Dist: python-pptx>=1.0.0
|
|
38
|
+
Requires-Dist: openpyxl>=3.1.0
|
|
39
|
+
Requires-Dist: xlsxwriter>=3.2.0
|
|
40
|
+
Requires-Dist: requests>=2.32.0
|
|
41
|
+
Requires-Dist: rich>=13.9.0
|
|
42
|
+
Requires-Dist: bcrypt==4.3.0
|
|
43
|
+
Requires-Dist: greenlet>=3.3.0
|
|
44
|
+
Requires-Dist: plotly>=6.5.1
|
|
45
|
+
Requires-Dist: litellm>=1.81.6
|
|
46
|
+
Requires-Dist: pypdf2>=3.0.1
|
|
47
|
+
Requires-Python: >=3.10
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
# 🧠 RAG Backend API
|
|
51
|
+
|
|
52
|
+
A production-ready Retrieval-Augmented Generation (RAG) system built with FastAPI, featuring document processing, vector search, and conversational AI capabilities.
|
|
53
|
+
|
|
54
|
+
## 📋 Table of Contents
|
|
55
|
+
|
|
56
|
+
- [Features](#-features)
|
|
57
|
+
- [Tech Stack](#-tech-stack)
|
|
58
|
+
- [Prerequisites](#-prerequisites)
|
|
59
|
+
- [Installation](#-installation)
|
|
60
|
+
- [Configuration](#-configuration)
|
|
61
|
+
- [Running the Application](#-running-the-application)
|
|
62
|
+
- [Testing](#-testing)
|
|
63
|
+
- [Development Workflow](#-development-workflow)
|
|
64
|
+
- [Project Structure](#-project-structure)
|
|
65
|
+
- [API Documentation](#-api-documentation)
|
|
66
|
+
- [ContrXuting](#-contrXuting)
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## ✨ Features
|
|
71
|
+
|
|
72
|
+
- 🔐 JWT-based authentication with refresh tokens
|
|
73
|
+
- 📄 PDF document processing and vectorization
|
|
74
|
+
- 🗃️ Vector similarity search using LanceDB
|
|
75
|
+
- 💬 Conversational AI with context retrieval
|
|
76
|
+
- 📊 Dataset management and metadata tracking
|
|
77
|
+
- ☁️ S3-compatXle storage integration
|
|
78
|
+
- 🐳 Full Docker support with health checks
|
|
79
|
+
- 🧪 Comprehensive test suite
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## 🛠️ Tech Stack
|
|
84
|
+
|
|
85
|
+
- **Language**: Python 3.12+
|
|
86
|
+
- **Framework**: FastAPI
|
|
87
|
+
- **Package Manager**: UV (modern Python package manager)
|
|
88
|
+
- **ORM**: SQLAlchemy 2.0
|
|
89
|
+
- **Database**: PostgreSQL
|
|
90
|
+
- **Vector Store**: LanceDB
|
|
91
|
+
- **Embeddings**: Sentence Transformers (gtr-t5-large)
|
|
92
|
+
- **LLM**: Mistral AI
|
|
93
|
+
- **Document Processing**: Docling
|
|
94
|
+
- **Storage**: S3-compatXle (Scaleway, AWS, etc.)
|
|
95
|
+
- **Containerization**: Docker + Docker Compose
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## 📦 Prerequisites
|
|
100
|
+
|
|
101
|
+
### System Requirements
|
|
102
|
+
|
|
103
|
+
#### Windows Users 🪟
|
|
104
|
+
|
|
105
|
+
```powershell
|
|
106
|
+
# 1. Install Chocolatey (Package Manager) - Run as Administrator
|
|
107
|
+
Set-ExecutionPolicy Bypass -Scope Process -Force
|
|
108
|
+
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
|
|
109
|
+
iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
|
|
110
|
+
|
|
111
|
+
# 2. Install Python 3.12
|
|
112
|
+
choco install python312 -y
|
|
113
|
+
|
|
114
|
+
# 3. Install PostgreSQL (for local development)
|
|
115
|
+
choco install postgresql15 -y
|
|
116
|
+
|
|
117
|
+
# 4. Install Visual C++ Build Tools (required for some Python packages)
|
|
118
|
+
choco install visualstudio2022buildtools -y
|
|
119
|
+
|
|
120
|
+
# 5. Restart your terminal
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
#### macOS Users 🍎
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
# 1. Install Homebrew (if not installed)
|
|
127
|
+
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
|
|
128
|
+
|
|
129
|
+
# 2. Install Python 3.12
|
|
130
|
+
brew install python@3.12
|
|
131
|
+
|
|
132
|
+
# 3. Install PostgreSQL (for local development)
|
|
133
|
+
brew install postgresql@15
|
|
134
|
+
|
|
135
|
+
# 4. Install OpenCV (required for document processing)
|
|
136
|
+
brew install opencv
|
|
137
|
+
|
|
138
|
+
# 5. Restart your terminal
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### Install UV Package Manager
|
|
142
|
+
|
|
143
|
+
#### Windows 🪟
|
|
144
|
+
|
|
145
|
+
```powershell
|
|
146
|
+
# Install UV using PowerShell (Run as Administrator)
|
|
147
|
+
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
|
148
|
+
|
|
149
|
+
# Close and reopen your terminal, then verify
|
|
150
|
+
uv --version
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
#### macOS/Linux 🍎🐧
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
# Install UV
|
|
157
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
158
|
+
|
|
159
|
+
# Reload shell configuration
|
|
160
|
+
source ~/.zshrc # or ~/.bashrc for bash users
|
|
161
|
+
|
|
162
|
+
# Verify installation
|
|
163
|
+
uv --version
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### Install Git (If Not Already Installed)
|
|
167
|
+
|
|
168
|
+
#### Windows 🪟
|
|
169
|
+
|
|
170
|
+
```powershell
|
|
171
|
+
# Install Git
|
|
172
|
+
choco install git -y
|
|
173
|
+
|
|
174
|
+
# Configure Git (replace with your info)
|
|
175
|
+
git config --global user.name "Your Name"
|
|
176
|
+
git config --global user.email "your.email@company.com"
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
#### macOS 🍎
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
# Install Git
|
|
183
|
+
brew install git
|
|
184
|
+
|
|
185
|
+
# Configure Git (replace with your info)
|
|
186
|
+
git config --global user.name "Your Name"
|
|
187
|
+
git config --global user.email "your.email@company.com"
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### Install Docker (Optional but Recommended)
|
|
191
|
+
|
|
192
|
+
- **Windows**: [Download Docker Desktop for Windows](https://www.docker.com/products/docker-desktop/)
|
|
193
|
+
- **macOS**: [Download Docker Desktop for Mac](https://www.docker.com/products/docker-desktop/)
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## 🚀 Installation
|
|
198
|
+
|
|
199
|
+
### 1. Clone the Repository
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
# Clone the repository
|
|
203
|
+
git clone <repository-url>
|
|
204
|
+
cd rag-backend-api
|
|
205
|
+
|
|
206
|
+
# Switch to development branch (if applicable)
|
|
207
|
+
git checkout develop
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
### 2. Create Virtual Environment
|
|
211
|
+
|
|
212
|
+
#### Windows 🪟
|
|
213
|
+
|
|
214
|
+
```powershell
|
|
215
|
+
# Create UV virtual environment with Python 3.12
|
|
216
|
+
uv venv --python 3.12
|
|
217
|
+
|
|
218
|
+
# Activate virtual environment
|
|
219
|
+
.venv\Scripts\activate
|
|
220
|
+
|
|
221
|
+
# Verify Python version
|
|
222
|
+
python --version # Should show Python 3.12.x
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
#### macOS/Linux 🍎🐧
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
# Create UV virtual environment with Python 3.12
|
|
229
|
+
uv venv --python 3.12
|
|
230
|
+
|
|
231
|
+
# Activate virtual environment
|
|
232
|
+
source .venv/bin/activate
|
|
233
|
+
|
|
234
|
+
# Verify Python version
|
|
235
|
+
python --version # Should show Python 3.12.x
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
### 3. Install Dependencies
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
# Install all dependencies (including dev dependencies)
|
|
242
|
+
uv sync
|
|
243
|
+
|
|
244
|
+
# For production only (without dev dependencies)
|
|
245
|
+
uv sync --no-dev
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
#### Platform-Specific Troubleshooting
|
|
249
|
+
|
|
250
|
+
**Windows 🪟**
|
|
251
|
+
|
|
252
|
+
```powershell
|
|
253
|
+
# If psycopg2-binary fails
|
|
254
|
+
# Make sure PostgreSQL is installed, then:
|
|
255
|
+
$env:PATH += ";C:\Program Files\PostgreSQL\15\bin"
|
|
256
|
+
uv pip install psycopg2-binary
|
|
257
|
+
|
|
258
|
+
# If Visual C++ errors occur
|
|
259
|
+
# Install Visual Studio Build Tools from:
|
|
260
|
+
# https://visualstudio.microsoft.com/visual-cpp-build-tools/
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
**macOS ARM (M1/M2/M3) 🍎**
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
# If PyTorch installation fails
|
|
267
|
+
uv pip install torch==2.6.0 torchvision==0.21.0
|
|
268
|
+
|
|
269
|
+
# If psycopg2 fails
|
|
270
|
+
brew install postgresql@15
|
|
271
|
+
export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
|
|
272
|
+
uv pip install psycopg2-binary
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
---
|
|
276
|
+
|
|
277
|
+
## ⚙️ Configuration
|
|
278
|
+
|
|
279
|
+
### 1. Create Environment File
|
|
280
|
+
|
|
281
|
+
#### Windows 🪟
|
|
282
|
+
|
|
283
|
+
```powershell
|
|
284
|
+
# Copy example configuration
|
|
285
|
+
Copy-Item .env.example .env
|
|
286
|
+
|
|
287
|
+
# Edit with notepad
|
|
288
|
+
notepad .env
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
#### macOS/Linux 🍎🐧
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
# Copy example configuration
|
|
295
|
+
cp .env.example .env
|
|
296
|
+
|
|
297
|
+
# Edit with your preferred editor
|
|
298
|
+
nano .env # or vim, code, etc.
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
### 2. Configure Environment Variables
|
|
302
|
+
|
|
303
|
+
Update the `.env` file with your actual values:
|
|
304
|
+
|
|
305
|
+
```env
|
|
306
|
+
# Application Mode
|
|
307
|
+
WORKING_MODE=development
|
|
308
|
+
|
|
309
|
+
# Database Configuration
|
|
310
|
+
DB_HOST=localhost
|
|
311
|
+
DB_PORT=5432
|
|
312
|
+
DB_NAME=rag_db
|
|
313
|
+
DB_USER=your_db_user
|
|
314
|
+
DB_PASSWORD=your_secure_password
|
|
315
|
+
DB_SCHEMA=public
|
|
316
|
+
|
|
317
|
+
# S3 Storage Configuration
|
|
318
|
+
S3_REGION=eu-west-3
|
|
319
|
+
S3_ACCESS_KEY=your_access_key
|
|
320
|
+
S3_ACCESS_KEY_SECRET=your_secret_key
|
|
321
|
+
S3_ENDPOINT=https://s3.eu-west-3.amazonaws.com
|
|
322
|
+
S3_BUCKET_NAME=your_bucket_name
|
|
323
|
+
|
|
324
|
+
# Mistral AI API
|
|
325
|
+
MISTRAL_API_KEY=your_mistral_api_key
|
|
326
|
+
MISTRAL_API_URL=https://api.mistral.ai/v1
|
|
327
|
+
|
|
328
|
+
# JWT Authentication
|
|
329
|
+
SECRET_KEY=your-super-secret-key-minimum-32-characters
|
|
330
|
+
ACCESS_TOKEN_EXPIRE_MINUTES=60
|
|
331
|
+
ALGORITHM=HS256
|
|
332
|
+
|
|
333
|
+
# Password Encryption
|
|
334
|
+
ENCRYPT_KEY=your-encryption-key
|
|
335
|
+
|
|
336
|
+
# Development Settings (optional)
|
|
337
|
+
ECHO_SQL=false
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
> ⚠️ **Security Warning**: Never commit `.env` files to version control!
|
|
341
|
+
|
|
342
|
+
### 3. Initialize Database
|
|
343
|
+
|
|
344
|
+
```bash
|
|
345
|
+
# Run database migrations
|
|
346
|
+
alembic upgrade head
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
---
|
|
350
|
+
|
|
351
|
+
## 🔐 Database Access (Important)
|
|
352
|
+
|
|
353
|
+
The PostgreSQL database is **IP-whitelisted** for security reasons.
|
|
354
|
+
|
|
355
|
+
➡️ **Before running the application**, you must ask the project owner/admin to add your public IP address to the database whitelist.
|
|
356
|
+
|
|
357
|
+
⚠️ If your IP is not whitelisted, the database connection will fail.
|
|
358
|
+
|
|
359
|
+
---
|
|
360
|
+
|
|
361
|
+
### 📡 How to Get Your Public IP Address
|
|
362
|
+
|
|
363
|
+
#### macOS / Linux 🍎🐧
|
|
364
|
+
```bash
|
|
365
|
+
curl ifconfig.me
|
|
366
|
+
```
|
|
367
|
+
or
|
|
368
|
+
```bash
|
|
369
|
+
curl https://api.ipify.org
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
#### Windows 🪟 (PowerShell)
|
|
373
|
+
```powershell
|
|
374
|
+
curl ifconfig.me
|
|
375
|
+
```
|
|
376
|
+
or
|
|
377
|
+
```powershell
|
|
378
|
+
(Invoke-WebRequest -uri "https://api.ipify.org").Content
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
---
|
|
382
|
+
|
|
383
|
+
### 📌 Next Steps
|
|
384
|
+
Send the resulting IP address to the administrator so it can be approved and added to the whitelist.
|
|
385
|
+
|
|
386
|
+
> ⚠️ **Note:** If you are on a dynamic network (VPN, mobile hotspot, etc.), your IP may change and need to be re-approved.
|
|
387
|
+
|
|
388
|
+
## 🎯 Running the Application
|
|
389
|
+
|
|
390
|
+
### Option 1: Local Development (Recommended) ⚡
|
|
391
|
+
|
|
392
|
+
#### Windows 🪟
|
|
393
|
+
|
|
394
|
+
```powershell
|
|
395
|
+
# Activate virtual environment
|
|
396
|
+
.venv\Scripts\activate
|
|
397
|
+
|
|
398
|
+
# Start the API server with auto-reload
|
|
399
|
+
uv run uvicorn th2rag.main:app --reload --host 0.0.0.0 --port 8000
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
#### macOS/Linux 🍎🐧
|
|
403
|
+
|
|
404
|
+
```bash
|
|
405
|
+
# Activate virtual environment
|
|
406
|
+
source .venv/bin/activate
|
|
407
|
+
|
|
408
|
+
# Start the API server with auto-reload
|
|
409
|
+
uv run uvicorn th2rag.main:app --reload --host 0.0.0.0 --port 8000
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
**Access Points:**
|
|
413
|
+
- 📚 Swagger UI: http://localhost:8000/docs
|
|
414
|
+
- 📖 ReDoc: http://localhost:8000/redoc
|
|
415
|
+
- ❤️ Health Check: http://localhost:8000/health
|
|
416
|
+
|
|
417
|
+
### Option 2: Docker (Production-like) 🐳
|
|
418
|
+
|
|
419
|
+
```bash
|
|
420
|
+
# Start all services
|
|
421
|
+
docker-compose up -d
|
|
422
|
+
|
|
423
|
+
# View logs
|
|
424
|
+
docker-compose logs -f api
|
|
425
|
+
|
|
426
|
+
# Stop services
|
|
427
|
+
docker-compose down
|
|
428
|
+
```
|
|
429
|
+
|
|
430
|
+
#### Updating Environment Variables in Docker
|
|
431
|
+
|
|
432
|
+
```bash
|
|
433
|
+
# Important: Restart does NOT reload .env!
|
|
434
|
+
# You must stop and recreate containers:
|
|
435
|
+
|
|
436
|
+
docker-compose down
|
|
437
|
+
docker-compose up -d
|
|
438
|
+
docker-compose logs -f api
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
---
|
|
442
|
+
|
|
443
|
+
## 🧪 Testing
|
|
444
|
+
|
|
445
|
+
### Run Tests Locally
|
|
446
|
+
|
|
447
|
+
#### Windows 🪟
|
|
448
|
+
|
|
449
|
+
```powershell
|
|
450
|
+
# Activate virtual environment
|
|
451
|
+
.venv\Scripts\activate
|
|
452
|
+
|
|
453
|
+
# Run all tests
|
|
454
|
+
uv run pytest
|
|
455
|
+
|
|
456
|
+
# Run with verbose output
|
|
457
|
+
uv run pytest -v
|
|
458
|
+
|
|
459
|
+
# Run specific test file
|
|
460
|
+
uv run pytest test\test_auth.py
|
|
461
|
+
|
|
462
|
+
# Run with coverage
|
|
463
|
+
uv run pytest --cov=src --cov-report=html
|
|
464
|
+
uv run pytest --cov=src --cov-report=term-missing
|
|
465
|
+
|
|
466
|
+
# Open coverage report
|
|
467
|
+
start htmlcov\index.html
|
|
468
|
+
```
|
|
469
|
+
|
|
470
|
+
#### macOS/Linux 🍎🐧
|
|
471
|
+
|
|
472
|
+
```bash
|
|
473
|
+
# Activate virtual environment
|
|
474
|
+
source .venv/bin/activate
|
|
475
|
+
|
|
476
|
+
# Run all tests
|
|
477
|
+
uv run pytest
|
|
478
|
+
|
|
479
|
+
# Run with verbose output
|
|
480
|
+
uv run pytest -v
|
|
481
|
+
|
|
482
|
+
# Run specific test file
|
|
483
|
+
uv run pytest test/test_auth.py
|
|
484
|
+
|
|
485
|
+
# Run with coverage
|
|
486
|
+
uv run pytest --cov=src --cov-report=html
|
|
487
|
+
uv run pytest --cov=src --cov-report=term-missing
|
|
488
|
+
|
|
489
|
+
# Open coverage report
|
|
490
|
+
open htmlcov/index.html
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
### Run Tests in Docker
|
|
494
|
+
|
|
495
|
+
```bash
|
|
496
|
+
# Ensure containers are running
|
|
497
|
+
docker-compose up -d
|
|
498
|
+
|
|
499
|
+
# Run tests inside container
|
|
500
|
+
docker-compose exec api pytest -v
|
|
501
|
+
```
|
|
502
|
+
|
|
503
|
+
---
|
|
504
|
+
|
|
505
|
+
## 🔄 Development Workflow
|
|
506
|
+
|
|
507
|
+
### Branch Strategy
|
|
508
|
+
|
|
509
|
+
This project follows a Git workflow with feature branches:
|
|
510
|
+
|
|
511
|
+
```bash
|
|
512
|
+
# 1. Create a new feature branch from main/develop
|
|
513
|
+
git checkout -b feature/your-feature-name
|
|
514
|
+
|
|
515
|
+
# 2. Make your changes and commit regularly
|
|
516
|
+
git add .
|
|
517
|
+
git commit -m "feat: description of your changes"
|
|
518
|
+
|
|
519
|
+
# 3. Push to remote
|
|
520
|
+
git push origin feature/your-feature-name
|
|
521
|
+
|
|
522
|
+
# 4. Create a Pull Request on GitHub
|
|
523
|
+
```
|
|
524
|
+
|
|
525
|
+
### Pre-commit Setup (Code Quality Checks)
|
|
526
|
+
|
|
527
|
+
Pre-commit hooks automatically check your code before each commit.
|
|
528
|
+
|
|
529
|
+
#### 1. Install Pre-commit
|
|
530
|
+
|
|
531
|
+
**Windows 🪟**
|
|
532
|
+
```powershell
|
|
533
|
+
uv pip install pre-commit
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
**macOS/Linux 🍎🐧**
|
|
537
|
+
```bash
|
|
538
|
+
uv pip install pre-commit
|
|
539
|
+
```
|
|
540
|
+
|
|
541
|
+
#### 2. Install Git Hooks
|
|
542
|
+
|
|
543
|
+
```bash
|
|
544
|
+
# Install the pre-commit hooks
|
|
545
|
+
pre-commit install
|
|
546
|
+
|
|
547
|
+
# (Optional) Run against all files to test
|
|
548
|
+
pre-commit run --all-files
|
|
549
|
+
```
|
|
550
|
+
|
|
551
|
+
#### 3. Create Pre-commit Configuration
|
|
552
|
+
|
|
553
|
+
Create `.pre-commit-config.yaml` in your project root:
|
|
554
|
+
|
|
555
|
+
```yaml
|
|
556
|
+
repos:
|
|
557
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
558
|
+
rev: v4.5.0
|
|
559
|
+
hooks:
|
|
560
|
+
- id: trailing-whitespace
|
|
561
|
+
- id: end-of-file-fixer
|
|
562
|
+
- id: check-yaml
|
|
563
|
+
- id: check-added-large-files
|
|
564
|
+
args: ['--maxkb=1000']
|
|
565
|
+
- id: check-json
|
|
566
|
+
- id: check-merge-conflict
|
|
567
|
+
- id: detect-private-key
|
|
568
|
+
|
|
569
|
+
- repo: https://github.com/psf/black
|
|
570
|
+
rev: 23.12.1
|
|
571
|
+
hooks:
|
|
572
|
+
- id: black
|
|
573
|
+
language_version: python3.12
|
|
574
|
+
|
|
575
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
576
|
+
rev: v0.1.9
|
|
577
|
+
hooks:
|
|
578
|
+
- id: ruff
|
|
579
|
+
args: [--fix, --exit-non-zero-on-fix]
|
|
580
|
+
|
|
581
|
+
- repo: https://github.com/pre-commit/mirrors-mypy
|
|
582
|
+
rev: v1.8.0
|
|
583
|
+
hooks:
|
|
584
|
+
- id: mypy
|
|
585
|
+
additional_dependencies: [types-all]
|
|
586
|
+
args: [--ignore-missing-imports]
|
|
587
|
+
```
|
|
588
|
+
|
|
589
|
+
### Commit Message Convention
|
|
590
|
+
|
|
591
|
+
Follow conventional commits for clear history:
|
|
592
|
+
|
|
593
|
+
```bash
|
|
594
|
+
# Format: <type>(<scope>): <subject>
|
|
595
|
+
|
|
596
|
+
git commit -m "feat(auth): add refresh token endpoint"
|
|
597
|
+
git commit -m "fix(database): resolve connection pool issue"
|
|
598
|
+
git commit -m "docs(readme): update installation instructions"
|
|
599
|
+
git commit -m "test(auth): add JWT validation tests"
|
|
600
|
+
git commit -m "chore(deps): update dependencies"
|
|
601
|
+
```
|
|
602
|
+
|
|
603
|
+
**Commit Types:**
|
|
604
|
+
- `feat`: New feature
|
|
605
|
+
- `fix`: Bug fix
|
|
606
|
+
- `docs`: Documentation changes
|
|
607
|
+
- `style`: Code style changes (formatting)
|
|
608
|
+
- `refactor`: Code refactoring
|
|
609
|
+
- `test`: Adding or updating tests
|
|
610
|
+
- `chore`: Maintenance tasks
|
|
611
|
+
|
|
612
|
+
### Pushing Your X Branch
|
|
613
|
+
|
|
614
|
+
#### Step-by-Step Guide
|
|
615
|
+
|
|
616
|
+
**1. Check your current status**
|
|
617
|
+
|
|
618
|
+
```bash
|
|
619
|
+
# See what files have changed
|
|
620
|
+
git status
|
|
621
|
+
|
|
622
|
+
# See the actual changes
|
|
623
|
+
git diff
|
|
624
|
+
```
|
|
625
|
+
|
|
626
|
+
**2. Stage your changes**
|
|
627
|
+
|
|
628
|
+
```bash
|
|
629
|
+
# Stage specific files
|
|
630
|
+
git add pyproject.toml
|
|
631
|
+
git add .pre-commit-config.yaml
|
|
632
|
+
git add docker-compose.yml
|
|
633
|
+
git add README.md
|
|
634
|
+
git add .env.example
|
|
635
|
+
|
|
636
|
+
# OR stage all changes
|
|
637
|
+
git add .
|
|
638
|
+
```
|
|
639
|
+
|
|
640
|
+
**3. Commit your changes**
|
|
641
|
+
|
|
642
|
+
```bash
|
|
643
|
+
# Commit with a descriptive message
|
|
644
|
+
git commit -m "feat(migration): migrate to UV package manager and update documentation"
|
|
645
|
+
```
|
|
646
|
+
|
|
647
|
+
**4. Create and push your branch**
|
|
648
|
+
|
|
649
|
+
**Windows 🪟**
|
|
650
|
+
```powershell
|
|
651
|
+
# Create and switch to X branch
|
|
652
|
+
git checkout -b X
|
|
653
|
+
|
|
654
|
+
# Push to remote repository
|
|
655
|
+
git push -u origin X
|
|
656
|
+
```
|
|
657
|
+
|
|
658
|
+
**macOS/Linux 🍎🐧**
|
|
659
|
+
```bash
|
|
660
|
+
# Create and switch to X branch
|
|
661
|
+
git checkout -b X
|
|
662
|
+
|
|
663
|
+
# Push to remote repository
|
|
664
|
+
git push -u origin X
|
|
665
|
+
```
|
|
666
|
+
|
|
667
|
+
**5. Create a Pull Request**
|
|
668
|
+
|
|
669
|
+
- Go to your GitHub repository
|
|
670
|
+
- Click "Compare & pull request" button
|
|
671
|
+
- Select base branch (usually `main` or `develop`)
|
|
672
|
+
- Add description of your changes
|
|
673
|
+
- Assign reviewers from your team
|
|
674
|
+
- Submit the pull request
|
|
675
|
+
|
|
676
|
+
### Daily Workflow Example
|
|
677
|
+
|
|
678
|
+
```bash
|
|
679
|
+
# 1. Start of day - update your branch
|
|
680
|
+
git checkout X
|
|
681
|
+
git pull origin X
|
|
682
|
+
|
|
683
|
+
# 2. Make changes to files
|
|
684
|
+
# ... edit code ...
|
|
685
|
+
|
|
686
|
+
# 3. Check what changed
|
|
687
|
+
git status
|
|
688
|
+
git diff
|
|
689
|
+
|
|
690
|
+
# 4. Run tests before committing
|
|
691
|
+
uv run pytest
|
|
692
|
+
|
|
693
|
+
# 5. Pre-commit will automatically run when you commit
|
|
694
|
+
git add .
|
|
695
|
+
git commit -m "feat(rag): improve document chunking algorithm"
|
|
696
|
+
|
|
697
|
+
# 6. Push changes
|
|
698
|
+
git push origin X
|
|
699
|
+
|
|
700
|
+
# 7. If you need to sync with main branch
|
|
701
|
+
git checkout main
|
|
702
|
+
git pull origin main
|
|
703
|
+
git checkout X
|
|
704
|
+
git merge main
|
|
705
|
+
# Resolve any conflicts if they occur
|
|
706
|
+
git push origin X
|
|
707
|
+
```
|
|
708
|
+
|
|
709
|
+
### Handling Pre-commit Failures
|
|
710
|
+
|
|
711
|
+
If pre-commit hooks fail:
|
|
712
|
+
|
|
713
|
+
```bash
|
|
714
|
+
# Pre-commit will show what failed
|
|
715
|
+
# Fix the issues, then:
|
|
716
|
+
|
|
717
|
+
git add .
|
|
718
|
+
git commit -m "your message"
|
|
719
|
+
|
|
720
|
+
# If automatic fixes were applied, stage them:
|
|
721
|
+
git add .
|
|
722
|
+
git commit --amend --no-edit
|
|
723
|
+
```
|
|
724
|
+
|
|
725
|
+
---
|
|
726
|
+
|
|
727
|
+
## 📁 Project Structure
|
|
728
|
+
|
|
729
|
+
```
|
|
730
|
+
.
|
|
731
|
+
├── .github/ # GitHub workflows and templates
|
|
732
|
+
├── alembic/ # Database migrations
|
|
733
|
+
│ ├── versions/ # Migration scripts
|
|
734
|
+
│ └── env.py # Alembic environment
|
|
735
|
+
├── data/ # LanceDB vector storage
|
|
736
|
+
├── src/ # Main application source
|
|
737
|
+
│ ├── auth/ # JWT authentication
|
|
738
|
+
│ ├── clients/ # S3 client integration
|
|
739
|
+
│ ├── conversations/ # Conversation management
|
|
740
|
+
│ ├── dataset/ # Dataset handling
|
|
741
|
+
│ ├── knowledge/ # Document management
|
|
742
|
+
│ ├── messages/ # Message operations
|
|
743
|
+
│ ├── rag/ # RAG pipeline
|
|
744
|
+
│ │ ├── chunkers/ # Document chunking
|
|
745
|
+
│ │ ├── converters/ # Format conversion
|
|
746
|
+
│ │ ├── embeddings/ # Vector embeddings
|
|
747
|
+
│ │ ├── generation/ # LLM generation
|
|
748
|
+
│ │ ├── retrieval/ # Vector search
|
|
749
|
+
│ │ ├── services/ # RAG orchestration
|
|
750
|
+
│ │ └── tasks/ # Background jobs
|
|
751
|
+
│ ├── users/ # User management
|
|
752
|
+
│ ├── utils/ # Utilities
|
|
753
|
+
│ └── main.py # Application entry
|
|
754
|
+
├── test/ # Test suite
|
|
755
|
+
├── .env.example # Environment template
|
|
756
|
+
├── .gitignore # Git ignore rules
|
|
757
|
+
├── .pre-commit-config.yaml # Pre-commit hooks
|
|
758
|
+
├── alembic.ini # Alembic config
|
|
759
|
+
├── docker-compose.yml # Docker services
|
|
760
|
+
├── Dockerfile # Container definition
|
|
761
|
+
├── pyproject.toml # Project dependencies
|
|
762
|
+
└── README.md # This file
|
|
763
|
+
```
|
|
764
|
+
|
|
765
|
+
---
|
|
766
|
+
|
|
767
|
+
## 📚 API Documentation
|
|
768
|
+
|
|
769
|
+
### Authentication
|
|
770
|
+
- `POST /auth/login` - Login and get tokens
|
|
771
|
+
- `POST /auth/refresh-token` - Refresh access token
|
|
772
|
+
|
|
773
|
+
### Users
|
|
774
|
+
- `GET /users` - List users (admin)
|
|
775
|
+
- `POST /users` - Create user
|
|
776
|
+
- `GET /users/me` - Get current user
|
|
777
|
+
- `GET /users/{id}` - Get user by ID
|
|
778
|
+
- `DELETE /users/{id}` - Delete user
|
|
779
|
+
|
|
780
|
+
### Conversations
|
|
781
|
+
- `GET /conversations` - List conversations
|
|
782
|
+
- `POST /conversations` - Create conversation
|
|
783
|
+
- `GET /conversations/{id}` - Get conversation details
|
|
784
|
+
- `DELETE /conversations/{id}` - Delete conversation
|
|
785
|
+
|
|
786
|
+
### Messages
|
|
787
|
+
- `GET /conversations/{id}/messages` - List messages
|
|
788
|
+
- `POST /conversations/{id}/messages` - Send message and get AI response
|
|
789
|
+
- `POST /conversations/{id}/messages/{message_id}/feedback` - Add feedback
|
|
790
|
+
|
|
791
|
+
### Knowledge Documents
|
|
792
|
+
- `GET /knowledge` - List documents
|
|
793
|
+
- `POST /knowledge` - Upload PDF document
|
|
794
|
+
- `GET /knowledge/{id}` - Get document details
|
|
795
|
+
- `DELETE /knowledge/{id}` - Delete document and vectors
|
|
796
|
+
|
|
797
|
+
### Datasets
|
|
798
|
+
- `GET /datasets` - List datasets
|
|
799
|
+
- `POST /datasets` - Create dataset
|
|
800
|
+
- `PUT /datasets/{id}` - Update dataset
|
|
801
|
+
- `DELETE /datasets/{id}` - Delete dataset
|
|
802
|
+
|
|
803
|
+
For detailed API schemas, visit: http://localhost:8000/docs
|
|
804
|
+
|
|
805
|
+
---
|
|
806
|
+
|
|
807
|
+
## 🧠 RAG Engine
|
|
808
|
+
|
|
809
|
+
The `rag/` module is responsible for the core Retrieval-Augmented Generation (RAG) pipeline. It orchestrates document conversion, chunking, embedding, retrieval, and final answer generation.
|
|
810
|
+
|
|
811
|
+
### 📄 1. Document Conversion
|
|
812
|
+
|
|
813
|
+
* **`converters/pdf_converter.py`**: Converts PDF files (local or S3) to an internal `Document` format using `docling`.
|
|
814
|
+
|
|
815
|
+
```python
|
|
816
|
+
convert_pdf(source: str) -> dict
|
|
817
|
+
```
|
|
818
|
+
|
|
819
|
+
### 🛫 2. Chunking
|
|
820
|
+
|
|
821
|
+
* **`chunkers/hybrid_chunker.py`**: Wraps `docling`'s `HybridChunker` to split documents into semantic chunks.
|
|
822
|
+
|
|
823
|
+
```python
|
|
824
|
+
HybridTextChunker.chunk_document(dl_doc) -> list
|
|
825
|
+
```
|
|
826
|
+
|
|
827
|
+
### 🖐️ 3. Embeddings
|
|
828
|
+
|
|
829
|
+
* **`embeddings/sentence_transformer.py`**: Uses `sentence-transformers/gtr-t5-large` to convert text chunks into dense vectors.
|
|
830
|
+
|
|
831
|
+
```python
|
|
832
|
+
SentenceTransformerEmbedding.encode(text: str) -> List[float]
|
|
833
|
+
```
|
|
834
|
+
|
|
835
|
+
### 🗆️ 4. Vector Store (LanceDB)
|
|
836
|
+
|
|
837
|
+
* **`retrieval/lancedb/lancedb_storage.py`**: Manages chunk storage/retrieval in LanceDB.
|
|
838
|
+
* **`retrieval/lancedb/schemas.py`**: Pydantic schema definitions for LanceDB.
|
|
839
|
+
|
|
840
|
+
```python
|
|
841
|
+
LanceDBStorage.add_chunks(chunks)
|
|
842
|
+
LanceDBStorage.query(query_vector, doc_id, limit)
|
|
843
|
+
```
|
|
844
|
+
|
|
845
|
+
### 🔍 5. Retrieval
|
|
846
|
+
|
|
847
|
+
* **`retrieval/lancedb/lancedb_retriever.py`**: Retrieves similar document chunks using LanceDB + embeddings.
|
|
848
|
+
|
|
849
|
+
```python
|
|
850
|
+
LanceDBRetriever.retrieve(query: str, doc_id: int, limit: int) -> List[Dict]
|
|
851
|
+
```
|
|
852
|
+
|
|
853
|
+
### 📏 6. Generation
|
|
854
|
+
|
|
855
|
+
* **`generation/mistral_generator.py`**: Calls Mistral API to generate responses using retrieved context.
|
|
856
|
+
|
|
857
|
+
```python
|
|
858
|
+
MistralGenerator.generate(question, retrieved_context, prompt, history) -> str
|
|
859
|
+
```
|
|
860
|
+
|
|
861
|
+
### ⚖️ 7. RAG Service
|
|
862
|
+
|
|
863
|
+
* **`services/rag_service.py`**: Orchestrates the full RAG pipeline.
|
|
864
|
+
* **`services/rag_service_builder.py`**: Builds the RAG service with configurable components.
|
|
865
|
+
|
|
866
|
+
```python
|
|
867
|
+
RAGService.answer_question(question, doc_id, limit, prompt, history)
|
|
868
|
+
```
|
|
869
|
+
|
|
870
|
+
### ⚙️ 8. Background Processing
|
|
871
|
+
|
|
872
|
+
* **`tasks/process_pdf.py`**: Asynchronously handles PDF-to-vector pipeline including download, convert, chunk, embed, and store.
|
|
873
|
+
|
|
874
|
+
```python
|
|
875
|
+
process_pdf_background(doc_id, source)
|
|
876
|
+
```
|
|
877
|
+
|
|
878
|
+
---
|
|
879
|
+
|
|
880
|
+
## 🤝 ContrXuting
|
|
881
|
+
|
|
882
|
+
### Code Review Checklist
|
|
883
|
+
|
|
884
|
+
Before submitting a PR, ensure:
|
|
885
|
+
|
|
886
|
+
- ✅ All tests pass (`uv run pytest`)
|
|
887
|
+
- ✅ Pre-commit hooks pass
|
|
888
|
+
- ✅ Code is documented with docstrings
|
|
889
|
+
- ✅ New features have tests
|
|
890
|
+
- ✅ README updated if needed
|
|
891
|
+
- ✅ Commit messages follow convention
|
|
892
|
+
- ✅ No sensitive data in commits
|
|
893
|
+
|
|
894
|
+
### Adding New Features
|
|
895
|
+
|
|
896
|
+
**1. Database Changes**
|
|
897
|
+
```bash
|
|
898
|
+
# Edit src/models.py
|
|
899
|
+
# Generate migration
|
|
900
|
+
alembic revision --autogenerate -m "add new feature"
|
|
901
|
+
# Review and apply
|
|
902
|
+
alembic upgrade head
|
|
903
|
+
```
|
|
904
|
+
|
|
905
|
+
**2. New Embedding Model**
|
|
906
|
+
```python
|
|
907
|
+
# src/rag/embeddings/custom_embedding.py
|
|
908
|
+
from th2rag.rag.embeddings.base import BaseEmbedding
|
|
909
|
+
|
|
910
|
+
class CustomEmbedding(BaseEmbedding):
|
|
911
|
+
def encode(self, text: str) -> List[float]:
|
|
912
|
+
# Implementation
|
|
913
|
+
pass
|
|
914
|
+
```
|
|
915
|
+
|
|
916
|
+
**3. New LLM Generator**
|
|
917
|
+
```python
|
|
918
|
+
# src/rag/generation/custom_generator.py
|
|
919
|
+
from th2rag.rag.generation.base import BaseGenerator
|
|
920
|
+
|
|
921
|
+
class CustomGenerator(BaseGenerator):
|
|
922
|
+
def generate(self, question: str, context: str) -> str:
|
|
923
|
+
# Implementation
|
|
924
|
+
pass
|
|
925
|
+
```
|
|
926
|
+
|
|
927
|
+
---
|
|
928
|
+
|
|
929
|
+
## 🆘 Troubleshooting
|
|
930
|
+
|
|
931
|
+
### Windows-Specific Issues 🪟
|
|
932
|
+
|
|
933
|
+
**Port already in use:**
|
|
934
|
+
```powershell
|
|
935
|
+
# Find process
|
|
936
|
+
netstat -ano | findstr :8000
|
|
937
|
+
# Kill process (replace PID)
|
|
938
|
+
taskkill /PID <PID> /F
|
|
939
|
+
```
|
|
940
|
+
|
|
941
|
+
**Python not found:**
|
|
942
|
+
```powershell
|
|
943
|
+
# Add Python to PATH
|
|
944
|
+
$env:Path += ";C:\Python312;C:\Python312\Scripts"
|
|
945
|
+
```
|
|
946
|
+
|
|
947
|
+
**Permission errors:**
|
|
948
|
+
```powershell
|
|
949
|
+
# Run PowerShell as Administrator
|
|
950
|
+
# Or adjust execution policy
|
|
951
|
+
Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
|
|
952
|
+
```
|
|
953
|
+
|
|
954
|
+
### macOS-Specific Issues 🍎
|
|
955
|
+
|
|
956
|
+
**Port already in use:**
|
|
957
|
+
```bash
|
|
958
|
+
# Find and kill process
|
|
959
|
+
lsof -ti:8000 | xargs kill -9
|
|
960
|
+
```
|
|
961
|
+
|
|
962
|
+
**SSL certificate errors:**
|
|
963
|
+
```bash
|
|
964
|
+
# Update certificates
|
|
965
|
+
pip install --upgrade certifi
|
|
966
|
+
```
|
|
967
|
+
|
|
968
|
+
### Common Issues (All Platforms)
|
|
969
|
+
|
|
970
|
+
**Database connection failed:**
|
|
971
|
+
- Verify PostgreSQL is running
|
|
972
|
+
- Check credentials in `.env`
|
|
973
|
+
- Ensure database exists
|
|
974
|
+
|
|
975
|
+
**Docker container won't start:**
|
|
976
|
+
```bash
|
|
977
|
+
docker-compose logs api
|
|
978
|
+
docker-compose down -v
|
|
979
|
+
docker-compose up -d --build
|
|
980
|
+
```
|
|
981
|
+
|
|
982
|
+
**Pre-commit hooks fail:**
|
|
983
|
+
```bash
|
|
984
|
+
# Update hooks
|
|
985
|
+
pre-commit autoupdate
|
|
986
|
+
pre-commit run --all-files
|
|
987
|
+
```
|
|
988
|
+
|
|
989
|
+
---
|
|
990
|
+
|
|
991
|
+
## 📞 Support
|
|
992
|
+
|
|
993
|
+
- 📧 Email: your-team@company.com
|
|
994
|
+
- 💬 Slack: #rag-backend-support
|
|
995
|
+
- 🐛 Issues: [GitHub Issues](link-to-issues)
|
|
996
|
+
- 📖 Wiki: [Project Wiki](link-to-wiki)
|
|
997
|
+
|
|
998
|
+
---
|
|
999
|
+
|
|
1000
|
+
## 📄 License
|
|
1001
|
+
|
|
1002
|
+
[Specify your license here]
|