th2rag 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- th2rag/__init__.py +0 -0
- th2rag/auth/__init__.py +0 -0
- th2rag/auth/exceptions.py +2 -0
- th2rag/auth/router.py +43 -0
- th2rag/auth/schemas.py +15 -0
- th2rag/auth/service.py +115 -0
- th2rag/clients/__init__.py +0 -0
- th2rag/clients/storage/config.py +13 -0
- th2rag/clients/storage/local.py +48 -0
- th2rag/clients/storage/s3.py +93 -0
- th2rag/config.py +267 -0
- th2rag/conversations/__init__.py +0 -0
- th2rag/conversations/dependencies.py +30 -0
- th2rag/conversations/exceptions.py +6 -0
- th2rag/conversations/router.py +47 -0
- th2rag/conversations/schemas.py +43 -0
- th2rag/conversations/service.py +51 -0
- th2rag/database.py +72 -0
- th2rag/dataset/__init__.py +0 -0
- th2rag/dataset/dependencies.py +25 -0
- th2rag/dataset/exceptions.py +2 -0
- th2rag/dataset/router.py +53 -0
- th2rag/dataset/schemas.py +23 -0
- th2rag/dataset/service.py +78 -0
- th2rag/dependencies.py +143 -0
- th2rag/knowledge/__init__.py +0 -0
- th2rag/knowledge/dependencies.py +25 -0
- th2rag/knowledge/exceptions.py +2 -0
- th2rag/knowledge/mage_client.py +505 -0
- th2rag/knowledge/router.py +714 -0
- th2rag/knowledge/schemas.py +108 -0
- th2rag/knowledge/service.py +392 -0
- th2rag/logger.py +21 -0
- th2rag/main.py +51 -0
- th2rag/messages/__init__.py +0 -0
- th2rag/messages/exceptions.py +2 -0
- th2rag/messages/router.py +63 -0
- th2rag/messages/schemas.py +68 -0
- th2rag/messages/service.py +210 -0
- th2rag/models.py +215 -0
- th2rag/pagination.py +68 -0
- th2rag/rag/__init__.py +0 -0
- th2rag/rag/chunkers/__init__.py +0 -0
- th2rag/rag/chunkers/hybrid_chunker.py +23 -0
- th2rag/rag/constants.py +15 -0
- th2rag/rag/converters/__init__.py +0 -0
- th2rag/rag/converters/documents_converter.py +123 -0
- th2rag/rag/converters/json_converter.py +521 -0
- th2rag/rag/converters/legacy_converter.py +124 -0
- th2rag/rag/converters/pdf_converter.py +384 -0
- th2rag/rag/converters/spreadsheet_converter.py +159 -0
- th2rag/rag/converters/txt_converter.py +98 -0
- th2rag/rag/embeddings/__init__.py +0 -0
- th2rag/rag/embeddings/base.py +21 -0
- th2rag/rag/embeddings/exceptions.py +2 -0
- th2rag/rag/embeddings/sentence_transformer.py +97 -0
- th2rag/rag/generation/base.py +22 -0
- th2rag/rag/generation/exceptions.py +2 -0
- th2rag/rag/generation/litellm_generator.py +143 -0
- th2rag/rag/generation/mistral_generator.py +269 -0
- th2rag/rag/generation/test_litellm.py +33 -0
- th2rag/rag/retrieval/__init__.py +0 -0
- th2rag/rag/retrieval/base.py +11 -0
- th2rag/rag/retrieval/exceptions.py +10 -0
- th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
- th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
- th2rag/rag/retrieval/lancedb/schemas.py +19 -0
- th2rag/rag/services/__init__.py +0 -0
- th2rag/rag/services/rag_service.py +99 -0
- th2rag/rag/services/rag_service_builder.py +42 -0
- th2rag/rag/tasks/process_pdf.py +586 -0
- th2rag/users/dependencies.py +44 -0
- th2rag/users/exceptions.py +6 -0
- th2rag/users/router.py +92 -0
- th2rag/users/schemas.py +61 -0
- th2rag/users/service.py +140 -0
- th2rag/utils/__init__.py +0 -0
- th2rag/utils/helpers.py +40 -0
- th2rag/utils/logger.py +15 -0
- th2rag/utils/security.py +76 -0
- th2rag/utils/welcome_router.py +98 -0
- th2rag/webhooks/__init__.py +0 -0
- th2rag/webhooks/sender.py +126 -0
- th2rag-0.0.2.dist-info/METADATA +1002 -0
- th2rag-0.0.2.dist-info/RECORD +87 -0
- th2rag-0.0.2.dist-info/WHEEL +4 -0
- th2rag-0.0.2.dist-info/entry_points.txt +3 -0
th2rag/__init__.py
ADDED
|
File without changes
|
th2rag/auth/__init__.py
ADDED
|
File without changes
|
th2rag/auth/router.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
|
2
|
+
from fastapi.security import OAuth2PasswordRequestForm
|
|
3
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
4
|
+
|
|
5
|
+
from th2rag.auth import exceptions, schemas, service
|
|
6
|
+
from th2rag.dependencies import get_db
|
|
7
|
+
|
|
8
|
+
router = APIRouter(prefix="/auth", tags=["auth"])
|
|
9
|
+
|
|
10
|
+
@router.post("/token", response_model=schemas.Token, status_code=status.HTTP_200_OK)
|
|
11
|
+
async def login(form_data: OAuth2PasswordRequestForm = Depends(), db: AsyncSession = Depends(get_db), response: Response = Response()):
|
|
12
|
+
# Dummy example — replace with real auth
|
|
13
|
+
# if form_data.username != "farid.azouaou@thaink2.com" or form_data.password != "secret":
|
|
14
|
+
# return {"error": "invalid credentials"}
|
|
15
|
+
request = schemas.LoginRequest(email=form_data.username, password=form_data.password)
|
|
16
|
+
try:
|
|
17
|
+
return await service.login(request, response, db)
|
|
18
|
+
except exceptions.InvalidCredentials as e:
|
|
19
|
+
raise HTTPException(
|
|
20
|
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
21
|
+
detail="Invalid email or password",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# @router.post("/login", response_model=schemas.Token, status_code=status.HTTP_200_OK)
|
|
26
|
+
# async def login(
|
|
27
|
+
# request: schemas.LoginRequest, response: Response, db: AsyncSession = Depends(get_db)
|
|
28
|
+
# ):
|
|
29
|
+
# try:
|
|
30
|
+
# return await service.login(request, response, db)
|
|
31
|
+
# except exceptions.InvalidCredentials as e:
|
|
32
|
+
# raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
|
|
33
|
+
|
|
34
|
+
# except UserNotFoundException as e:
|
|
35
|
+
# raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@router.post("/refresh-token", response_model=schemas.Token, status_code=status.HTTP_200_OK)
|
|
39
|
+
async def refresh_token(request: Request, db: AsyncSession = Depends(get_db)):
|
|
40
|
+
try:
|
|
41
|
+
return await service.refresh(request, db)
|
|
42
|
+
except exceptions.InvalidCredentials as e:
|
|
43
|
+
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=str(e))
|
th2rag/auth/schemas.py
ADDED
th2rag/auth/service.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
from datetime import timedelta
|
|
2
|
+
|
|
3
|
+
import jwt
|
|
4
|
+
from fastapi import Request, Response
|
|
5
|
+
from sqlalchemy import select
|
|
6
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
7
|
+
|
|
8
|
+
from th2rag.auth import exceptions, schemas
|
|
9
|
+
from th2rag.config import settings
|
|
10
|
+
from th2rag.models import User
|
|
11
|
+
from th2rag.users.schemas import User as UserSchema
|
|
12
|
+
from th2rag.utils.security import create_access_token, verify_password
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
async def get_current_user_from_token(token: str, db: AsyncSession) -> UserSchema:
|
|
16
|
+
"""
|
|
17
|
+
Validate a Bearer JWT token and return the corresponding user.
|
|
18
|
+
Reuses the same JWT settings as login/refresh.
|
|
19
|
+
"""
|
|
20
|
+
try:
|
|
21
|
+
payload = jwt.decode(token, settings.encrypt_key, algorithms=[settings.algorithm])
|
|
22
|
+
if payload.get("type") != "access":
|
|
23
|
+
raise exceptions.InvalidCredentials("Invalid token type")
|
|
24
|
+
except Exception:
|
|
25
|
+
raise exceptions.InvalidCredentials("Invalid or expired token") from None
|
|
26
|
+
|
|
27
|
+
user_email = payload.get("sub")
|
|
28
|
+
if not user_email:
|
|
29
|
+
raise exceptions.InvalidCredentials("Invalid token payload")
|
|
30
|
+
|
|
31
|
+
stmt = select(User).where(User.email == user_email)
|
|
32
|
+
result = await db.execute(stmt)
|
|
33
|
+
user = result.scalar_one_or_none()
|
|
34
|
+
|
|
35
|
+
if user is None:
|
|
36
|
+
raise exceptions.InvalidCredentials("User not found")
|
|
37
|
+
|
|
38
|
+
return UserSchema(
|
|
39
|
+
user_id=user.user_id,
|
|
40
|
+
email=user.email,
|
|
41
|
+
role=user.role.value,
|
|
42
|
+
first_name=user.first_name,
|
|
43
|
+
last_name=user.last_name,
|
|
44
|
+
created_at=user.created_at,
|
|
45
|
+
updated_at=user.updated_at,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
async def login(request: schemas.LoginRequest, response: Response, db: AsyncSession):
|
|
50
|
+
stmt = select(User).where(User.email == request.email)
|
|
51
|
+
result = await db.execute(stmt)
|
|
52
|
+
user = result.scalar_one_or_none()
|
|
53
|
+
if user is None:
|
|
54
|
+
raise exceptions.InvalidCredentials("Invalid email or password")
|
|
55
|
+
|
|
56
|
+
if user.password is None:
|
|
57
|
+
raise exceptions.InvalidCredentials("Invalid email or password")
|
|
58
|
+
|
|
59
|
+
if verify_password(request.password, user.password) is False:
|
|
60
|
+
raise exceptions.InvalidCredentials("Invalid email or password")
|
|
61
|
+
|
|
62
|
+
access_token_expires = timedelta(minutes=settings.access_token_expire_minutes)
|
|
63
|
+
access_token = create_access_token(
|
|
64
|
+
data={"sub": user.email, "role": user.role.name, "type": "access"},
|
|
65
|
+
expires_delta=access_token_expires,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
refresh_token_expires = timedelta(days=30)
|
|
69
|
+
refresh_token = create_access_token(
|
|
70
|
+
data={"sub": user.email, "role": "USER", "type": "refresh"},
|
|
71
|
+
expires_delta=refresh_token_expires,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
response.set_cookie(
|
|
75
|
+
key="refresh_token",
|
|
76
|
+
value=refresh_token,
|
|
77
|
+
httponly=True,
|
|
78
|
+
max_age=30 * 24 * 60 * 60,
|
|
79
|
+
secure=(settings.working_mode != "development"),
|
|
80
|
+
samesite="lax",
|
|
81
|
+
path="/",
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
return schemas.Token(access_token=access_token, token_type="bearer")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
async def refresh(request: Request, db: AsyncSession):
|
|
88
|
+
refresh_token = request.cookies.get("refresh_token")
|
|
89
|
+
if not refresh_token:
|
|
90
|
+
raise exceptions.InvalidCredentials("Missing refresh token")
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
payload = jwt.decode(refresh_token, settings.encrypt_key, algorithms=[settings.algorithm])
|
|
94
|
+
if payload.get("type") != "refresh":
|
|
95
|
+
raise exceptions.InvalidCredentials("Invalid token type")
|
|
96
|
+
except Exception:
|
|
97
|
+
raise exceptions.InvalidCredentials("Invalid or expired token") from None
|
|
98
|
+
|
|
99
|
+
user_email = payload.get("sub")
|
|
100
|
+
if not user_email:
|
|
101
|
+
raise exceptions.InvalidCredentials("Invalid token")
|
|
102
|
+
|
|
103
|
+
stmt = select(User).where(User.email == user_email)
|
|
104
|
+
result = await db.execute(stmt)
|
|
105
|
+
user = result.scalar_one_or_none()
|
|
106
|
+
if user is None:
|
|
107
|
+
raise exceptions.InvalidCredentials("User not found")
|
|
108
|
+
|
|
109
|
+
access_token_expires = timedelta(minutes=settings.access_token_expire_minutes)
|
|
110
|
+
access_token = create_access_token(
|
|
111
|
+
data={"sub": user.email, "role": "USER", "type": "access"},
|
|
112
|
+
expires_delta=access_token_expires,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
return schemas.Token(access_token=access_token, token_type="bearer")
|
|
File without changes
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from dotenv import load_dotenv
|
|
4
|
+
|
|
5
|
+
dotenv_path = os.getenv("WORK_DIR", default=os.getcwd()) + "/.env"
|
|
6
|
+
|
|
7
|
+
load_dotenv(dotenv_path)
|
|
8
|
+
|
|
9
|
+
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY")
|
|
10
|
+
S3_ACCESS_KEY_SECRET = os.getenv("S3_ACCESS_KEY_SECRET")
|
|
11
|
+
S3_REGION = os.getenv("S3_REGION")
|
|
12
|
+
S3_ENDPOINT = os.getenv("S3_ENDPOINT")
|
|
13
|
+
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME")
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
# Create local storage directory
|
|
6
|
+
LOCAL_STORAGE_DIR = Path("./local_storage/pdfs")
|
|
7
|
+
LOCAL_STORAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def upload_file_locally(file_bytes: bytes, file_name: str) -> str:
|
|
11
|
+
"""
|
|
12
|
+
Saves file bytes locally and returns the file path.
|
|
13
|
+
"""
|
|
14
|
+
try:
|
|
15
|
+
now = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
16
|
+
base, ext = os.path.splitext(file_name)
|
|
17
|
+
unique_filename = f"{base}_{now}{ext}"
|
|
18
|
+
|
|
19
|
+
file_path = LOCAL_STORAGE_DIR / unique_filename
|
|
20
|
+
|
|
21
|
+
with open(file_path, "wb") as f:
|
|
22
|
+
f.write(file_bytes)
|
|
23
|
+
|
|
24
|
+
return str(file_path.absolute())
|
|
25
|
+
except Exception as e:
|
|
26
|
+
raise Exception(f"Failed to save file '{file_name}': {e}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def download_file_locally(file_path: str) -> bytes:
|
|
30
|
+
"""
|
|
31
|
+
Reads file bytes from local storage.
|
|
32
|
+
"""
|
|
33
|
+
try:
|
|
34
|
+
with open(file_path, "rb") as f:
|
|
35
|
+
return f.read()
|
|
36
|
+
except Exception as e:
|
|
37
|
+
raise Exception(f"Failed to read file from '{file_path}': {e}")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def delete_file_locally(file_path: str) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Deletes a file from local storage.
|
|
43
|
+
"""
|
|
44
|
+
try:
|
|
45
|
+
if os.path.exists(file_path):
|
|
46
|
+
os.remove(file_path)
|
|
47
|
+
except Exception as e:
|
|
48
|
+
raise RuntimeError(f"Failed to delete file {file_path}: {e}")
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import os
|
|
3
|
+
from datetime import timezone, datetime
|
|
4
|
+
from urllib.parse import urlparse
|
|
5
|
+
|
|
6
|
+
import boto3
|
|
7
|
+
from th2rag.config import settings
|
|
8
|
+
|
|
9
|
+
endpoint_url = settings.s3_endpoint
|
|
10
|
+
|
|
11
|
+
s3_client = boto3.client(
|
|
12
|
+
"s3",
|
|
13
|
+
aws_access_key_id=settings.s3_access_key,
|
|
14
|
+
aws_secret_access_key=settings.s3_access_key_secret,
|
|
15
|
+
region_name=settings.s3_region,
|
|
16
|
+
endpoint_url=settings.s3_endpoint, # Custom endpoint for Scaleway's S3 service
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def upload_file_to_s3(
|
|
21
|
+
file_bytes: bytes, file_name: str, doc_id: int, bucket_name: str = settings.s3_bucket_name
|
|
22
|
+
) -> str:
|
|
23
|
+
"""
|
|
24
|
+
Uploads file bytes to S3 and returns the URL of the uploaded file.
|
|
25
|
+
|
|
26
|
+
:param file_bytes: The file content in bytes.
|
|
27
|
+
:param file_name: The key name to be used for the file in the bucket.
|
|
28
|
+
:param doc_id: The document ID to use as folder prefix in S3.
|
|
29
|
+
:param bucket_name: The S3 bucket name.
|
|
30
|
+
:return: The URL of the uploaded file.
|
|
31
|
+
:raises Exception: If the upload fails.
|
|
32
|
+
"""
|
|
33
|
+
try:
|
|
34
|
+
now = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
35
|
+
base, ext = os.path.splitext(file_name)
|
|
36
|
+
unique_key = f"{doc_id}/{base}_{now}{ext}"
|
|
37
|
+
file_obj = io.BytesIO(file_bytes)
|
|
38
|
+
file_obj.seek(0)
|
|
39
|
+
|
|
40
|
+
s3_client.upload_fileobj(
|
|
41
|
+
Fileobj=file_obj,
|
|
42
|
+
Bucket=bucket_name,
|
|
43
|
+
Key=unique_key,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return f"{endpoint_url}/{bucket_name}/{unique_key}"
|
|
47
|
+
|
|
48
|
+
except Exception as e:
|
|
49
|
+
raise RuntimeError(
|
|
50
|
+
f"Failed to upload file '{file_name}'"
|
|
51
|
+
) from e
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def download_file_from_s3(url: str) -> bytes:
|
|
56
|
+
"""
|
|
57
|
+
Downloads file bytes from S3 given the full URL and returns them.
|
|
58
|
+
|
|
59
|
+
:param url: The full S3 URL.
|
|
60
|
+
:return: The file content in bytes.
|
|
61
|
+
:raises Exception: If the download fails.
|
|
62
|
+
"""
|
|
63
|
+
try:
|
|
64
|
+
parsed_url = urlparse(url)
|
|
65
|
+
|
|
66
|
+
path_parts = parsed_url.path.lstrip("/").split("/")
|
|
67
|
+
if len(path_parts) < 2:
|
|
68
|
+
raise Exception("URL path is not valid. It must include bucket and file key.")
|
|
69
|
+
bucket = path_parts[0]
|
|
70
|
+
file_key = "/".join(path_parts[1:])
|
|
71
|
+
|
|
72
|
+
file_obj = io.BytesIO()
|
|
73
|
+
s3_client.download_fileobj(bucket, file_key, file_obj)
|
|
74
|
+
file_obj.seek(0)
|
|
75
|
+
return file_obj.read()
|
|
76
|
+
except Exception as e:
|
|
77
|
+
raise Exception(f"Failed to download file from URL '{url}': {e}")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def delete_file_from_s3(url: str) -> None:
|
|
81
|
+
"""
|
|
82
|
+
Deletes an object from S3 given its full URL.
|
|
83
|
+
"""
|
|
84
|
+
parsed = urlparse(url)
|
|
85
|
+
# path is "/bucket/key…"
|
|
86
|
+
parts = parsed.path.lstrip("/").split("/", 1)
|
|
87
|
+
if len(parts) != 2:
|
|
88
|
+
raise ValueError(f"Invalid S3 URL: {url}")
|
|
89
|
+
bucket, key = parts
|
|
90
|
+
try:
|
|
91
|
+
s3_client.delete_object(Bucket=bucket, Key=key)
|
|
92
|
+
except Exception as e:
|
|
93
|
+
raise RuntimeError(f"Failed to delete S3 object {bucket}/{key}: {e}")
|
th2rag/config.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from urllib.parse import quote_plus
|
|
3
|
+
|
|
4
|
+
from dotenv import load_dotenv
|
|
5
|
+
from pydantic import ConfigDict, Field
|
|
6
|
+
from pydantic_settings import BaseSettings
|
|
7
|
+
|
|
8
|
+
dotenv_path = os.getenv("WORK_DIR", default=os.getcwd()) + "/.env"
|
|
9
|
+
load_dotenv(dotenv_path)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Settings(BaseSettings):
|
|
14
|
+
|
|
15
|
+
# fast api
|
|
16
|
+
root_path: str = Field(
|
|
17
|
+
default="http://localhost:8000",
|
|
18
|
+
description="Backendenpoint"
|
|
19
|
+
)
|
|
20
|
+
# database variables
|
|
21
|
+
db_host: str
|
|
22
|
+
db_port: str
|
|
23
|
+
db_name: str
|
|
24
|
+
db_user: str
|
|
25
|
+
db_password: str
|
|
26
|
+
db_schema: str
|
|
27
|
+
db_schema_demo: str = "th2_llm"
|
|
28
|
+
bypass_auth: bool = False
|
|
29
|
+
|
|
30
|
+
# security variables
|
|
31
|
+
algorithm: str
|
|
32
|
+
access_token_expire_minutes: int
|
|
33
|
+
encrypt_key: str
|
|
34
|
+
|
|
35
|
+
# sqlalchemy config
|
|
36
|
+
echo_sql: bool = False
|
|
37
|
+
|
|
38
|
+
# Mistral Config
|
|
39
|
+
mistral_api_key: str
|
|
40
|
+
mistral_api_url: str = Field(
|
|
41
|
+
default="https://api.mistral.ai/v1",
|
|
42
|
+
description="Mistral API base URL (or OVH endpoint)"
|
|
43
|
+
)
|
|
44
|
+
mistral_model_name: str = Field(
|
|
45
|
+
default="mistral-nemo-instruct-2407",
|
|
46
|
+
description="Mistral model to use for generation"
|
|
47
|
+
)
|
|
48
|
+
mistral_max_tokens: int = Field(
|
|
49
|
+
default=512,
|
|
50
|
+
description="Maximum tokens for Mistral generation"
|
|
51
|
+
)
|
|
52
|
+
mistral_temperature: float = Field(
|
|
53
|
+
default=0.0,
|
|
54
|
+
ge=0.0,
|
|
55
|
+
le=2.0,
|
|
56
|
+
description="Temperature for generation (0.0-2.0)"
|
|
57
|
+
)
|
|
58
|
+
mistral_top_p: float = Field(
|
|
59
|
+
default=1.0,
|
|
60
|
+
ge=0.0,
|
|
61
|
+
le=1.0,
|
|
62
|
+
description="Top-p sampling parameter"
|
|
63
|
+
)
|
|
64
|
+
mistral_presence_penalty: float = Field(
|
|
65
|
+
default=0.0,
|
|
66
|
+
ge=-2.0,
|
|
67
|
+
le=2.0,
|
|
68
|
+
description="Presence penalty (-2.0 to 2.0)"
|
|
69
|
+
)
|
|
70
|
+
mistral_stream: bool = Field(
|
|
71
|
+
default=True,
|
|
72
|
+
description="Whether to stream responses"
|
|
73
|
+
)
|
|
74
|
+
#Lightllm config
|
|
75
|
+
litellm_provider: str = Field(
|
|
76
|
+
default="mistral",
|
|
77
|
+
description="LLM provider: 'mistral', 'anthropic', etc."
|
|
78
|
+
)
|
|
79
|
+
litellm_model: str = Field(
|
|
80
|
+
default="Mistral-Small-3.2-24B-Instruct-2506",
|
|
81
|
+
description="Model name (without provider prefix)"
|
|
82
|
+
)
|
|
83
|
+
litellm_api_key: str | None = Field(
|
|
84
|
+
default=None,
|
|
85
|
+
description="API key for the LiteLLM provider"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
anthropic_api_key: str | None = Field(
|
|
90
|
+
default=None,
|
|
91
|
+
description="Anthropic API key"
|
|
92
|
+
)
|
|
93
|
+
anthropic_model: str = Field(
|
|
94
|
+
default="claude-3-haiku-20240307",
|
|
95
|
+
description="Anthropic model name"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# Embedding Model Configuration
|
|
99
|
+
embedding_model_name: str = Field(
|
|
100
|
+
default="sentence-transformers/gtr-t5-large",
|
|
101
|
+
description="Sentence transformer model for embeddings"
|
|
102
|
+
)
|
|
103
|
+
embedding_cache_folder: str = Field(
|
|
104
|
+
default="./models/embeddings",
|
|
105
|
+
description="Local cache folder for embedding models"
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# S3 Storage Configuration
|
|
109
|
+
s3_region: str
|
|
110
|
+
s3_access_key_secret: str
|
|
111
|
+
s3_access_key: str
|
|
112
|
+
s3_endpoint: str
|
|
113
|
+
s3_bucket_name: str
|
|
114
|
+
storage_mode: str = "local"
|
|
115
|
+
|
|
116
|
+
# Application Configuration
|
|
117
|
+
working_mode: str = Field(default="development")
|
|
118
|
+
log_level: str = Field(default="INFO")
|
|
119
|
+
|
|
120
|
+
# RAG Configuration
|
|
121
|
+
rag_top_k: int = Field(
|
|
122
|
+
default=5,
|
|
123
|
+
description="Default number of chunks to retrieve"
|
|
124
|
+
)
|
|
125
|
+
lancedb_table_name: str = Field(
|
|
126
|
+
default="docling",
|
|
127
|
+
description="LanceDB table name"
|
|
128
|
+
)
|
|
129
|
+
# Mage AI Configuration
|
|
130
|
+
mage_base_url: str = Field(
|
|
131
|
+
default="http://localhost:6789",
|
|
132
|
+
alias="BASE_URL",
|
|
133
|
+
description="Mage AI base URL"
|
|
134
|
+
)
|
|
135
|
+
mage_api_key: str | None = Field(
|
|
136
|
+
default=None,
|
|
137
|
+
alias="API_KEY",
|
|
138
|
+
description="Mage AI API key for authentication"
|
|
139
|
+
)
|
|
140
|
+
mage_oauth_token: str | None = Field(
|
|
141
|
+
default=None,
|
|
142
|
+
alias="OAUTH_TOKEN",
|
|
143
|
+
description="Mage AI OAuth token for authentication"
|
|
144
|
+
)
|
|
145
|
+
mage_pipeline_uuid: str = Field(
|
|
146
|
+
default="process_pdf",
|
|
147
|
+
description="Mage AI pipeline UUID to trigger"
|
|
148
|
+
)
|
|
149
|
+
mage_trigger_name: str = Field(
|
|
150
|
+
default="multi_runner_trigger",
|
|
151
|
+
description="Mage AI trigger name"
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
model_config = ConfigDict(extra="ignore")
|
|
155
|
+
|
|
156
|
+
# PDF Processing Configuration
|
|
157
|
+
pdf_processing_mode: str = Field(
|
|
158
|
+
default="async",
|
|
159
|
+
description="PDF processing mode: 'async' (background) or 'sync' (wait for completion)"
|
|
160
|
+
)
|
|
161
|
+
pdf_processing_timeout: int = Field(
|
|
162
|
+
default=300,
|
|
163
|
+
gt=0,
|
|
164
|
+
description="Maximum time (seconds) to wait for synchronous PDF processing"
|
|
165
|
+
)
|
|
166
|
+
pdf_max_file_size_mb: int = Field(
|
|
167
|
+
default=30,
|
|
168
|
+
gt=0,
|
|
169
|
+
description="Maximum allowed PDF file size in megabytes"
|
|
170
|
+
)
|
|
171
|
+
pdf_auto_async_threshold_mb: int = Field(
|
|
172
|
+
default=5,
|
|
173
|
+
gt=0,
|
|
174
|
+
description="Auto-switch to async processing if PDF exceeds this size (MB)"
|
|
175
|
+
)
|
|
176
|
+
pdf_enable_auto_mode: bool = Field(
|
|
177
|
+
default=True,
|
|
178
|
+
description="Automatically choose sync/async based on file size"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
# Knowledge Timeout Monitor Configuration
|
|
182
|
+
knowledge_pending_timeout_minutes: int = Field(
|
|
183
|
+
default=60,
|
|
184
|
+
gt=0,
|
|
185
|
+
description="Minutes before a PENDING knowledge document is auto-failed"
|
|
186
|
+
)
|
|
187
|
+
knowledge_timeout_check_interval_seconds: int = Field(
|
|
188
|
+
default=60,
|
|
189
|
+
gt=0,
|
|
190
|
+
description="How often to check for stale PENDING documents"
|
|
191
|
+
)
|
|
192
|
+
knowledge_timeout_monitor_enabled: bool = Field(
|
|
193
|
+
default=True,
|
|
194
|
+
description="Enable/disable the automatic timeout monitor"
|
|
195
|
+
)
|
|
196
|
+
# Webhook Configuration (for notifying th2agent when indexation completes)
|
|
197
|
+
webhook_secret: str = Field(
|
|
198
|
+
default="th2-webhook-default-secret",
|
|
199
|
+
description="HMAC-SHA256 secret used to sign webhook payloads (must match th2agent RAG_WEBHOOK_SECRET)"
|
|
200
|
+
)
|
|
201
|
+
webhook_timeout_seconds: int = Field(
|
|
202
|
+
default=10,
|
|
203
|
+
gt=0,
|
|
204
|
+
description="HTTP timeout for webhook delivery attempts"
|
|
205
|
+
)
|
|
206
|
+
webhook_max_retries: int = Field(
|
|
207
|
+
default=3,
|
|
208
|
+
gt=0,
|
|
209
|
+
description="Maximum number of webhook delivery attempts"
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
# IMAGE HANDLING SWITCH
|
|
213
|
+
# "extract" (Default).
|
|
214
|
+
# "describe" using a vlm
|
|
215
|
+
handle_img_mode: str = "extract"
|
|
216
|
+
|
|
217
|
+
# Webhook Configuration
|
|
218
|
+
webhook_secret: str = Field(
|
|
219
|
+
default="th2-webhook-default-secret",
|
|
220
|
+
description="HMAC-SHA256 secret used to sign webhook payloads (must match th2agent RAG_WEBHOOK_SECRET)"
|
|
221
|
+
)
|
|
222
|
+
webhook_timeout_seconds: int = Field(
|
|
223
|
+
default=30,
|
|
224
|
+
gt=0,
|
|
225
|
+
description="HTTP timeout in seconds for webhook delivery"
|
|
226
|
+
)
|
|
227
|
+
webhook_max_retries: int = Field(
|
|
228
|
+
default=3,
|
|
229
|
+
ge=1,
|
|
230
|
+
description="Maximum number of delivery attempts for a webhook"
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def database_url(self) -> str:
|
|
236
|
+
encoded_password = quote_plus(self.db_password)
|
|
237
|
+
return f"postgresql+asyncpg://{self.db_user}:{encoded_password}@{self.db_host}:{self.db_port}/{self.db_name}"
|
|
238
|
+
|
|
239
|
+
@property
|
|
240
|
+
def alembic_url(self) -> str:
|
|
241
|
+
encoded_password = quote_plus(self.db_password)
|
|
242
|
+
return f"postgresql+psycopg2://{self.db_user}:{encoded_password}@{self.db_host}:{self.db_port}/{self.db_name}"
|
|
243
|
+
|
|
244
|
+
@property
|
|
245
|
+
def lancedb_url(self) -> str:
|
|
246
|
+
"""LanceDB storage path based on working mode"""
|
|
247
|
+
if self.working_mode == "dev":
|
|
248
|
+
return "tempdata/lancedb"
|
|
249
|
+
else:
|
|
250
|
+
return "data/lancedb"
|
|
251
|
+
|
|
252
|
+
def get_pdf_processing_mode(self, file_size_mb: float = None) -> str:
|
|
253
|
+
"""
|
|
254
|
+
Determine the appropriate PDF processing mode.
|
|
255
|
+
"""
|
|
256
|
+
if not self.pdf_enable_auto_mode:
|
|
257
|
+
return self.pdf_processing_mode
|
|
258
|
+
|
|
259
|
+
if file_size_mb is None:
|
|
260
|
+
return self.pdf_processing_mode
|
|
261
|
+
|
|
262
|
+
# Auto-determine based on file size
|
|
263
|
+
if file_size_mb > self.pdf_auto_async_threshold_mb:
|
|
264
|
+
return "async"
|
|
265
|
+
|
|
266
|
+
return "sync"
|
|
267
|
+
settings = Settings()
|
|
File without changes
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from fastapi import Depends, HTTPException, Path, status
|
|
2
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
3
|
+
|
|
4
|
+
from th2rag.conversations import exceptions, schemas, service
|
|
5
|
+
from th2rag.database import get_db
|
|
6
|
+
from th2rag.dependencies import get_current_user
|
|
7
|
+
from th2rag.users.schemas import User
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
async def get_conversation_or_404(
|
|
11
|
+
conversation_id: int = Path(..., description="The ID of the conversation"),
|
|
12
|
+
db: AsyncSession = Depends(get_db),
|
|
13
|
+
) -> schemas.Conversation:
|
|
14
|
+
try:
|
|
15
|
+
return await service.get_conversation_by_id(conversation_id, db)
|
|
16
|
+
except exceptions.ConversationNotFoundException:
|
|
17
|
+
raise HTTPException(status.HTTP_404_NOT_FOUND, "Conversation not found")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
async def require_conversation_owner_or_admin(
|
|
21
|
+
conversation: schemas.Conversation = Depends(get_conversation_or_404),
|
|
22
|
+
current_user: User = Depends(get_current_user),
|
|
23
|
+
) -> schemas.Conversation:
|
|
24
|
+
if (
|
|
25
|
+
current_user.role != "ADMIN"
|
|
26
|
+
and conversation.user_id is not None
|
|
27
|
+
and conversation.user_id != current_user.user_id
|
|
28
|
+
):
|
|
29
|
+
raise HTTPException(status.HTTP_403_FORBIDDEN, detail="Not authorized")
|
|
30
|
+
return conversation
|