th2rag 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- th2rag/__init__.py +0 -0
- th2rag/auth/__init__.py +0 -0
- th2rag/auth/exceptions.py +2 -0
- th2rag/auth/router.py +43 -0
- th2rag/auth/schemas.py +15 -0
- th2rag/auth/service.py +115 -0
- th2rag/clients/__init__.py +0 -0
- th2rag/clients/storage/config.py +13 -0
- th2rag/clients/storage/local.py +48 -0
- th2rag/clients/storage/s3.py +93 -0
- th2rag/config.py +267 -0
- th2rag/conversations/__init__.py +0 -0
- th2rag/conversations/dependencies.py +30 -0
- th2rag/conversations/exceptions.py +6 -0
- th2rag/conversations/router.py +47 -0
- th2rag/conversations/schemas.py +43 -0
- th2rag/conversations/service.py +51 -0
- th2rag/database.py +72 -0
- th2rag/dataset/__init__.py +0 -0
- th2rag/dataset/dependencies.py +25 -0
- th2rag/dataset/exceptions.py +2 -0
- th2rag/dataset/router.py +53 -0
- th2rag/dataset/schemas.py +23 -0
- th2rag/dataset/service.py +78 -0
- th2rag/dependencies.py +143 -0
- th2rag/knowledge/__init__.py +0 -0
- th2rag/knowledge/dependencies.py +25 -0
- th2rag/knowledge/exceptions.py +2 -0
- th2rag/knowledge/mage_client.py +505 -0
- th2rag/knowledge/router.py +714 -0
- th2rag/knowledge/schemas.py +108 -0
- th2rag/knowledge/service.py +392 -0
- th2rag/logger.py +21 -0
- th2rag/main.py +51 -0
- th2rag/messages/__init__.py +0 -0
- th2rag/messages/exceptions.py +2 -0
- th2rag/messages/router.py +63 -0
- th2rag/messages/schemas.py +68 -0
- th2rag/messages/service.py +210 -0
- th2rag/models.py +215 -0
- th2rag/pagination.py +68 -0
- th2rag/rag/__init__.py +0 -0
- th2rag/rag/chunkers/__init__.py +0 -0
- th2rag/rag/chunkers/hybrid_chunker.py +23 -0
- th2rag/rag/constants.py +15 -0
- th2rag/rag/converters/__init__.py +0 -0
- th2rag/rag/converters/documents_converter.py +123 -0
- th2rag/rag/converters/json_converter.py +521 -0
- th2rag/rag/converters/legacy_converter.py +124 -0
- th2rag/rag/converters/pdf_converter.py +384 -0
- th2rag/rag/converters/spreadsheet_converter.py +159 -0
- th2rag/rag/converters/txt_converter.py +98 -0
- th2rag/rag/embeddings/__init__.py +0 -0
- th2rag/rag/embeddings/base.py +21 -0
- th2rag/rag/embeddings/exceptions.py +2 -0
- th2rag/rag/embeddings/sentence_transformer.py +97 -0
- th2rag/rag/generation/base.py +22 -0
- th2rag/rag/generation/exceptions.py +2 -0
- th2rag/rag/generation/litellm_generator.py +143 -0
- th2rag/rag/generation/mistral_generator.py +269 -0
- th2rag/rag/generation/test_litellm.py +33 -0
- th2rag/rag/retrieval/__init__.py +0 -0
- th2rag/rag/retrieval/base.py +11 -0
- th2rag/rag/retrieval/exceptions.py +10 -0
- th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
- th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
- th2rag/rag/retrieval/lancedb/schemas.py +19 -0
- th2rag/rag/services/__init__.py +0 -0
- th2rag/rag/services/rag_service.py +99 -0
- th2rag/rag/services/rag_service_builder.py +42 -0
- th2rag/rag/tasks/process_pdf.py +586 -0
- th2rag/users/dependencies.py +44 -0
- th2rag/users/exceptions.py +6 -0
- th2rag/users/router.py +92 -0
- th2rag/users/schemas.py +61 -0
- th2rag/users/service.py +140 -0
- th2rag/utils/__init__.py +0 -0
- th2rag/utils/helpers.py +40 -0
- th2rag/utils/logger.py +15 -0
- th2rag/utils/security.py +76 -0
- th2rag/utils/welcome_router.py +98 -0
- th2rag/webhooks/__init__.py +0 -0
- th2rag/webhooks/sender.py +126 -0
- th2rag-0.0.2.dist-info/METADATA +1002 -0
- th2rag-0.0.2.dist-info/RECORD +87 -0
- th2rag-0.0.2.dist-info/WHEEL +4 -0
- th2rag-0.0.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Sender(str, Enum):
|
|
9
|
+
USER = "USER"
|
|
10
|
+
SYSTEM = "SYSTEM"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class MessageType(str, Enum):
|
|
14
|
+
TEXT = "TEXT"
|
|
15
|
+
SCRIPT = "SCRIPT"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FeedbackType(str, Enum):
|
|
19
|
+
A = "A"
|
|
20
|
+
B = "B"
|
|
21
|
+
OTHER = "OTHER"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class MessageBase(BaseModel):
|
|
29
|
+
conversation_id: int | None = None
|
|
30
|
+
content: str
|
|
31
|
+
sender: Sender
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class MessageCreate(MessageBase):
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Message(MessageBase):
|
|
39
|
+
message_id: int
|
|
40
|
+
type: MessageType
|
|
41
|
+
created_at: datetime
|
|
42
|
+
updated_at: datetime
|
|
43
|
+
# feedback: Optional["Feedback"] = Field(default=None)
|
|
44
|
+
|
|
45
|
+
model_config = ConfigDict(from_attributes=True)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class FeedbackBase(BaseModel):
|
|
49
|
+
message_id: int | None = None
|
|
50
|
+
feedback_type: FeedbackType | None
|
|
51
|
+
like: bool | None = None
|
|
52
|
+
reason: str | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class FeedbackCreate(FeedbackBase):
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Feedback(FeedbackBase):
|
|
60
|
+
feedback_id: int
|
|
61
|
+
created_at: datetime
|
|
62
|
+
updated_at: datetime
|
|
63
|
+
|
|
64
|
+
model_config = ConfigDict(from_attributes=True)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
Message.model_rebuild()
|
|
68
|
+
Feedback.model_rebuild()
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from sqlalchemy import select
|
|
4
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
5
|
+
from sqlalchemy.orm import selectinload
|
|
6
|
+
from sqlalchemy.orm.attributes import set_committed_value
|
|
7
|
+
|
|
8
|
+
from th2rag.conversations import (
|
|
9
|
+
exceptions as conversations_exceptions,
|
|
10
|
+
)
|
|
11
|
+
from th2rag.conversations import (
|
|
12
|
+
service as conversations_service,
|
|
13
|
+
)
|
|
14
|
+
from th2rag.dataset import service as dataset_service
|
|
15
|
+
from th2rag.messages import exceptions, schemas
|
|
16
|
+
from th2rag.models import Feedback, Knowledge, Message
|
|
17
|
+
from th2rag.pagination import PageParams, PageResponse, paginate
|
|
18
|
+
from th2rag.rag.services.rag_service import RAGService
|
|
19
|
+
from th2rag.utils.helpers import build_script_generation_prompt, construct_history
|
|
20
|
+
from th2rag.dependencies import get_rag_service
|
|
21
|
+
from fastapi import HTTPException
|
|
22
|
+
from th2rag.rag.services.rag_service import RAGService
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
async def get_all_messages(
|
|
27
|
+
conversation_id: int, page_params: PageParams, db: AsyncSession
|
|
28
|
+
) -> PageResponse[schemas.Message]:
|
|
29
|
+
stmt = (
|
|
30
|
+
select(Message)
|
|
31
|
+
.where(Message.conversation_id == conversation_id)
|
|
32
|
+
.order_by(Message.created_at.desc())
|
|
33
|
+
.options(selectinload(Message.conversation))
|
|
34
|
+
.options(selectinload(Message.feedback))
|
|
35
|
+
)
|
|
36
|
+
return await paginate(db, stmt, page_params, schemas.Message)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
async def get_message_by_id(
|
|
40
|
+
conversation_id: int, message_id: int, db: AsyncSession
|
|
41
|
+
) -> schemas.Message:
|
|
42
|
+
stmt = (
|
|
43
|
+
select(Message)
|
|
44
|
+
.where(
|
|
45
|
+
Message.message_id == message_id,
|
|
46
|
+
Message.conversation_id == conversation_id,
|
|
47
|
+
)
|
|
48
|
+
.options(selectinload(Message.feedback))
|
|
49
|
+
)
|
|
50
|
+
result = await db.execute(stmt)
|
|
51
|
+
message = result.scalar_one_or_none()
|
|
52
|
+
if not message:
|
|
53
|
+
raise exceptions.MessageNotFoundException(
|
|
54
|
+
"Message not found or does not belong to the conversation"
|
|
55
|
+
)
|
|
56
|
+
return schemas.Message.model_validate(message)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def create_message(
|
|
60
|
+
conversation_id: int, message_in: schemas.MessageCreate, db: AsyncSession
|
|
61
|
+
) -> schemas.Message:
|
|
62
|
+
try:
|
|
63
|
+
current_conversation = await conversations_service.get_conversation_by_id(
|
|
64
|
+
conversation_id, db
|
|
65
|
+
)
|
|
66
|
+
new_message = Message(
|
|
67
|
+
conversation_id=conversation_id, content=message_in.content, sender=message_in.sender
|
|
68
|
+
)
|
|
69
|
+
db.add(new_message)
|
|
70
|
+
await db.commit()
|
|
71
|
+
await db.refresh(new_message)
|
|
72
|
+
set_committed_value(new_message, "feedback", None)
|
|
73
|
+
return schemas.Message.model_validate(new_message)
|
|
74
|
+
|
|
75
|
+
except conversations_exceptions.ConversationNotFoundException as e:
|
|
76
|
+
raise e
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def create_and_respond_message(
|
|
82
|
+
conversation_id: int,
|
|
83
|
+
message_in: schemas.MessageCreate,
|
|
84
|
+
db: AsyncSession,
|
|
85
|
+
rag_service: RAGService,
|
|
86
|
+
) -> schemas.Message:
|
|
87
|
+
# Récupération de la conversation
|
|
88
|
+
conversation = await conversations_service.get_conversation_by_id(
|
|
89
|
+
conversation_id=conversation_id, db=db
|
|
90
|
+
)
|
|
91
|
+
if not conversation:
|
|
92
|
+
raise HTTPException(status_code=404, detail="Conversation not found")
|
|
93
|
+
|
|
94
|
+
# Check knowledge status BEFORE loading models or processing anything
|
|
95
|
+
if conversation.knowledge_id:
|
|
96
|
+
stmt = select(Knowledge).where(Knowledge.knowledge_id == conversation.knowledge_id)
|
|
97
|
+
result = await db.execute(stmt)
|
|
98
|
+
knowledge = result.scalar_one_or_none()
|
|
99
|
+
if knowledge and knowledge.status.value in ["FAILED"]:
|
|
100
|
+
raise HTTPException(
|
|
101
|
+
status_code=422,
|
|
102
|
+
detail="The knowledge base could not be processed and is unavailable. Please try again or create a new knowledge base."
|
|
103
|
+
)
|
|
104
|
+
if knowledge and knowledge.status.value in ["PENDING"]:
|
|
105
|
+
raise HTTPException(
|
|
106
|
+
status_code=202,
|
|
107
|
+
detail="The knowledge base is currently being processed. Please wait a moment and try again."
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# Récupération de l'historique
|
|
111
|
+
stmt = (
|
|
112
|
+
select(Message)
|
|
113
|
+
.where(Message.conversation_id == conversation_id)
|
|
114
|
+
.order_by(Message.created_at.asc())
|
|
115
|
+
)
|
|
116
|
+
result = await db.execute(stmt)
|
|
117
|
+
messages = result.scalars().all()
|
|
118
|
+
history = construct_history(messages=messages)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
if conversation.dataset_id:
|
|
123
|
+
dataset = await dataset_service.get_dataset_by_id(dataset_id=conversation.dataset_id, db=db)
|
|
124
|
+
if not dataset:
|
|
125
|
+
raise HTTPException(status_code=404, detail="Dataset not found")
|
|
126
|
+
|
|
127
|
+
prompt = build_script_generation_prompt(message_in.content, dataset.column_types)
|
|
128
|
+
|
|
129
|
+
try:
|
|
130
|
+
llm_output = rag_service.generate_code_snippet(
|
|
131
|
+
question=message_in.content, prompt=prompt, history=history
|
|
132
|
+
)
|
|
133
|
+
parsed_output = json.loads(llm_output)
|
|
134
|
+
generated_text = parsed_output.get("script")
|
|
135
|
+
except json.JSONDecodeError:
|
|
136
|
+
raise HTTPException(status_code=500, detail="LLM did not return valid JSON")
|
|
137
|
+
except Exception as e:
|
|
138
|
+
raise HTTPException(status_code=500, detail=f"Script generation failed: {str(e)}")
|
|
139
|
+
|
|
140
|
+
message_type = schemas.MessageType.SCRIPT
|
|
141
|
+
|
|
142
|
+
else:
|
|
143
|
+
# MODE = text
|
|
144
|
+
doc_id = conversation.knowledge_id
|
|
145
|
+
prompt = "You are a helpful assistant"
|
|
146
|
+
if doc_id:
|
|
147
|
+
stmt = select(Knowledge).where(Knowledge.knowledge_id == doc_id)
|
|
148
|
+
result = await db.execute(stmt)
|
|
149
|
+
knowledge = result.scalar_one_or_none()
|
|
150
|
+
if knowledge:
|
|
151
|
+
prompt = knowledge.prompt
|
|
152
|
+
|
|
153
|
+
try:
|
|
154
|
+
generated_text = rag_service.answer_question(
|
|
155
|
+
question=message_in.content, doc_id=doc_id, limit=3, prompt=prompt, history=history
|
|
156
|
+
)
|
|
157
|
+
except Exception as e:
|
|
158
|
+
raise HTTPException(status_code=500, detail=f"Response generation failed: {str(e)}")
|
|
159
|
+
|
|
160
|
+
message_type = schemas.MessageType.TEXT
|
|
161
|
+
|
|
162
|
+
# Créer le message utilisateur
|
|
163
|
+
user_message = Message(
|
|
164
|
+
conversation_id=conversation_id,
|
|
165
|
+
content=message_in.content,
|
|
166
|
+
sender=message_in.sender,
|
|
167
|
+
type=schemas.MessageType.TEXT,
|
|
168
|
+
)
|
|
169
|
+
db.add(user_message)
|
|
170
|
+
|
|
171
|
+
# Créer le message système
|
|
172
|
+
system_message = Message(
|
|
173
|
+
conversation_id=conversation_id,
|
|
174
|
+
content=generated_text,
|
|
175
|
+
sender=schemas.Sender.SYSTEM,
|
|
176
|
+
type=message_type,
|
|
177
|
+
)
|
|
178
|
+
db.add(system_message)
|
|
179
|
+
|
|
180
|
+
await db.commit()
|
|
181
|
+
await db.refresh(system_message)
|
|
182
|
+
return schemas.Message.model_validate(system_message)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
async def add_feedback_to_message(
|
|
186
|
+
conversation_id: int, message_id: int, feedback_in: schemas.FeedbackCreate, db: AsyncSession
|
|
187
|
+
) -> schemas.Message:
|
|
188
|
+
stmt = select(Message).where(
|
|
189
|
+
Message.message_id == message_id, Message.conversation_id == conversation_id
|
|
190
|
+
)
|
|
191
|
+
result = await db.execute(stmt)
|
|
192
|
+
message = result.scalar_one_or_none()
|
|
193
|
+
if not message:
|
|
194
|
+
raise exceptions.MessageNotFoundException(
|
|
195
|
+
"Message not found or does not belong to the conversation"
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
new_feedback = Feedback(
|
|
199
|
+
feedback_id=message.message_id,
|
|
200
|
+
feedback_type=feedback_in.feedback_type,
|
|
201
|
+
like=feedback_in.like,
|
|
202
|
+
reason=feedback_in.reason,
|
|
203
|
+
)
|
|
204
|
+
message.feedback = new_feedback
|
|
205
|
+
|
|
206
|
+
db.add(new_feedback)
|
|
207
|
+
await db.commit()
|
|
208
|
+
await db.refresh(message)
|
|
209
|
+
|
|
210
|
+
return schemas.Message.model_validate(message)
|
th2rag/models.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import enum
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
|
|
4
|
+
from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, String, Text
|
|
5
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
|
6
|
+
from sqlalchemy.orm import relationship
|
|
7
|
+
|
|
8
|
+
from th2rag.config import settings
|
|
9
|
+
from th2rag.database import Base
|
|
10
|
+
from th2rag.rag.constants import DEFAULT_PROMPT
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Sender(enum.Enum):
|
|
14
|
+
USER = "USER"
|
|
15
|
+
SYSTEM = "SYSTEM"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FeebackType(enum.Enum):
|
|
19
|
+
A = "A"
|
|
20
|
+
B = "B"
|
|
21
|
+
OTHER = "OTHER"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class UserRole(enum.Enum):
|
|
25
|
+
ADMIN = "ADMIN"
|
|
26
|
+
USER = "USER"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Status(enum.Enum):
|
|
30
|
+
PENDING = "PENDING"
|
|
31
|
+
COMPLETED = "COMPLETED"
|
|
32
|
+
FAILED = "FAILED"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MessageType(enum.Enum):
|
|
36
|
+
TEXT = "TEXT"
|
|
37
|
+
SCRIPT = "SCRIPT"
|
|
38
|
+
|
|
39
|
+
class User(Base):
|
|
40
|
+
__tablename__ = "user"
|
|
41
|
+
|
|
42
|
+
user_id = Column(Integer, primary_key=True)
|
|
43
|
+
first_name = Column(String(100), nullable=False)
|
|
44
|
+
last_name = Column(String(100), nullable=False)
|
|
45
|
+
email = Column(String(255), unique=True, nullable=False)
|
|
46
|
+
password = Column(String(255), nullable=True)
|
|
47
|
+
role = Column(
|
|
48
|
+
Enum(UserRole, name="role_enum", schema=settings.db_schema),
|
|
49
|
+
nullable=False,
|
|
50
|
+
default=UserRole.USER,
|
|
51
|
+
)
|
|
52
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
53
|
+
updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
|
54
|
+
|
|
55
|
+
# Relationships
|
|
56
|
+
conversations = relationship("Conversation", back_populates="user")
|
|
57
|
+
knowledges = relationship("Knowledge", back_populates="user")
|
|
58
|
+
datasets = relationship("Dataset", back_populates="user")
|
|
59
|
+
llm_config=relationship("UserLLMConfig", back_populates="user", uselist=False)
|
|
60
|
+
|
|
61
|
+
def __repr__(self):
|
|
62
|
+
return f"<User(user_id={self.user_id}, email={self.email})>"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Conversation(Base):
|
|
66
|
+
__tablename__ = "conversation"
|
|
67
|
+
|
|
68
|
+
conversation_id = Column(Integer, primary_key=True)
|
|
69
|
+
knowledge_id = Column(
|
|
70
|
+
Integer, ForeignKey("knowledge.knowledge_id", ondelete="SET NULL"), nullable=True
|
|
71
|
+
)
|
|
72
|
+
dataset_id = Column(
|
|
73
|
+
Integer, ForeignKey("Dataset.dataset_id", ondelete="SET NULL"), nullable=True
|
|
74
|
+
)
|
|
75
|
+
user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
|
|
76
|
+
title = Column(String(255), nullable=False)
|
|
77
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
78
|
+
updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
|
79
|
+
|
|
80
|
+
# Relationships
|
|
81
|
+
user = relationship("User", back_populates="conversations")
|
|
82
|
+
messages = relationship(
|
|
83
|
+
"Message", back_populates="conversation", cascade="all, delete-orphan", passive_deletes=True
|
|
84
|
+
)
|
|
85
|
+
knowledge = relationship("Knowledge", back_populates="conversations")
|
|
86
|
+
dataset = relationship("Dataset", back_populates="conversations")
|
|
87
|
+
|
|
88
|
+
def __repr__(self):
|
|
89
|
+
return f"<Conversation(conversation_id={self.conversation_id})>"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class Message(Base):
|
|
93
|
+
__tablename__ = "message"
|
|
94
|
+
|
|
95
|
+
message_id = Column(Integer, primary_key=True)
|
|
96
|
+
conversation_id = Column(
|
|
97
|
+
Integer, ForeignKey("conversation.conversation_id", ondelete="CASCADE"), nullable=False
|
|
98
|
+
)
|
|
99
|
+
content = Column(Text, nullable=False)
|
|
100
|
+
sender = Column(Enum(Sender, name="sender_enum", schema=settings.db_schema), nullable=False)
|
|
101
|
+
type = Column(Enum(MessageType, name="type_enum", schema=settings.db_schema), nullable=False)
|
|
102
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
103
|
+
updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
|
104
|
+
|
|
105
|
+
# Relationships
|
|
106
|
+
conversation = relationship("Conversation", back_populates="messages")
|
|
107
|
+
feedback = relationship("Feedback", uselist=False, back_populates="message")
|
|
108
|
+
sources = relationship("Source", back_populates="message")
|
|
109
|
+
|
|
110
|
+
def __repr__(self):
|
|
111
|
+
return f"<Message(message_id={self.message_id})>"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class Source(Base):
|
|
115
|
+
__tablename__ = "source"
|
|
116
|
+
|
|
117
|
+
source_id = Column(Integer, primary_key=True)
|
|
118
|
+
message_id = Column(
|
|
119
|
+
Integer, ForeignKey("message.message_id", ondelete="CASCADE"), nullable=False
|
|
120
|
+
)
|
|
121
|
+
source_path = Column(String(255))
|
|
122
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
123
|
+
|
|
124
|
+
# Relationships
|
|
125
|
+
message = relationship("Message", back_populates="sources")
|
|
126
|
+
|
|
127
|
+
def __repr__(self):
|
|
128
|
+
return f"<Source(source_id={self.source_id}, name={self.source_path})>"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class Feedback(Base):
|
|
132
|
+
__tablename__ = "feedback"
|
|
133
|
+
|
|
134
|
+
feedback_id = Column(
|
|
135
|
+
Integer, ForeignKey("message.message_id", ondelete="CASCADE"), primary_key=True
|
|
136
|
+
)
|
|
137
|
+
feedback_type = Column(
|
|
138
|
+
Enum(FeebackType, name="feedback_enum", schema=settings.db_schema), nullable=True
|
|
139
|
+
)
|
|
140
|
+
like = Column(Boolean, nullable=False)
|
|
141
|
+
reason = Column(Text, nullable=True)
|
|
142
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
143
|
+
updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
|
144
|
+
|
|
145
|
+
# Relationships
|
|
146
|
+
message = relationship("Message", back_populates="feedback")
|
|
147
|
+
|
|
148
|
+
def __repr__(self):
|
|
149
|
+
return f"<Feedback(feedback_id={self.feedback_id})>"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class Knowledge(Base):
|
|
153
|
+
__tablename__ = "knowledge"
|
|
154
|
+
|
|
155
|
+
knowledge_id = Column(Integer, primary_key=True)
|
|
156
|
+
name = Column(String(255), nullable=False)
|
|
157
|
+
description = Column(Text, nullable=True)
|
|
158
|
+
status = Column(
|
|
159
|
+
Enum(Status, name="status_enum", schema=settings.db_schema),
|
|
160
|
+
nullable=False,
|
|
161
|
+
default=Status.PENDING,
|
|
162
|
+
)
|
|
163
|
+
prompt = Column(Text, default=DEFAULT_PROMPT)
|
|
164
|
+
knowledge_path = Column(String(255), nullable=False)
|
|
165
|
+
# Changed from Text to JSONB for better performance and querying
|
|
166
|
+
knowledge_paths = Column(JSONB, nullable=True)
|
|
167
|
+
filenames = Column(JSONB, nullable=True)
|
|
168
|
+
user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
|
|
169
|
+
callback_url = Column(String(2048), nullable=True)
|
|
170
|
+
error_message = Column(Text, nullable=True)
|
|
171
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
172
|
+
updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
|
173
|
+
|
|
174
|
+
# Relationships
|
|
175
|
+
conversations = relationship("Conversation", back_populates="knowledge")
|
|
176
|
+
user = relationship("User", back_populates="knowledges")
|
|
177
|
+
|
|
178
|
+
def __repr__(self):
|
|
179
|
+
return f"<Knowledge(knowledge_id={self.knowledge_id}, name={self.name})>"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class Dataset(Base):
|
|
183
|
+
__tablename__ = "Dataset"
|
|
184
|
+
|
|
185
|
+
dataset_id = Column(Integer, primary_key=True)
|
|
186
|
+
name = Column(String(255), nullable=False)
|
|
187
|
+
n_columns = Column(Integer, nullable=False)
|
|
188
|
+
n_rows = Column(Integer, nullable=False)
|
|
189
|
+
user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
|
|
190
|
+
column_types = Column(JSONB, nullable=False)
|
|
191
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
192
|
+
|
|
193
|
+
conversations = relationship("Conversation", back_populates="dataset")
|
|
194
|
+
user = relationship("User", back_populates="datasets")
|
|
195
|
+
|
|
196
|
+
def __repr__(self):
|
|
197
|
+
return f"<Dataset(dataset_id={self.dataset_id}, name={self.name})>"
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class UserLLMConfig(Base):
|
|
202
|
+
__tablename__ = "user_llm_config"
|
|
203
|
+
|
|
204
|
+
config_id = Column(Integer, primary_key=True)
|
|
205
|
+
user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
|
|
206
|
+
provider = Column(String(100), nullable=False)
|
|
207
|
+
model_name = Column(String(100), nullable=False)
|
|
208
|
+
api_key = Column(String(255), nullable=False)
|
|
209
|
+
created_at = Column(DateTime, default=datetime.now)
|
|
210
|
+
updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
|
|
211
|
+
|
|
212
|
+
user = relationship("User", back_populates="llm_config")
|
|
213
|
+
|
|
214
|
+
def __repr__(self):
|
|
215
|
+
return f"<UserLLMConfig(config_id={self.config_id}, user_id={self.user_id})>"
|
th2rag/pagination.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from typing import Generic, TypeVar
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
from sqlalchemy import func, select
|
|
5
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
6
|
+
from sqlalchemy.sql import Select
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PageParams(BaseModel):
|
|
10
|
+
"""Request query params for paginated API."""
|
|
11
|
+
|
|
12
|
+
page: int = Field(default=1, ge=1)
|
|
13
|
+
size: int = Field(default=20, ge=1, le=100)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
T = TypeVar("T")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PageResponse(BaseModel, Generic[T]):
|
|
20
|
+
total: int
|
|
21
|
+
page: int
|
|
22
|
+
size: int
|
|
23
|
+
has_next: bool
|
|
24
|
+
has_previous: bool
|
|
25
|
+
results: list[T]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# def paginate(query: Query, page_params: PageParams, ResponseSchema: BaseModel) -> PageResponse[T]:
|
|
29
|
+
|
|
30
|
+
# paginated_query = query.offset((page_params.page - 1) * page_params.size).limit(page_params.size).all()
|
|
31
|
+
# total = query.count()
|
|
32
|
+
# has_next = (page_params.page * page_params.size) < total
|
|
33
|
+
# has_previous = page_params.page > 1
|
|
34
|
+
|
|
35
|
+
# return PageResponse(
|
|
36
|
+
# total=total,
|
|
37
|
+
# page=page_params.page,
|
|
38
|
+
# size=page_params.size,
|
|
39
|
+
# has_next=has_next,
|
|
40
|
+
# has_previous=has_previous,
|
|
41
|
+
# results=[ResponseSchema.model_validate(item) for item in paginated_query]
|
|
42
|
+
# )
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def paginate(
|
|
46
|
+
session: AsyncSession, stmt: Select, page_params: PageParams, ResponseSchema: BaseModel
|
|
47
|
+
) -> PageResponse[T]:
|
|
48
|
+
offset_val = (page_params.page - 1) * page_params.size
|
|
49
|
+
paginated_stmt = stmt.offset(offset_val).limit(page_params.size)
|
|
50
|
+
|
|
51
|
+
result = await session.execute(paginated_stmt)
|
|
52
|
+
items = result.scalars().all()
|
|
53
|
+
|
|
54
|
+
count_stmt = select(func.count()).select_from(stmt.subquery())
|
|
55
|
+
count_result = await session.execute(count_stmt)
|
|
56
|
+
total = count_result.scalar_one()
|
|
57
|
+
|
|
58
|
+
has_next = (page_params.page * page_params.size) < total
|
|
59
|
+
has_previous = page_params.page > 1
|
|
60
|
+
|
|
61
|
+
return PageResponse(
|
|
62
|
+
total=total,
|
|
63
|
+
page=page_params.page,
|
|
64
|
+
size=page_params.size,
|
|
65
|
+
has_next=has_next,
|
|
66
|
+
has_previous=has_previous,
|
|
67
|
+
results=[ResponseSchema.model_validate(item) for item in items],
|
|
68
|
+
)
|
th2rag/rag/__init__.py
ADDED
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from docling.chunking import HybridChunker
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class HybridTextChunker:
|
|
5
|
+
"""
|
|
6
|
+
Wraps the HybridChunker from docling to split documents into chunks.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
def __init__(self, tokenizer: str = "bert-base-uncased", merge_peers: bool = True):
|
|
10
|
+
self.chunker = HybridChunker(tokenizer=tokenizer, merge_peers=merge_peers, max_tokens=512)
|
|
11
|
+
|
|
12
|
+
def chunk_document(self, dl_doc) -> list:
|
|
13
|
+
"""
|
|
14
|
+
Chunks the given document into smaller pieces.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
dl_doc: Document object from DocumentConverter.
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
list: A list of chunk objects.
|
|
21
|
+
"""
|
|
22
|
+
chunk_iter = self.chunker.chunk(dl_doc=dl_doc)
|
|
23
|
+
return list(chunk_iter)
|
th2rag/rag/constants.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
DEFAULT_PROMPT = """
|
|
2
|
+
# Role and Purpose
|
|
3
|
+
You are an AI assistant with access to a dynamic knowledge base. Users have uploaded various documents
|
|
4
|
+
to create a personalized repository. Your task is to answer the user's question based solely on the provided context.
|
|
5
|
+
|
|
6
|
+
# Guidelines:
|
|
7
|
+
1. Provide a clear and concise answer to the question.
|
|
8
|
+
2. Use only the information from the relevant context to support your answer.
|
|
9
|
+
3. The context is retrieved based on similarity search, so some information might be missing or irrelevant.
|
|
10
|
+
4. Be transparent when there is insufficient information to fully answer the question.
|
|
11
|
+
5. Do not make up or infer information not present in the provided context.
|
|
12
|
+
6. If you cannot answer the question based on the given context, clearly state that.
|
|
13
|
+
7. Maintain a helpful and professional tone.
|
|
14
|
+
8. Respond in the same language of the question.
|
|
15
|
+
"""
|
|
File without changes
|