th2rag 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. th2rag/__init__.py +0 -0
  2. th2rag/auth/__init__.py +0 -0
  3. th2rag/auth/exceptions.py +2 -0
  4. th2rag/auth/router.py +43 -0
  5. th2rag/auth/schemas.py +15 -0
  6. th2rag/auth/service.py +115 -0
  7. th2rag/clients/__init__.py +0 -0
  8. th2rag/clients/storage/config.py +13 -0
  9. th2rag/clients/storage/local.py +48 -0
  10. th2rag/clients/storage/s3.py +93 -0
  11. th2rag/config.py +267 -0
  12. th2rag/conversations/__init__.py +0 -0
  13. th2rag/conversations/dependencies.py +30 -0
  14. th2rag/conversations/exceptions.py +6 -0
  15. th2rag/conversations/router.py +47 -0
  16. th2rag/conversations/schemas.py +43 -0
  17. th2rag/conversations/service.py +51 -0
  18. th2rag/database.py +72 -0
  19. th2rag/dataset/__init__.py +0 -0
  20. th2rag/dataset/dependencies.py +25 -0
  21. th2rag/dataset/exceptions.py +2 -0
  22. th2rag/dataset/router.py +53 -0
  23. th2rag/dataset/schemas.py +23 -0
  24. th2rag/dataset/service.py +78 -0
  25. th2rag/dependencies.py +143 -0
  26. th2rag/knowledge/__init__.py +0 -0
  27. th2rag/knowledge/dependencies.py +25 -0
  28. th2rag/knowledge/exceptions.py +2 -0
  29. th2rag/knowledge/mage_client.py +505 -0
  30. th2rag/knowledge/router.py +714 -0
  31. th2rag/knowledge/schemas.py +108 -0
  32. th2rag/knowledge/service.py +392 -0
  33. th2rag/logger.py +21 -0
  34. th2rag/main.py +51 -0
  35. th2rag/messages/__init__.py +0 -0
  36. th2rag/messages/exceptions.py +2 -0
  37. th2rag/messages/router.py +63 -0
  38. th2rag/messages/schemas.py +68 -0
  39. th2rag/messages/service.py +210 -0
  40. th2rag/models.py +215 -0
  41. th2rag/pagination.py +68 -0
  42. th2rag/rag/__init__.py +0 -0
  43. th2rag/rag/chunkers/__init__.py +0 -0
  44. th2rag/rag/chunkers/hybrid_chunker.py +23 -0
  45. th2rag/rag/constants.py +15 -0
  46. th2rag/rag/converters/__init__.py +0 -0
  47. th2rag/rag/converters/documents_converter.py +123 -0
  48. th2rag/rag/converters/json_converter.py +521 -0
  49. th2rag/rag/converters/legacy_converter.py +124 -0
  50. th2rag/rag/converters/pdf_converter.py +384 -0
  51. th2rag/rag/converters/spreadsheet_converter.py +159 -0
  52. th2rag/rag/converters/txt_converter.py +98 -0
  53. th2rag/rag/embeddings/__init__.py +0 -0
  54. th2rag/rag/embeddings/base.py +21 -0
  55. th2rag/rag/embeddings/exceptions.py +2 -0
  56. th2rag/rag/embeddings/sentence_transformer.py +97 -0
  57. th2rag/rag/generation/base.py +22 -0
  58. th2rag/rag/generation/exceptions.py +2 -0
  59. th2rag/rag/generation/litellm_generator.py +143 -0
  60. th2rag/rag/generation/mistral_generator.py +269 -0
  61. th2rag/rag/generation/test_litellm.py +33 -0
  62. th2rag/rag/retrieval/__init__.py +0 -0
  63. th2rag/rag/retrieval/base.py +11 -0
  64. th2rag/rag/retrieval/exceptions.py +10 -0
  65. th2rag/rag/retrieval/lancedb/lancedb_retriever.py +25 -0
  66. th2rag/rag/retrieval/lancedb/lancedb_storage.py +49 -0
  67. th2rag/rag/retrieval/lancedb/schemas.py +19 -0
  68. th2rag/rag/services/__init__.py +0 -0
  69. th2rag/rag/services/rag_service.py +99 -0
  70. th2rag/rag/services/rag_service_builder.py +42 -0
  71. th2rag/rag/tasks/process_pdf.py +586 -0
  72. th2rag/users/dependencies.py +44 -0
  73. th2rag/users/exceptions.py +6 -0
  74. th2rag/users/router.py +92 -0
  75. th2rag/users/schemas.py +61 -0
  76. th2rag/users/service.py +140 -0
  77. th2rag/utils/__init__.py +0 -0
  78. th2rag/utils/helpers.py +40 -0
  79. th2rag/utils/logger.py +15 -0
  80. th2rag/utils/security.py +76 -0
  81. th2rag/utils/welcome_router.py +98 -0
  82. th2rag/webhooks/__init__.py +0 -0
  83. th2rag/webhooks/sender.py +126 -0
  84. th2rag-0.0.2.dist-info/METADATA +1002 -0
  85. th2rag-0.0.2.dist-info/RECORD +87 -0
  86. th2rag-0.0.2.dist-info/WHEEL +4 -0
  87. th2rag-0.0.2.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,68 @@
1
+ from datetime import datetime
2
+ from enum import Enum
3
+ from typing import TYPE_CHECKING
4
+
5
+ from pydantic import BaseModel, ConfigDict
6
+
7
+
8
+ class Sender(str, Enum):
9
+ USER = "USER"
10
+ SYSTEM = "SYSTEM"
11
+
12
+
13
+ class MessageType(str, Enum):
14
+ TEXT = "TEXT"
15
+ SCRIPT = "SCRIPT"
16
+
17
+
18
+ class FeedbackType(str, Enum):
19
+ A = "A"
20
+ B = "B"
21
+ OTHER = "OTHER"
22
+
23
+
24
+ if TYPE_CHECKING:
25
+ pass
26
+
27
+
28
+ class MessageBase(BaseModel):
29
+ conversation_id: int | None = None
30
+ content: str
31
+ sender: Sender
32
+
33
+
34
+ class MessageCreate(MessageBase):
35
+ pass
36
+
37
+
38
+ class Message(MessageBase):
39
+ message_id: int
40
+ type: MessageType
41
+ created_at: datetime
42
+ updated_at: datetime
43
+ # feedback: Optional["Feedback"] = Field(default=None)
44
+
45
+ model_config = ConfigDict(from_attributes=True)
46
+
47
+
48
+ class FeedbackBase(BaseModel):
49
+ message_id: int | None = None
50
+ feedback_type: FeedbackType | None
51
+ like: bool | None = None
52
+ reason: str | None = None
53
+
54
+
55
+ class FeedbackCreate(FeedbackBase):
56
+ pass
57
+
58
+
59
+ class Feedback(FeedbackBase):
60
+ feedback_id: int
61
+ created_at: datetime
62
+ updated_at: datetime
63
+
64
+ model_config = ConfigDict(from_attributes=True)
65
+
66
+
67
+ Message.model_rebuild()
68
+ Feedback.model_rebuild()
@@ -0,0 +1,210 @@
1
+ import json
2
+
3
+ from sqlalchemy import select
4
+ from sqlalchemy.ext.asyncio import AsyncSession
5
+ from sqlalchemy.orm import selectinload
6
+ from sqlalchemy.orm.attributes import set_committed_value
7
+
8
+ from th2rag.conversations import (
9
+ exceptions as conversations_exceptions,
10
+ )
11
+ from th2rag.conversations import (
12
+ service as conversations_service,
13
+ )
14
+ from th2rag.dataset import service as dataset_service
15
+ from th2rag.messages import exceptions, schemas
16
+ from th2rag.models import Feedback, Knowledge, Message
17
+ from th2rag.pagination import PageParams, PageResponse, paginate
18
+ from th2rag.rag.services.rag_service import RAGService
19
+ from th2rag.utils.helpers import build_script_generation_prompt, construct_history
20
+ from th2rag.dependencies import get_rag_service
21
+ from fastapi import HTTPException
22
+ from th2rag.rag.services.rag_service import RAGService
23
+
24
+
25
+
26
+ async def get_all_messages(
27
+ conversation_id: int, page_params: PageParams, db: AsyncSession
28
+ ) -> PageResponse[schemas.Message]:
29
+ stmt = (
30
+ select(Message)
31
+ .where(Message.conversation_id == conversation_id)
32
+ .order_by(Message.created_at.desc())
33
+ .options(selectinload(Message.conversation))
34
+ .options(selectinload(Message.feedback))
35
+ )
36
+ return await paginate(db, stmt, page_params, schemas.Message)
37
+
38
+
39
+ async def get_message_by_id(
40
+ conversation_id: int, message_id: int, db: AsyncSession
41
+ ) -> schemas.Message:
42
+ stmt = (
43
+ select(Message)
44
+ .where(
45
+ Message.message_id == message_id,
46
+ Message.conversation_id == conversation_id,
47
+ )
48
+ .options(selectinload(Message.feedback))
49
+ )
50
+ result = await db.execute(stmt)
51
+ message = result.scalar_one_or_none()
52
+ if not message:
53
+ raise exceptions.MessageNotFoundException(
54
+ "Message not found or does not belong to the conversation"
55
+ )
56
+ return schemas.Message.model_validate(message)
57
+
58
+
59
+ async def create_message(
60
+ conversation_id: int, message_in: schemas.MessageCreate, db: AsyncSession
61
+ ) -> schemas.Message:
62
+ try:
63
+ current_conversation = await conversations_service.get_conversation_by_id(
64
+ conversation_id, db
65
+ )
66
+ new_message = Message(
67
+ conversation_id=conversation_id, content=message_in.content, sender=message_in.sender
68
+ )
69
+ db.add(new_message)
70
+ await db.commit()
71
+ await db.refresh(new_message)
72
+ set_committed_value(new_message, "feedback", None)
73
+ return schemas.Message.model_validate(new_message)
74
+
75
+ except conversations_exceptions.ConversationNotFoundException as e:
76
+ raise e
77
+
78
+
79
+
80
+
81
+ async def create_and_respond_message(
82
+ conversation_id: int,
83
+ message_in: schemas.MessageCreate,
84
+ db: AsyncSession,
85
+ rag_service: RAGService,
86
+ ) -> schemas.Message:
87
+ # Récupération de la conversation
88
+ conversation = await conversations_service.get_conversation_by_id(
89
+ conversation_id=conversation_id, db=db
90
+ )
91
+ if not conversation:
92
+ raise HTTPException(status_code=404, detail="Conversation not found")
93
+
94
+ # Check knowledge status BEFORE loading models or processing anything
95
+ if conversation.knowledge_id:
96
+ stmt = select(Knowledge).where(Knowledge.knowledge_id == conversation.knowledge_id)
97
+ result = await db.execute(stmt)
98
+ knowledge = result.scalar_one_or_none()
99
+ if knowledge and knowledge.status.value in ["FAILED"]:
100
+ raise HTTPException(
101
+ status_code=422,
102
+ detail="The knowledge base could not be processed and is unavailable. Please try again or create a new knowledge base."
103
+ )
104
+ if knowledge and knowledge.status.value in ["PENDING"]:
105
+ raise HTTPException(
106
+ status_code=202,
107
+ detail="The knowledge base is currently being processed. Please wait a moment and try again."
108
+ )
109
+
110
+ # Récupération de l'historique
111
+ stmt = (
112
+ select(Message)
113
+ .where(Message.conversation_id == conversation_id)
114
+ .order_by(Message.created_at.asc())
115
+ )
116
+ result = await db.execute(stmt)
117
+ messages = result.scalars().all()
118
+ history = construct_history(messages=messages)
119
+
120
+
121
+
122
+ if conversation.dataset_id:
123
+ dataset = await dataset_service.get_dataset_by_id(dataset_id=conversation.dataset_id, db=db)
124
+ if not dataset:
125
+ raise HTTPException(status_code=404, detail="Dataset not found")
126
+
127
+ prompt = build_script_generation_prompt(message_in.content, dataset.column_types)
128
+
129
+ try:
130
+ llm_output = rag_service.generate_code_snippet(
131
+ question=message_in.content, prompt=prompt, history=history
132
+ )
133
+ parsed_output = json.loads(llm_output)
134
+ generated_text = parsed_output.get("script")
135
+ except json.JSONDecodeError:
136
+ raise HTTPException(status_code=500, detail="LLM did not return valid JSON")
137
+ except Exception as e:
138
+ raise HTTPException(status_code=500, detail=f"Script generation failed: {str(e)}")
139
+
140
+ message_type = schemas.MessageType.SCRIPT
141
+
142
+ else:
143
+ # MODE = text
144
+ doc_id = conversation.knowledge_id
145
+ prompt = "You are a helpful assistant"
146
+ if doc_id:
147
+ stmt = select(Knowledge).where(Knowledge.knowledge_id == doc_id)
148
+ result = await db.execute(stmt)
149
+ knowledge = result.scalar_one_or_none()
150
+ if knowledge:
151
+ prompt = knowledge.prompt
152
+
153
+ try:
154
+ generated_text = rag_service.answer_question(
155
+ question=message_in.content, doc_id=doc_id, limit=3, prompt=prompt, history=history
156
+ )
157
+ except Exception as e:
158
+ raise HTTPException(status_code=500, detail=f"Response generation failed: {str(e)}")
159
+
160
+ message_type = schemas.MessageType.TEXT
161
+
162
+ # Créer le message utilisateur
163
+ user_message = Message(
164
+ conversation_id=conversation_id,
165
+ content=message_in.content,
166
+ sender=message_in.sender,
167
+ type=schemas.MessageType.TEXT,
168
+ )
169
+ db.add(user_message)
170
+
171
+ # Créer le message système
172
+ system_message = Message(
173
+ conversation_id=conversation_id,
174
+ content=generated_text,
175
+ sender=schemas.Sender.SYSTEM,
176
+ type=message_type,
177
+ )
178
+ db.add(system_message)
179
+
180
+ await db.commit()
181
+ await db.refresh(system_message)
182
+ return schemas.Message.model_validate(system_message)
183
+
184
+
185
+ async def add_feedback_to_message(
186
+ conversation_id: int, message_id: int, feedback_in: schemas.FeedbackCreate, db: AsyncSession
187
+ ) -> schemas.Message:
188
+ stmt = select(Message).where(
189
+ Message.message_id == message_id, Message.conversation_id == conversation_id
190
+ )
191
+ result = await db.execute(stmt)
192
+ message = result.scalar_one_or_none()
193
+ if not message:
194
+ raise exceptions.MessageNotFoundException(
195
+ "Message not found or does not belong to the conversation"
196
+ )
197
+
198
+ new_feedback = Feedback(
199
+ feedback_id=message.message_id,
200
+ feedback_type=feedback_in.feedback_type,
201
+ like=feedback_in.like,
202
+ reason=feedback_in.reason,
203
+ )
204
+ message.feedback = new_feedback
205
+
206
+ db.add(new_feedback)
207
+ await db.commit()
208
+ await db.refresh(message)
209
+
210
+ return schemas.Message.model_validate(message)
th2rag/models.py ADDED
@@ -0,0 +1,215 @@
1
+ import enum
2
+ from datetime import datetime
3
+
4
+ from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, String, Text
5
+ from sqlalchemy.dialects.postgresql import JSONB
6
+ from sqlalchemy.orm import relationship
7
+
8
+ from th2rag.config import settings
9
+ from th2rag.database import Base
10
+ from th2rag.rag.constants import DEFAULT_PROMPT
11
+
12
+
13
+ class Sender(enum.Enum):
14
+ USER = "USER"
15
+ SYSTEM = "SYSTEM"
16
+
17
+
18
+ class FeebackType(enum.Enum):
19
+ A = "A"
20
+ B = "B"
21
+ OTHER = "OTHER"
22
+
23
+
24
+ class UserRole(enum.Enum):
25
+ ADMIN = "ADMIN"
26
+ USER = "USER"
27
+
28
+
29
+ class Status(enum.Enum):
30
+ PENDING = "PENDING"
31
+ COMPLETED = "COMPLETED"
32
+ FAILED = "FAILED"
33
+
34
+
35
+ class MessageType(enum.Enum):
36
+ TEXT = "TEXT"
37
+ SCRIPT = "SCRIPT"
38
+
39
+ class User(Base):
40
+ __tablename__ = "user"
41
+
42
+ user_id = Column(Integer, primary_key=True)
43
+ first_name = Column(String(100), nullable=False)
44
+ last_name = Column(String(100), nullable=False)
45
+ email = Column(String(255), unique=True, nullable=False)
46
+ password = Column(String(255), nullable=True)
47
+ role = Column(
48
+ Enum(UserRole, name="role_enum", schema=settings.db_schema),
49
+ nullable=False,
50
+ default=UserRole.USER,
51
+ )
52
+ created_at = Column(DateTime, default=datetime.now)
53
+ updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
54
+
55
+ # Relationships
56
+ conversations = relationship("Conversation", back_populates="user")
57
+ knowledges = relationship("Knowledge", back_populates="user")
58
+ datasets = relationship("Dataset", back_populates="user")
59
+ llm_config=relationship("UserLLMConfig", back_populates="user", uselist=False)
60
+
61
+ def __repr__(self):
62
+ return f"<User(user_id={self.user_id}, email={self.email})>"
63
+
64
+
65
+ class Conversation(Base):
66
+ __tablename__ = "conversation"
67
+
68
+ conversation_id = Column(Integer, primary_key=True)
69
+ knowledge_id = Column(
70
+ Integer, ForeignKey("knowledge.knowledge_id", ondelete="SET NULL"), nullable=True
71
+ )
72
+ dataset_id = Column(
73
+ Integer, ForeignKey("Dataset.dataset_id", ondelete="SET NULL"), nullable=True
74
+ )
75
+ user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
76
+ title = Column(String(255), nullable=False)
77
+ created_at = Column(DateTime, default=datetime.now)
78
+ updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
79
+
80
+ # Relationships
81
+ user = relationship("User", back_populates="conversations")
82
+ messages = relationship(
83
+ "Message", back_populates="conversation", cascade="all, delete-orphan", passive_deletes=True
84
+ )
85
+ knowledge = relationship("Knowledge", back_populates="conversations")
86
+ dataset = relationship("Dataset", back_populates="conversations")
87
+
88
+ def __repr__(self):
89
+ return f"<Conversation(conversation_id={self.conversation_id})>"
90
+
91
+
92
+ class Message(Base):
93
+ __tablename__ = "message"
94
+
95
+ message_id = Column(Integer, primary_key=True)
96
+ conversation_id = Column(
97
+ Integer, ForeignKey("conversation.conversation_id", ondelete="CASCADE"), nullable=False
98
+ )
99
+ content = Column(Text, nullable=False)
100
+ sender = Column(Enum(Sender, name="sender_enum", schema=settings.db_schema), nullable=False)
101
+ type = Column(Enum(MessageType, name="type_enum", schema=settings.db_schema), nullable=False)
102
+ created_at = Column(DateTime, default=datetime.now)
103
+ updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
104
+
105
+ # Relationships
106
+ conversation = relationship("Conversation", back_populates="messages")
107
+ feedback = relationship("Feedback", uselist=False, back_populates="message")
108
+ sources = relationship("Source", back_populates="message")
109
+
110
+ def __repr__(self):
111
+ return f"<Message(message_id={self.message_id})>"
112
+
113
+
114
+ class Source(Base):
115
+ __tablename__ = "source"
116
+
117
+ source_id = Column(Integer, primary_key=True)
118
+ message_id = Column(
119
+ Integer, ForeignKey("message.message_id", ondelete="CASCADE"), nullable=False
120
+ )
121
+ source_path = Column(String(255))
122
+ created_at = Column(DateTime, default=datetime.now)
123
+
124
+ # Relationships
125
+ message = relationship("Message", back_populates="sources")
126
+
127
+ def __repr__(self):
128
+ return f"<Source(source_id={self.source_id}, name={self.source_path})>"
129
+
130
+
131
+ class Feedback(Base):
132
+ __tablename__ = "feedback"
133
+
134
+ feedback_id = Column(
135
+ Integer, ForeignKey("message.message_id", ondelete="CASCADE"), primary_key=True
136
+ )
137
+ feedback_type = Column(
138
+ Enum(FeebackType, name="feedback_enum", schema=settings.db_schema), nullable=True
139
+ )
140
+ like = Column(Boolean, nullable=False)
141
+ reason = Column(Text, nullable=True)
142
+ created_at = Column(DateTime, default=datetime.now)
143
+ updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
144
+
145
+ # Relationships
146
+ message = relationship("Message", back_populates="feedback")
147
+
148
+ def __repr__(self):
149
+ return f"<Feedback(feedback_id={self.feedback_id})>"
150
+
151
+
152
+ class Knowledge(Base):
153
+ __tablename__ = "knowledge"
154
+
155
+ knowledge_id = Column(Integer, primary_key=True)
156
+ name = Column(String(255), nullable=False)
157
+ description = Column(Text, nullable=True)
158
+ status = Column(
159
+ Enum(Status, name="status_enum", schema=settings.db_schema),
160
+ nullable=False,
161
+ default=Status.PENDING,
162
+ )
163
+ prompt = Column(Text, default=DEFAULT_PROMPT)
164
+ knowledge_path = Column(String(255), nullable=False)
165
+ # Changed from Text to JSONB for better performance and querying
166
+ knowledge_paths = Column(JSONB, nullable=True)
167
+ filenames = Column(JSONB, nullable=True)
168
+ user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
169
+ callback_url = Column(String(2048), nullable=True)
170
+ error_message = Column(Text, nullable=True)
171
+ created_at = Column(DateTime, default=datetime.now)
172
+ updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
173
+
174
+ # Relationships
175
+ conversations = relationship("Conversation", back_populates="knowledge")
176
+ user = relationship("User", back_populates="knowledges")
177
+
178
+ def __repr__(self):
179
+ return f"<Knowledge(knowledge_id={self.knowledge_id}, name={self.name})>"
180
+
181
+
182
+ class Dataset(Base):
183
+ __tablename__ = "Dataset"
184
+
185
+ dataset_id = Column(Integer, primary_key=True)
186
+ name = Column(String(255), nullable=False)
187
+ n_columns = Column(Integer, nullable=False)
188
+ n_rows = Column(Integer, nullable=False)
189
+ user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
190
+ column_types = Column(JSONB, nullable=False)
191
+ created_at = Column(DateTime, default=datetime.now)
192
+
193
+ conversations = relationship("Conversation", back_populates="dataset")
194
+ user = relationship("User", back_populates="datasets")
195
+
196
+ def __repr__(self):
197
+ return f"<Dataset(dataset_id={self.dataset_id}, name={self.name})>"
198
+
199
+
200
+
201
+ class UserLLMConfig(Base):
202
+ __tablename__ = "user_llm_config"
203
+
204
+ config_id = Column(Integer, primary_key=True)
205
+ user_id = Column(Integer, ForeignKey("user.user_id", ondelete="CASCADE"), nullable=False)
206
+ provider = Column(String(100), nullable=False)
207
+ model_name = Column(String(100), nullable=False)
208
+ api_key = Column(String(255), nullable=False)
209
+ created_at = Column(DateTime, default=datetime.now)
210
+ updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now)
211
+
212
+ user = relationship("User", back_populates="llm_config")
213
+
214
+ def __repr__(self):
215
+ return f"<UserLLMConfig(config_id={self.config_id}, user_id={self.user_id})>"
th2rag/pagination.py ADDED
@@ -0,0 +1,68 @@
1
+ from typing import Generic, TypeVar
2
+
3
+ from pydantic import BaseModel, Field
4
+ from sqlalchemy import func, select
5
+ from sqlalchemy.ext.asyncio import AsyncSession
6
+ from sqlalchemy.sql import Select
7
+
8
+
9
+ class PageParams(BaseModel):
10
+ """Request query params for paginated API."""
11
+
12
+ page: int = Field(default=1, ge=1)
13
+ size: int = Field(default=20, ge=1, le=100)
14
+
15
+
16
+ T = TypeVar("T")
17
+
18
+
19
+ class PageResponse(BaseModel, Generic[T]):
20
+ total: int
21
+ page: int
22
+ size: int
23
+ has_next: bool
24
+ has_previous: bool
25
+ results: list[T]
26
+
27
+
28
+ # def paginate(query: Query, page_params: PageParams, ResponseSchema: BaseModel) -> PageResponse[T]:
29
+
30
+ # paginated_query = query.offset((page_params.page - 1) * page_params.size).limit(page_params.size).all()
31
+ # total = query.count()
32
+ # has_next = (page_params.page * page_params.size) < total
33
+ # has_previous = page_params.page > 1
34
+
35
+ # return PageResponse(
36
+ # total=total,
37
+ # page=page_params.page,
38
+ # size=page_params.size,
39
+ # has_next=has_next,
40
+ # has_previous=has_previous,
41
+ # results=[ResponseSchema.model_validate(item) for item in paginated_query]
42
+ # )
43
+
44
+
45
+ async def paginate(
46
+ session: AsyncSession, stmt: Select, page_params: PageParams, ResponseSchema: BaseModel
47
+ ) -> PageResponse[T]:
48
+ offset_val = (page_params.page - 1) * page_params.size
49
+ paginated_stmt = stmt.offset(offset_val).limit(page_params.size)
50
+
51
+ result = await session.execute(paginated_stmt)
52
+ items = result.scalars().all()
53
+
54
+ count_stmt = select(func.count()).select_from(stmt.subquery())
55
+ count_result = await session.execute(count_stmt)
56
+ total = count_result.scalar_one()
57
+
58
+ has_next = (page_params.page * page_params.size) < total
59
+ has_previous = page_params.page > 1
60
+
61
+ return PageResponse(
62
+ total=total,
63
+ page=page_params.page,
64
+ size=page_params.size,
65
+ has_next=has_next,
66
+ has_previous=has_previous,
67
+ results=[ResponseSchema.model_validate(item) for item in items],
68
+ )
th2rag/rag/__init__.py ADDED
File without changes
File without changes
@@ -0,0 +1,23 @@
1
+ from docling.chunking import HybridChunker
2
+
3
+
4
+ class HybridTextChunker:
5
+ """
6
+ Wraps the HybridChunker from docling to split documents into chunks.
7
+ """
8
+
9
+ def __init__(self, tokenizer: str = "bert-base-uncased", merge_peers: bool = True):
10
+ self.chunker = HybridChunker(tokenizer=tokenizer, merge_peers=merge_peers, max_tokens=512)
11
+
12
+ def chunk_document(self, dl_doc) -> list:
13
+ """
14
+ Chunks the given document into smaller pieces.
15
+
16
+ Args:
17
+ dl_doc: Document object from DocumentConverter.
18
+
19
+ Returns:
20
+ list: A list of chunk objects.
21
+ """
22
+ chunk_iter = self.chunker.chunk(dl_doc=dl_doc)
23
+ return list(chunk_iter)
@@ -0,0 +1,15 @@
1
+ DEFAULT_PROMPT = """
2
+ # Role and Purpose
3
+ You are an AI assistant with access to a dynamic knowledge base. Users have uploaded various documents
4
+ to create a personalized repository. Your task is to answer the user's question based solely on the provided context.
5
+
6
+ # Guidelines:
7
+ 1. Provide a clear and concise answer to the question.
8
+ 2. Use only the information from the relevant context to support your answer.
9
+ 3. The context is retrieved based on similarity search, so some information might be missing or irrelevant.
10
+ 4. Be transparent when there is insufficient information to fully answer the question.
11
+ 5. Do not make up or infer information not present in the provided context.
12
+ 6. If you cannot answer the question based on the given context, clearly state that.
13
+ 7. Maintain a helpful and professional tone.
14
+ 8. Respond in the same language of the question.
15
+ """
File without changes