rag-agent 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rag_agent/__init__.py ADDED
@@ -0,0 +1,9 @@
1
+ from .pipeline import RAGPipeline, RAGPipelineConfig
2
+ from .scrapers.ix import IXScraper, IXScraperConfig
3
+
4
+ __all__ = [k for k, v in globals().items() if v in (
5
+ RAGPipeline,
6
+ RAGPipelineConfig,
7
+ IXScraper,
8
+ IXScraperConfig,
9
+ )]
@@ -0,0 +1,4 @@
1
+ from .config import Settings as RAGPipelineConfig
2
+ from .pipeline import RAGPipeline
3
+
4
+ __all__ = [k for k, v in globals().items() if v in (RAGPipeline, RAGPipelineConfig)]
@@ -0,0 +1,33 @@
1
+ import asyncio
2
+ import argparse
3
+ from .pipeline import RAGPipeline
4
+
5
+
6
+ async def amain(args: argparse.Namespace):
7
+ async with RAGPipeline() as pipeline:
8
+ if args.setup:
9
+ documents = await pipeline.load_documents()
10
+ processed_docs = await pipeline.process_documents(documents)
11
+ await pipeline.update_vectorstore(processed_docs)
12
+
13
+ await pipeline.setup_retrieval_chain()
14
+
15
+ answer = await pipeline.run(args.question)
16
+ print("Answer:", answer, end="\n\n")
17
+
18
+
19
+ def main():
20
+ parser = argparse.ArgumentParser(
21
+ prog="rag_agent.pipeline",
22
+ description="RAG Pipeline CLI",
23
+ add_help=True
24
+ )
25
+ parser.add_argument('--question', '-q', required=True, type=str, help='Question to ask the RAG system')
26
+ parser.add_argument('--setup', '-s', action='store_true', help='Setup knowledge base (load and process documents)')
27
+ args = parser.parse_args()
28
+
29
+ asyncio.run(amain(args))
30
+
31
+
32
+ if __name__ == "__main__":
33
+ main()
@@ -0,0 +1,209 @@
1
+ from pydantic import Field, field_validator, ValidationError, BaseModel
2
+ from typing import Optional, Literal, Union
3
+ from pydantic_settings import BaseSettings, SettingsConfigDict
4
+ import re
5
+ import os
6
+ import json
7
+
8
+
9
+ class DocumentSource(BaseModel):
10
+ source_type: Literal["txt", "pdf", "html"] = Field(default="txt", description="Type of source")
11
+ meta_pattern: re.Pattern = Field(default_factory=lambda: re.compile(
12
+ r"^.*/(?P<journal>[^/]+)/"
13
+ r"(?P<year>\d+)/"
14
+ r"(?P<month>\d+)/"
15
+ r"(?P<page>\d+)(?:\.\w+)?$"
16
+ ),
17
+ description="Regex pattern for extracting metadata from the source path"
18
+ )
19
+ glob_pattern: str = Field(
20
+ default="**/*",
21
+ description="Glob pattern for matching files in directory"
22
+ )
23
+
24
+ @field_validator("meta_pattern", mode="before")
25
+ @classmethod
26
+ def parse_meta_pattern(cls, v: str) -> re.Pattern:
27
+ try:
28
+ return re.compile(v) if not isinstance(v, re.Pattern) else v
29
+ except re.error:
30
+ raise ValidationError(f"Invalid regex pattern: {v}")
31
+
32
+
33
+ PDF_SOURCE = DocumentSource(
34
+ source_type="pdf",
35
+ glob_pattern="**/*.pdf",
36
+ )
37
+
38
+ TXT_SOURCE = DocumentSource(
39
+ source_type="txt",
40
+ glob_pattern="**/*.txt",
41
+ )
42
+
43
+ MHTML_SOURCE = DocumentSource(
44
+ source_type="html",
45
+ glob_pattern="**/*.mhtml",
46
+ )
47
+
48
+ HTML_SOURCE = DocumentSource(
49
+ source_type="html",
50
+ glob_pattern="**/*.html",
51
+ )
52
+
53
+
54
+ class Settings(BaseSettings):
55
+ """Configuration for RAGPipeline."""
56
+ model_config = SettingsConfigDict(
57
+ env_file=".env",
58
+ env_file_encoding="utf-8",
59
+ env_nested_delimiter="_",
60
+ frozen=True,
61
+ extra="ignore",
62
+ case_sensitive=False,
63
+ )
64
+
65
+ pipeline_max_threads: int = Field(default=10, description="Maximum number of threads")
66
+
67
+ # Embedding model settings
68
+ pipeline_embedding_model: str = Field(
69
+ default="all-MiniLM-L6-v2",
70
+ description="Name of the sentence-transformers model to use"
71
+ )
72
+ pipeline_embedding_model_kwargs: dict = Field(
73
+ default={"device": "cuda"},
74
+ description="Keyword arguments for the embedding model"
75
+ )
76
+
77
+ # Document source settings
78
+ pipeline_sources: dict[str, list[Union[str, DocumentSource]]] = Field(
79
+ default_factory=lambda: {
80
+ os.getcwd(): [PDF_SOURCE, MHTML_SOURCE]
81
+ },
82
+ description="Sources to be used for the pipeline"
83
+ )
84
+
85
+ # Text splitting settings
86
+ pipeline_chunk_size: int = Field(
87
+ default=1000,
88
+ ge=100,
89
+ description="Size of text chunks for splitting documents"
90
+ )
91
+ pipeline_chunk_overlap: int = Field(
92
+ default=200,
93
+ ge=0,
94
+ description="Overlap between chunks"
95
+ )
96
+
97
+ # Vector store settings
98
+ pipeline_persist_directory: str = Field(
99
+ default="chroma_db",
100
+ description="Directory to store the vector database"
101
+ )
102
+ pipeline_collection_name: str = Field(
103
+ default="default_collection",
104
+ description="Name of the ChromaDB collection to use",
105
+ min_length=3,
106
+ max_length=512
107
+ )
108
+
109
+ # Retrieval settings
110
+ pipeline_search_type: Literal["similarity", "mmr", "similarity_score_threshold"] = Field(
111
+ default="similarity",
112
+ description="Type of search to perform"
113
+ )
114
+ pipeline_k: int = Field(
115
+ default=5,
116
+ ge=1,
117
+ description="Number of documents to retrieve"
118
+ )
119
+ pipeline_score_threshold: Optional[float] = Field(
120
+ default=0.5,
121
+ ge=0.0,
122
+ le=1.0,
123
+ description="Minimum similarity score (for similarity_score_threshold)"
124
+ )
125
+ pipeline_fetch_k: Optional[int] = Field(
126
+ default=20,
127
+ ge=1,
128
+ description="Number of documents to fetch before filtering (for MMR)"
129
+ )
130
+ pipeline_lambda_mult: Optional[float] = Field(
131
+ default=0.5,
132
+ ge=0.0,
133
+ le=1.0,
134
+ description="Diversity factor (for MMR, 0-1)"
135
+ )
136
+
137
+ # Prompt template
138
+ pipeline_prompt_template: Optional[str] = Field(
139
+ default="""You are a helpful assistant that can answer questions about the given context.
140
+ You have access to the iX-Magazine for professional IT experts.
141
+ If you don't have enough information to answer the question, say so.
142
+
143
+ For each piece of information you use, please provide the source (Title, Journal, Year, Month and Page).
144
+ Answer in the language of the question.
145
+
146
+ Answer the question based on the following context:
147
+
148
+ Context: {context}
149
+
150
+ Question: {question}
151
+
152
+ Answer:""",
153
+ description="Optional custom prompt template for the RAG chain"
154
+ )
155
+
156
+ # LLM settings
157
+ pipeline_llm_provider: Literal["ollama", "huggingface"] = Field(
158
+ default="ollama",
159
+ description="Provider of the LLM model"
160
+ )
161
+ pipeline_llm_model: str = Field(
162
+ default="mistral",
163
+ description="Name of the LLM model to use"
164
+ )
165
+ pipeline_llm_model_kwargs: dict = Field(
166
+ default={"temperature": 0.3},
167
+ description="Keyword arguments for the LLM model"
168
+ )
169
+ pipeline_llm_api_key: Optional[str] = Field(
170
+ default=None,
171
+ description="API key for the LLM provider (required for Hugging Face)",
172
+ repr=False
173
+ )
174
+
175
+ @field_validator("pipeline_sources", mode="before")
176
+ @classmethod
177
+ def parse_sources(
178
+ cls,
179
+ v: Union[str, dict[str, list[Union[str, DocumentSource]]]]
180
+ ) -> dict[str, list[DocumentSource]]:
181
+ if not isinstance(v, (str, dict)):
182
+ return v
183
+
184
+ sources = {}
185
+ for path, source_list in (json.loads(v) if isinstance(v, str) else v).items():
186
+ sources[path] = []
187
+ for source in source_list:
188
+ if isinstance(source, str):
189
+ match source.lower():
190
+ case "pdf":
191
+ sources[path].append(PDF_SOURCE)
192
+ case "mhtml":
193
+ sources[path].append(MHTML_SOURCE)
194
+ case "html":
195
+ sources[path].append(HTML_SOURCE)
196
+ case "txt":
197
+ sources[path].append(TXT_SOURCE)
198
+ case _:
199
+ raise ValidationError(f"Invalid source type: {source}")
200
+ elif isinstance(source, DocumentSource):
201
+ sources[path].append(source)
202
+ elif isinstance(source, dict):
203
+ try:
204
+ sources[path].append(DocumentSource(**source))
205
+ except ValidationError:
206
+ raise ValidationError(f"Invalid source: {source}")
207
+ else:
208
+ raise ValidationError(f"Invalid source: {source}")
209
+ return sources
@@ -0,0 +1,351 @@
1
+ from typing import List, TypedDict, Dict, Any, Optional, Literal
2
+ import logging
3
+ import asyncio
4
+ from concurrent.futures import ThreadPoolExecutor
5
+ from langchain_community.document_loaders import (
6
+ TextLoader,
7
+ DirectoryLoader,
8
+ PyMuPDFLoader,
9
+ UnstructuredHTMLLoader,
10
+ )
11
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
12
+ from langgraph.graph import START, StateGraph
13
+ from langchain_chroma import Chroma
14
+ from langchain_core.prompts import ChatPromptTemplate
15
+ from langchain_ollama import OllamaLLM
16
+ from langchain_huggingface import HuggingFaceEmbeddings, HuggingFaceEndpoint
17
+ from langchain_community.vectorstores.utils import filter_complex_metadata
18
+ from langchain_core.documents import Document
19
+ from chromadb.config import Settings as ChromaSettings
20
+ import re
21
+ from tqdm import tqdm
22
+ import json
23
+
24
+ from .config import Settings
25
+
26
+ try:
27
+ from importlib import import_module
28
+ LOADERS_AVAILABLE = all([
29
+ import_module("pymupdf") is not None,
30
+ import_module("unstructured") is not None,
31
+ ])
32
+ except (ImportError, ModuleNotFoundError):
33
+ LOADERS_AVAILABLE = False
34
+
35
+ class NotImported:
36
+ def __getattr__(self, item):
37
+ raise ModuleNotFoundError(
38
+ "Loader dependencies are not installed. "
39
+ "Please install them using: pip install 'rag-agent[loaders]'"
40
+ )
41
+
42
+ def __call__(self, *args, **kwargs):
43
+ raise ModuleNotFoundError(
44
+ "Loader dependencies are not installed. "
45
+ "Please install them using: pip install 'rag-agent[loaders]'"
46
+ )
47
+
48
+ globals().update(dict.fromkeys(
49
+ [
50
+ "PyMuPDFLoader",
51
+ "UnstructuredHTMLLoader",
52
+ ],
53
+ NotImported()
54
+ ))
55
+
56
+
57
+ logging.basicConfig(
58
+ level=logging.INFO,
59
+ style='{',
60
+ format='{asctime} - {levelname} - {message}',
61
+ )
62
+
63
+ logger = logging.getLogger("rag_agent.pipeline")
64
+
65
+
66
+ class RAGPipeline:
67
+ """RAG pipeline implementation using LangChain."""
68
+
69
+ def __init__(self, config: Optional[Settings] = None):
70
+ """Initialize the RAG pipeline.
71
+
72
+ Args:
73
+ config: Optional configuration object. If not provided, defaults will be used.
74
+ """
75
+ self._config = config or Settings()
76
+ logger.info(f"Initializing RAG pipeline with config: {self._config.model_dump_json(indent=2)}")
77
+ self._thread_pool = ThreadPoolExecutor(max_workers=self._config.pipeline_max_threads)
78
+
79
+ try:
80
+ self._loop = asyncio.get_event_loop()
81
+ except RuntimeError:
82
+ self._loop = asyncio.new_event_loop()
83
+ asyncio.set_event_loop(self._loop)
84
+
85
+ self._embedding_model = HuggingFaceEmbeddings(
86
+ model_name=self._config.pipeline_embedding_model,
87
+ model_kwargs=self._config.pipeline_embedding_model_kwargs,
88
+ encode_kwargs={"normalize_embeddings": True}
89
+ )
90
+ self._text_splitter = RecursiveCharacterTextSplitter(
91
+ chunk_size=self._config.pipeline_chunk_size,
92
+ chunk_overlap=self._config.pipeline_chunk_overlap
93
+ )
94
+ self._vectorstore = Chroma(
95
+ embedding_function=self._embedding_model,
96
+ persist_directory=self._config.pipeline_persist_directory,
97
+ collection_name=self._config.pipeline_collection_name,
98
+ client_settings=ChromaSettings(
99
+ anonymized_telemetry=False
100
+ )
101
+ )
102
+
103
+ match self._config.pipeline_llm_provider:
104
+ case "ollama":
105
+ self._llm = OllamaLLM(
106
+ model=self._config.pipeline_llm_model,
107
+ **self._config.pipeline_llm_model_kwargs
108
+ )
109
+ case "huggingface":
110
+ if not self._config.pipeline_llm_api_key:
111
+ raise ValueError("Hugging Face API key is required")
112
+ self._llm = HuggingFaceEndpoint(
113
+ repo_id=self._config.pipeline_llm_model,
114
+ task="text-generation",
115
+ huggingface_api_token=self._config.pipeline_llm_api_key,
116
+ **self._config.pipeline_llm_model_kwargs
117
+ )
118
+ case _:
119
+ raise ValueError(f"Unsupported LLM provider: {self._config.pipeline_llm_provider}")
120
+
121
+ self._rag_chain = None
122
+
123
+ async def load_documents(self) -> List[Document]:
124
+ """Load documents from various sources asynchronously.
125
+
126
+ Returns:
127
+ List of loaded documents
128
+ """
129
+ try:
130
+ loader_cls = None
131
+ loader_kwargs = {}
132
+
133
+ documents = []
134
+ for path, sources in self._config.pipeline_sources.items():
135
+ for source in sources:
136
+ logger.info(f"Loading documents from {path} using {source.source_type} loader")
137
+ match source.source_type:
138
+ case "txt":
139
+ loader_cls = TextLoader
140
+ loader_kwargs = {"autodetect_encoding": True}
141
+ case "pdf":
142
+ loader_cls = PyMuPDFLoader
143
+ loader_kwargs = {
144
+ "mode": "single"
145
+ }
146
+ case "html":
147
+ loader_cls = UnstructuredHTMLLoader
148
+ loader_kwargs = {"mode": "single", "strategy": "fast"}
149
+ case _:
150
+ raise ValueError(f"Unsupported source type: {source.source_type}")
151
+
152
+ loader = DirectoryLoader(
153
+ path,
154
+ glob=source.glob_pattern,
155
+ loader_cls=loader_cls,
156
+ loader_kwargs=loader_kwargs,
157
+ use_multithreading=True,
158
+ max_concurrency=self._config.pipeline_max_threads,
159
+ show_progress=True,
160
+ ) if source.glob_pattern else loader_cls(
161
+ path,
162
+ loader_kwargs=loader_kwargs
163
+ )
164
+
165
+ for doc in await loader.aload():
166
+ meta_pattern = (
167
+ source.meta_pattern
168
+ if isinstance(source.meta_pattern, re.Pattern)
169
+ else re.compile(source.meta_pattern)
170
+ )
171
+ doc.metadata.update({
172
+ "meta_pattern": meta_pattern.pattern,
173
+ **((
174
+ match.groupdict() or
175
+ dict(((str(i), g) for i, g in enumerate(match.groups())))
176
+ ) if (match := meta_pattern.match(doc.metadata.get('source', ''))) else {})
177
+ })
178
+ documents.append(doc)
179
+
180
+ logger.info(f"Loaded {len(documents)} documents from {path} using {source.source_type} loader")
181
+
182
+ return documents
183
+
184
+ except Exception as e:
185
+ logging.error(f"Error loading documents: {str(e)}")
186
+ raise
187
+
188
+ async def process_documents(self, documents: List[Document]) -> List[Document]:
189
+ """Process and split documents into chunks asynchronously.
190
+
191
+ Args:
192
+ documents: List of documents to process
193
+
194
+ Returns:
195
+ List of processed document chunks
196
+ """
197
+ logger.info(f"Processing {len(documents)} documents")
198
+ processed_docs = []
199
+
200
+ with tqdm(total=len(documents), desc="Processing documents") as pbar:
201
+ for doc in documents:
202
+ chunks = await self._loop.run_in_executor(
203
+ self._thread_pool,
204
+ self._text_splitter.split_documents,
205
+ [doc]
206
+ )
207
+ processed_docs.extend(chunks)
208
+ pbar.update(1)
209
+ pbar.set_postfix({"chunks": len(processed_docs)})
210
+
211
+ logger.info(f"Total processed chunks: {len(processed_docs)}")
212
+ return processed_docs
213
+
214
+ async def update_vectorstore(self, documents: List[Document]) -> List[str]:
215
+ """Update the vector store with new documents asynchronously.
216
+
217
+ Args:
218
+ documents: List of processed document chunks
219
+
220
+ Returns:
221
+ List of document ids
222
+ """
223
+
224
+ documents = list(filter_complex_metadata(documents))
225
+ logger.info(f"Updating vector store with {len(documents)} documents")
226
+
227
+ with tqdm(total=len(documents), desc="Updating vector store") as pbar:
228
+ ids = []
229
+ batch_size = 100
230
+ for i in range(0, len(documents), batch_size):
231
+ batch = documents[i:i + batch_size]
232
+ batch_ids = await self._vectorstore.aadd_documents(batch)
233
+ ids.extend(batch_ids)
234
+ pbar.update(len(batch))
235
+ pbar.set_postfix({"processed": len(ids)})
236
+
237
+ logger.info(f"Added {len(ids)} documents to vector store")
238
+ return ids
239
+
240
+ async def list_vectorstore(self) -> Dict[str, Any]:
241
+ """List all documents in the vector store asynchronously.
242
+
243
+ Returns:
244
+ List of document ids
245
+ """
246
+ return await self._loop.run_in_executor(
247
+ self._thread_pool,
248
+ lambda: self._vectorstore.get(
249
+ include=["documents", "metadatas"]
250
+ )
251
+ )
252
+
253
+ async def setup_retrieval_chain(self, context_format: Literal["json", "markdown"] = "json"):
254
+ """Set up the retrieval-augmented generation chain asynchronously.
255
+
256
+ Args:
257
+ context_format: Format of the context to be used in the prompt
258
+ """
259
+ prompt = ChatPromptTemplate.from_template(self._config.pipeline_prompt_template)
260
+
261
+ retrieval_settings = {
262
+ "search_type": self._config.pipeline_search_type,
263
+ "k": self._config.pipeline_k,
264
+ }
265
+
266
+ match self._config.pipeline_search_type:
267
+ case "similarity":
268
+ pass
269
+ case "similarity_score_threshold":
270
+ retrieval_settings["score_threshold"] = self._config.pipeline_score_threshold
271
+ case "mmr":
272
+ retrieval_settings.update({
273
+ "fetch_k": self._config.pipeline_fetch_k,
274
+ "lambda_mult": self._config.pipeline_lambda_mult
275
+ })
276
+ case _:
277
+ raise ValueError(f"Unsupported search type: {self._config.pipeline_search_type}")
278
+
279
+ retriever = self._vectorstore.as_retriever(**retrieval_settings)
280
+
281
+ class State(TypedDict):
282
+ context: str
283
+ question: str
284
+ answer: str
285
+
286
+ def retrieve(state: State) -> State:
287
+ results = retriever.invoke(state["question"])
288
+ return {"context": results}
289
+
290
+ def generate(state: State) -> State:
291
+ match context_format:
292
+ case "json":
293
+ formatted_context = json.dumps([
294
+ {
295
+ **doc.metadata,
296
+ "Content": doc.page_content
297
+ }
298
+ for doc in state["context"]
299
+ ])
300
+ case "markdown":
301
+ keys = {key for doc in state["context"] for key in doc.metadata.keys()}
302
+ formatted_context = "\n".join([
303
+ "| " + " | ".join(keys) + " |",
304
+ "| " + " | ".join(["---"] * len(keys)) + " |",
305
+ *[
306
+ "| " + " | ".join([str(doc.get(key, "")) for key in keys]) + " |"
307
+ for doc in state["context"]
308
+ ],
309
+ ])
310
+ case _:
311
+ raise ValueError(f"Unsupported context format: {context_format}")
312
+
313
+ prompt_template = prompt.invoke({
314
+ "context": f"```{context_format}\n\n{formatted_context}\n\n```",
315
+ "question": state["question"]
316
+ })
317
+ response = self._llm.invoke(prompt_template)
318
+ return {"answer": response, "context": formatted_context}
319
+
320
+ graph = StateGraph(State)
321
+ graph.add_node("retrieve", retrieve)
322
+ graph.add_edge(START, "retrieve")
323
+ graph.add_node("generate", generate)
324
+ graph.add_edge("retrieve", "generate")
325
+
326
+ self._rag_chain = graph.compile()
327
+
328
+ async def run(self, question: str) -> str:
329
+ """Run the RAG pipeline asynchronously.
330
+
331
+ Args:
332
+ question: Question to answer
333
+
334
+ Returns:
335
+ Answer to the question
336
+ """
337
+ if not self._rag_chain:
338
+ raise ValueError("Retrieval chain not initialized. Call setup_retrieval_chain() first.")
339
+
340
+ response = {}
341
+ async for chunk in self._rag_chain.astream({"question": question}, stream_mode="updates"):
342
+ response.update(chunk)
343
+ return response
344
+
345
+ async def __aenter__(self):
346
+ """Async context manager entry."""
347
+ return self
348
+
349
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
350
+ """Async context manager exit."""
351
+ self._thread_pool.shutdown(wait=True)
@@ -0,0 +1,23 @@
1
+ from abc import ABC, abstractmethod
2
+
3
+
4
+ class Scraper(ABC):
5
+ """Abstract base class for scrapers."""
6
+
7
+ @abstractmethod
8
+ async def run(self):
9
+ """Run the scraper."""
10
+ pass
11
+
12
+ @abstractmethod
13
+ async def __aenter__(self):
14
+ """Async context manager entry."""
15
+ return self
16
+
17
+ @abstractmethod
18
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
19
+ """Async context manager exit."""
20
+ pass
21
+
22
+
23
+ __all__ = [k for k, v in globals().items() if v in (Scraper,)]
@@ -0,0 +1,4 @@
1
+ from .scraper import IXScraper
2
+ from .config import Settings as IXScraperConfig
3
+
4
+ __all__ = [k for k, v in globals().items() if v in (IXScraper, IXScraperConfig)]
@@ -0,0 +1,23 @@
1
+ import asyncio
2
+ import argparse
3
+
4
+ from .scraper import IXScraper
5
+
6
+
7
+ async def amain(args: argparse.Namespace):
8
+ async with IXScraper() as scraper:
9
+ await scraper.run()
10
+
11
+
12
+ def main():
13
+ parser = argparse.ArgumentParser(
14
+ prog="rag_agent.scrapers.ix",
15
+ description="IX Scraper CLI",
16
+ add_help=True
17
+ )
18
+ args = parser.parse_args()
19
+ asyncio.run(amain(args))
20
+
21
+
22
+ if __name__ == "__main__":
23
+ main()