rag-agent 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rag_agent/__init__.py +9 -0
- rag_agent/pipeline/__init__.py +4 -0
- rag_agent/pipeline/__main__.py +33 -0
- rag_agent/pipeline/config.py +209 -0
- rag_agent/pipeline/pipeline.py +351 -0
- rag_agent/scrapers/__init__.py +23 -0
- rag_agent/scrapers/ix/__init__.py +4 -0
- rag_agent/scrapers/ix/__main__.py +23 -0
- rag_agent/scrapers/ix/article.py +96 -0
- rag_agent/scrapers/ix/config.py +155 -0
- rag_agent/scrapers/ix/scraper.py +739 -0
- rag_agent/utils/__init__.py +0 -0
- rag_agent/utils/merge/__init__.py +0 -0
- rag_agent/utils/merge/__main__.py +155 -0
- rag_agent-0.1.0.dist-info/METADATA +310 -0
- rag_agent-0.1.0.dist-info/RECORD +19 -0
- rag_agent-0.1.0.dist-info/WHEEL +5 -0
- rag_agent-0.1.0.dist-info/licenses/LICENSE +21 -0
- rag_agent-0.1.0.dist-info/top_level.txt +1 -0
rag_agent/__init__.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import argparse
|
|
3
|
+
from .pipeline import RAGPipeline
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
async def amain(args: argparse.Namespace):
|
|
7
|
+
async with RAGPipeline() as pipeline:
|
|
8
|
+
if args.setup:
|
|
9
|
+
documents = await pipeline.load_documents()
|
|
10
|
+
processed_docs = await pipeline.process_documents(documents)
|
|
11
|
+
await pipeline.update_vectorstore(processed_docs)
|
|
12
|
+
|
|
13
|
+
await pipeline.setup_retrieval_chain()
|
|
14
|
+
|
|
15
|
+
answer = await pipeline.run(args.question)
|
|
16
|
+
print("Answer:", answer, end="\n\n")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def main():
|
|
20
|
+
parser = argparse.ArgumentParser(
|
|
21
|
+
prog="rag_agent.pipeline",
|
|
22
|
+
description="RAG Pipeline CLI",
|
|
23
|
+
add_help=True
|
|
24
|
+
)
|
|
25
|
+
parser.add_argument('--question', '-q', required=True, type=str, help='Question to ask the RAG system')
|
|
26
|
+
parser.add_argument('--setup', '-s', action='store_true', help='Setup knowledge base (load and process documents)')
|
|
27
|
+
args = parser.parse_args()
|
|
28
|
+
|
|
29
|
+
asyncio.run(amain(args))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
if __name__ == "__main__":
|
|
33
|
+
main()
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
from pydantic import Field, field_validator, ValidationError, BaseModel
|
|
2
|
+
from typing import Optional, Literal, Union
|
|
3
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
4
|
+
import re
|
|
5
|
+
import os
|
|
6
|
+
import json
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class DocumentSource(BaseModel):
|
|
10
|
+
source_type: Literal["txt", "pdf", "html"] = Field(default="txt", description="Type of source")
|
|
11
|
+
meta_pattern: re.Pattern = Field(default_factory=lambda: re.compile(
|
|
12
|
+
r"^.*/(?P<journal>[^/]+)/"
|
|
13
|
+
r"(?P<year>\d+)/"
|
|
14
|
+
r"(?P<month>\d+)/"
|
|
15
|
+
r"(?P<page>\d+)(?:\.\w+)?$"
|
|
16
|
+
),
|
|
17
|
+
description="Regex pattern for extracting metadata from the source path"
|
|
18
|
+
)
|
|
19
|
+
glob_pattern: str = Field(
|
|
20
|
+
default="**/*",
|
|
21
|
+
description="Glob pattern for matching files in directory"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
@field_validator("meta_pattern", mode="before")
|
|
25
|
+
@classmethod
|
|
26
|
+
def parse_meta_pattern(cls, v: str) -> re.Pattern:
|
|
27
|
+
try:
|
|
28
|
+
return re.compile(v) if not isinstance(v, re.Pattern) else v
|
|
29
|
+
except re.error:
|
|
30
|
+
raise ValidationError(f"Invalid regex pattern: {v}")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
PDF_SOURCE = DocumentSource(
|
|
34
|
+
source_type="pdf",
|
|
35
|
+
glob_pattern="**/*.pdf",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
TXT_SOURCE = DocumentSource(
|
|
39
|
+
source_type="txt",
|
|
40
|
+
glob_pattern="**/*.txt",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
MHTML_SOURCE = DocumentSource(
|
|
44
|
+
source_type="html",
|
|
45
|
+
glob_pattern="**/*.mhtml",
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
HTML_SOURCE = DocumentSource(
|
|
49
|
+
source_type="html",
|
|
50
|
+
glob_pattern="**/*.html",
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Settings(BaseSettings):
|
|
55
|
+
"""Configuration for RAGPipeline."""
|
|
56
|
+
model_config = SettingsConfigDict(
|
|
57
|
+
env_file=".env",
|
|
58
|
+
env_file_encoding="utf-8",
|
|
59
|
+
env_nested_delimiter="_",
|
|
60
|
+
frozen=True,
|
|
61
|
+
extra="ignore",
|
|
62
|
+
case_sensitive=False,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
pipeline_max_threads: int = Field(default=10, description="Maximum number of threads")
|
|
66
|
+
|
|
67
|
+
# Embedding model settings
|
|
68
|
+
pipeline_embedding_model: str = Field(
|
|
69
|
+
default="all-MiniLM-L6-v2",
|
|
70
|
+
description="Name of the sentence-transformers model to use"
|
|
71
|
+
)
|
|
72
|
+
pipeline_embedding_model_kwargs: dict = Field(
|
|
73
|
+
default={"device": "cuda"},
|
|
74
|
+
description="Keyword arguments for the embedding model"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
# Document source settings
|
|
78
|
+
pipeline_sources: dict[str, list[Union[str, DocumentSource]]] = Field(
|
|
79
|
+
default_factory=lambda: {
|
|
80
|
+
os.getcwd(): [PDF_SOURCE, MHTML_SOURCE]
|
|
81
|
+
},
|
|
82
|
+
description="Sources to be used for the pipeline"
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# Text splitting settings
|
|
86
|
+
pipeline_chunk_size: int = Field(
|
|
87
|
+
default=1000,
|
|
88
|
+
ge=100,
|
|
89
|
+
description="Size of text chunks for splitting documents"
|
|
90
|
+
)
|
|
91
|
+
pipeline_chunk_overlap: int = Field(
|
|
92
|
+
default=200,
|
|
93
|
+
ge=0,
|
|
94
|
+
description="Overlap between chunks"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Vector store settings
|
|
98
|
+
pipeline_persist_directory: str = Field(
|
|
99
|
+
default="chroma_db",
|
|
100
|
+
description="Directory to store the vector database"
|
|
101
|
+
)
|
|
102
|
+
pipeline_collection_name: str = Field(
|
|
103
|
+
default="default_collection",
|
|
104
|
+
description="Name of the ChromaDB collection to use",
|
|
105
|
+
min_length=3,
|
|
106
|
+
max_length=512
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
# Retrieval settings
|
|
110
|
+
pipeline_search_type: Literal["similarity", "mmr", "similarity_score_threshold"] = Field(
|
|
111
|
+
default="similarity",
|
|
112
|
+
description="Type of search to perform"
|
|
113
|
+
)
|
|
114
|
+
pipeline_k: int = Field(
|
|
115
|
+
default=5,
|
|
116
|
+
ge=1,
|
|
117
|
+
description="Number of documents to retrieve"
|
|
118
|
+
)
|
|
119
|
+
pipeline_score_threshold: Optional[float] = Field(
|
|
120
|
+
default=0.5,
|
|
121
|
+
ge=0.0,
|
|
122
|
+
le=1.0,
|
|
123
|
+
description="Minimum similarity score (for similarity_score_threshold)"
|
|
124
|
+
)
|
|
125
|
+
pipeline_fetch_k: Optional[int] = Field(
|
|
126
|
+
default=20,
|
|
127
|
+
ge=1,
|
|
128
|
+
description="Number of documents to fetch before filtering (for MMR)"
|
|
129
|
+
)
|
|
130
|
+
pipeline_lambda_mult: Optional[float] = Field(
|
|
131
|
+
default=0.5,
|
|
132
|
+
ge=0.0,
|
|
133
|
+
le=1.0,
|
|
134
|
+
description="Diversity factor (for MMR, 0-1)"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# Prompt template
|
|
138
|
+
pipeline_prompt_template: Optional[str] = Field(
|
|
139
|
+
default="""You are a helpful assistant that can answer questions about the given context.
|
|
140
|
+
You have access to the iX-Magazine for professional IT experts.
|
|
141
|
+
If you don't have enough information to answer the question, say so.
|
|
142
|
+
|
|
143
|
+
For each piece of information you use, please provide the source (Title, Journal, Year, Month and Page).
|
|
144
|
+
Answer in the language of the question.
|
|
145
|
+
|
|
146
|
+
Answer the question based on the following context:
|
|
147
|
+
|
|
148
|
+
Context: {context}
|
|
149
|
+
|
|
150
|
+
Question: {question}
|
|
151
|
+
|
|
152
|
+
Answer:""",
|
|
153
|
+
description="Optional custom prompt template for the RAG chain"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
# LLM settings
|
|
157
|
+
pipeline_llm_provider: Literal["ollama", "huggingface"] = Field(
|
|
158
|
+
default="ollama",
|
|
159
|
+
description="Provider of the LLM model"
|
|
160
|
+
)
|
|
161
|
+
pipeline_llm_model: str = Field(
|
|
162
|
+
default="mistral",
|
|
163
|
+
description="Name of the LLM model to use"
|
|
164
|
+
)
|
|
165
|
+
pipeline_llm_model_kwargs: dict = Field(
|
|
166
|
+
default={"temperature": 0.3},
|
|
167
|
+
description="Keyword arguments for the LLM model"
|
|
168
|
+
)
|
|
169
|
+
pipeline_llm_api_key: Optional[str] = Field(
|
|
170
|
+
default=None,
|
|
171
|
+
description="API key for the LLM provider (required for Hugging Face)",
|
|
172
|
+
repr=False
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
@field_validator("pipeline_sources", mode="before")
|
|
176
|
+
@classmethod
|
|
177
|
+
def parse_sources(
|
|
178
|
+
cls,
|
|
179
|
+
v: Union[str, dict[str, list[Union[str, DocumentSource]]]]
|
|
180
|
+
) -> dict[str, list[DocumentSource]]:
|
|
181
|
+
if not isinstance(v, (str, dict)):
|
|
182
|
+
return v
|
|
183
|
+
|
|
184
|
+
sources = {}
|
|
185
|
+
for path, source_list in (json.loads(v) if isinstance(v, str) else v).items():
|
|
186
|
+
sources[path] = []
|
|
187
|
+
for source in source_list:
|
|
188
|
+
if isinstance(source, str):
|
|
189
|
+
match source.lower():
|
|
190
|
+
case "pdf":
|
|
191
|
+
sources[path].append(PDF_SOURCE)
|
|
192
|
+
case "mhtml":
|
|
193
|
+
sources[path].append(MHTML_SOURCE)
|
|
194
|
+
case "html":
|
|
195
|
+
sources[path].append(HTML_SOURCE)
|
|
196
|
+
case "txt":
|
|
197
|
+
sources[path].append(TXT_SOURCE)
|
|
198
|
+
case _:
|
|
199
|
+
raise ValidationError(f"Invalid source type: {source}")
|
|
200
|
+
elif isinstance(source, DocumentSource):
|
|
201
|
+
sources[path].append(source)
|
|
202
|
+
elif isinstance(source, dict):
|
|
203
|
+
try:
|
|
204
|
+
sources[path].append(DocumentSource(**source))
|
|
205
|
+
except ValidationError:
|
|
206
|
+
raise ValidationError(f"Invalid source: {source}")
|
|
207
|
+
else:
|
|
208
|
+
raise ValidationError(f"Invalid source: {source}")
|
|
209
|
+
return sources
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
from typing import List, TypedDict, Dict, Any, Optional, Literal
|
|
2
|
+
import logging
|
|
3
|
+
import asyncio
|
|
4
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
5
|
+
from langchain_community.document_loaders import (
|
|
6
|
+
TextLoader,
|
|
7
|
+
DirectoryLoader,
|
|
8
|
+
PyMuPDFLoader,
|
|
9
|
+
UnstructuredHTMLLoader,
|
|
10
|
+
)
|
|
11
|
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
12
|
+
from langgraph.graph import START, StateGraph
|
|
13
|
+
from langchain_chroma import Chroma
|
|
14
|
+
from langchain_core.prompts import ChatPromptTemplate
|
|
15
|
+
from langchain_ollama import OllamaLLM
|
|
16
|
+
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFaceEndpoint
|
|
17
|
+
from langchain_community.vectorstores.utils import filter_complex_metadata
|
|
18
|
+
from langchain_core.documents import Document
|
|
19
|
+
from chromadb.config import Settings as ChromaSettings
|
|
20
|
+
import re
|
|
21
|
+
from tqdm import tqdm
|
|
22
|
+
import json
|
|
23
|
+
|
|
24
|
+
from .config import Settings
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
from importlib import import_module
|
|
28
|
+
LOADERS_AVAILABLE = all([
|
|
29
|
+
import_module("pymupdf") is not None,
|
|
30
|
+
import_module("unstructured") is not None,
|
|
31
|
+
])
|
|
32
|
+
except (ImportError, ModuleNotFoundError):
|
|
33
|
+
LOADERS_AVAILABLE = False
|
|
34
|
+
|
|
35
|
+
class NotImported:
|
|
36
|
+
def __getattr__(self, item):
|
|
37
|
+
raise ModuleNotFoundError(
|
|
38
|
+
"Loader dependencies are not installed. "
|
|
39
|
+
"Please install them using: pip install 'rag-agent[loaders]'"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
def __call__(self, *args, **kwargs):
|
|
43
|
+
raise ModuleNotFoundError(
|
|
44
|
+
"Loader dependencies are not installed. "
|
|
45
|
+
"Please install them using: pip install 'rag-agent[loaders]'"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
globals().update(dict.fromkeys(
|
|
49
|
+
[
|
|
50
|
+
"PyMuPDFLoader",
|
|
51
|
+
"UnstructuredHTMLLoader",
|
|
52
|
+
],
|
|
53
|
+
NotImported()
|
|
54
|
+
))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
logging.basicConfig(
|
|
58
|
+
level=logging.INFO,
|
|
59
|
+
style='{',
|
|
60
|
+
format='{asctime} - {levelname} - {message}',
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
logger = logging.getLogger("rag_agent.pipeline")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class RAGPipeline:
|
|
67
|
+
"""RAG pipeline implementation using LangChain."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, config: Optional[Settings] = None):
|
|
70
|
+
"""Initialize the RAG pipeline.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
config: Optional configuration object. If not provided, defaults will be used.
|
|
74
|
+
"""
|
|
75
|
+
self._config = config or Settings()
|
|
76
|
+
logger.info(f"Initializing RAG pipeline with config: {self._config.model_dump_json(indent=2)}")
|
|
77
|
+
self._thread_pool = ThreadPoolExecutor(max_workers=self._config.pipeline_max_threads)
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
self._loop = asyncio.get_event_loop()
|
|
81
|
+
except RuntimeError:
|
|
82
|
+
self._loop = asyncio.new_event_loop()
|
|
83
|
+
asyncio.set_event_loop(self._loop)
|
|
84
|
+
|
|
85
|
+
self._embedding_model = HuggingFaceEmbeddings(
|
|
86
|
+
model_name=self._config.pipeline_embedding_model,
|
|
87
|
+
model_kwargs=self._config.pipeline_embedding_model_kwargs,
|
|
88
|
+
encode_kwargs={"normalize_embeddings": True}
|
|
89
|
+
)
|
|
90
|
+
self._text_splitter = RecursiveCharacterTextSplitter(
|
|
91
|
+
chunk_size=self._config.pipeline_chunk_size,
|
|
92
|
+
chunk_overlap=self._config.pipeline_chunk_overlap
|
|
93
|
+
)
|
|
94
|
+
self._vectorstore = Chroma(
|
|
95
|
+
embedding_function=self._embedding_model,
|
|
96
|
+
persist_directory=self._config.pipeline_persist_directory,
|
|
97
|
+
collection_name=self._config.pipeline_collection_name,
|
|
98
|
+
client_settings=ChromaSettings(
|
|
99
|
+
anonymized_telemetry=False
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
match self._config.pipeline_llm_provider:
|
|
104
|
+
case "ollama":
|
|
105
|
+
self._llm = OllamaLLM(
|
|
106
|
+
model=self._config.pipeline_llm_model,
|
|
107
|
+
**self._config.pipeline_llm_model_kwargs
|
|
108
|
+
)
|
|
109
|
+
case "huggingface":
|
|
110
|
+
if not self._config.pipeline_llm_api_key:
|
|
111
|
+
raise ValueError("Hugging Face API key is required")
|
|
112
|
+
self._llm = HuggingFaceEndpoint(
|
|
113
|
+
repo_id=self._config.pipeline_llm_model,
|
|
114
|
+
task="text-generation",
|
|
115
|
+
huggingface_api_token=self._config.pipeline_llm_api_key,
|
|
116
|
+
**self._config.pipeline_llm_model_kwargs
|
|
117
|
+
)
|
|
118
|
+
case _:
|
|
119
|
+
raise ValueError(f"Unsupported LLM provider: {self._config.pipeline_llm_provider}")
|
|
120
|
+
|
|
121
|
+
self._rag_chain = None
|
|
122
|
+
|
|
123
|
+
async def load_documents(self) -> List[Document]:
|
|
124
|
+
"""Load documents from various sources asynchronously.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
List of loaded documents
|
|
128
|
+
"""
|
|
129
|
+
try:
|
|
130
|
+
loader_cls = None
|
|
131
|
+
loader_kwargs = {}
|
|
132
|
+
|
|
133
|
+
documents = []
|
|
134
|
+
for path, sources in self._config.pipeline_sources.items():
|
|
135
|
+
for source in sources:
|
|
136
|
+
logger.info(f"Loading documents from {path} using {source.source_type} loader")
|
|
137
|
+
match source.source_type:
|
|
138
|
+
case "txt":
|
|
139
|
+
loader_cls = TextLoader
|
|
140
|
+
loader_kwargs = {"autodetect_encoding": True}
|
|
141
|
+
case "pdf":
|
|
142
|
+
loader_cls = PyMuPDFLoader
|
|
143
|
+
loader_kwargs = {
|
|
144
|
+
"mode": "single"
|
|
145
|
+
}
|
|
146
|
+
case "html":
|
|
147
|
+
loader_cls = UnstructuredHTMLLoader
|
|
148
|
+
loader_kwargs = {"mode": "single", "strategy": "fast"}
|
|
149
|
+
case _:
|
|
150
|
+
raise ValueError(f"Unsupported source type: {source.source_type}")
|
|
151
|
+
|
|
152
|
+
loader = DirectoryLoader(
|
|
153
|
+
path,
|
|
154
|
+
glob=source.glob_pattern,
|
|
155
|
+
loader_cls=loader_cls,
|
|
156
|
+
loader_kwargs=loader_kwargs,
|
|
157
|
+
use_multithreading=True,
|
|
158
|
+
max_concurrency=self._config.pipeline_max_threads,
|
|
159
|
+
show_progress=True,
|
|
160
|
+
) if source.glob_pattern else loader_cls(
|
|
161
|
+
path,
|
|
162
|
+
loader_kwargs=loader_kwargs
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
for doc in await loader.aload():
|
|
166
|
+
meta_pattern = (
|
|
167
|
+
source.meta_pattern
|
|
168
|
+
if isinstance(source.meta_pattern, re.Pattern)
|
|
169
|
+
else re.compile(source.meta_pattern)
|
|
170
|
+
)
|
|
171
|
+
doc.metadata.update({
|
|
172
|
+
"meta_pattern": meta_pattern.pattern,
|
|
173
|
+
**((
|
|
174
|
+
match.groupdict() or
|
|
175
|
+
dict(((str(i), g) for i, g in enumerate(match.groups())))
|
|
176
|
+
) if (match := meta_pattern.match(doc.metadata.get('source', ''))) else {})
|
|
177
|
+
})
|
|
178
|
+
documents.append(doc)
|
|
179
|
+
|
|
180
|
+
logger.info(f"Loaded {len(documents)} documents from {path} using {source.source_type} loader")
|
|
181
|
+
|
|
182
|
+
return documents
|
|
183
|
+
|
|
184
|
+
except Exception as e:
|
|
185
|
+
logging.error(f"Error loading documents: {str(e)}")
|
|
186
|
+
raise
|
|
187
|
+
|
|
188
|
+
async def process_documents(self, documents: List[Document]) -> List[Document]:
|
|
189
|
+
"""Process and split documents into chunks asynchronously.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
documents: List of documents to process
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
List of processed document chunks
|
|
196
|
+
"""
|
|
197
|
+
logger.info(f"Processing {len(documents)} documents")
|
|
198
|
+
processed_docs = []
|
|
199
|
+
|
|
200
|
+
with tqdm(total=len(documents), desc="Processing documents") as pbar:
|
|
201
|
+
for doc in documents:
|
|
202
|
+
chunks = await self._loop.run_in_executor(
|
|
203
|
+
self._thread_pool,
|
|
204
|
+
self._text_splitter.split_documents,
|
|
205
|
+
[doc]
|
|
206
|
+
)
|
|
207
|
+
processed_docs.extend(chunks)
|
|
208
|
+
pbar.update(1)
|
|
209
|
+
pbar.set_postfix({"chunks": len(processed_docs)})
|
|
210
|
+
|
|
211
|
+
logger.info(f"Total processed chunks: {len(processed_docs)}")
|
|
212
|
+
return processed_docs
|
|
213
|
+
|
|
214
|
+
async def update_vectorstore(self, documents: List[Document]) -> List[str]:
|
|
215
|
+
"""Update the vector store with new documents asynchronously.
|
|
216
|
+
|
|
217
|
+
Args:
|
|
218
|
+
documents: List of processed document chunks
|
|
219
|
+
|
|
220
|
+
Returns:
|
|
221
|
+
List of document ids
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
documents = list(filter_complex_metadata(documents))
|
|
225
|
+
logger.info(f"Updating vector store with {len(documents)} documents")
|
|
226
|
+
|
|
227
|
+
with tqdm(total=len(documents), desc="Updating vector store") as pbar:
|
|
228
|
+
ids = []
|
|
229
|
+
batch_size = 100
|
|
230
|
+
for i in range(0, len(documents), batch_size):
|
|
231
|
+
batch = documents[i:i + batch_size]
|
|
232
|
+
batch_ids = await self._vectorstore.aadd_documents(batch)
|
|
233
|
+
ids.extend(batch_ids)
|
|
234
|
+
pbar.update(len(batch))
|
|
235
|
+
pbar.set_postfix({"processed": len(ids)})
|
|
236
|
+
|
|
237
|
+
logger.info(f"Added {len(ids)} documents to vector store")
|
|
238
|
+
return ids
|
|
239
|
+
|
|
240
|
+
async def list_vectorstore(self) -> Dict[str, Any]:
|
|
241
|
+
"""List all documents in the vector store asynchronously.
|
|
242
|
+
|
|
243
|
+
Returns:
|
|
244
|
+
List of document ids
|
|
245
|
+
"""
|
|
246
|
+
return await self._loop.run_in_executor(
|
|
247
|
+
self._thread_pool,
|
|
248
|
+
lambda: self._vectorstore.get(
|
|
249
|
+
include=["documents", "metadatas"]
|
|
250
|
+
)
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
async def setup_retrieval_chain(self, context_format: Literal["json", "markdown"] = "json"):
|
|
254
|
+
"""Set up the retrieval-augmented generation chain asynchronously.
|
|
255
|
+
|
|
256
|
+
Args:
|
|
257
|
+
context_format: Format of the context to be used in the prompt
|
|
258
|
+
"""
|
|
259
|
+
prompt = ChatPromptTemplate.from_template(self._config.pipeline_prompt_template)
|
|
260
|
+
|
|
261
|
+
retrieval_settings = {
|
|
262
|
+
"search_type": self._config.pipeline_search_type,
|
|
263
|
+
"k": self._config.pipeline_k,
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
match self._config.pipeline_search_type:
|
|
267
|
+
case "similarity":
|
|
268
|
+
pass
|
|
269
|
+
case "similarity_score_threshold":
|
|
270
|
+
retrieval_settings["score_threshold"] = self._config.pipeline_score_threshold
|
|
271
|
+
case "mmr":
|
|
272
|
+
retrieval_settings.update({
|
|
273
|
+
"fetch_k": self._config.pipeline_fetch_k,
|
|
274
|
+
"lambda_mult": self._config.pipeline_lambda_mult
|
|
275
|
+
})
|
|
276
|
+
case _:
|
|
277
|
+
raise ValueError(f"Unsupported search type: {self._config.pipeline_search_type}")
|
|
278
|
+
|
|
279
|
+
retriever = self._vectorstore.as_retriever(**retrieval_settings)
|
|
280
|
+
|
|
281
|
+
class State(TypedDict):
|
|
282
|
+
context: str
|
|
283
|
+
question: str
|
|
284
|
+
answer: str
|
|
285
|
+
|
|
286
|
+
def retrieve(state: State) -> State:
|
|
287
|
+
results = retriever.invoke(state["question"])
|
|
288
|
+
return {"context": results}
|
|
289
|
+
|
|
290
|
+
def generate(state: State) -> State:
|
|
291
|
+
match context_format:
|
|
292
|
+
case "json":
|
|
293
|
+
formatted_context = json.dumps([
|
|
294
|
+
{
|
|
295
|
+
**doc.metadata,
|
|
296
|
+
"Content": doc.page_content
|
|
297
|
+
}
|
|
298
|
+
for doc in state["context"]
|
|
299
|
+
])
|
|
300
|
+
case "markdown":
|
|
301
|
+
keys = {key for doc in state["context"] for key in doc.metadata.keys()}
|
|
302
|
+
formatted_context = "\n".join([
|
|
303
|
+
"| " + " | ".join(keys) + " |",
|
|
304
|
+
"| " + " | ".join(["---"] * len(keys)) + " |",
|
|
305
|
+
*[
|
|
306
|
+
"| " + " | ".join([str(doc.get(key, "")) for key in keys]) + " |"
|
|
307
|
+
for doc in state["context"]
|
|
308
|
+
],
|
|
309
|
+
])
|
|
310
|
+
case _:
|
|
311
|
+
raise ValueError(f"Unsupported context format: {context_format}")
|
|
312
|
+
|
|
313
|
+
prompt_template = prompt.invoke({
|
|
314
|
+
"context": f"```{context_format}\n\n{formatted_context}\n\n```",
|
|
315
|
+
"question": state["question"]
|
|
316
|
+
})
|
|
317
|
+
response = self._llm.invoke(prompt_template)
|
|
318
|
+
return {"answer": response, "context": formatted_context}
|
|
319
|
+
|
|
320
|
+
graph = StateGraph(State)
|
|
321
|
+
graph.add_node("retrieve", retrieve)
|
|
322
|
+
graph.add_edge(START, "retrieve")
|
|
323
|
+
graph.add_node("generate", generate)
|
|
324
|
+
graph.add_edge("retrieve", "generate")
|
|
325
|
+
|
|
326
|
+
self._rag_chain = graph.compile()
|
|
327
|
+
|
|
328
|
+
async def run(self, question: str) -> str:
|
|
329
|
+
"""Run the RAG pipeline asynchronously.
|
|
330
|
+
|
|
331
|
+
Args:
|
|
332
|
+
question: Question to answer
|
|
333
|
+
|
|
334
|
+
Returns:
|
|
335
|
+
Answer to the question
|
|
336
|
+
"""
|
|
337
|
+
if not self._rag_chain:
|
|
338
|
+
raise ValueError("Retrieval chain not initialized. Call setup_retrieval_chain() first.")
|
|
339
|
+
|
|
340
|
+
response = {}
|
|
341
|
+
async for chunk in self._rag_chain.astream({"question": question}, stream_mode="updates"):
|
|
342
|
+
response.update(chunk)
|
|
343
|
+
return response
|
|
344
|
+
|
|
345
|
+
async def __aenter__(self):
|
|
346
|
+
"""Async context manager entry."""
|
|
347
|
+
return self
|
|
348
|
+
|
|
349
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
350
|
+
"""Async context manager exit."""
|
|
351
|
+
self._thread_pool.shutdown(wait=True)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Scraper(ABC):
|
|
5
|
+
"""Abstract base class for scrapers."""
|
|
6
|
+
|
|
7
|
+
@abstractmethod
|
|
8
|
+
async def run(self):
|
|
9
|
+
"""Run the scraper."""
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
async def __aenter__(self):
|
|
14
|
+
"""Async context manager entry."""
|
|
15
|
+
return self
|
|
16
|
+
|
|
17
|
+
@abstractmethod
|
|
18
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
19
|
+
"""Async context manager exit."""
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
__all__ = [k for k, v in globals().items() if v in (Scraper,)]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import argparse
|
|
3
|
+
|
|
4
|
+
from .scraper import IXScraper
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
async def amain(args: argparse.Namespace):
|
|
8
|
+
async with IXScraper() as scraper:
|
|
9
|
+
await scraper.run()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def main():
|
|
13
|
+
parser = argparse.ArgumentParser(
|
|
14
|
+
prog="rag_agent.scrapers.ix",
|
|
15
|
+
description="IX Scraper CLI",
|
|
16
|
+
add_help=True
|
|
17
|
+
)
|
|
18
|
+
args = parser.parse_args()
|
|
19
|
+
asyncio.run(amain(args))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
if __name__ == "__main__":
|
|
23
|
+
main()
|