rag-agent 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rag_agent-0.1.0/LICENSE +21 -0
- rag_agent-0.1.0/PKG-INFO +310 -0
- rag_agent-0.1.0/README.md +273 -0
- rag_agent-0.1.0/pyproject.toml +61 -0
- rag_agent-0.1.0/setup.cfg +4 -0
- rag_agent-0.1.0/src/rag_agent/__init__.py +9 -0
- rag_agent-0.1.0/src/rag_agent/pipeline/__init__.py +4 -0
- rag_agent-0.1.0/src/rag_agent/pipeline/__main__.py +33 -0
- rag_agent-0.1.0/src/rag_agent/pipeline/config.py +209 -0
- rag_agent-0.1.0/src/rag_agent/pipeline/pipeline.py +351 -0
- rag_agent-0.1.0/src/rag_agent/scrapers/__init__.py +23 -0
- rag_agent-0.1.0/src/rag_agent/scrapers/ix/__init__.py +4 -0
- rag_agent-0.1.0/src/rag_agent/scrapers/ix/__main__.py +23 -0
- rag_agent-0.1.0/src/rag_agent/scrapers/ix/article.py +96 -0
- rag_agent-0.1.0/src/rag_agent/scrapers/ix/config.py +155 -0
- rag_agent-0.1.0/src/rag_agent/scrapers/ix/scraper.py +739 -0
- rag_agent-0.1.0/src/rag_agent/utils/__init__.py +0 -0
- rag_agent-0.1.0/src/rag_agent/utils/merge/__init__.py +0 -0
- rag_agent-0.1.0/src/rag_agent/utils/merge/__main__.py +155 -0
- rag_agent-0.1.0/src/rag_agent.egg-info/PKG-INFO +310 -0
- rag_agent-0.1.0/src/rag_agent.egg-info/SOURCES.txt +22 -0
- rag_agent-0.1.0/src/rag_agent.egg-info/dependency_links.txt +1 -0
- rag_agent-0.1.0/src/rag_agent.egg-info/requires.txt +26 -0
- rag_agent-0.1.0/src/rag_agent.egg-info/top_level.txt +1 -0
rag_agent-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Dawid Ciepiela
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
rag_agent-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rag-agent
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: RAG (Retrieval-Augmented Generation) pipeline implementation
|
|
5
|
+
Author-email: Dawid Ciepiela <71898979+sarumaj@users.noreply.github.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Requires-Python: >=3.12
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: aiohttp==3.11.18
|
|
14
|
+
Requires-Dist: asyncio==3.4.3
|
|
15
|
+
Requires-Dist: beautifulsoup4==4.13.4
|
|
16
|
+
Requires-Dist: langchain==0.3.25
|
|
17
|
+
Requires-Dist: langchain-chroma==0.2.3
|
|
18
|
+
Requires-Dist: langchain-community==0.3.23
|
|
19
|
+
Requires-Dist: langchain-core==0.3.58
|
|
20
|
+
Requires-Dist: langchain-huggingface==0.1.2
|
|
21
|
+
Requires-Dist: langchain-ollama==0.3.2
|
|
22
|
+
Requires-Dist: langchain-text-splitters==0.3.8
|
|
23
|
+
Requires-Dist: langgraph==0.4.1
|
|
24
|
+
Requires-Dist: pydantic==2.11.4
|
|
25
|
+
Requires-Dist: pydantic-settings==2.9.1
|
|
26
|
+
Requires-Dist: tqdm==4.67.1
|
|
27
|
+
Provides-Extra: loaders
|
|
28
|
+
Requires-Dist: pymupdf==1.25.5; extra == "loaders"
|
|
29
|
+
Requires-Dist: unstructured==0.17.2; extra == "loaders"
|
|
30
|
+
Provides-Extra: scraper
|
|
31
|
+
Requires-Dist: selenium==4.32.0; extra == "scraper"
|
|
32
|
+
Requires-Dist: webdriver_manager==4.0.2; extra == "scraper"
|
|
33
|
+
Provides-Extra: test
|
|
34
|
+
Requires-Dist: pytest==8.0.0; extra == "test"
|
|
35
|
+
Requires-Dist: pytest-asyncio==0.23.5; extra == "test"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
[](https://github.com/sarumaj/rag-agent/actions/workflows/release.yml)
|
|
39
|
+
[](https://github.com/sarumaj/rag-agent/releases/latest)
|
|
40
|
+
[](https://github.com/sarumaj/rag-agent/blob/main/pyproject.toml)
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
# RAG Pipeline
|
|
45
|
+
|
|
46
|
+
A powerful Retrieval-Augmented Generation (RAG) pipeline implementation that supports multiple document types, embedding models, and LLM providers.
|
|
47
|
+
|
|
48
|
+
## Features
|
|
49
|
+
|
|
50
|
+
- **Multiple Document Support**
|
|
51
|
+
- PDF documents
|
|
52
|
+
- Text files
|
|
53
|
+
- HTML content
|
|
54
|
+
|
|
55
|
+
- **Flexible LLM Integration**
|
|
56
|
+
- Ollama support (local models)
|
|
57
|
+
- Hugging Face integration
|
|
58
|
+
- Configurable model parameters
|
|
59
|
+
|
|
60
|
+
- **Advanced Document Processing**
|
|
61
|
+
- Metadata extraction from file paths
|
|
62
|
+
- Configurable text chunking
|
|
63
|
+
- Duplicate detection
|
|
64
|
+
- Progress tracking
|
|
65
|
+
|
|
66
|
+
- **Vector Store Features**
|
|
67
|
+
- ChromaDB integration
|
|
68
|
+
- Configurable embedding models
|
|
69
|
+
- Multiple search strategies
|
|
70
|
+
- Persistent storage
|
|
71
|
+
|
|
72
|
+
## Installation
|
|
73
|
+
|
|
74
|
+
1. Clone the repository:
|
|
75
|
+
```bash
|
|
76
|
+
git clone https://github.com/yourusername/rag-pipeline.git
|
|
77
|
+
cd rag-pipeline
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
2. Create and activate a virtual environment:
|
|
81
|
+
```bash
|
|
82
|
+
python -m venv venv
|
|
83
|
+
source venv/bin/activate # On Windows: venv\Scripts\activate
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
3. Install dependencies:
|
|
87
|
+
```bash
|
|
88
|
+
pip install -e .
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
4. Install Ollama (if using local models):
|
|
92
|
+
```bash
|
|
93
|
+
curl -fsSL https://ollama.com/install.sh | sh
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Configuration
|
|
97
|
+
|
|
98
|
+
Create a `.env` file in the project root with your configuration:
|
|
99
|
+
|
|
100
|
+
```env
|
|
101
|
+
# Document source settings
|
|
102
|
+
PIPELINE_SOURCES='{"/path/to/your/documents": ["pdf", "mhtml"]}'
|
|
103
|
+
|
|
104
|
+
# Text splitting settings
|
|
105
|
+
PIPELINE_CHUNK_SIZE=1000
|
|
106
|
+
PIPELINE_CHUNK_OVERLAP=200
|
|
107
|
+
|
|
108
|
+
# Vector store settings
|
|
109
|
+
PIPELINE_PERSIST_DIRECTORY=chroma_db
|
|
110
|
+
PIPELINE_COLLECTION_NAME=default_collection
|
|
111
|
+
|
|
112
|
+
# Embedding model settings
|
|
113
|
+
PIPELINE_EMBEDDING_MODEL=all-MiniLM-L6-v2
|
|
114
|
+
PIPELINE_EMBEDDING_MODEL_KWARGS={"device": "cuda"}
|
|
115
|
+
|
|
116
|
+
# LLM settings
|
|
117
|
+
PIPELINE_LLM_PROVIDER=ollama # or 'huggingface'
|
|
118
|
+
PIPELINE_LLM_MODEL=mistral
|
|
119
|
+
PIPELINE_LLM_MODEL_KWARGS={"temperature": 0.3}
|
|
120
|
+
PIPELINE_LLM_API_KEY=your_api_key # Required for Hugging Face
|
|
121
|
+
|
|
122
|
+
# Retrieval settings
|
|
123
|
+
PIPELINE_SEARCH_TYPE=similarity # or 'mmr', 'similarity_score_threshold'
|
|
124
|
+
PIPELINE_K=5
|
|
125
|
+
PIPELINE_SCORE_THRESHOLD=0.5
|
|
126
|
+
PIPELINE_FETCH_K=20
|
|
127
|
+
PIPELINE_LAMBDA_MULT=0.5
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Usage
|
|
131
|
+
|
|
132
|
+
### Basic Usage
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from pipeline import RAGPipeline, Settings
|
|
136
|
+
|
|
137
|
+
async def main():
|
|
138
|
+
# Initialize with default settings
|
|
139
|
+
async with RAGPipeline() as pipeline:
|
|
140
|
+
# Load and process documents
|
|
141
|
+
documents = await pipeline.load_documents()
|
|
142
|
+
processed_docs = await pipeline.process_documents(documents)
|
|
143
|
+
await pipeline.update_vectorstore(processed_docs)
|
|
144
|
+
|
|
145
|
+
# Setup and run query
|
|
146
|
+
await pipeline.setup_retrieval_chain()
|
|
147
|
+
answer = await pipeline.run("Your question here")
|
|
148
|
+
print(answer)
|
|
149
|
+
|
|
150
|
+
# Run the pipeline
|
|
151
|
+
import asyncio
|
|
152
|
+
asyncio.run(main())
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
### Custom Configuration
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
config = Settings(
|
|
159
|
+
pipeline_source="/path/to/documents",
|
|
160
|
+
pipeline_source_type="pdf",
|
|
161
|
+
pipeline_llm_provider="ollama",
|
|
162
|
+
pipeline_llm_model="mixtral",
|
|
163
|
+
pipeline_llm_model_kwargs={"temperature": 0.3}
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
async with RAGPipeline(config) as pipeline:
|
|
167
|
+
# ... rest of the code
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Supported Models
|
|
171
|
+
|
|
172
|
+
### Ollama Models
|
|
173
|
+
- `mistral` (7B parameters)
|
|
174
|
+
- `mixtral` (8x7B parameters)
|
|
175
|
+
- `llama2` (7B parameters)
|
|
176
|
+
- `codellama` (Code specialized)
|
|
177
|
+
- `neural-chat` (Chat optimized)
|
|
178
|
+
- `dolphin-mixtral` (Chat optimized)
|
|
179
|
+
- And many more...
|
|
180
|
+
|
|
181
|
+
### Hugging Face Models
|
|
182
|
+
- `mistralai/Mistral-7B-Instruct-v0.2`
|
|
183
|
+
- `meta-llama/Llama-2-7b-chat-hf`
|
|
184
|
+
- `malteos/gpt2-wechsel-german`
|
|
185
|
+
- And many more...
|
|
186
|
+
|
|
187
|
+
## Document Processing
|
|
188
|
+
|
|
189
|
+
The pipeline supports various document types and processing options:
|
|
190
|
+
|
|
191
|
+
### PDF Processing
|
|
192
|
+
- Uses PyMuPDF for efficient PDF processing
|
|
193
|
+
- Extracts text and metadata
|
|
194
|
+
- Configurable processing modes
|
|
195
|
+
|
|
196
|
+
### Text Processing
|
|
197
|
+
- Configurable chunk sizes
|
|
198
|
+
- Overlap control
|
|
199
|
+
- Metadata extraction
|
|
200
|
+
|
|
201
|
+
### HTML Processing
|
|
202
|
+
- Clean HTML extraction
|
|
203
|
+
- Structured content handling
|
|
204
|
+
- Metadata preservation
|
|
205
|
+
|
|
206
|
+
## IX Archive Scraper
|
|
207
|
+
|
|
208
|
+
The project includes a specialized scraper for the IX archive with the following features:
|
|
209
|
+
|
|
210
|
+
### Features
|
|
211
|
+
- **Automated Article Download**
|
|
212
|
+
- Downloads articles in multiple formats (PDF, MHTML)
|
|
213
|
+
- Preserves article metadata and structure
|
|
214
|
+
- Handles authentication and session management
|
|
215
|
+
|
|
216
|
+
- **Parallel Processing**
|
|
217
|
+
- Concurrent article processing
|
|
218
|
+
- Configurable thread pool
|
|
219
|
+
- Progress tracking with tqdm
|
|
220
|
+
|
|
221
|
+
- **Robust Error Handling**
|
|
222
|
+
- Automatic retries for failed downloads
|
|
223
|
+
- Graceful cleanup on interruption
|
|
224
|
+
- Detailed logging
|
|
225
|
+
|
|
226
|
+
### Configuration
|
|
227
|
+
|
|
228
|
+
Add the following to your `.env` file for IX scraper configuration:
|
|
229
|
+
|
|
230
|
+
```env
|
|
231
|
+
# IX Scraper settings
|
|
232
|
+
IX_SCRAPER_BASE_URL=https://www.heise.de
|
|
233
|
+
IX_SCRAPER_SIGN_IN_URL=https://www.heise.de/sso/login/
|
|
234
|
+
IX_SCRAPER_ARCHIVE_URL=https://www.heise.de/select/ix/archiv/
|
|
235
|
+
IX_SCRAPER_MAX_THREADS=10
|
|
236
|
+
IX_SCRAPER_MAX_CONCURRENT=10
|
|
237
|
+
IX_SCRAPER_TIMEOUT=30
|
|
238
|
+
IX_SCRAPER_RETRY_ATTEMPTS=5
|
|
239
|
+
IX_SCRAPER_OUTPUT_DIR=~/Downloads/ix
|
|
240
|
+
IX_SCRAPER_USERNAME=your_username
|
|
241
|
+
IX_SCRAPER_PASSWORD=your_password
|
|
242
|
+
IX_SCRAPER_OVERWRITE=false
|
|
243
|
+
IX_SCRAPER_EXPORT_FORMATS=["pdf"]
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
### Usage
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
# pip install -e .[scraper]
|
|
250
|
+
from scrappers.ix import IXScraper, Settings
|
|
251
|
+
|
|
252
|
+
async def main():
|
|
253
|
+
# Initialize with default settings
|
|
254
|
+
async with IXScraper() as scraper:
|
|
255
|
+
# Run the scraper
|
|
256
|
+
await scraper.run()
|
|
257
|
+
|
|
258
|
+
# Run the scraper
|
|
259
|
+
import asyncio
|
|
260
|
+
asyncio.run(main())
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
### Export Formats
|
|
264
|
+
|
|
265
|
+
The scraper supports multiple export formats:
|
|
266
|
+
|
|
267
|
+
1. **PDF Export**
|
|
268
|
+
- High-quality PDF output
|
|
269
|
+
- Configurable page settings
|
|
270
|
+
- Base64 encoded transfer
|
|
271
|
+
|
|
272
|
+
2. **MHTML Export**
|
|
273
|
+
- Preserves web page structure
|
|
274
|
+
- Includes all resources
|
|
275
|
+
- Suitable for archival
|
|
276
|
+
|
|
277
|
+
### WebDriver Configuration
|
|
278
|
+
|
|
279
|
+
The scraper uses Selenium WebDriver with configurable options:
|
|
280
|
+
- Headless mode support
|
|
281
|
+
- Custom user agent
|
|
282
|
+
- Resource optimization
|
|
283
|
+
- Security settings
|
|
284
|
+
|
|
285
|
+
## Vector Store
|
|
286
|
+
|
|
287
|
+
The pipeline uses ChromaDB for vector storage with features like:
|
|
288
|
+
- Configurable embedding models
|
|
289
|
+
- Multiple search strategies
|
|
290
|
+
- Persistent storage
|
|
291
|
+
- Duplicate detection
|
|
292
|
+
|
|
293
|
+
## Contributing
|
|
294
|
+
|
|
295
|
+
1. Fork the repository
|
|
296
|
+
2. Create a feature branch
|
|
297
|
+
3. Commit your changes
|
|
298
|
+
4. Push to the branch
|
|
299
|
+
5. Create a Pull Request
|
|
300
|
+
|
|
301
|
+
## License
|
|
302
|
+
|
|
303
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
304
|
+
|
|
305
|
+
## Acknowledgments
|
|
306
|
+
|
|
307
|
+
- LangChain for the RAG framework
|
|
308
|
+
- ChromaDB for vector storage
|
|
309
|
+
- Ollama for local LLM support
|
|
310
|
+
- Hugging Face for model hosting
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
[](https://github.com/sarumaj/rag-agent/actions/workflows/release.yml)
|
|
2
|
+
[](https://github.com/sarumaj/rag-agent/releases/latest)
|
|
3
|
+
[](https://github.com/sarumaj/rag-agent/blob/main/pyproject.toml)
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# RAG Pipeline
|
|
8
|
+
|
|
9
|
+
A powerful Retrieval-Augmented Generation (RAG) pipeline implementation that supports multiple document types, embedding models, and LLM providers.
|
|
10
|
+
|
|
11
|
+
## Features
|
|
12
|
+
|
|
13
|
+
- **Multiple Document Support**
|
|
14
|
+
- PDF documents
|
|
15
|
+
- Text files
|
|
16
|
+
- HTML content
|
|
17
|
+
|
|
18
|
+
- **Flexible LLM Integration**
|
|
19
|
+
- Ollama support (local models)
|
|
20
|
+
- Hugging Face integration
|
|
21
|
+
- Configurable model parameters
|
|
22
|
+
|
|
23
|
+
- **Advanced Document Processing**
|
|
24
|
+
- Metadata extraction from file paths
|
|
25
|
+
- Configurable text chunking
|
|
26
|
+
- Duplicate detection
|
|
27
|
+
- Progress tracking
|
|
28
|
+
|
|
29
|
+
- **Vector Store Features**
|
|
30
|
+
- ChromaDB integration
|
|
31
|
+
- Configurable embedding models
|
|
32
|
+
- Multiple search strategies
|
|
33
|
+
- Persistent storage
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
1. Clone the repository:
|
|
38
|
+
```bash
|
|
39
|
+
git clone https://github.com/yourusername/rag-pipeline.git
|
|
40
|
+
cd rag-pipeline
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
2. Create and activate a virtual environment:
|
|
44
|
+
```bash
|
|
45
|
+
python -m venv venv
|
|
46
|
+
source venv/bin/activate # On Windows: venv\Scripts\activate
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
3. Install dependencies:
|
|
50
|
+
```bash
|
|
51
|
+
pip install -e .
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
4. Install Ollama (if using local models):
|
|
55
|
+
```bash
|
|
56
|
+
curl -fsSL https://ollama.com/install.sh | sh
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Configuration
|
|
60
|
+
|
|
61
|
+
Create a `.env` file in the project root with your configuration:
|
|
62
|
+
|
|
63
|
+
```env
|
|
64
|
+
# Document source settings
|
|
65
|
+
PIPELINE_SOURCES='{"/path/to/your/documents": ["pdf", "mhtml"]}'
|
|
66
|
+
|
|
67
|
+
# Text splitting settings
|
|
68
|
+
PIPELINE_CHUNK_SIZE=1000
|
|
69
|
+
PIPELINE_CHUNK_OVERLAP=200
|
|
70
|
+
|
|
71
|
+
# Vector store settings
|
|
72
|
+
PIPELINE_PERSIST_DIRECTORY=chroma_db
|
|
73
|
+
PIPELINE_COLLECTION_NAME=default_collection
|
|
74
|
+
|
|
75
|
+
# Embedding model settings
|
|
76
|
+
PIPELINE_EMBEDDING_MODEL=all-MiniLM-L6-v2
|
|
77
|
+
PIPELINE_EMBEDDING_MODEL_KWARGS={"device": "cuda"}
|
|
78
|
+
|
|
79
|
+
# LLM settings
|
|
80
|
+
PIPELINE_LLM_PROVIDER=ollama # or 'huggingface'
|
|
81
|
+
PIPELINE_LLM_MODEL=mistral
|
|
82
|
+
PIPELINE_LLM_MODEL_KWARGS={"temperature": 0.3}
|
|
83
|
+
PIPELINE_LLM_API_KEY=your_api_key # Required for Hugging Face
|
|
84
|
+
|
|
85
|
+
# Retrieval settings
|
|
86
|
+
PIPELINE_SEARCH_TYPE=similarity # or 'mmr', 'similarity_score_threshold'
|
|
87
|
+
PIPELINE_K=5
|
|
88
|
+
PIPELINE_SCORE_THRESHOLD=0.5
|
|
89
|
+
PIPELINE_FETCH_K=20
|
|
90
|
+
PIPELINE_LAMBDA_MULT=0.5
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Usage
|
|
94
|
+
|
|
95
|
+
### Basic Usage
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from pipeline import RAGPipeline, Settings
|
|
99
|
+
|
|
100
|
+
async def main():
|
|
101
|
+
# Initialize with default settings
|
|
102
|
+
async with RAGPipeline() as pipeline:
|
|
103
|
+
# Load and process documents
|
|
104
|
+
documents = await pipeline.load_documents()
|
|
105
|
+
processed_docs = await pipeline.process_documents(documents)
|
|
106
|
+
await pipeline.update_vectorstore(processed_docs)
|
|
107
|
+
|
|
108
|
+
# Setup and run query
|
|
109
|
+
await pipeline.setup_retrieval_chain()
|
|
110
|
+
answer = await pipeline.run("Your question here")
|
|
111
|
+
print(answer)
|
|
112
|
+
|
|
113
|
+
# Run the pipeline
|
|
114
|
+
import asyncio
|
|
115
|
+
asyncio.run(main())
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### Custom Configuration
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
config = Settings(
|
|
122
|
+
pipeline_source="/path/to/documents",
|
|
123
|
+
pipeline_source_type="pdf",
|
|
124
|
+
pipeline_llm_provider="ollama",
|
|
125
|
+
pipeline_llm_model="mixtral",
|
|
126
|
+
pipeline_llm_model_kwargs={"temperature": 0.3}
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
async with RAGPipeline(config) as pipeline:
|
|
130
|
+
# ... rest of the code
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Supported Models
|
|
134
|
+
|
|
135
|
+
### Ollama Models
|
|
136
|
+
- `mistral` (7B parameters)
|
|
137
|
+
- `mixtral` (8x7B parameters)
|
|
138
|
+
- `llama2` (7B parameters)
|
|
139
|
+
- `codellama` (Code specialized)
|
|
140
|
+
- `neural-chat` (Chat optimized)
|
|
141
|
+
- `dolphin-mixtral` (Chat optimized)
|
|
142
|
+
- And many more...
|
|
143
|
+
|
|
144
|
+
### Hugging Face Models
|
|
145
|
+
- `mistralai/Mistral-7B-Instruct-v0.2`
|
|
146
|
+
- `meta-llama/Llama-2-7b-chat-hf`
|
|
147
|
+
- `malteos/gpt2-wechsel-german`
|
|
148
|
+
- And many more...
|
|
149
|
+
|
|
150
|
+
## Document Processing
|
|
151
|
+
|
|
152
|
+
The pipeline supports various document types and processing options:
|
|
153
|
+
|
|
154
|
+
### PDF Processing
|
|
155
|
+
- Uses PyMuPDF for efficient PDF processing
|
|
156
|
+
- Extracts text and metadata
|
|
157
|
+
- Configurable processing modes
|
|
158
|
+
|
|
159
|
+
### Text Processing
|
|
160
|
+
- Configurable chunk sizes
|
|
161
|
+
- Overlap control
|
|
162
|
+
- Metadata extraction
|
|
163
|
+
|
|
164
|
+
### HTML Processing
|
|
165
|
+
- Clean HTML extraction
|
|
166
|
+
- Structured content handling
|
|
167
|
+
- Metadata preservation
|
|
168
|
+
|
|
169
|
+
## IX Archive Scraper
|
|
170
|
+
|
|
171
|
+
The project includes a specialized scraper for the IX archive with the following features:
|
|
172
|
+
|
|
173
|
+
### Features
|
|
174
|
+
- **Automated Article Download**
|
|
175
|
+
- Downloads articles in multiple formats (PDF, MHTML)
|
|
176
|
+
- Preserves article metadata and structure
|
|
177
|
+
- Handles authentication and session management
|
|
178
|
+
|
|
179
|
+
- **Parallel Processing**
|
|
180
|
+
- Concurrent article processing
|
|
181
|
+
- Configurable thread pool
|
|
182
|
+
- Progress tracking with tqdm
|
|
183
|
+
|
|
184
|
+
- **Robust Error Handling**
|
|
185
|
+
- Automatic retries for failed downloads
|
|
186
|
+
- Graceful cleanup on interruption
|
|
187
|
+
- Detailed logging
|
|
188
|
+
|
|
189
|
+
### Configuration
|
|
190
|
+
|
|
191
|
+
Add the following to your `.env` file for IX scraper configuration:
|
|
192
|
+
|
|
193
|
+
```env
|
|
194
|
+
# IX Scraper settings
|
|
195
|
+
IX_SCRAPER_BASE_URL=https://www.heise.de
|
|
196
|
+
IX_SCRAPER_SIGN_IN_URL=https://www.heise.de/sso/login/
|
|
197
|
+
IX_SCRAPER_ARCHIVE_URL=https://www.heise.de/select/ix/archiv/
|
|
198
|
+
IX_SCRAPER_MAX_THREADS=10
|
|
199
|
+
IX_SCRAPER_MAX_CONCURRENT=10
|
|
200
|
+
IX_SCRAPER_TIMEOUT=30
|
|
201
|
+
IX_SCRAPER_RETRY_ATTEMPTS=5
|
|
202
|
+
IX_SCRAPER_OUTPUT_DIR=~/Downloads/ix
|
|
203
|
+
IX_SCRAPER_USERNAME=your_username
|
|
204
|
+
IX_SCRAPER_PASSWORD=your_password
|
|
205
|
+
IX_SCRAPER_OVERWRITE=false
|
|
206
|
+
IX_SCRAPER_EXPORT_FORMATS=["pdf"]
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Usage
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
# pip install -e .[scraper]
|
|
213
|
+
from scrappers.ix import IXScraper, Settings
|
|
214
|
+
|
|
215
|
+
async def main():
|
|
216
|
+
# Initialize with default settings
|
|
217
|
+
async with IXScraper() as scraper:
|
|
218
|
+
# Run the scraper
|
|
219
|
+
await scraper.run()
|
|
220
|
+
|
|
221
|
+
# Run the scraper
|
|
222
|
+
import asyncio
|
|
223
|
+
asyncio.run(main())
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
### Export Formats
|
|
227
|
+
|
|
228
|
+
The scraper supports multiple export formats:
|
|
229
|
+
|
|
230
|
+
1. **PDF Export**
|
|
231
|
+
- High-quality PDF output
|
|
232
|
+
- Configurable page settings
|
|
233
|
+
- Base64 encoded transfer
|
|
234
|
+
|
|
235
|
+
2. **MHTML Export**
|
|
236
|
+
- Preserves web page structure
|
|
237
|
+
- Includes all resources
|
|
238
|
+
- Suitable for archival
|
|
239
|
+
|
|
240
|
+
### WebDriver Configuration
|
|
241
|
+
|
|
242
|
+
The scraper uses Selenium WebDriver with configurable options:
|
|
243
|
+
- Headless mode support
|
|
244
|
+
- Custom user agent
|
|
245
|
+
- Resource optimization
|
|
246
|
+
- Security settings
|
|
247
|
+
|
|
248
|
+
## Vector Store
|
|
249
|
+
|
|
250
|
+
The pipeline uses ChromaDB for vector storage with features like:
|
|
251
|
+
- Configurable embedding models
|
|
252
|
+
- Multiple search strategies
|
|
253
|
+
- Persistent storage
|
|
254
|
+
- Duplicate detection
|
|
255
|
+
|
|
256
|
+
## Contributing
|
|
257
|
+
|
|
258
|
+
1. Fork the repository
|
|
259
|
+
2. Create a feature branch
|
|
260
|
+
3. Commit your changes
|
|
261
|
+
4. Push to the branch
|
|
262
|
+
5. Create a Pull Request
|
|
263
|
+
|
|
264
|
+
## License
|
|
265
|
+
|
|
266
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
267
|
+
|
|
268
|
+
## Acknowledgments
|
|
269
|
+
|
|
270
|
+
- LangChain for the RAG framework
|
|
271
|
+
- ChromaDB for vector storage
|
|
272
|
+
- Ollama for local LLM support
|
|
273
|
+
- Hugging Face for model hosting
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=42", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "rag-agent"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "RAG (Retrieval-Augmented Generation) pipeline implementation"
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
authors = [
|
|
11
|
+
{ name="Dawid Ciepiela", email="71898979+sarumaj@users.noreply.github.com" }
|
|
12
|
+
]
|
|
13
|
+
readme = "README.md"
|
|
14
|
+
license = { text = "MIT" }
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Operating System :: OS Independent"
|
|
19
|
+
]
|
|
20
|
+
dependencies = [
|
|
21
|
+
"aiohttp==3.11.18",
|
|
22
|
+
"asyncio==3.4.3",
|
|
23
|
+
"beautifulsoup4==4.13.4",
|
|
24
|
+
"langchain==0.3.25",
|
|
25
|
+
"langchain-chroma==0.2.3",
|
|
26
|
+
"langchain-community==0.3.23",
|
|
27
|
+
"langchain-core==0.3.58",
|
|
28
|
+
"langchain-huggingface==0.1.2",
|
|
29
|
+
"langchain-ollama==0.3.2",
|
|
30
|
+
"langchain-text-splitters==0.3.8",
|
|
31
|
+
"langgraph==0.4.1",
|
|
32
|
+
"pydantic==2.11.4",
|
|
33
|
+
"pydantic-settings==2.9.1",
|
|
34
|
+
"tqdm==4.67.1"
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
loaders = [
|
|
39
|
+
"pymupdf==1.25.5",
|
|
40
|
+
"unstructured==0.17.2"
|
|
41
|
+
]
|
|
42
|
+
scraper = [
|
|
43
|
+
"selenium==4.32.0",
|
|
44
|
+
"webdriver_manager==4.0.2"
|
|
45
|
+
]
|
|
46
|
+
test = [
|
|
47
|
+
"pytest==8.0.0",
|
|
48
|
+
"pytest-asyncio==0.23.5"
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
exclude = ["tests*"]
|
|
54
|
+
|
|
55
|
+
[tool.pytest.ini_options]
|
|
56
|
+
pythonpath = ["src/rag_agent"]
|
|
57
|
+
testpaths = ["tests"]
|
|
58
|
+
addopts = [
|
|
59
|
+
"-v",
|
|
60
|
+
"--asyncio-mode=auto"
|
|
61
|
+
]
|