ragfoundry 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragfoundry/__init__.py +36 -0
- ragfoundry/_utils.py +88 -0
- ragfoundry/chunk_store.py +109 -0
- ragfoundry/chunking/__init__.py +23 -0
- ragfoundry/chunking/_text.py +20 -0
- ragfoundry/chunking/base.py +171 -0
- ragfoundry/chunking/llm.py +88 -0
- ragfoundry/chunking/semantic.py +67 -0
- ragfoundry/cli.py +162 -0
- ragfoundry/config.py +248 -0
- ragfoundry/errors.py +19 -0
- ragfoundry/evaluation/__init__.py +35 -0
- ragfoundry/evaluation/dataset.py +46 -0
- ragfoundry/evaluation/generation.py +116 -0
- ragfoundry/evaluation/ragas_adapter.py +84 -0
- ragfoundry/evaluation/retrieval.py +71 -0
- ragfoundry/evaluation/runner.py +190 -0
- ragfoundry/graphs/__init__.py +14 -0
- ragfoundry/graphs/ingest.py +99 -0
- ragfoundry/graphs/query.py +197 -0
- ragfoundry/loaders/__init__.py +15 -0
- ragfoundry/loaders/base.py +162 -0
- ragfoundry/loaders/excel.py +64 -0
- ragfoundry/loaders/web.py +136 -0
- ragfoundry/models/__init__.py +64 -0
- ragfoundry/observability/__init__.py +65 -0
- ragfoundry/pipeline.py +353 -0
- ragfoundry/prompts.py +90 -0
- ragfoundry/retrieval/__init__.py +19 -0
- ragfoundry/retrieval/fusion.py +32 -0
- ragfoundry/retrieval/rerank.py +62 -0
- ragfoundry/retrieval/retriever.py +100 -0
- ragfoundry/retrieval/transforms.py +53 -0
- ragfoundry/vectorstores/__init__.py +8 -0
- ragfoundry/vectorstores/factory.py +232 -0
- ragfoundry/vectorstores/index.py +59 -0
- ragfoundry-0.1.0.dist-info/METADATA +802 -0
- ragfoundry-0.1.0.dist-info/RECORD +40 -0
- ragfoundry-0.1.0.dist-info/WHEEL +4 -0
- ragfoundry-0.1.0.dist-info/entry_points.txt +2 -0
ragfoundry/cli.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Command line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated, Literal
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
|
|
11
|
+
from . import RAGPipeline
|
|
12
|
+
from .errors import RAGFoundryError
|
|
13
|
+
|
|
14
|
+
app = typer.Typer(no_args_is_help=True, add_completion=False, help="Build and run RAG pipelines.")
|
|
15
|
+
|
|
16
|
+
ConfigOption = Annotated[Path, typer.Option("--config", "-c", help="Path to the YAML config.")]
|
|
17
|
+
|
|
18
|
+
TEMPLATE = """\
|
|
19
|
+
# ragfoundry configuration. ${VAR} reads from the environment.
|
|
20
|
+
persist_dir: .ragfoundry
|
|
21
|
+
|
|
22
|
+
chunking:
|
|
23
|
+
strategy: recursive # fixed | recursive | semantic | llm
|
|
24
|
+
chunk_size: 1000
|
|
25
|
+
chunk_overlap: 200
|
|
26
|
+
length_unit: characters # characters | tokens
|
|
27
|
+
|
|
28
|
+
embeddings:
|
|
29
|
+
provider: huggingface # huggingface | openai | gemini
|
|
30
|
+
model: sentence-transformers/all-MiniLM-L6-v2
|
|
31
|
+
|
|
32
|
+
vector_store:
|
|
33
|
+
provider: chroma # chroma | faiss | pinecone | qdrant | weaviate | milvus | pgvector
|
|
34
|
+
collection_name: my_docs
|
|
35
|
+
|
|
36
|
+
llm:
|
|
37
|
+
provider: ollama # groq | openai | gemini | claude | ollama
|
|
38
|
+
model: llama3.2
|
|
39
|
+
|
|
40
|
+
retrieval:
|
|
41
|
+
top_k: 5
|
|
42
|
+
fetch_k: 20
|
|
43
|
+
hybrid:
|
|
44
|
+
enabled: true
|
|
45
|
+
# rerank:
|
|
46
|
+
# provider: huggingface # huggingface | cohere
|
|
47
|
+
# model: cross-encoder/ms-marco-MiniLM-L-6-v2
|
|
48
|
+
query_transform:
|
|
49
|
+
strategies: [] # rewrite, multi_query, hyde
|
|
50
|
+
corrective:
|
|
51
|
+
enabled: false
|
|
52
|
+
|
|
53
|
+
observability:
|
|
54
|
+
provider: none # none | langsmith | langfuse
|
|
55
|
+
# api_key: ${LANGSMITH_API_KEY} # LangSmith
|
|
56
|
+
# public_key: ${LANGFUSE_PUBLIC_KEY} # Langfuse
|
|
57
|
+
# secret_key: ${LANGFUSE_SECRET_KEY} # Langfuse
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _fail(error: Exception) -> typer.Exit:
|
|
62
|
+
typer.secho(f"Error: {error}", fg=typer.colors.RED, err=True)
|
|
63
|
+
return typer.Exit(code=1)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@app.command()
|
|
67
|
+
def init(path: Path = Path("rag.yaml")) -> None:
|
|
68
|
+
"""Write a starter configuration file."""
|
|
69
|
+
if path.exists():
|
|
70
|
+
raise _fail(FileExistsError(f"{path} already exists"))
|
|
71
|
+
path.write_text(TEMPLATE, encoding="utf-8")
|
|
72
|
+
typer.echo(f"Wrote {path}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@app.command()
|
|
76
|
+
def ingest(
|
|
77
|
+
sources: list[str],
|
|
78
|
+
config: ConfigOption = Path("rag.yaml"),
|
|
79
|
+
) -> None:
|
|
80
|
+
"""Load, chunk and index files, directories or URLs."""
|
|
81
|
+
try:
|
|
82
|
+
with RAGPipeline.from_yaml(config) as pipeline:
|
|
83
|
+
report = pipeline.ingest(sources)
|
|
84
|
+
except RAGFoundryError as error:
|
|
85
|
+
raise _fail(error) from error
|
|
86
|
+
typer.echo(
|
|
87
|
+
f"Documents: {report.documents_loaded} Chunks: {report.chunks_total} "
|
|
88
|
+
f"Added: {report.chunks_added} Unchanged: {report.chunks_unchanged} "
|
|
89
|
+
f"Removed: {report.chunks_removed}"
|
|
90
|
+
)
|
|
91
|
+
for failure in report.failures:
|
|
92
|
+
typer.secho(f"Failed: {failure.source}: {failure.error}", fg=typer.colors.YELLOW, err=True)
|
|
93
|
+
if report.failures:
|
|
94
|
+
raise typer.Exit(code=1)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@app.command()
|
|
98
|
+
def query(
|
|
99
|
+
question: str,
|
|
100
|
+
config: ConfigOption = Path("rag.yaml"),
|
|
101
|
+
stream: Annotated[bool, typer.Option(help="Print the answer as it is generated.")] = False,
|
|
102
|
+
as_json: Annotated[bool, typer.Option("--json", help="Print the full result.")] = False,
|
|
103
|
+
) -> None:
|
|
104
|
+
"""Answer a question from the indexed documents."""
|
|
105
|
+
try:
|
|
106
|
+
with RAGPipeline.from_yaml(config) as pipeline:
|
|
107
|
+
if stream:
|
|
108
|
+
for text in pipeline.stream(question):
|
|
109
|
+
typer.echo(text, nl=False)
|
|
110
|
+
typer.echo()
|
|
111
|
+
return
|
|
112
|
+
result = pipeline.query(question)
|
|
113
|
+
except RAGFoundryError as error:
|
|
114
|
+
raise _fail(error) from error
|
|
115
|
+
if as_json:
|
|
116
|
+
typer.echo(json.dumps(result.to_dict(), indent=2, ensure_ascii=False, default=str))
|
|
117
|
+
return
|
|
118
|
+
typer.echo(result.answer)
|
|
119
|
+
for source in result.sources:
|
|
120
|
+
typer.echo(f" - {source}")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@app.command("eval")
|
|
124
|
+
def evaluate(
|
|
125
|
+
dataset: Path,
|
|
126
|
+
config: ConfigOption = Path("rag.yaml"),
|
|
127
|
+
mode: Literal["retrieval", "generation", "all"] = "all",
|
|
128
|
+
backend: Literal["native", "ragas"] = "native",
|
|
129
|
+
output: Annotated[Path | None, typer.Option(help="Write the full report as JSON.")] = None,
|
|
130
|
+
) -> None:
|
|
131
|
+
"""Evaluate retrieval and generation against a .json or .jsonl dataset."""
|
|
132
|
+
try:
|
|
133
|
+
with RAGPipeline.from_yaml(config) as pipeline:
|
|
134
|
+
report = pipeline.evaluate(dataset, mode=mode, backend=backend)
|
|
135
|
+
except RAGFoundryError as error:
|
|
136
|
+
raise _fail(error) from error
|
|
137
|
+
for name, stats in report.summary.items():
|
|
138
|
+
typer.echo(f"{name:<24} {stats['mean']:.3f} (n={stats['count']})")
|
|
139
|
+
if output:
|
|
140
|
+
report.save(output)
|
|
141
|
+
typer.echo(f"Report written to {output}")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@app.command("sources")
|
|
145
|
+
def list_sources(config: ConfigOption = Path("rag.yaml")) -> None:
|
|
146
|
+
"""List indexed sources and their chunk counts."""
|
|
147
|
+
try:
|
|
148
|
+
with RAGPipeline.from_yaml(config) as pipeline:
|
|
149
|
+
for source, count in pipeline.sources().items():
|
|
150
|
+
typer.echo(f"{count:>6} {source}")
|
|
151
|
+
except RAGFoundryError as error:
|
|
152
|
+
raise _fail(error) from error
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@app.command("delete-source")
|
|
156
|
+
def delete_source(source: str, config: ConfigOption = Path("rag.yaml")) -> None:
|
|
157
|
+
"""Remove one source from the index."""
|
|
158
|
+
try:
|
|
159
|
+
with RAGPipeline.from_yaml(config) as pipeline:
|
|
160
|
+
typer.echo(f"Removed {pipeline.delete_source(source)} chunks")
|
|
161
|
+
except RAGFoundryError as error:
|
|
162
|
+
raise _fail(error) from error
|
ragfoundry/config.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"""Typed configuration for a RAG pipeline, loadable from Python or YAML."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated, Any, Literal
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
|
|
12
|
+
|
|
13
|
+
from .errors import ConfigurationError
|
|
14
|
+
|
|
15
|
+
_ENV_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _Model(BaseModel):
|
|
19
|
+
model_config = ConfigDict(extra="forbid", protected_namespaces=())
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# --------------------------------------------------------------------------- loading
|
|
23
|
+
class WebConfig(_Model):
|
|
24
|
+
max_depth: int = Field(0, ge=0, description="Link depth to follow; 0 fetches only the URL.")
|
|
25
|
+
max_pages: int = Field(50, ge=1)
|
|
26
|
+
same_domain: bool = True
|
|
27
|
+
respect_robots: bool = True
|
|
28
|
+
delay_seconds: float = Field(0.5, ge=0)
|
|
29
|
+
timeout_seconds: float = Field(30, gt=0)
|
|
30
|
+
max_bytes: int = Field(10_000_000, gt=0)
|
|
31
|
+
user_agent: str = "ragfoundry/0.1"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class LoadingConfig(_Model):
|
|
35
|
+
on_error: Literal["raise", "skip"] = "raise"
|
|
36
|
+
encoding: str = "utf-8"
|
|
37
|
+
glob: str = "**/*"
|
|
38
|
+
web: WebConfig = Field(default_factory=WebConfig)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# -------------------------------------------------------------------------- chunking
|
|
42
|
+
class SemanticChunkingConfig(_Model):
|
|
43
|
+
breakpoint_percentile: float = Field(95, gt=0, le=100)
|
|
44
|
+
buffer_size: int = Field(1, ge=0, description="Neighbouring sentences embedded with each one.")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class LLMChunkingConfig(_Model):
|
|
48
|
+
window_chars: int = Field(8000, gt=0, description="Text sent to the LLM per request.")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class ChunkingConfig(_Model):
|
|
52
|
+
strategy: str = Field("recursive", description="fixed, recursive, semantic, llm, or custom.")
|
|
53
|
+
chunk_size: int = Field(1000, gt=0)
|
|
54
|
+
chunk_overlap: int = Field(200, ge=0)
|
|
55
|
+
length_unit: Literal["characters", "tokens"] = "characters"
|
|
56
|
+
encoding_name: str = "cl100k_base"
|
|
57
|
+
separators: list[str] | None = None
|
|
58
|
+
semantic: SemanticChunkingConfig = Field(default_factory=SemanticChunkingConfig)
|
|
59
|
+
llm: LLMChunkingConfig = Field(default_factory=LLMChunkingConfig)
|
|
60
|
+
|
|
61
|
+
@model_validator(mode="after")
|
|
62
|
+
def _overlap_below_size(self) -> ChunkingConfig:
|
|
63
|
+
if self.chunk_overlap >= self.chunk_size:
|
|
64
|
+
raise ValueError("chunk_overlap must be smaller than chunk_size")
|
|
65
|
+
return self
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# ------------------------------------------------------------------- models
|
|
69
|
+
class EmbeddingConfig(_Model):
|
|
70
|
+
provider: Literal["huggingface", "openai", "gemini"]
|
|
71
|
+
model: str
|
|
72
|
+
api_key: SecretStr | None = None
|
|
73
|
+
base_url: str | None = None
|
|
74
|
+
options: dict[str, Any] = Field(default_factory=dict)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class LLMConfig(_Model):
|
|
78
|
+
provider: Literal["groq", "openai", "gemini", "claude", "ollama"]
|
|
79
|
+
model: str
|
|
80
|
+
api_key: SecretStr | None = None
|
|
81
|
+
base_url: str | None = None
|
|
82
|
+
temperature: float | None = Field(None, description="Omitted from requests when unset.")
|
|
83
|
+
max_tokens: int | None = None
|
|
84
|
+
options: dict[str, Any] = Field(default_factory=dict)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# -------------------------------------------------------------------- vector stores
|
|
88
|
+
class _VectorStoreBase(_Model):
|
|
89
|
+
collection_name: str = "ragfoundry"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class ChromaConfig(_VectorStoreBase):
|
|
93
|
+
provider: Literal["chroma"] = "chroma"
|
|
94
|
+
persist_directory: str | None = None
|
|
95
|
+
host: str | None = None
|
|
96
|
+
port: int | None = None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class FAISSConfig(_VectorStoreBase):
|
|
100
|
+
provider: Literal["faiss"] = "faiss"
|
|
101
|
+
path: str | None = None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class PineconeConfig(_VectorStoreBase):
|
|
105
|
+
provider: Literal["pinecone"] = "pinecone"
|
|
106
|
+
index_name: str
|
|
107
|
+
api_key: SecretStr | None = None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class QdrantConfig(_VectorStoreBase):
|
|
111
|
+
provider: Literal["qdrant"] = "qdrant"
|
|
112
|
+
url: str | None = None
|
|
113
|
+
path: str | None = None
|
|
114
|
+
api_key: SecretStr | None = None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class WeaviateConfig(_VectorStoreBase):
|
|
118
|
+
provider: Literal["weaviate"] = "weaviate"
|
|
119
|
+
url: str = "http://localhost:8080"
|
|
120
|
+
grpc_port: int = 50051
|
|
121
|
+
api_key: SecretStr | None = None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class MilvusConfig(_VectorStoreBase):
|
|
125
|
+
provider: Literal["milvus"] = "milvus"
|
|
126
|
+
uri: str | None = None
|
|
127
|
+
token: SecretStr | None = None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class PGVectorConfig(_VectorStoreBase):
|
|
131
|
+
provider: Literal["pgvector"] = "pgvector"
|
|
132
|
+
connection: SecretStr
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
VectorStoreConfig = Annotated[
|
|
136
|
+
ChromaConfig
|
|
137
|
+
| FAISSConfig
|
|
138
|
+
| PineconeConfig
|
|
139
|
+
| QdrantConfig
|
|
140
|
+
| WeaviateConfig
|
|
141
|
+
| MilvusConfig
|
|
142
|
+
| PGVectorConfig,
|
|
143
|
+
Field(discriminator="provider"),
|
|
144
|
+
]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# ------------------------------------------------------------------------ retrieval
|
|
148
|
+
class HybridConfig(_Model):
|
|
149
|
+
enabled: bool = False
|
|
150
|
+
rrf_k: int = Field(60, gt=0)
|
|
151
|
+
vector_weight: float = Field(1.0, ge=0)
|
|
152
|
+
bm25_weight: float = Field(1.0, ge=0)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class RerankConfig(_Model):
|
|
156
|
+
provider: Literal["huggingface", "cohere"]
|
|
157
|
+
model: str
|
|
158
|
+
api_key: SecretStr | None = None
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class QueryTransformConfig(_Model):
|
|
162
|
+
strategies: list[Literal["rewrite", "multi_query", "hyde"]] = Field(default_factory=list)
|
|
163
|
+
num_queries: int = Field(3, ge=1, description="Alternative queries for multi_query.")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class CorrectiveConfig(_Model):
|
|
167
|
+
enabled: bool = False
|
|
168
|
+
max_retries: int = Field(1, ge=0)
|
|
169
|
+
min_relevant_docs: int = Field(1, ge=1)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class RetrievalConfig(_Model):
|
|
173
|
+
top_k: int = Field(5, gt=0, description="Documents returned to the generator.")
|
|
174
|
+
fetch_k: int = Field(20, gt=0, description="Candidates per query before fusion and reranking.")
|
|
175
|
+
hybrid: HybridConfig = Field(default_factory=HybridConfig)
|
|
176
|
+
rerank: RerankConfig | None = None
|
|
177
|
+
query_transform: QueryTransformConfig = Field(default_factory=QueryTransformConfig)
|
|
178
|
+
corrective: CorrectiveConfig = Field(default_factory=CorrectiveConfig)
|
|
179
|
+
|
|
180
|
+
@model_validator(mode="after")
|
|
181
|
+
def _fetch_covers_top(self) -> RetrievalConfig:
|
|
182
|
+
if self.fetch_k < self.top_k:
|
|
183
|
+
raise ValueError("fetch_k must be greater than or equal to top_k")
|
|
184
|
+
return self
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class GenerationConfig(_Model):
|
|
188
|
+
system_prompt: str | None = Field(None, description="Replaces the default system prompt.")
|
|
189
|
+
abstain_message: str = "I don't have enough information in the provided documents to answer."
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
# --------------------------------------------------------------------- observability
|
|
193
|
+
class ObservabilityConfig(_Model):
|
|
194
|
+
provider: Literal["none", "langsmith", "langfuse"] = "none"
|
|
195
|
+
project: str = "ragfoundry"
|
|
196
|
+
api_key: SecretStr | None = Field(None, description="LangSmith API key.")
|
|
197
|
+
public_key: str | None = Field(None, description="Langfuse public key.")
|
|
198
|
+
secret_key: SecretStr | None = Field(None, description="Langfuse secret key.")
|
|
199
|
+
endpoint: str | None = Field(None, description="Self-hosted or regional server URL.")
|
|
200
|
+
tags: list[str] = Field(default_factory=list)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class EvaluationConfig(_Model):
|
|
204
|
+
llm: LLMConfig | None = Field(None, description="Judge model; defaults to the pipeline LLM.")
|
|
205
|
+
k_values: list[int] = Field(default_factory=lambda: [1, 3, 5])
|
|
206
|
+
max_concurrency: int = Field(1, ge=1)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# ------------------------------------------------------------------------------ root
|
|
210
|
+
class RAGConfig(_Model):
|
|
211
|
+
persist_dir: str = ".ragfoundry"
|
|
212
|
+
batch_size: int = Field(64, gt=0, description="Chunks written to the vector store per batch.")
|
|
213
|
+
loading: LoadingConfig = Field(default_factory=LoadingConfig)
|
|
214
|
+
chunking: ChunkingConfig = Field(default_factory=ChunkingConfig)
|
|
215
|
+
embeddings: EmbeddingConfig | None = None
|
|
216
|
+
vector_store: VectorStoreConfig | None = None
|
|
217
|
+
llm: LLMConfig | None = None
|
|
218
|
+
retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig)
|
|
219
|
+
generation: GenerationConfig = Field(default_factory=GenerationConfig)
|
|
220
|
+
observability: ObservabilityConfig = Field(default_factory=ObservabilityConfig)
|
|
221
|
+
evaluation: EvaluationConfig = Field(default_factory=EvaluationConfig)
|
|
222
|
+
|
|
223
|
+
@classmethod
|
|
224
|
+
def from_yaml(cls, path: str | Path) -> RAGConfig:
|
|
225
|
+
"""Read a YAML file, expanding ``${VAR}`` and ``${VAR:-default}`` in string values."""
|
|
226
|
+
try:
|
|
227
|
+
raw = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
|
|
228
|
+
except (OSError, yaml.YAMLError) as exc:
|
|
229
|
+
raise ConfigurationError(f"Could not read config '{path}': {exc}") from exc
|
|
230
|
+
return cls.model_validate(_expand_env(raw or {}))
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _expand_env(value: Any) -> Any:
|
|
234
|
+
if isinstance(value, dict):
|
|
235
|
+
return {key: _expand_env(item) for key, item in value.items()}
|
|
236
|
+
if isinstance(value, list):
|
|
237
|
+
return [_expand_env(item) for item in value]
|
|
238
|
+
if isinstance(value, str):
|
|
239
|
+
return _ENV_PATTERN.sub(_env_value, value)
|
|
240
|
+
return value
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _env_value(match: re.Match[str]) -> str:
|
|
244
|
+
name, default = match.group(1), match.group(2)
|
|
245
|
+
resolved = os.environ.get(name, default)
|
|
246
|
+
if resolved is None:
|
|
247
|
+
raise ConfigurationError(f"Environment variable '{name}' is referenced but not set")
|
|
248
|
+
return resolved
|
ragfoundry/errors.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Exceptions raised by ragfoundry."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RAGFoundryError(Exception):
|
|
7
|
+
"""Base class for all ragfoundry errors."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ConfigurationError(RAGFoundryError):
|
|
11
|
+
"""The supplied configuration is invalid or incomplete."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MissingDependencyError(RAGFoundryError, ImportError):
|
|
15
|
+
"""An optional dependency is required for the selected feature."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class IngestionError(RAGFoundryError):
|
|
19
|
+
"""A document could not be loaded or indexed."""
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Retrieval and generation evaluation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .dataset import EvalSample, load_dataset
|
|
6
|
+
from .generation import (
|
|
7
|
+
answer_correctness,
|
|
8
|
+
answer_relevance,
|
|
9
|
+
context_recall,
|
|
10
|
+
context_relevance,
|
|
11
|
+
faithfulness,
|
|
12
|
+
)
|
|
13
|
+
from .ragas_adapter import evaluate_with_ragas
|
|
14
|
+
from .retrieval import average_precision, hit_rate, mrr, ndcg_at_k, precision_at_k, recall_at_k
|
|
15
|
+
from .runner import EvalReport, evaluate_generation_rows, evaluate_pipeline
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"EvalReport",
|
|
19
|
+
"EvalSample",
|
|
20
|
+
"answer_correctness",
|
|
21
|
+
"answer_relevance",
|
|
22
|
+
"average_precision",
|
|
23
|
+
"context_recall",
|
|
24
|
+
"context_relevance",
|
|
25
|
+
"evaluate_generation_rows",
|
|
26
|
+
"evaluate_pipeline",
|
|
27
|
+
"evaluate_with_ragas",
|
|
28
|
+
"faithfulness",
|
|
29
|
+
"hit_rate",
|
|
30
|
+
"load_dataset",
|
|
31
|
+
"mrr",
|
|
32
|
+
"ndcg_at_k",
|
|
33
|
+
"precision_at_k",
|
|
34
|
+
"recall_at_k",
|
|
35
|
+
]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Evaluation dataset schema and file loading."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
9
|
+
|
|
10
|
+
from ..errors import ConfigurationError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class EvalSample(BaseModel):
|
|
14
|
+
"""One test question. Retrieval metrics need at least one of the three relevance fields."""
|
|
15
|
+
|
|
16
|
+
model_config = ConfigDict(extra="forbid")
|
|
17
|
+
|
|
18
|
+
question: str
|
|
19
|
+
reference_answer: str | None = None
|
|
20
|
+
relevant_ids: list[str] = Field(default_factory=list, description="Exact chunk ids.")
|
|
21
|
+
relevant_sources: list[str] = Field(
|
|
22
|
+
default_factory=list, description="Source URLs, paths, or path suffixes such as a filename."
|
|
23
|
+
)
|
|
24
|
+
reference_contexts: list[str] = Field(
|
|
25
|
+
default_factory=list, description="Passages that a relevant chunk should overlap."
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def has_relevance_labels(self) -> bool:
|
|
30
|
+
return bool(self.relevant_ids or self.relevant_sources or self.reference_contexts)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def load_dataset(path: str | Path) -> list[EvalSample]:
|
|
34
|
+
"""Read samples from a ``.json`` array or ``.jsonl`` file."""
|
|
35
|
+
file = Path(path)
|
|
36
|
+
try:
|
|
37
|
+
text = file.read_text(encoding="utf-8")
|
|
38
|
+
if file.suffix.lower() == ".jsonl":
|
|
39
|
+
records = [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
40
|
+
else:
|
|
41
|
+
records = json.loads(text)
|
|
42
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
43
|
+
raise ConfigurationError(f"Could not read dataset '{path}': {exc}") from exc
|
|
44
|
+
if not isinstance(records, list) or not records:
|
|
45
|
+
raise ConfigurationError(f"Dataset '{path}' must contain a non-empty list of samples")
|
|
46
|
+
return [EvalSample.model_validate(record) for record in records]
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""LLM-judged generation metrics.
|
|
2
|
+
|
|
3
|
+
Each function returns a score from 0 to 1, or ``None`` when the metric does not apply or the judge
|
|
4
|
+
reply could not be interpreted. ``None`` is excluded from averages, never counted as zero.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from langchain_core.language_models import BaseChatModel, LanguageModelInput
|
|
13
|
+
|
|
14
|
+
from .._utils import message_text, parse_json, string_items
|
|
15
|
+
from ..prompts import (
|
|
16
|
+
ANSWER_CORRECTNESS_PROMPT,
|
|
17
|
+
ANSWER_RELEVANCE_PROMPT,
|
|
18
|
+
EXTRACT_CLAIMS_PROMPT,
|
|
19
|
+
GRADE_PROMPT,
|
|
20
|
+
VERIFY_CLAIM_PROMPT,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _yes_fraction(
|
|
25
|
+
llm: BaseChatModel, prompts: list[LanguageModelInput], config: Any
|
|
26
|
+
) -> float | None:
|
|
27
|
+
votes = []
|
|
28
|
+
for reply in llm.batch(prompts, config=config):
|
|
29
|
+
verdict = message_text(reply).strip().lower()
|
|
30
|
+
if verdict.startswith("yes"):
|
|
31
|
+
votes.append(True)
|
|
32
|
+
elif verdict.startswith("no"):
|
|
33
|
+
votes.append(False)
|
|
34
|
+
return sum(votes) / len(votes) if votes else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _rubric_score(llm: BaseChatModel, prompt: str, config: Any) -> float | None:
|
|
38
|
+
parsed = parse_json(message_text(llm.invoke(prompt, config=config)))
|
|
39
|
+
score = parsed.get("score") if isinstance(parsed, dict) else None
|
|
40
|
+
if isinstance(score, bool) or not isinstance(score, (int, float)):
|
|
41
|
+
return None
|
|
42
|
+
return max(0.0, min(1.0, float(score)))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _claims(llm: BaseChatModel, question: str, text: str, config: Any) -> list[str] | None:
|
|
46
|
+
prompt = EXTRACT_CLAIMS_PROMPT.format(question=question or "(not given)", answer=text)
|
|
47
|
+
reply = llm.invoke(prompt, config=config)
|
|
48
|
+
return string_items(parse_json(message_text(reply)))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def faithfulness(
|
|
52
|
+
llm: BaseChatModel,
|
|
53
|
+
answer: str,
|
|
54
|
+
contexts: Sequence[str],
|
|
55
|
+
question: str = "",
|
|
56
|
+
config: Any = None,
|
|
57
|
+
) -> float | None:
|
|
58
|
+
"""Fraction of the answer's factual claims supported by the retrieved contexts."""
|
|
59
|
+
if not contexts:
|
|
60
|
+
return None
|
|
61
|
+
claims = _claims(llm, question, answer, config)
|
|
62
|
+
if not claims:
|
|
63
|
+
return None
|
|
64
|
+
context = "\n\n".join(contexts)
|
|
65
|
+
prompts: list[LanguageModelInput] = [
|
|
66
|
+
VERIFY_CLAIM_PROMPT.format(context=context, claim=claim) for claim in claims
|
|
67
|
+
]
|
|
68
|
+
return _yes_fraction(llm, prompts, config)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def answer_relevance(
|
|
72
|
+
llm: BaseChatModel, question: str, answer: str, config: Any = None
|
|
73
|
+
) -> float | None:
|
|
74
|
+
"""How directly the answer addresses the question."""
|
|
75
|
+
prompt = ANSWER_RELEVANCE_PROMPT.format(question=question, answer=answer)
|
|
76
|
+
return _rubric_score(llm, prompt, config)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def answer_correctness(
|
|
80
|
+
llm: BaseChatModel, question: str, answer: str, reference: str, config: Any = None
|
|
81
|
+
) -> float | None:
|
|
82
|
+
"""Factual agreement between the answer and a reference answer."""
|
|
83
|
+
prompt = ANSWER_CORRECTNESS_PROMPT.format(question=question, reference=reference, answer=answer)
|
|
84
|
+
return _rubric_score(llm, prompt, config)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def context_relevance(
|
|
88
|
+
llm: BaseChatModel, question: str, contexts: Sequence[str], config: Any = None
|
|
89
|
+
) -> float | None:
|
|
90
|
+
"""Fraction of retrieved contexts useful for the question."""
|
|
91
|
+
if not contexts:
|
|
92
|
+
return None
|
|
93
|
+
prompts: list[LanguageModelInput] = [
|
|
94
|
+
GRADE_PROMPT.format(question=question, passage=context) for context in contexts
|
|
95
|
+
]
|
|
96
|
+
return _yes_fraction(llm, prompts, config)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def context_recall(
|
|
100
|
+
llm: BaseChatModel,
|
|
101
|
+
reference: str,
|
|
102
|
+
contexts: Sequence[str],
|
|
103
|
+
question: str = "",
|
|
104
|
+
config: Any = None,
|
|
105
|
+
) -> float | None:
|
|
106
|
+
"""Fraction of the reference answer's claims that the retrieved contexts support."""
|
|
107
|
+
if not contexts:
|
|
108
|
+
return None
|
|
109
|
+
claims = _claims(llm, question, reference, config)
|
|
110
|
+
if not claims:
|
|
111
|
+
return None
|
|
112
|
+
context = "\n\n".join(contexts)
|
|
113
|
+
prompts: list[LanguageModelInput] = [
|
|
114
|
+
VERIFY_CLAIM_PROMPT.format(context=context, claim=claim) for claim in claims
|
|
115
|
+
]
|
|
116
|
+
return _yes_fraction(llm, prompts, config)
|