ragfoundry 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. ragfoundry/__init__.py +36 -0
  2. ragfoundry/_utils.py +88 -0
  3. ragfoundry/chunk_store.py +109 -0
  4. ragfoundry/chunking/__init__.py +23 -0
  5. ragfoundry/chunking/_text.py +20 -0
  6. ragfoundry/chunking/base.py +171 -0
  7. ragfoundry/chunking/llm.py +88 -0
  8. ragfoundry/chunking/semantic.py +67 -0
  9. ragfoundry/cli.py +162 -0
  10. ragfoundry/config.py +248 -0
  11. ragfoundry/errors.py +19 -0
  12. ragfoundry/evaluation/__init__.py +35 -0
  13. ragfoundry/evaluation/dataset.py +46 -0
  14. ragfoundry/evaluation/generation.py +116 -0
  15. ragfoundry/evaluation/ragas_adapter.py +84 -0
  16. ragfoundry/evaluation/retrieval.py +71 -0
  17. ragfoundry/evaluation/runner.py +190 -0
  18. ragfoundry/graphs/__init__.py +14 -0
  19. ragfoundry/graphs/ingest.py +99 -0
  20. ragfoundry/graphs/query.py +197 -0
  21. ragfoundry/loaders/__init__.py +15 -0
  22. ragfoundry/loaders/base.py +162 -0
  23. ragfoundry/loaders/excel.py +64 -0
  24. ragfoundry/loaders/web.py +136 -0
  25. ragfoundry/models/__init__.py +64 -0
  26. ragfoundry/observability/__init__.py +65 -0
  27. ragfoundry/pipeline.py +353 -0
  28. ragfoundry/prompts.py +90 -0
  29. ragfoundry/retrieval/__init__.py +19 -0
  30. ragfoundry/retrieval/fusion.py +32 -0
  31. ragfoundry/retrieval/rerank.py +62 -0
  32. ragfoundry/retrieval/retriever.py +100 -0
  33. ragfoundry/retrieval/transforms.py +53 -0
  34. ragfoundry/vectorstores/__init__.py +8 -0
  35. ragfoundry/vectorstores/factory.py +232 -0
  36. ragfoundry/vectorstores/index.py +59 -0
  37. ragfoundry-0.1.0.dist-info/METADATA +802 -0
  38. ragfoundry-0.1.0.dist-info/RECORD +40 -0
  39. ragfoundry-0.1.0.dist-info/WHEEL +4 -0
  40. ragfoundry-0.1.0.dist-info/entry_points.txt +2 -0
ragfoundry/cli.py ADDED
@@ -0,0 +1,162 @@
1
+ """Command line interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Annotated, Literal
8
+
9
+ import typer
10
+
11
+ from . import RAGPipeline
12
+ from .errors import RAGFoundryError
13
+
14
+ app = typer.Typer(no_args_is_help=True, add_completion=False, help="Build and run RAG pipelines.")
15
+
16
+ ConfigOption = Annotated[Path, typer.Option("--config", "-c", help="Path to the YAML config.")]
17
+
18
+ TEMPLATE = """\
19
+ # ragfoundry configuration. ${VAR} reads from the environment.
20
+ persist_dir: .ragfoundry
21
+
22
+ chunking:
23
+ strategy: recursive # fixed | recursive | semantic | llm
24
+ chunk_size: 1000
25
+ chunk_overlap: 200
26
+ length_unit: characters # characters | tokens
27
+
28
+ embeddings:
29
+ provider: huggingface # huggingface | openai | gemini
30
+ model: sentence-transformers/all-MiniLM-L6-v2
31
+
32
+ vector_store:
33
+ provider: chroma # chroma | faiss | pinecone | qdrant | weaviate | milvus | pgvector
34
+ collection_name: my_docs
35
+
36
+ llm:
37
+ provider: ollama # groq | openai | gemini | claude | ollama
38
+ model: llama3.2
39
+
40
+ retrieval:
41
+ top_k: 5
42
+ fetch_k: 20
43
+ hybrid:
44
+ enabled: true
45
+ # rerank:
46
+ # provider: huggingface # huggingface | cohere
47
+ # model: cross-encoder/ms-marco-MiniLM-L-6-v2
48
+ query_transform:
49
+ strategies: [] # rewrite, multi_query, hyde
50
+ corrective:
51
+ enabled: false
52
+
53
+ observability:
54
+ provider: none # none | langsmith | langfuse
55
+ # api_key: ${LANGSMITH_API_KEY} # LangSmith
56
+ # public_key: ${LANGFUSE_PUBLIC_KEY} # Langfuse
57
+ # secret_key: ${LANGFUSE_SECRET_KEY} # Langfuse
58
+ """
59
+
60
+
61
+ def _fail(error: Exception) -> typer.Exit:
62
+ typer.secho(f"Error: {error}", fg=typer.colors.RED, err=True)
63
+ return typer.Exit(code=1)
64
+
65
+
66
+ @app.command()
67
+ def init(path: Path = Path("rag.yaml")) -> None:
68
+ """Write a starter configuration file."""
69
+ if path.exists():
70
+ raise _fail(FileExistsError(f"{path} already exists"))
71
+ path.write_text(TEMPLATE, encoding="utf-8")
72
+ typer.echo(f"Wrote {path}")
73
+
74
+
75
+ @app.command()
76
+ def ingest(
77
+ sources: list[str],
78
+ config: ConfigOption = Path("rag.yaml"),
79
+ ) -> None:
80
+ """Load, chunk and index files, directories or URLs."""
81
+ try:
82
+ with RAGPipeline.from_yaml(config) as pipeline:
83
+ report = pipeline.ingest(sources)
84
+ except RAGFoundryError as error:
85
+ raise _fail(error) from error
86
+ typer.echo(
87
+ f"Documents: {report.documents_loaded} Chunks: {report.chunks_total} "
88
+ f"Added: {report.chunks_added} Unchanged: {report.chunks_unchanged} "
89
+ f"Removed: {report.chunks_removed}"
90
+ )
91
+ for failure in report.failures:
92
+ typer.secho(f"Failed: {failure.source}: {failure.error}", fg=typer.colors.YELLOW, err=True)
93
+ if report.failures:
94
+ raise typer.Exit(code=1)
95
+
96
+
97
+ @app.command()
98
+ def query(
99
+ question: str,
100
+ config: ConfigOption = Path("rag.yaml"),
101
+ stream: Annotated[bool, typer.Option(help="Print the answer as it is generated.")] = False,
102
+ as_json: Annotated[bool, typer.Option("--json", help="Print the full result.")] = False,
103
+ ) -> None:
104
+ """Answer a question from the indexed documents."""
105
+ try:
106
+ with RAGPipeline.from_yaml(config) as pipeline:
107
+ if stream:
108
+ for text in pipeline.stream(question):
109
+ typer.echo(text, nl=False)
110
+ typer.echo()
111
+ return
112
+ result = pipeline.query(question)
113
+ except RAGFoundryError as error:
114
+ raise _fail(error) from error
115
+ if as_json:
116
+ typer.echo(json.dumps(result.to_dict(), indent=2, ensure_ascii=False, default=str))
117
+ return
118
+ typer.echo(result.answer)
119
+ for source in result.sources:
120
+ typer.echo(f" - {source}")
121
+
122
+
123
+ @app.command("eval")
124
+ def evaluate(
125
+ dataset: Path,
126
+ config: ConfigOption = Path("rag.yaml"),
127
+ mode: Literal["retrieval", "generation", "all"] = "all",
128
+ backend: Literal["native", "ragas"] = "native",
129
+ output: Annotated[Path | None, typer.Option(help="Write the full report as JSON.")] = None,
130
+ ) -> None:
131
+ """Evaluate retrieval and generation against a .json or .jsonl dataset."""
132
+ try:
133
+ with RAGPipeline.from_yaml(config) as pipeline:
134
+ report = pipeline.evaluate(dataset, mode=mode, backend=backend)
135
+ except RAGFoundryError as error:
136
+ raise _fail(error) from error
137
+ for name, stats in report.summary.items():
138
+ typer.echo(f"{name:<24} {stats['mean']:.3f} (n={stats['count']})")
139
+ if output:
140
+ report.save(output)
141
+ typer.echo(f"Report written to {output}")
142
+
143
+
144
+ @app.command("sources")
145
+ def list_sources(config: ConfigOption = Path("rag.yaml")) -> None:
146
+ """List indexed sources and their chunk counts."""
147
+ try:
148
+ with RAGPipeline.from_yaml(config) as pipeline:
149
+ for source, count in pipeline.sources().items():
150
+ typer.echo(f"{count:>6} {source}")
151
+ except RAGFoundryError as error:
152
+ raise _fail(error) from error
153
+
154
+
155
+ @app.command("delete-source")
156
+ def delete_source(source: str, config: ConfigOption = Path("rag.yaml")) -> None:
157
+ """Remove one source from the index."""
158
+ try:
159
+ with RAGPipeline.from_yaml(config) as pipeline:
160
+ typer.echo(f"Removed {pipeline.delete_source(source)} chunks")
161
+ except RAGFoundryError as error:
162
+ raise _fail(error) from error
ragfoundry/config.py ADDED
@@ -0,0 +1,248 @@
1
+ """Typed configuration for a RAG pipeline, loadable from Python or YAML."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import re
7
+ from pathlib import Path
8
+ from typing import Annotated, Any, Literal
9
+
10
+ import yaml
11
+ from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
12
+
13
+ from .errors import ConfigurationError
14
+
15
+ _ENV_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
16
+
17
+
18
+ class _Model(BaseModel):
19
+ model_config = ConfigDict(extra="forbid", protected_namespaces=())
20
+
21
+
22
+ # --------------------------------------------------------------------------- loading
23
+ class WebConfig(_Model):
24
+ max_depth: int = Field(0, ge=0, description="Link depth to follow; 0 fetches only the URL.")
25
+ max_pages: int = Field(50, ge=1)
26
+ same_domain: bool = True
27
+ respect_robots: bool = True
28
+ delay_seconds: float = Field(0.5, ge=0)
29
+ timeout_seconds: float = Field(30, gt=0)
30
+ max_bytes: int = Field(10_000_000, gt=0)
31
+ user_agent: str = "ragfoundry/0.1"
32
+
33
+
34
+ class LoadingConfig(_Model):
35
+ on_error: Literal["raise", "skip"] = "raise"
36
+ encoding: str = "utf-8"
37
+ glob: str = "**/*"
38
+ web: WebConfig = Field(default_factory=WebConfig)
39
+
40
+
41
+ # -------------------------------------------------------------------------- chunking
42
+ class SemanticChunkingConfig(_Model):
43
+ breakpoint_percentile: float = Field(95, gt=0, le=100)
44
+ buffer_size: int = Field(1, ge=0, description="Neighbouring sentences embedded with each one.")
45
+
46
+
47
+ class LLMChunkingConfig(_Model):
48
+ window_chars: int = Field(8000, gt=0, description="Text sent to the LLM per request.")
49
+
50
+
51
+ class ChunkingConfig(_Model):
52
+ strategy: str = Field("recursive", description="fixed, recursive, semantic, llm, or custom.")
53
+ chunk_size: int = Field(1000, gt=0)
54
+ chunk_overlap: int = Field(200, ge=0)
55
+ length_unit: Literal["characters", "tokens"] = "characters"
56
+ encoding_name: str = "cl100k_base"
57
+ separators: list[str] | None = None
58
+ semantic: SemanticChunkingConfig = Field(default_factory=SemanticChunkingConfig)
59
+ llm: LLMChunkingConfig = Field(default_factory=LLMChunkingConfig)
60
+
61
+ @model_validator(mode="after")
62
+ def _overlap_below_size(self) -> ChunkingConfig:
63
+ if self.chunk_overlap >= self.chunk_size:
64
+ raise ValueError("chunk_overlap must be smaller than chunk_size")
65
+ return self
66
+
67
+
68
+ # ------------------------------------------------------------------- models
69
+ class EmbeddingConfig(_Model):
70
+ provider: Literal["huggingface", "openai", "gemini"]
71
+ model: str
72
+ api_key: SecretStr | None = None
73
+ base_url: str | None = None
74
+ options: dict[str, Any] = Field(default_factory=dict)
75
+
76
+
77
+ class LLMConfig(_Model):
78
+ provider: Literal["groq", "openai", "gemini", "claude", "ollama"]
79
+ model: str
80
+ api_key: SecretStr | None = None
81
+ base_url: str | None = None
82
+ temperature: float | None = Field(None, description="Omitted from requests when unset.")
83
+ max_tokens: int | None = None
84
+ options: dict[str, Any] = Field(default_factory=dict)
85
+
86
+
87
+ # -------------------------------------------------------------------- vector stores
88
+ class _VectorStoreBase(_Model):
89
+ collection_name: str = "ragfoundry"
90
+
91
+
92
+ class ChromaConfig(_VectorStoreBase):
93
+ provider: Literal["chroma"] = "chroma"
94
+ persist_directory: str | None = None
95
+ host: str | None = None
96
+ port: int | None = None
97
+
98
+
99
+ class FAISSConfig(_VectorStoreBase):
100
+ provider: Literal["faiss"] = "faiss"
101
+ path: str | None = None
102
+
103
+
104
+ class PineconeConfig(_VectorStoreBase):
105
+ provider: Literal["pinecone"] = "pinecone"
106
+ index_name: str
107
+ api_key: SecretStr | None = None
108
+
109
+
110
+ class QdrantConfig(_VectorStoreBase):
111
+ provider: Literal["qdrant"] = "qdrant"
112
+ url: str | None = None
113
+ path: str | None = None
114
+ api_key: SecretStr | None = None
115
+
116
+
117
+ class WeaviateConfig(_VectorStoreBase):
118
+ provider: Literal["weaviate"] = "weaviate"
119
+ url: str = "http://localhost:8080"
120
+ grpc_port: int = 50051
121
+ api_key: SecretStr | None = None
122
+
123
+
124
+ class MilvusConfig(_VectorStoreBase):
125
+ provider: Literal["milvus"] = "milvus"
126
+ uri: str | None = None
127
+ token: SecretStr | None = None
128
+
129
+
130
+ class PGVectorConfig(_VectorStoreBase):
131
+ provider: Literal["pgvector"] = "pgvector"
132
+ connection: SecretStr
133
+
134
+
135
+ VectorStoreConfig = Annotated[
136
+ ChromaConfig
137
+ | FAISSConfig
138
+ | PineconeConfig
139
+ | QdrantConfig
140
+ | WeaviateConfig
141
+ | MilvusConfig
142
+ | PGVectorConfig,
143
+ Field(discriminator="provider"),
144
+ ]
145
+
146
+
147
+ # ------------------------------------------------------------------------ retrieval
148
+ class HybridConfig(_Model):
149
+ enabled: bool = False
150
+ rrf_k: int = Field(60, gt=0)
151
+ vector_weight: float = Field(1.0, ge=0)
152
+ bm25_weight: float = Field(1.0, ge=0)
153
+
154
+
155
+ class RerankConfig(_Model):
156
+ provider: Literal["huggingface", "cohere"]
157
+ model: str
158
+ api_key: SecretStr | None = None
159
+
160
+
161
+ class QueryTransformConfig(_Model):
162
+ strategies: list[Literal["rewrite", "multi_query", "hyde"]] = Field(default_factory=list)
163
+ num_queries: int = Field(3, ge=1, description="Alternative queries for multi_query.")
164
+
165
+
166
+ class CorrectiveConfig(_Model):
167
+ enabled: bool = False
168
+ max_retries: int = Field(1, ge=0)
169
+ min_relevant_docs: int = Field(1, ge=1)
170
+
171
+
172
+ class RetrievalConfig(_Model):
173
+ top_k: int = Field(5, gt=0, description="Documents returned to the generator.")
174
+ fetch_k: int = Field(20, gt=0, description="Candidates per query before fusion and reranking.")
175
+ hybrid: HybridConfig = Field(default_factory=HybridConfig)
176
+ rerank: RerankConfig | None = None
177
+ query_transform: QueryTransformConfig = Field(default_factory=QueryTransformConfig)
178
+ corrective: CorrectiveConfig = Field(default_factory=CorrectiveConfig)
179
+
180
+ @model_validator(mode="after")
181
+ def _fetch_covers_top(self) -> RetrievalConfig:
182
+ if self.fetch_k < self.top_k:
183
+ raise ValueError("fetch_k must be greater than or equal to top_k")
184
+ return self
185
+
186
+
187
+ class GenerationConfig(_Model):
188
+ system_prompt: str | None = Field(None, description="Replaces the default system prompt.")
189
+ abstain_message: str = "I don't have enough information in the provided documents to answer."
190
+
191
+
192
+ # --------------------------------------------------------------------- observability
193
+ class ObservabilityConfig(_Model):
194
+ provider: Literal["none", "langsmith", "langfuse"] = "none"
195
+ project: str = "ragfoundry"
196
+ api_key: SecretStr | None = Field(None, description="LangSmith API key.")
197
+ public_key: str | None = Field(None, description="Langfuse public key.")
198
+ secret_key: SecretStr | None = Field(None, description="Langfuse secret key.")
199
+ endpoint: str | None = Field(None, description="Self-hosted or regional server URL.")
200
+ tags: list[str] = Field(default_factory=list)
201
+
202
+
203
+ class EvaluationConfig(_Model):
204
+ llm: LLMConfig | None = Field(None, description="Judge model; defaults to the pipeline LLM.")
205
+ k_values: list[int] = Field(default_factory=lambda: [1, 3, 5])
206
+ max_concurrency: int = Field(1, ge=1)
207
+
208
+
209
+ # ------------------------------------------------------------------------------ root
210
+ class RAGConfig(_Model):
211
+ persist_dir: str = ".ragfoundry"
212
+ batch_size: int = Field(64, gt=0, description="Chunks written to the vector store per batch.")
213
+ loading: LoadingConfig = Field(default_factory=LoadingConfig)
214
+ chunking: ChunkingConfig = Field(default_factory=ChunkingConfig)
215
+ embeddings: EmbeddingConfig | None = None
216
+ vector_store: VectorStoreConfig | None = None
217
+ llm: LLMConfig | None = None
218
+ retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig)
219
+ generation: GenerationConfig = Field(default_factory=GenerationConfig)
220
+ observability: ObservabilityConfig = Field(default_factory=ObservabilityConfig)
221
+ evaluation: EvaluationConfig = Field(default_factory=EvaluationConfig)
222
+
223
+ @classmethod
224
+ def from_yaml(cls, path: str | Path) -> RAGConfig:
225
+ """Read a YAML file, expanding ``${VAR}`` and ``${VAR:-default}`` in string values."""
226
+ try:
227
+ raw = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
228
+ except (OSError, yaml.YAMLError) as exc:
229
+ raise ConfigurationError(f"Could not read config '{path}': {exc}") from exc
230
+ return cls.model_validate(_expand_env(raw or {}))
231
+
232
+
233
+ def _expand_env(value: Any) -> Any:
234
+ if isinstance(value, dict):
235
+ return {key: _expand_env(item) for key, item in value.items()}
236
+ if isinstance(value, list):
237
+ return [_expand_env(item) for item in value]
238
+ if isinstance(value, str):
239
+ return _ENV_PATTERN.sub(_env_value, value)
240
+ return value
241
+
242
+
243
+ def _env_value(match: re.Match[str]) -> str:
244
+ name, default = match.group(1), match.group(2)
245
+ resolved = os.environ.get(name, default)
246
+ if resolved is None:
247
+ raise ConfigurationError(f"Environment variable '{name}' is referenced but not set")
248
+ return resolved
ragfoundry/errors.py ADDED
@@ -0,0 +1,19 @@
1
+ """Exceptions raised by ragfoundry."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class RAGFoundryError(Exception):
7
+ """Base class for all ragfoundry errors."""
8
+
9
+
10
+ class ConfigurationError(RAGFoundryError):
11
+ """The supplied configuration is invalid or incomplete."""
12
+
13
+
14
+ class MissingDependencyError(RAGFoundryError, ImportError):
15
+ """An optional dependency is required for the selected feature."""
16
+
17
+
18
+ class IngestionError(RAGFoundryError):
19
+ """A document could not be loaded or indexed."""
@@ -0,0 +1,35 @@
1
+ """Retrieval and generation evaluation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .dataset import EvalSample, load_dataset
6
+ from .generation import (
7
+ answer_correctness,
8
+ answer_relevance,
9
+ context_recall,
10
+ context_relevance,
11
+ faithfulness,
12
+ )
13
+ from .ragas_adapter import evaluate_with_ragas
14
+ from .retrieval import average_precision, hit_rate, mrr, ndcg_at_k, precision_at_k, recall_at_k
15
+ from .runner import EvalReport, evaluate_generation_rows, evaluate_pipeline
16
+
17
+ __all__ = [
18
+ "EvalReport",
19
+ "EvalSample",
20
+ "answer_correctness",
21
+ "answer_relevance",
22
+ "average_precision",
23
+ "context_recall",
24
+ "context_relevance",
25
+ "evaluate_generation_rows",
26
+ "evaluate_pipeline",
27
+ "evaluate_with_ragas",
28
+ "faithfulness",
29
+ "hit_rate",
30
+ "load_dataset",
31
+ "mrr",
32
+ "ndcg_at_k",
33
+ "precision_at_k",
34
+ "recall_at_k",
35
+ ]
@@ -0,0 +1,46 @@
1
+ """Evaluation dataset schema and file loading."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ from pydantic import BaseModel, ConfigDict, Field
9
+
10
+ from ..errors import ConfigurationError
11
+
12
+
13
+ class EvalSample(BaseModel):
14
+ """One test question. Retrieval metrics need at least one of the three relevance fields."""
15
+
16
+ model_config = ConfigDict(extra="forbid")
17
+
18
+ question: str
19
+ reference_answer: str | None = None
20
+ relevant_ids: list[str] = Field(default_factory=list, description="Exact chunk ids.")
21
+ relevant_sources: list[str] = Field(
22
+ default_factory=list, description="Source URLs, paths, or path suffixes such as a filename."
23
+ )
24
+ reference_contexts: list[str] = Field(
25
+ default_factory=list, description="Passages that a relevant chunk should overlap."
26
+ )
27
+
28
+ @property
29
+ def has_relevance_labels(self) -> bool:
30
+ return bool(self.relevant_ids or self.relevant_sources or self.reference_contexts)
31
+
32
+
33
+ def load_dataset(path: str | Path) -> list[EvalSample]:
34
+ """Read samples from a ``.json`` array or ``.jsonl`` file."""
35
+ file = Path(path)
36
+ try:
37
+ text = file.read_text(encoding="utf-8")
38
+ if file.suffix.lower() == ".jsonl":
39
+ records = [json.loads(line) for line in text.splitlines() if line.strip()]
40
+ else:
41
+ records = json.loads(text)
42
+ except (OSError, json.JSONDecodeError) as exc:
43
+ raise ConfigurationError(f"Could not read dataset '{path}': {exc}") from exc
44
+ if not isinstance(records, list) or not records:
45
+ raise ConfigurationError(f"Dataset '{path}' must contain a non-empty list of samples")
46
+ return [EvalSample.model_validate(record) for record in records]
@@ -0,0 +1,116 @@
1
+ """LLM-judged generation metrics.
2
+
3
+ Each function returns a score from 0 to 1, or ``None`` when the metric does not apply or the judge
4
+ reply could not be interpreted. ``None`` is excluded from averages, never counted as zero.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Sequence
10
+ from typing import Any
11
+
12
+ from langchain_core.language_models import BaseChatModel, LanguageModelInput
13
+
14
+ from .._utils import message_text, parse_json, string_items
15
+ from ..prompts import (
16
+ ANSWER_CORRECTNESS_PROMPT,
17
+ ANSWER_RELEVANCE_PROMPT,
18
+ EXTRACT_CLAIMS_PROMPT,
19
+ GRADE_PROMPT,
20
+ VERIFY_CLAIM_PROMPT,
21
+ )
22
+
23
+
24
+ def _yes_fraction(
25
+ llm: BaseChatModel, prompts: list[LanguageModelInput], config: Any
26
+ ) -> float | None:
27
+ votes = []
28
+ for reply in llm.batch(prompts, config=config):
29
+ verdict = message_text(reply).strip().lower()
30
+ if verdict.startswith("yes"):
31
+ votes.append(True)
32
+ elif verdict.startswith("no"):
33
+ votes.append(False)
34
+ return sum(votes) / len(votes) if votes else None
35
+
36
+
37
+ def _rubric_score(llm: BaseChatModel, prompt: str, config: Any) -> float | None:
38
+ parsed = parse_json(message_text(llm.invoke(prompt, config=config)))
39
+ score = parsed.get("score") if isinstance(parsed, dict) else None
40
+ if isinstance(score, bool) or not isinstance(score, (int, float)):
41
+ return None
42
+ return max(0.0, min(1.0, float(score)))
43
+
44
+
45
+ def _claims(llm: BaseChatModel, question: str, text: str, config: Any) -> list[str] | None:
46
+ prompt = EXTRACT_CLAIMS_PROMPT.format(question=question or "(not given)", answer=text)
47
+ reply = llm.invoke(prompt, config=config)
48
+ return string_items(parse_json(message_text(reply)))
49
+
50
+
51
+ def faithfulness(
52
+ llm: BaseChatModel,
53
+ answer: str,
54
+ contexts: Sequence[str],
55
+ question: str = "",
56
+ config: Any = None,
57
+ ) -> float | None:
58
+ """Fraction of the answer's factual claims supported by the retrieved contexts."""
59
+ if not contexts:
60
+ return None
61
+ claims = _claims(llm, question, answer, config)
62
+ if not claims:
63
+ return None
64
+ context = "\n\n".join(contexts)
65
+ prompts: list[LanguageModelInput] = [
66
+ VERIFY_CLAIM_PROMPT.format(context=context, claim=claim) for claim in claims
67
+ ]
68
+ return _yes_fraction(llm, prompts, config)
69
+
70
+
71
+ def answer_relevance(
72
+ llm: BaseChatModel, question: str, answer: str, config: Any = None
73
+ ) -> float | None:
74
+ """How directly the answer addresses the question."""
75
+ prompt = ANSWER_RELEVANCE_PROMPT.format(question=question, answer=answer)
76
+ return _rubric_score(llm, prompt, config)
77
+
78
+
79
+ def answer_correctness(
80
+ llm: BaseChatModel, question: str, answer: str, reference: str, config: Any = None
81
+ ) -> float | None:
82
+ """Factual agreement between the answer and a reference answer."""
83
+ prompt = ANSWER_CORRECTNESS_PROMPT.format(question=question, reference=reference, answer=answer)
84
+ return _rubric_score(llm, prompt, config)
85
+
86
+
87
+ def context_relevance(
88
+ llm: BaseChatModel, question: str, contexts: Sequence[str], config: Any = None
89
+ ) -> float | None:
90
+ """Fraction of retrieved contexts useful for the question."""
91
+ if not contexts:
92
+ return None
93
+ prompts: list[LanguageModelInput] = [
94
+ GRADE_PROMPT.format(question=question, passage=context) for context in contexts
95
+ ]
96
+ return _yes_fraction(llm, prompts, config)
97
+
98
+
99
+ def context_recall(
100
+ llm: BaseChatModel,
101
+ reference: str,
102
+ contexts: Sequence[str],
103
+ question: str = "",
104
+ config: Any = None,
105
+ ) -> float | None:
106
+ """Fraction of the reference answer's claims that the retrieved contexts support."""
107
+ if not contexts:
108
+ return None
109
+ claims = _claims(llm, question, reference, config)
110
+ if not claims:
111
+ return None
112
+ context = "\n\n".join(contexts)
113
+ prompts: list[LanguageModelInput] = [
114
+ VERIFY_CLAIM_PROMPT.format(context=context, claim=claim) for claim in claims
115
+ ]
116
+ return _yes_fraction(llm, prompts, config)