spindle-eval 0.1.0__py3-none-any.whl

spindle_eval/__init__.py

@@ -0,0 +1,3 @@
+"""spindle-eval: Experimentation infrastructure for Graph RAG pipelines."""
+
+__version__ = "0.1.0"

spindle_eval/baselines/__init__.py

@@ -0,0 +1,5 @@
+"""Baseline implementations for fair Graph RAG comparisons."""
+
+from spindle_eval.baselines.base import BaselineRunner, BaselineResult
+
+__all__ = ["BaselineRunner", "BaselineResult"]

spindle_eval/baselines/base.py

@@ -0,0 +1,21 @@
+"""Baseline protocol and shared result structures."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+
+@dataclass
+class BaselineResult:
+    answer: str
+    contexts: list[str]
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class BaselineRunner(Protocol):
+    """Common interface across baseline systems."""
+
+    name: str
+
+    def run(self, question: str, **kwargs: Any) -> BaselineResult: ...

spindle_eval/baselines/bm25_baseline.py

@@ -0,0 +1,42 @@
+"""BM25 lexical retrieval baseline."""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from rank_bm25 import BM25Okapi
+
+from spindle_eval.baselines.base import BaselineResult
+
+
+class BM25Baseline:
+    name = "bm25"
+
+    def __init__(
+        self,
+        corpus: list[str],
+        llm_callable: Callable[[str], str],
+        top_k: int = 5,
+    ) -> None:
+        self._corpus = corpus
+        self._tokenized = [doc.lower().split() for doc in corpus]
+        self._bm25 = BM25Okapi(self._tokenized)
+        self._llm_callable = llm_callable
+        self._top_k = top_k
+
+    def run(self, question: str, **kwargs: Any) -> BaselineResult:
+        top_k = int(kwargs.get("top_k", self._top_k))
+        scores = self._bm25.get_scores(question.lower().split())
+        ranked = sorted(
+            zip(self._corpus, scores, strict=False),
+            key=lambda item: item[1],
+            reverse=True,
+        )
+        contexts = [doc for doc, _ in ranked[:top_k]]
+        prompt = f"Question: {question}\n\nContext:\n" + "\n".join(contexts)
+        answer = self._llm_callable(prompt)
+        return BaselineResult(
+            answer=answer,
+            contexts=contexts,
+            metadata={"baseline": self.name, "top_k": top_k},
+        )

spindle_eval/baselines/hybrid_search.py

@@ -0,0 +1,69 @@
+"""Hybrid baseline: weighted fusion of BM25 and dense retrieval."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any, Callable, Protocol
+
+from rank_bm25 import BM25Okapi
+
+from spindle_eval.baselines.base import BaselineResult
+
+
+class DenseRetriever(Protocol):
+    def retrieve_with_scores(self, query: str, top_k: int) -> list[tuple[str, float]]: ...
+
+
+class HybridSearchBaseline:
+    name = "hybrid_search"
+
+    def __init__(
+        self,
+        corpus: list[str],
+        dense_retriever: DenseRetriever,
+        llm_callable: Callable[[str], str],
+        bm25_weight: float = 0.70,
+        dense_weight: float = 0.30,
+        top_k: int = 5,
+    ) -> None:
+        self._corpus = corpus
+        self._bm25 = BM25Okapi([doc.lower().split() for doc in corpus])
+        self._dense_retriever = dense_retriever
+        self._llm_callable = llm_callable
+        self._bm25_weight = bm25_weight
+        self._dense_weight = dense_weight
+        self._top_k = top_k
+
+    def run(self, question: str, **kwargs: Any) -> BaselineResult:
+        top_k = int(kwargs.get("top_k", self._top_k))
+        bm25_scores = self._bm25.get_scores(question.lower().split())
+        bm25_ranked = sorted(
+            zip(self._corpus, bm25_scores, strict=False),
+            key=lambda item: item[1],
+            reverse=True,
+        )[: max(top_k * 2, 10)]
+
+        dense_ranked = self._dense_retriever.retrieve_with_scores(
+            question,
+            top_k=max(top_k * 2, 10),
+        )
+
+        fused: dict[str, float] = defaultdict(float)
+        for doc, score in bm25_ranked:
+            fused[doc] += self._bm25_weight * float(score)
+        for doc, score in dense_ranked:
+            fused[doc] += self._dense_weight * float(score)
+
+        contexts = [doc for doc, _ in sorted(fused.items(), key=lambda x: x[1], reverse=True)[:top_k]]
+        prompt = f"Question: {question}\n\nContext:\n" + "\n".join(contexts)
+        answer = self._llm_callable(prompt)
+        return BaselineResult(
+            answer=answer,
+            contexts=contexts,
+            metadata={
+                "baseline": self.name,
+                "top_k": top_k,
+                "bm25_weight": self._bm25_weight,
+                "dense_weight": self._dense_weight,
+            },
+        )

spindle_eval/baselines/naive_rag.py

@@ -0,0 +1,36 @@
+"""Naive vector top-k RAG baseline."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Protocol
+
+from spindle_eval.baselines.base import BaselineResult
+
+
+class VectorRetriever(Protocol):
+    def retrieve(self, query: str, top_k: int) -> list[str]: ...
+
+
+class NaiveRAGBaseline:
+    name = "naive_rag"
+
+    def __init__(
+        self,
+        retriever: VectorRetriever,
+        llm_callable: Callable[[str], str],
+        top_k: int = 5,
+    ) -> None:
+        self._retriever = retriever
+        self._llm_callable = llm_callable
+        self._top_k = top_k
+
+    def run(self, question: str, **kwargs: Any) -> BaselineResult:
+        top_k = int(kwargs.get("top_k", self._top_k))
+        contexts = self._retriever.retrieve(question, top_k=top_k)
+        prompt = f"Question: {question}\n\nContext:\n" + "\n".join(contexts)
+        answer = self._llm_callable(prompt)
+        return BaselineResult(
+            answer=answer,
+            contexts=contexts,
+            metadata={"baseline": self.name, "top_k": top_k},
+        )

spindle_eval/baselines/no_rag.py

@@ -0,0 +1,22 @@
+"""No-RAG baseline: answer from model parametric memory only."""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from spindle_eval.baselines.base import BaselineResult
+
+
+class NoRAGBaseline:
+    name = "no_rag"
+
+    def __init__(self, llm_callable: Callable[[str], str]) -> None:
+        self._llm_callable = llm_callable
+
+    def run(self, question: str, **kwargs: Any) -> BaselineResult:
+        answer = self._llm_callable(question)
+        return BaselineResult(
+            answer=answer,
+            contexts=[],
+            metadata={"baseline": self.name},
+        )

spindle_eval/baselines/oracle.py

@@ -0,0 +1,24 @@
+"""Oracle baseline: answer generation with ground-truth contexts."""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from spindle_eval.baselines.base import BaselineResult
+
+
+class OracleBaseline:
+    name = "oracle"
+
+    def __init__(self, llm_callable: Callable[[str], str]) -> None:
+        self._llm_callable = llm_callable
+
+    def run(self, question: str, **kwargs: Any) -> BaselineResult:
+        contexts = list(kwargs.get("contexts", []))
+        prompt = f"Question: {question}\n\nGround truth context:\n" + "\n".join(contexts)
+        answer = self._llm_callable(prompt)
+        return BaselineResult(
+            answer=answer,
+            contexts=contexts,
+            metadata={"baseline": self.name},
+        )

spindle_eval/ci/__init__.py

@@ -0,0 +1 @@
+"""CI utilities for regression detection and reporting."""

spindle_eval/ci/regression.py

@@ -0,0 +1,76 @@
+"""Regression detection with tolerance thresholds and confidence intervals."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from spindle_eval.metrics.statistical import bootstrap_ci
+
+
+DEFAULT_TOLERANCES = {
+    "faithfulness": -0.02,
+    "answer_relevancy": -0.03,
+    "context_precision": -0.05,
+}
+
+
+@dataclass
+class RegressionResult:
+    passed: bool
+    deltas: dict[str, float]
+    violations: dict[str, float]
+    confidence_intervals: dict[str, dict[str, float]]
+
+
+def load_baseline_metrics(path: str | Path) -> dict[str, float]:
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    return {k: float(v) for k, v in payload.items() if not k.startswith("_")}
+
+
+def compare_against_baseline(
+    current_metrics: dict[str, float],
+    baseline_metrics: dict[str, float],
+    tolerances: dict[str, float] | None = None,
+    per_query_deltas: dict[str, list[float]] | None = None,
+) -> RegressionResult:
+    """Compare metrics with tolerance checks and optional bootstrap intervals."""
+    limits = {**DEFAULT_TOLERANCES, **(tolerances or {})}
+    deltas: dict[str, float] = {}
+    violations: dict[str, float] = {}
+    intervals: dict[str, dict[str, float]] = {}
+
+    for metric, baseline in baseline_metrics.items():
+        if metric not in current_metrics:
+            continue
+        delta = float(current_metrics[metric] - baseline)
+        deltas[metric] = delta
+
+        if metric in limits and delta < limits[metric]:
+            violations[metric] = delta
+
+        if per_query_deltas and metric in per_query_deltas:
+            ci = bootstrap_ci(per_query_deltas[metric], confidence_level=0.95)
+            intervals[metric] = {
+                "lower": ci.lower,
+                "upper": ci.upper,
+                "point_estimate": ci.point_estimate,
+            }
+
+    return RegressionResult(
+        passed=len(violations) == 0,
+        deltas=deltas,
+        violations=violations,
+        confidence_intervals=intervals,
+    )
+
+
+def to_dict(result: RegressionResult) -> dict[str, Any]:
+    return {
+        "passed": result.passed,
+        "deltas": result.deltas,
+        "violations": result.violations,
+        "confidence_intervals": result.confidence_intervals,
+    }

spindle_eval/ci/reporter.py

@@ -0,0 +1,40 @@
+"""PR comment/report generation for evaluation results."""
+
+from __future__ import annotations
+
+from spindle_eval.ci.regression import RegressionResult
+
+
+def _fmt(value: float) -> str:
+    return f"{value:+.4f}"
+
+
+def render_regression_report(result: RegressionResult) -> str:
+    """Render markdown report suitable for GitHub PR comments."""
+    lines: list[str] = []
+    status = "PASS" if result.passed else "FAIL"
+    lines.append(f"## spindle-eval Regression Check: {status}")
+    lines.append("")
+    lines.append("| Metric | Delta vs Baseline | Status |")
+    lines.append("|---|---:|---|")
+    for metric, delta in sorted(result.deltas.items()):
+        violated = metric in result.violations
+        label = "REGRESSION" if violated else "OK"
+        lines.append(f"| `{metric}` | `{_fmt(delta)}` | {label} |")
+
+    if result.confidence_intervals:
+        lines.append("")
+        lines.append("### Bootstrap 95% CI")
+        lines.append("| Metric | Lower | Upper | Point Estimate |")
+        lines.append("|---|---:|---:|---:|")
+        for metric, ci in sorted(result.confidence_intervals.items()):
+            lines.append(
+                f"| `{metric}` | `{ci['lower']:.4f}` | `{ci['upper']:.4f}` | `{ci['point_estimate']:.4f}` |"
+            )
+
+    if result.violations:
+        lines.append("")
+        lines.append("### Violations")
+        for metric, delta in sorted(result.violations.items()):
+            lines.append(f"- `{metric}` dropped by `{_fmt(delta)}`")
+    return "\n".join(lines)

spindle_eval/compat.py

@@ -0,0 +1,153 @@
+"""Compatibility layer for wrapping legacy component dicts as Stage pipelines.
+
+Bridges the old 6-component dict interface (preprocessor, ontology_extractor,
+triple_extractor, graph_store, retriever, generator) to the new StageDef/Stage
+pipeline model.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from spindle_eval.protocols import StageDef, StageResult
+
+
+class WrappedCallableStage:
+    """Wraps a legacy callable component as a Stage.
+
+    Each stage type has its own calling convention, so we dispatch based
+    on the stage name to unpack the correct arguments.
+    """
+
+    def __init__(self, name: str, component: Any, **extras: Any) -> None:
+        self.name = name
+        self._component = component
+        self._extras = extras
+
+    def run(self, inputs: dict[str, Any], cfg: Any) -> StageResult:
+        if self.name == "preprocessing":
+            return self._run_preprocessor(cfg)
+        if self.name == "ontology_extraction":
+            return self._run_ontology(inputs, cfg)
+        if self.name == "triple_extraction":
+            return self._run_triple_extraction(inputs, cfg)
+        if self.name == "retrieval":
+            return self._run_retrieval(inputs, cfg)
+        if self.name == "generation":
+            return self._run_generation(inputs, cfg)
+        # Generic fallback: call component with (inputs, cfg)
+        result = self._component(inputs, cfg)
+        return StageResult(outputs={"result": result})
+
+    def _run_preprocessor(self, cfg: Any) -> StageResult:
+        chunks = self._component(cfg)
+        return StageResult(
+            outputs={"chunks": chunks},
+            metrics={"num_chunks": float(len(chunks))},
+        )
+
+    def _run_ontology(self, inputs: dict[str, Any], cfg: Any) -> StageResult:
+        chunks = inputs["chunks"]
+        ontology = self._component(chunks, cfg)
+        return StageResult(
+            outputs={"ontology": ontology},
+            metrics={
+                "entity_type_count": float(len(ontology.entity_types)),
+                "relation_type_count": float(len(ontology.relation_types)),
+            },
+        )
+
+    def _run_triple_extraction(
+        self, inputs: dict[str, Any], cfg: Any
+    ) -> StageResult:
+        chunks = inputs["chunks"]
+        ontology = inputs["ontology"]
+        triples = self._component(chunks, ontology, cfg)
+        graph_store = self._extras.get("graph_store")
+        if graph_store is not None:
+            graph_store.ingest(triples)
+        return StageResult(
+            outputs={"triples": triples},
+            metrics={"triple_count": float(len(triples))},
+        )
+
+    def _run_retrieval(self, inputs: dict[str, Any], cfg: Any) -> StageResult:
+        component = self._component
+        query = inputs.get("query", "default query")
+        if callable(component):
+            contexts = component(query, cfg)
+        elif hasattr(component, "retrieve"):
+            contexts = component.retrieve(query, cfg)
+        else:
+            raise TypeError(
+                "Retriever must be callable or expose a retrieve() method."
+            )
+        return StageResult(
+            outputs={"contexts": contexts},
+            metrics={"num_contexts": float(len(contexts))},
+        )
+
+    def _run_generation(self, inputs: dict[str, Any], cfg: Any) -> StageResult:
+        component = self._component
+        contexts = inputs["contexts"]
+        query = inputs.get("query", "default query")
+        if callable(component):
+            answer = component(query, contexts, cfg)
+        elif hasattr(component, "generate"):
+            answer = component.generate(query, contexts, cfg)
+        else:
+            raise TypeError(
+                "Generator must be callable or expose a generate() method."
+            )
+        return StageResult(
+            outputs={"answer": answer},
+            metrics={"answer_length": float(len(answer))},
+        )
+
+
+def from_component_dict(
+    components: dict[str, Any],
+    *,
+    tracker: Any = None,
+) -> list[StageDef]:
+    """Convert a legacy component dict to a list of StageDefs.
+
+    Expected keys: preprocessor, ontology_extractor, triple_extractor,
+    graph_store, retriever, generator.
+    """
+    graph_store = components.get("graph_store")
+
+    return [
+        StageDef(
+            name="preprocessing",
+            stage=WrappedCallableStage("preprocessing", components["preprocessor"]),
+        ),
+        StageDef(
+            name="ontology_extraction",
+            stage=WrappedCallableStage(
+                "ontology_extraction", components["ontology_extractor"]
+            ),
+            input_keys={"chunks": "preprocessing.chunks"},
+        ),
+        StageDef(
+            name="triple_extraction",
+            stage=WrappedCallableStage(
+                "triple_extraction",
+                components["triple_extractor"],
+                graph_store=graph_store,
+            ),
+            input_keys={
+                "chunks": "preprocessing.chunks",
+                "ontology": "ontology_extraction.ontology",
+            },
+        ),
+        StageDef(
+            name="retrieval",
+            stage=WrappedCallableStage("retrieval", components["retriever"]),
+        ),
+        StageDef(
+            name="generation",
+            stage=WrappedCallableStage("generation", components["generator"]),
+            input_keys={"contexts": "retrieval.contexts"},
+        ),
+    ]

spindle_eval/conf/config.yaml

@@ -0,0 +1,28 @@
+defaults:
+  - preprocessing: default
+  - ontology: hybrid
+  - extraction: llm
+  - retrieval: hybrid
+  - generation: gpt4
+  - evaluation: quick
+  - sweep: none
+  - _self_
+
+run_name: spindle-eval-default
+seed: 7
+dataset_path: golden_data/questions.jsonl
+oracle:
+  enabled: false
+  stage: ""
+
+tracking:
+  backend: mlflow
+  mlflow_tracking_uri: ${oc.env:MLFLOW_TRACKING_URI,http://mlflow.internal:5000}
+  experiment_name: ${oc.env:MLFLOW_EXPERIMENT_NAME,graph-rag-sweeps}
+  enable_langfuse: false
+  langfuse_endpoint: ${oc.env:LANGFUSE_OTEL_ENDPOINT,http://langfuse.internal:4318/api/public/otel}
+
+runner:
+  use_spindle_when_available: true
+  allow_mock_fallback: true
+  artifact_output_dir: ${oc.env:SPINDLE_EVAL_ARTIFACT_DIR,artifacts}

spindle_eval/conf/evaluation/full.yaml

@@ -0,0 +1,8 @@
+dataset_split: full
+max_examples: 0
+metrics:
+  - faithfulness
+  - context_precision
+  - context_recall
+  - answer_relevancy
+  - answer_correctness

spindle_eval/conf/evaluation/quick.yaml

@@ -0,0 +1,6 @@
+dataset_split: quick
+max_examples: 50
+metrics:
+  - faithfulness
+  - context_precision
+  - answer_relevancy

spindle_eval/conf/extraction/finetuned.yaml

@@ -0,0 +1,4 @@
+backend: finetuned
+model: local-finetuned
+max_gleanings: 2
+entity_resolution_threshold: 0.9

spindle_eval/conf/extraction/llm.yaml

@@ -0,0 +1,4 @@
+backend: llm
+model: gpt-4o-mini
+max_gleanings: 1
+entity_resolution_threshold: 0.85

spindle_eval/conf/extraction/nlp.yaml

@@ -0,0 +1,4 @@
+backend: nlp
+model: spacy
+max_gleanings: 1
+entity_resolution_threshold: 0.85

spindle_eval/conf/generation/claude.yaml

@@ -0,0 +1,3 @@
+model: claude-3-5-sonnet
+temperature: 0.2
+max_tokens: 1024

spindle_eval/conf/generation/gemini.yaml

@@ -0,0 +1,3 @@
+model: gemini-1.5-pro
+temperature: 0.2
+max_tokens: 1024

spindle_eval/conf/generation/gpt4.yaml

@@ -0,0 +1,3 @@
+model: gpt-4o
+temperature: 0.2
+max_tokens: 1024

spindle_eval/conf/ontology/hybrid.yaml

@@ -0,0 +1,12 @@
+mode: hybrid
+discover_entity_types: true
+entity_types:
+  - person
+  - organization
+  - location
+  - event
+relation_types:
+  - WORKS_AT
+  - LOCATED_IN
+  - FOUNDED
+  - PARTICIPATED_IN

spindle_eval/conf/ontology/schema_first.yaml

@@ -0,0 +1,12 @@
+mode: schema_first
+discover_entity_types: false
+entity_types:
+  - person
+  - organization
+  - location
+  - event
+relation_types:
+  - WORKS_AT
+  - LOCATED_IN
+  - FOUNDED
+  - PARTICIPATED_IN

spindle_eval/conf/ontology/schema_free.yaml

@@ -0,0 +1,4 @@
+mode: schema_free
+discover_entity_types: true
+entity_types: []
+relation_types: []

spindle_eval/conf/preprocessing/default.yaml

@@ -0,0 +1,4 @@
+chunk_size: 600
+overlap: 0.10
+strategy: recursive
+num_chunks: 0

spindle_eval/conf/preprocessing/large_chunks.yaml

@@ -0,0 +1,4 @@
+chunk_size: 1200
+overlap: 0.10
+strategy: fixed
+num_chunks: 0

spindle_eval/conf/preprocessing/small_chunks.yaml

@@ -0,0 +1,4 @@
+chunk_size: 300
+overlap: 0.20
+strategy: recursive
+num_chunks: 0

spindle_eval/conf/retrieval/drift.yaml

@@ -0,0 +1,4 @@
+strategy: drift
+top_k: 10
+traversal_depth: 2
+community_level: C1

spindle_eval/conf/retrieval/global.yaml

@@ -0,0 +1,4 @@
+strategy: global
+top_k: 10
+traversal_depth: 1
+community_level: C0

spindle_eval/conf/retrieval/hybrid.yaml

@@ -0,0 +1,4 @@
+strategy: hybrid
+top_k: 10
+traversal_depth: 2
+community_level: C0

spindle_eval/conf/retrieval/local.yaml

@@ -0,0 +1,4 @@
+strategy: local
+top_k: 10
+traversal_depth: 1
+community_level: C1

spindle_eval/conf/sweep/chunk_size.yaml

@@ -0,0 +1,11 @@
+enabled: true
+hydra:
+  sweeper:
+    direction: maximize
+    n_trials: 50
+    sampler:
+      _target_: optuna.samplers.TPESampler
+      seed: 7
+    params:
+      preprocessing.chunk_size: choice(128,256,384,512,640,768,896,1024,1280,1536,1792,2048)
+      preprocessing.overlap: choice(0.0,0.1,0.2)