mare-retrieval 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mare/__init__.py +21 -0
- mare/api.py +65 -0
- mare/ask.py +56 -0
- mare/demo.py +59 -0
- mare/engine.py +44 -0
- mare/fusion.py +61 -0
- mare/ingest.py +152 -0
- mare/retrievers/__init__.py +2 -0
- mare/retrievers/base.py +17 -0
- mare/retrievers/image.py +70 -0
- mare/retrievers/text.py +76 -0
- mare/router.py +85 -0
- mare/streamlit_app.py +100 -0
- mare/types.py +52 -0
- mare_retrieval-0.2.0.dist-info/METADATA +352 -0
- mare_retrieval-0.2.0.dist-info/RECORD +20 -0
- mare_retrieval-0.2.0.dist-info/WHEEL +5 -0
- mare_retrieval-0.2.0.dist-info/entry_points.txt +5 -0
- mare_retrieval-0.2.0.dist-info/licenses/LICENSE +22 -0
- mare_retrieval-0.2.0.dist-info/top_level.txt +1 -0
mare/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""MARE: Modality-Aware Retrieval Engine."""
|
|
2
|
+
|
|
3
|
+
from .api import MAREApp, load_corpus, load_pdf
|
|
4
|
+
from .engine import MAREngine
|
|
5
|
+
from .fusion import WeightedScoreFusion
|
|
6
|
+
from .router import HeuristicModalityRouter
|
|
7
|
+
from .types import Document, Modality, QueryPlan, RetrievalExplanation, RetrievalHit
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"Document",
|
|
11
|
+
"HeuristicModalityRouter",
|
|
12
|
+
"load_corpus",
|
|
13
|
+
"load_pdf",
|
|
14
|
+
"MAREApp",
|
|
15
|
+
"MAREngine",
|
|
16
|
+
"Modality",
|
|
17
|
+
"QueryPlan",
|
|
18
|
+
"RetrievalExplanation",
|
|
19
|
+
"RetrievalHit",
|
|
20
|
+
"WeightedScoreFusion",
|
|
21
|
+
]
|
mare/api.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from mare.demo import load_documents
|
|
7
|
+
from mare.engine import MAREngine
|
|
8
|
+
from mare.ingest import ingest_pdf
|
|
9
|
+
from mare.types import Document, RetrievalExplanation, RetrievalHit
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class MAREApp:
|
|
14
|
+
documents: list[Document]
|
|
15
|
+
corpus_path: Path | None = None
|
|
16
|
+
source_pdf: Path | None = None
|
|
17
|
+
engine: MAREngine = field(init=False)
|
|
18
|
+
|
|
19
|
+
def __post_init__(self) -> None:
|
|
20
|
+
self.engine = MAREngine(self.documents)
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def from_documents(cls, documents: list[Document]) -> "MAREApp":
|
|
24
|
+
return cls(documents=documents)
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def from_corpus(cls, corpus_path: str | Path) -> "MAREApp":
|
|
28
|
+
path = Path(corpus_path)
|
|
29
|
+
documents = load_documents(path)
|
|
30
|
+
return cls(documents=documents, corpus_path=path)
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def from_pdf(
|
|
34
|
+
cls,
|
|
35
|
+
pdf_path: str | Path,
|
|
36
|
+
output_path: str | Path | None = None,
|
|
37
|
+
reuse: bool = False,
|
|
38
|
+
) -> "MAREApp":
|
|
39
|
+
pdf_file = Path(pdf_path)
|
|
40
|
+
corpus_path = Path(output_path) if output_path is not None else Path("generated") / f"{pdf_file.stem}.json"
|
|
41
|
+
corpus_path.parent.mkdir(parents=True, exist_ok=True)
|
|
42
|
+
|
|
43
|
+
if not reuse or not corpus_path.exists():
|
|
44
|
+
ingest_pdf(pdf_path=pdf_file, output_path=corpus_path)
|
|
45
|
+
|
|
46
|
+
documents = load_documents(corpus_path)
|
|
47
|
+
return cls(documents=documents, corpus_path=corpus_path, source_pdf=pdf_file)
|
|
48
|
+
|
|
49
|
+
def explain(self, query: str, top_k: int = 3) -> RetrievalExplanation:
|
|
50
|
+
return self.engine.explain(query, top_k=top_k)
|
|
51
|
+
|
|
52
|
+
def retrieve(self, query: str, top_k: int = 3) -> list[RetrievalHit]:
|
|
53
|
+
return self.engine.retrieve(query, top_k=top_k)
|
|
54
|
+
|
|
55
|
+
def best_match(self, query: str, top_k: int = 3) -> RetrievalHit | None:
|
|
56
|
+
results = self.retrieve(query=query, top_k=top_k)
|
|
57
|
+
return results[0] if results else None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def load_corpus(corpus_path: str | Path) -> MAREApp:
|
|
61
|
+
return MAREApp.from_corpus(corpus_path)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def load_pdf(pdf_path: str | Path, output_path: str | Path | None = None, reuse: bool = False) -> MAREApp:
|
|
65
|
+
return MAREApp.from_pdf(pdf_path=pdf_path, output_path=output_path, reuse=reuse)
|
mare/ask.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from mare.api import load_pdf
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _default_output_path(pdf_path: Path) -> Path:
|
|
10
|
+
return Path("generated") / f"{pdf_path.stem}.json"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _print_answer_block(query: str, corpus_path: Path, explanation) -> None:
|
|
14
|
+
print(f"Question: {query}")
|
|
15
|
+
print(f"Corpus: {corpus_path}")
|
|
16
|
+
print("")
|
|
17
|
+
|
|
18
|
+
if not explanation.fused_results:
|
|
19
|
+
print("Answer: No matching page found.")
|
|
20
|
+
return
|
|
21
|
+
|
|
22
|
+
best = explanation.fused_results[0]
|
|
23
|
+
print("Best Match")
|
|
24
|
+
print(f"Page: {best.page}")
|
|
25
|
+
print(f"Score: {best.score}")
|
|
26
|
+
print(f"Reason: {best.reason}")
|
|
27
|
+
print(f"Snippet: {best.snippet or '[no snippet available]'}")
|
|
28
|
+
print(f"Image: {best.page_image_path or '[no page image available]'}")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def ask_pdf(pdf_path: Path, query: str, top_k: int = 3, reuse: bool = False):
|
|
32
|
+
app = load_pdf(pdf_path=pdf_path, output_path=_default_output_path(pdf_path), reuse=reuse)
|
|
33
|
+
explanation = app.explain(query, top_k=top_k)
|
|
34
|
+
output_path = app.corpus_path or _default_output_path(pdf_path)
|
|
35
|
+
return output_path, explanation
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main() -> None:
|
|
39
|
+
parser = argparse.ArgumentParser(description="Ask a PDF a question with MARE")
|
|
40
|
+
parser.add_argument("pdf", help="Path to a PDF file")
|
|
41
|
+
parser.add_argument("query", help="Question to ask about the PDF")
|
|
42
|
+
parser.add_argument("--top-k", type=int, default=3, help="How many results to retrieve")
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--reuse",
|
|
45
|
+
action="store_true",
|
|
46
|
+
help="Reuse an existing generated corpus if present instead of re-ingesting",
|
|
47
|
+
)
|
|
48
|
+
args = parser.parse_args()
|
|
49
|
+
|
|
50
|
+
pdf_path = Path(args.pdf)
|
|
51
|
+
output_path, explanation = ask_pdf(pdf_path=pdf_path, query=args.query, top_k=args.top_k, reuse=args.reuse)
|
|
52
|
+
_print_answer_block(args.query, output_path, explanation)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
if __name__ == "__main__":
|
|
56
|
+
main()
|
mare/demo.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from mare.engine import MAREngine
|
|
8
|
+
from mare.types import Document
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def load_documents(path: Path) -> list[Document]:
|
|
12
|
+
payload = json.loads(path.read_text())
|
|
13
|
+
return [Document(**item) for item in payload["documents"]]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main() -> None:
|
|
17
|
+
parser = argparse.ArgumentParser(description="Run a MARE retrieval demo")
|
|
18
|
+
parser.add_argument("--query", required=True, help="User query to route and retrieve against")
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
"--corpus",
|
|
21
|
+
default="examples/sample_corpus.json",
|
|
22
|
+
help="Path to a JSON corpus with text/image/layout fields",
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument("--top-k", type=int, default=3, help="How many results to return")
|
|
25
|
+
args = parser.parse_args()
|
|
26
|
+
|
|
27
|
+
documents = load_documents(Path(args.corpus))
|
|
28
|
+
engine = MAREngine(documents)
|
|
29
|
+
explanation = engine.explain(args.query, top_k=args.top_k)
|
|
30
|
+
|
|
31
|
+
print(
|
|
32
|
+
json.dumps(
|
|
33
|
+
{
|
|
34
|
+
"query": explanation.plan.query,
|
|
35
|
+
"intent": explanation.plan.intent,
|
|
36
|
+
"selected_modalities": [item.value for item in explanation.plan.selected_modalities],
|
|
37
|
+
"discarded_modalities": [item.value for item in explanation.plan.discarded_modalities],
|
|
38
|
+
"confidence": explanation.plan.confidence,
|
|
39
|
+
"rationale": explanation.plan.rationale,
|
|
40
|
+
"results": [
|
|
41
|
+
{
|
|
42
|
+
"doc_id": hit.doc_id,
|
|
43
|
+
"title": hit.title,
|
|
44
|
+
"page": hit.page,
|
|
45
|
+
"score": hit.score,
|
|
46
|
+
"reason": hit.reason,
|
|
47
|
+
"snippet": hit.snippet,
|
|
48
|
+
"page_image_path": hit.page_image_path,
|
|
49
|
+
}
|
|
50
|
+
for hit in explanation.fused_results
|
|
51
|
+
],
|
|
52
|
+
},
|
|
53
|
+
indent=2,
|
|
54
|
+
)
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == "__main__":
|
|
59
|
+
main()
|
mare/engine.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from mare.fusion import WeightedScoreFusion
|
|
4
|
+
from mare.retrievers.base import BaseRetriever
|
|
5
|
+
from mare.retrievers.image import ImageRetriever, LayoutRetriever
|
|
6
|
+
from mare.retrievers.text import TextRetriever
|
|
7
|
+
from mare.router import HeuristicModalityRouter
|
|
8
|
+
from mare.types import Document, Modality, RetrievalExplanation
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MAREngine:
|
|
12
|
+
"""Routes a query to modality-specific retrievers, then fuses their results."""
|
|
13
|
+
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
documents: list[Document],
|
|
17
|
+
router: HeuristicModalityRouter | None = None,
|
|
18
|
+
fusion: WeightedScoreFusion | None = None,
|
|
19
|
+
) -> None:
|
|
20
|
+
self.documents = documents
|
|
21
|
+
self.router = router or HeuristicModalityRouter()
|
|
22
|
+
self.fusion = fusion or WeightedScoreFusion()
|
|
23
|
+
self.retrievers: dict[Modality, BaseRetriever] = {
|
|
24
|
+
Modality.TEXT: TextRetriever(documents),
|
|
25
|
+
Modality.IMAGE: ImageRetriever(documents),
|
|
26
|
+
Modality.LAYOUT: LayoutRetriever(documents),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
def explain(self, query: str, top_k: int = 5) -> RetrievalExplanation:
|
|
30
|
+
plan = self.router.route(query)
|
|
31
|
+
per_modality_results = {
|
|
32
|
+
modality: self.retrievers[modality].retrieve(query=query, top_k=top_k)
|
|
33
|
+
for modality in plan.selected_modalities
|
|
34
|
+
}
|
|
35
|
+
fused_results = self.fusion.fuse(per_modality_results, top_k=top_k)
|
|
36
|
+
return RetrievalExplanation(
|
|
37
|
+
plan=plan,
|
|
38
|
+
per_modality_results=per_modality_results,
|
|
39
|
+
fused_results=fused_results,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
def retrieve(self, query: str, top_k: int = 5):
|
|
43
|
+
return self.explain(query=query, top_k=top_k).fused_results
|
|
44
|
+
|
mare/fusion.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
|
|
5
|
+
from mare.types import Modality, RetrievalHit
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class WeightedScoreFusion:
|
|
9
|
+
"""Simple late fusion with modality-aware weights."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, weights: dict[Modality, float] | None = None) -> None:
|
|
12
|
+
self.weights = weights or {
|
|
13
|
+
Modality.TEXT: 1.0,
|
|
14
|
+
Modality.IMAGE: 1.1,
|
|
15
|
+
Modality.LAYOUT: 0.9,
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
def fuse(self, results: dict[Modality, list[RetrievalHit]], top_k: int = 5) -> list[RetrievalHit]:
|
|
19
|
+
merged: dict[str, dict[str, object]] = defaultdict(
|
|
20
|
+
lambda: {
|
|
21
|
+
"score": 0.0,
|
|
22
|
+
"title": "",
|
|
23
|
+
"page": 0,
|
|
24
|
+
"reasons": [],
|
|
25
|
+
"snippet": "",
|
|
26
|
+
"page_image_path": "",
|
|
27
|
+
"metadata": {},
|
|
28
|
+
}
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
for modality, hits in results.items():
|
|
32
|
+
weight = self.weights.get(modality, 1.0)
|
|
33
|
+
for hit in hits:
|
|
34
|
+
bucket = merged[hit.doc_id]
|
|
35
|
+
bucket["score"] = float(bucket["score"]) + (hit.score * weight)
|
|
36
|
+
bucket["title"] = hit.title
|
|
37
|
+
bucket["page"] = hit.page
|
|
38
|
+
bucket["reasons"].append(f"{modality.value}:{hit.reason}")
|
|
39
|
+
if not bucket["snippet"] and hit.snippet:
|
|
40
|
+
bucket["snippet"] = hit.snippet
|
|
41
|
+
if not bucket["page_image_path"] and hit.page_image_path:
|
|
42
|
+
bucket["page_image_path"] = hit.page_image_path
|
|
43
|
+
bucket["metadata"] = hit.metadata
|
|
44
|
+
|
|
45
|
+
fused: list[RetrievalHit] = []
|
|
46
|
+
for doc_id, payload in merged.items():
|
|
47
|
+
fused.append(
|
|
48
|
+
RetrievalHit(
|
|
49
|
+
doc_id=doc_id,
|
|
50
|
+
title=str(payload["title"]),
|
|
51
|
+
page=int(payload["page"]),
|
|
52
|
+
modality=Modality.TEXT,
|
|
53
|
+
score=round(float(payload["score"]), 4),
|
|
54
|
+
reason=" | ".join(payload["reasons"]),
|
|
55
|
+
snippet=str(payload["snippet"]),
|
|
56
|
+
page_image_path=str(payload["page_image_path"]),
|
|
57
|
+
metadata=dict(payload["metadata"]),
|
|
58
|
+
)
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
return sorted(fused, key=lambda hit: hit.score, reverse=True)[:top_k]
|
mare/ingest.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from mare.types import Document
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _require_pypdf():
|
|
13
|
+
try:
|
|
14
|
+
from pypdf import PdfReader
|
|
15
|
+
except ImportError as exc:
|
|
16
|
+
raise RuntimeError(
|
|
17
|
+
"pypdf is required for PDF ingestion. Install dependencies with `pip install -e .` "
|
|
18
|
+
"or `pip install pypdf`."
|
|
19
|
+
) from exc
|
|
20
|
+
return PdfReader
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _require_pdf_renderer():
|
|
24
|
+
try:
|
|
25
|
+
import pypdfium2 as pdfium
|
|
26
|
+
except ImportError as exc:
|
|
27
|
+
raise RuntimeError(
|
|
28
|
+
"pypdfium2 is required for page image rendering. Install it with `pip install pypdfium2`."
|
|
29
|
+
) from exc
|
|
30
|
+
return pdfium
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _normalize_text(value: str) -> str:
|
|
34
|
+
return re.sub(r"\s+", " ", value).strip()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _infer_layout_hints(text: str) -> str:
|
|
38
|
+
hints: list[str] = []
|
|
39
|
+
lowered = text.lower()
|
|
40
|
+
|
|
41
|
+
if "table" in lowered:
|
|
42
|
+
hints.append("table")
|
|
43
|
+
if "figure" in lowered or "fig." in lowered:
|
|
44
|
+
hints.append("figure")
|
|
45
|
+
if "abstract" in lowered:
|
|
46
|
+
hints.append("abstract")
|
|
47
|
+
if "references" in lowered:
|
|
48
|
+
hints.append("references")
|
|
49
|
+
|
|
50
|
+
return " ".join(hints)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _render_page_images(pdf_path: Path, image_dir: Path, scale: float = 1.5) -> list[str]:
|
|
54
|
+
pdfium = _require_pdf_renderer()
|
|
55
|
+
image_dir.mkdir(parents=True, exist_ok=True)
|
|
56
|
+
pdf = pdfium.PdfDocument(str(pdf_path))
|
|
57
|
+
image_paths: list[str] = []
|
|
58
|
+
|
|
59
|
+
for page_index in range(len(pdf)):
|
|
60
|
+
page = pdf[page_index]
|
|
61
|
+
bitmap = page.render(scale=scale)
|
|
62
|
+
image = bitmap.to_pil()
|
|
63
|
+
image_path = image_dir / f"page-{page_index + 1}.png"
|
|
64
|
+
image.save(image_path)
|
|
65
|
+
image_paths.append(str(image_path))
|
|
66
|
+
|
|
67
|
+
return image_paths
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def ingest_pdf(pdf_path: str | Path, output_path: str | Path | None = None) -> dict[str, Any]:
|
|
71
|
+
PdfReader = _require_pypdf()
|
|
72
|
+
pdf_file = Path(pdf_path)
|
|
73
|
+
reader = PdfReader(str(pdf_file))
|
|
74
|
+
title = pdf_file.stem
|
|
75
|
+
output = Path(output_path) if output_path is not None else Path("generated") / f"{pdf_file.stem}.json"
|
|
76
|
+
page_images = _render_page_images(pdf_file, output.with_suffix(""))
|
|
77
|
+
documents: list[Document] = []
|
|
78
|
+
|
|
79
|
+
for idx, page in enumerate(reader.pages, start=1):
|
|
80
|
+
raw_text = page.extract_text() or ""
|
|
81
|
+
text = _normalize_text(raw_text)
|
|
82
|
+
if not text:
|
|
83
|
+
text = f"[No extractable text found on page {idx}]"
|
|
84
|
+
|
|
85
|
+
documents.append(
|
|
86
|
+
Document(
|
|
87
|
+
doc_id=f"{pdf_file.stem.lower().replace(' ', '-')}-p{idx}",
|
|
88
|
+
title=title,
|
|
89
|
+
page=idx,
|
|
90
|
+
text=text,
|
|
91
|
+
image_caption="",
|
|
92
|
+
layout_hints=_infer_layout_hints(text),
|
|
93
|
+
page_image_path=page_images[idx - 1] if idx - 1 < len(page_images) else "",
|
|
94
|
+
metadata={
|
|
95
|
+
"source": str(pdf_file),
|
|
96
|
+
"collection": "pdf-ingest",
|
|
97
|
+
},
|
|
98
|
+
)
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
payload = {
|
|
102
|
+
"source_pdf": str(pdf_file),
|
|
103
|
+
"documents": [
|
|
104
|
+
{
|
|
105
|
+
"doc_id": doc.doc_id,
|
|
106
|
+
"title": doc.title,
|
|
107
|
+
"page": doc.page,
|
|
108
|
+
"text": doc.text,
|
|
109
|
+
"image_caption": doc.image_caption,
|
|
110
|
+
"layout_hints": doc.layout_hints,
|
|
111
|
+
"page_image_path": doc.page_image_path,
|
|
112
|
+
"metadata": doc.metadata,
|
|
113
|
+
}
|
|
114
|
+
for doc in documents
|
|
115
|
+
],
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
output.write_text(json.dumps(payload, indent=2))
|
|
119
|
+
|
|
120
|
+
return payload
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def main() -> None:
|
|
124
|
+
parser = argparse.ArgumentParser(description="Ingest a PDF into a MARE JSON corpus")
|
|
125
|
+
parser.add_argument("pdf", help="Path to a PDF file")
|
|
126
|
+
parser.add_argument(
|
|
127
|
+
"--output",
|
|
128
|
+
"-o",
|
|
129
|
+
help="Output JSON file path. Defaults to generated/<pdf-stem>.json",
|
|
130
|
+
)
|
|
131
|
+
args = parser.parse_args()
|
|
132
|
+
|
|
133
|
+
pdf_path = Path(args.pdf)
|
|
134
|
+
output = Path(args.output) if args.output else Path("generated") / f"{pdf_path.stem}.json"
|
|
135
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
136
|
+
payload = ingest_pdf(pdf_path=pdf_path, output_path=output)
|
|
137
|
+
|
|
138
|
+
print(
|
|
139
|
+
json.dumps(
|
|
140
|
+
{
|
|
141
|
+
"source_pdf": payload["source_pdf"],
|
|
142
|
+
"pages_indexed": len(payload["documents"]),
|
|
143
|
+
"output": str(output),
|
|
144
|
+
"page_images_dir": str(output.with_suffix("")),
|
|
145
|
+
},
|
|
146
|
+
indent=2,
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
if __name__ == "__main__":
|
|
152
|
+
main()
|
mare/retrievers/base.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
|
|
5
|
+
from mare.types import Document, Modality, RetrievalHit
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BaseRetriever(ABC):
|
|
9
|
+
modality: Modality
|
|
10
|
+
|
|
11
|
+
def __init__(self, documents: list[Document]) -> None:
|
|
12
|
+
self.documents = documents
|
|
13
|
+
|
|
14
|
+
@abstractmethod
|
|
15
|
+
def retrieve(self, query: str, top_k: int = 5) -> list[RetrievalHit]:
|
|
16
|
+
raise NotImplementedError
|
|
17
|
+
|
mare/retrievers/image.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from mare.retrievers.base import BaseRetriever
|
|
6
|
+
from mare.types import Modality, RetrievalHit
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _shared_terms(query: str, field: str) -> set[str]:
|
|
10
|
+
query_terms = set(re.findall(r"[a-z0-9]+", query.lower()))
|
|
11
|
+
field_terms = set(re.findall(r"[a-z0-9]+", field.lower()))
|
|
12
|
+
return query_terms & field_terms
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ImageRetriever(BaseRetriever):
|
|
16
|
+
modality = Modality.IMAGE
|
|
17
|
+
|
|
18
|
+
def retrieve(self, query: str, top_k: int = 5) -> list[RetrievalHit]:
|
|
19
|
+
hits: list[RetrievalHit] = []
|
|
20
|
+
|
|
21
|
+
for document in self.documents:
|
|
22
|
+
overlap = _shared_terms(query, document.image_caption)
|
|
23
|
+
if not overlap:
|
|
24
|
+
continue
|
|
25
|
+
|
|
26
|
+
score = min(1.0, 0.2 * len(overlap))
|
|
27
|
+
hits.append(
|
|
28
|
+
RetrievalHit(
|
|
29
|
+
doc_id=document.doc_id,
|
|
30
|
+
title=document.title,
|
|
31
|
+
page=document.page,
|
|
32
|
+
modality=self.modality,
|
|
33
|
+
score=score,
|
|
34
|
+
reason=f"Matched visual cues: {', '.join(sorted(overlap)[:5])}",
|
|
35
|
+
snippet=document.image_caption,
|
|
36
|
+
page_image_path=document.page_image_path,
|
|
37
|
+
metadata=document.metadata,
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
return sorted(hits, key=lambda hit: hit.score, reverse=True)[:top_k]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class LayoutRetriever(BaseRetriever):
|
|
45
|
+
modality = Modality.LAYOUT
|
|
46
|
+
|
|
47
|
+
def retrieve(self, query: str, top_k: int = 5) -> list[RetrievalHit]:
|
|
48
|
+
hits: list[RetrievalHit] = []
|
|
49
|
+
|
|
50
|
+
for document in self.documents:
|
|
51
|
+
overlap = _shared_terms(query, document.layout_hints)
|
|
52
|
+
if not overlap:
|
|
53
|
+
continue
|
|
54
|
+
|
|
55
|
+
score = min(1.0, 0.25 * len(overlap))
|
|
56
|
+
hits.append(
|
|
57
|
+
RetrievalHit(
|
|
58
|
+
doc_id=document.doc_id,
|
|
59
|
+
title=document.title,
|
|
60
|
+
page=document.page,
|
|
61
|
+
modality=self.modality,
|
|
62
|
+
score=score,
|
|
63
|
+
reason=f"Matched layout cues: {', '.join(sorted(overlap)[:5])}",
|
|
64
|
+
snippet=document.layout_hints,
|
|
65
|
+
page_image_path=document.page_image_path,
|
|
66
|
+
metadata=document.metadata,
|
|
67
|
+
)
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
return sorted(hits, key=lambda hit: hit.score, reverse=True)[:top_k]
|
mare/retrievers/text.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import re
|
|
5
|
+
from collections import Counter
|
|
6
|
+
|
|
7
|
+
from mare.retrievers.base import BaseRetriever
|
|
8
|
+
from mare.types import Modality, RetrievalHit
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _tokenize(value: str) -> list[str]:
|
|
12
|
+
return re.findall(r"[a-z0-9]+", value.lower())
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _best_snippet(text: str, query: str, window: int = 220) -> str:
|
|
16
|
+
normalized = re.sub(r"\s+", " ", text).strip()
|
|
17
|
+
if not normalized:
|
|
18
|
+
return ""
|
|
19
|
+
|
|
20
|
+
query_terms = [term for term in _tokenize(query) if len(term) > 1]
|
|
21
|
+
if not query_terms:
|
|
22
|
+
return normalized[:window]
|
|
23
|
+
|
|
24
|
+
lowered = normalized.lower()
|
|
25
|
+
best_index = -1
|
|
26
|
+
for term in query_terms:
|
|
27
|
+
found = lowered.find(term)
|
|
28
|
+
if found != -1:
|
|
29
|
+
best_index = found if best_index == -1 else min(best_index, found)
|
|
30
|
+
|
|
31
|
+
if best_index == -1:
|
|
32
|
+
return normalized[:window]
|
|
33
|
+
|
|
34
|
+
start = max(0, best_index - 60)
|
|
35
|
+
end = min(len(normalized), start + window)
|
|
36
|
+
snippet = normalized[start:end].strip()
|
|
37
|
+
if start > 0:
|
|
38
|
+
snippet = "..." + snippet
|
|
39
|
+
if end < len(normalized):
|
|
40
|
+
snippet = snippet + "..."
|
|
41
|
+
return snippet
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class TextRetriever(BaseRetriever):
|
|
45
|
+
modality = Modality.TEXT
|
|
46
|
+
|
|
47
|
+
def retrieve(self, query: str, top_k: int = 5) -> list[RetrievalHit]:
|
|
48
|
+
query_counts = Counter(_tokenize(query))
|
|
49
|
+
hits: list[RetrievalHit] = []
|
|
50
|
+
|
|
51
|
+
for document in self.documents:
|
|
52
|
+
doc_counts = Counter(_tokenize(document.text))
|
|
53
|
+
overlap = set(query_counts) & set(doc_counts)
|
|
54
|
+
if not overlap:
|
|
55
|
+
continue
|
|
56
|
+
|
|
57
|
+
numerator = sum(query_counts[token] * doc_counts[token] for token in overlap)
|
|
58
|
+
norm = math.sqrt(sum(v * v for v in query_counts.values())) * math.sqrt(
|
|
59
|
+
sum(v * v for v in doc_counts.values())
|
|
60
|
+
)
|
|
61
|
+
score = numerator / norm if norm else 0.0
|
|
62
|
+
hits.append(
|
|
63
|
+
RetrievalHit(
|
|
64
|
+
doc_id=document.doc_id,
|
|
65
|
+
title=document.title,
|
|
66
|
+
page=document.page,
|
|
67
|
+
modality=self.modality,
|
|
68
|
+
score=score,
|
|
69
|
+
reason=f"Matched text terms: {', '.join(sorted(overlap)[:5])}",
|
|
70
|
+
snippet=_best_snippet(document.text, query),
|
|
71
|
+
page_image_path=document.page_image_path,
|
|
72
|
+
metadata=document.metadata,
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
return sorted(hits, key=lambda hit: hit.score, reverse=True)[:top_k]
|
mare/router.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from .types import Modality, QueryPlan
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class HeuristicModalityRouter:
|
|
7
|
+
"""A small baseline router that can later be replaced with an LLM or classifier."""
|
|
8
|
+
|
|
9
|
+
IMAGE_HINTS = {
|
|
10
|
+
"diagram",
|
|
11
|
+
"figure",
|
|
12
|
+
"image",
|
|
13
|
+
"visual",
|
|
14
|
+
"architecture",
|
|
15
|
+
"screenshot",
|
|
16
|
+
"flowchart",
|
|
17
|
+
}
|
|
18
|
+
LAYOUT_HINTS = {
|
|
19
|
+
"table",
|
|
20
|
+
"layout",
|
|
21
|
+
"section",
|
|
22
|
+
"page",
|
|
23
|
+
"compare",
|
|
24
|
+
"comparison",
|
|
25
|
+
"column",
|
|
26
|
+
}
|
|
27
|
+
TEXT_HINTS = {
|
|
28
|
+
"what",
|
|
29
|
+
"why",
|
|
30
|
+
"how",
|
|
31
|
+
"formula",
|
|
32
|
+
"equation",
|
|
33
|
+
"method",
|
|
34
|
+
"instruction",
|
|
35
|
+
"definition",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
def route(self, query: str) -> QueryPlan:
|
|
39
|
+
lowered = query.lower()
|
|
40
|
+
tokens = set(lowered.replace("-", " ").split())
|
|
41
|
+
|
|
42
|
+
scores = {
|
|
43
|
+
Modality.TEXT: len(tokens & self.TEXT_HINTS),
|
|
44
|
+
Modality.IMAGE: len(tokens & self.IMAGE_HINTS),
|
|
45
|
+
Modality.LAYOUT: len(tokens & self.LAYOUT_HINTS),
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if max(scores.values()) == 0:
|
|
49
|
+
selected = [Modality.TEXT]
|
|
50
|
+
discarded = [Modality.IMAGE, Modality.LAYOUT]
|
|
51
|
+
return QueryPlan(
|
|
52
|
+
query=query,
|
|
53
|
+
selected_modalities=selected,
|
|
54
|
+
discarded_modalities=discarded,
|
|
55
|
+
confidence=0.55,
|
|
56
|
+
intent="general_lookup",
|
|
57
|
+
rationale="No strong visual or layout cues were detected, so the router defaulted to text retrieval.",
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
ranked = sorted(scores.items(), key=lambda item: item[1], reverse=True)
|
|
61
|
+
top_score = ranked[0][1]
|
|
62
|
+
selected = [modality for modality, score in ranked if score > 0 and score >= top_score - 1]
|
|
63
|
+
discarded = [modality for modality in Modality if modality not in selected]
|
|
64
|
+
|
|
65
|
+
intent = "visual_lookup" if Modality.IMAGE in selected else "structured_lookup"
|
|
66
|
+
if selected == [Modality.TEXT]:
|
|
67
|
+
intent = "semantic_lookup"
|
|
68
|
+
elif Modality.TEXT in selected and len(selected) > 1:
|
|
69
|
+
intent = "hybrid_lookup"
|
|
70
|
+
|
|
71
|
+
confidence = min(0.95, 0.55 + (0.1 * top_score) + (0.05 * len(selected)))
|
|
72
|
+
rationale = (
|
|
73
|
+
f"Detected modality cues in query tokens. Selected {', '.join(mod.value for mod in selected)} "
|
|
74
|
+
f"based on keyword overlap with routing hints."
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
return QueryPlan(
|
|
78
|
+
query=query,
|
|
79
|
+
selected_modalities=selected,
|
|
80
|
+
discarded_modalities=discarded,
|
|
81
|
+
confidence=confidence,
|
|
82
|
+
intent=intent,
|
|
83
|
+
rationale=rationale,
|
|
84
|
+
)
|
|
85
|
+
|
mare/streamlit_app.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import tempfile
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _require_streamlit():
|
|
8
|
+
try:
|
|
9
|
+
import streamlit as st
|
|
10
|
+
except ImportError as exc:
|
|
11
|
+
raise RuntimeError(
|
|
12
|
+
"streamlit is required for the visual demo. Install it with `pip install -e '.[ui]'` "
|
|
13
|
+
"or `pip install streamlit`."
|
|
14
|
+
) from exc
|
|
15
|
+
return st
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main() -> None:
|
|
19
|
+
st = _require_streamlit()
|
|
20
|
+
|
|
21
|
+
from mare.ask import ask_pdf
|
|
22
|
+
|
|
23
|
+
st.set_page_config(page_title="MARE Demo", layout="wide")
|
|
24
|
+
st.title("MARE")
|
|
25
|
+
st.caption("Ask a PDF a question and see the best page, snippet, and evidence image.")
|
|
26
|
+
|
|
27
|
+
uploaded_pdf = st.file_uploader("Upload a PDF", type=["pdf"])
|
|
28
|
+
with st.form("mare_query_form"):
|
|
29
|
+
query = st.text_input("Ask a question about the document")
|
|
30
|
+
top_k = st.slider("How many results to show", min_value=1, max_value=5, value=3)
|
|
31
|
+
submitted = st.form_submit_button("Ask MARE")
|
|
32
|
+
|
|
33
|
+
if uploaded_pdf is None:
|
|
34
|
+
st.info("Upload a PDF to start.")
|
|
35
|
+
return
|
|
36
|
+
|
|
37
|
+
temp_dir = Path(tempfile.gettempdir()) / "mare_streamlit"
|
|
38
|
+
temp_dir.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
pdf_path = temp_dir / uploaded_pdf.name
|
|
40
|
+
pdf_path.write_bytes(uploaded_pdf.getvalue())
|
|
41
|
+
|
|
42
|
+
if submitted:
|
|
43
|
+
if not query.strip():
|
|
44
|
+
st.warning("Enter a question first.")
|
|
45
|
+
return
|
|
46
|
+
|
|
47
|
+
with st.spinner("Ingesting PDF and retrieving best pages..."):
|
|
48
|
+
corpus_path, explanation = ask_pdf(pdf_path=pdf_path, query=query, top_k=top_k, reuse=False)
|
|
49
|
+
|
|
50
|
+
if not explanation.fused_results:
|
|
51
|
+
st.error("No matching page found.")
|
|
52
|
+
return
|
|
53
|
+
|
|
54
|
+
best = explanation.fused_results[0]
|
|
55
|
+
left, right = st.columns([1, 1])
|
|
56
|
+
|
|
57
|
+
with left:
|
|
58
|
+
st.subheader("Best Match")
|
|
59
|
+
st.markdown(f"**Page:** {best.page}")
|
|
60
|
+
st.markdown(f"**Score:** {best.score}")
|
|
61
|
+
st.markdown(f"**Reason:** {best.reason}")
|
|
62
|
+
st.markdown("**Snippet:**")
|
|
63
|
+
st.write(best.snippet or "[no snippet available]")
|
|
64
|
+
|
|
65
|
+
with right:
|
|
66
|
+
st.subheader("Page Image")
|
|
67
|
+
image_path = Path(best.page_image_path)
|
|
68
|
+
if best.page_image_path and image_path.exists():
|
|
69
|
+
st.image(str(image_path), caption=f"Page {best.page}")
|
|
70
|
+
else:
|
|
71
|
+
st.warning("No page image available.")
|
|
72
|
+
|
|
73
|
+
if len(explanation.fused_results) > 1:
|
|
74
|
+
st.subheader("Other Candidate Pages")
|
|
75
|
+
for hit in explanation.fused_results[1:]:
|
|
76
|
+
st.write(
|
|
77
|
+
{
|
|
78
|
+
"page": hit.page,
|
|
79
|
+
"score": hit.score,
|
|
80
|
+
"reason": hit.reason,
|
|
81
|
+
"snippet": hit.snippet,
|
|
82
|
+
"page_image_path": hit.page_image_path,
|
|
83
|
+
}
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
with st.expander("Debug details"):
|
|
87
|
+
st.write(
|
|
88
|
+
{
|
|
89
|
+
"intent": explanation.plan.intent,
|
|
90
|
+
"selected_modalities": [item.value for item in explanation.plan.selected_modalities],
|
|
91
|
+
"discarded_modalities": [item.value for item in explanation.plan.discarded_modalities],
|
|
92
|
+
"confidence": explanation.plan.confidence,
|
|
93
|
+
"rationale": explanation.plan.rationale,
|
|
94
|
+
"corpus": str(corpus_path),
|
|
95
|
+
}
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
if __name__ == "__main__":
|
|
100
|
+
main()
|
mare/types.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from enum import Enum
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Modality(str, Enum):
|
|
8
|
+
TEXT = "text"
|
|
9
|
+
IMAGE = "image"
|
|
10
|
+
LAYOUT = "layout"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class Document:
|
|
15
|
+
doc_id: str
|
|
16
|
+
title: str
|
|
17
|
+
page: int
|
|
18
|
+
text: str = ""
|
|
19
|
+
image_caption: str = ""
|
|
20
|
+
layout_hints: str = ""
|
|
21
|
+
page_image_path: str = ""
|
|
22
|
+
metadata: dict[str, str] = field(default_factory=dict)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class RetrievalHit:
|
|
27
|
+
doc_id: str
|
|
28
|
+
title: str
|
|
29
|
+
page: int
|
|
30
|
+
modality: Modality
|
|
31
|
+
score: float
|
|
32
|
+
reason: str
|
|
33
|
+
snippet: str = ""
|
|
34
|
+
page_image_path: str = ""
|
|
35
|
+
metadata: dict[str, str] = field(default_factory=dict)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class QueryPlan:
|
|
40
|
+
query: str
|
|
41
|
+
selected_modalities: list[Modality]
|
|
42
|
+
discarded_modalities: list[Modality]
|
|
43
|
+
confidence: float
|
|
44
|
+
intent: str
|
|
45
|
+
rationale: str
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class RetrievalExplanation:
|
|
50
|
+
plan: QueryPlan
|
|
51
|
+
per_modality_results: dict[Modality, list[RetrievalHit]]
|
|
52
|
+
fused_results: list[RetrievalHit]
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mare-retrieval
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Modality-Aware Retrieval Engine inspired by IRPAPERS-style multimodal retrieval.
|
|
5
|
+
Author: Saisandeep Kantareddy
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/SaiSandeepKantareddy/MARE
|
|
8
|
+
Project-URL: Repository, https://github.com/SaiSandeepKantareddy/MARE
|
|
9
|
+
Project-URL: Issues, https://github.com/SaiSandeepKantareddy/MARE/issues
|
|
10
|
+
Keywords: retrieval,rag,pdf,document-ai,multimodal
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: pypdf>=4.0
|
|
22
|
+
Requires-Dist: pypdfium2>=4.30.0
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
25
|
+
Provides-Extra: publish
|
|
26
|
+
Requires-Dist: build>=1.2.0; extra == "publish"
|
|
27
|
+
Requires-Dist: twine>=5.0.0; extra == "publish"
|
|
28
|
+
Provides-Extra: ui
|
|
29
|
+
Requires-Dist: streamlit<2.0,>=1.12; extra == "ui"
|
|
30
|
+
Requires-Dist: altair<5; extra == "ui"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
Dynamic: requires-python
|
|
33
|
+
|
|
34
|
+
# MARE
|
|
35
|
+
|
|
36
|
+
MARE is an open-source Python library for evidence-first document retrieval.
|
|
37
|
+
|
|
38
|
+
It is inspired by the direction highlighted in the IRPAPERS paper, which shows that page-image retrieval and text retrieval have complementary failure modes on scientific documents. Instead of flattening everything into one retrieval path, MARE treats routing, retrieval, fusion, and observability as separate system concerns.
|
|
39
|
+
|
|
40
|
+
## What this repo is
|
|
41
|
+
|
|
42
|
+
- A lightweight Python package between a query and modality-specific indexes
|
|
43
|
+
- A baseline router that decides whether a query should hit text, image, layout, or a hybrid path
|
|
44
|
+
- A late-fusion layer that combines modality-specific scores
|
|
45
|
+
- An explainable debug surface that tells you why a modality was selected
|
|
46
|
+
|
|
47
|
+
## What this repo is not
|
|
48
|
+
|
|
49
|
+
- Not a chatbot wrapper
|
|
50
|
+
- Not a full PDF parsing stack yet
|
|
51
|
+
- Not a claim that heuristic routing is state of the art
|
|
52
|
+
|
|
53
|
+
## Why now
|
|
54
|
+
|
|
55
|
+
IRPAPERS asks a useful systems question: when should we retrieve over OCR text, page images, layout structure, or some combination? The paper reports that text-based and image-based retrieval each solve queries the other misses, and that fusion improves retrieval quality over either modality alone.
|
|
56
|
+
|
|
57
|
+
This repo turns that observation into an MVP developer layer.
|
|
58
|
+
|
|
59
|
+
Paper: https://arxiv.org/pdf/2602.17687
|
|
60
|
+
|
|
61
|
+
## Architecture
|
|
62
|
+
|
|
63
|
+
```text
|
|
64
|
+
query
|
|
65
|
+
-> router
|
|
66
|
+
-> modality-specific retrievers
|
|
67
|
+
-> text index
|
|
68
|
+
-> image index
|
|
69
|
+
-> layout index
|
|
70
|
+
-> fusion
|
|
71
|
+
-> explainable results
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Current implementation choices:
|
|
75
|
+
|
|
76
|
+
- Router: keyword heuristic baseline
|
|
77
|
+
- Text retrieval: token-overlap cosine baseline
|
|
78
|
+
- Image retrieval: caption and visual-tag overlap baseline
|
|
79
|
+
- Layout retrieval: layout-hint overlap baseline
|
|
80
|
+
- Fusion: weighted late fusion
|
|
81
|
+
|
|
82
|
+
The point of `v0.1` is not raw benchmark quality. It is to package the control plane cleanly enough that stronger models can drop in later.
|
|
83
|
+
|
|
84
|
+
## Repo layout
|
|
85
|
+
|
|
86
|
+
```text
|
|
87
|
+
src/mare/
|
|
88
|
+
engine.py
|
|
89
|
+
router.py
|
|
90
|
+
fusion.py
|
|
91
|
+
types.py
|
|
92
|
+
retrievers/
|
|
93
|
+
examples/
|
|
94
|
+
tests/
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Quickstart
|
|
98
|
+
|
|
99
|
+
Clone and install:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
git clone https://github.com/SaiSandeepKantareddy/MARE.git
|
|
103
|
+
cd MARE
|
|
104
|
+
python3 -m venv .venv
|
|
105
|
+
source .venv/bin/activate
|
|
106
|
+
pip install -e ".[dev]"
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Or install directly from GitHub:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
pip install "git+https://github.com/SaiSandeepKantareddy/MARE.git"
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The intended package install after PyPI release is:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pip install mare-retrieval
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Then use it as a library:
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from mare import MAREApp
|
|
125
|
+
|
|
126
|
+
app = MAREApp.from_pdf("manual.pdf", reuse=True)
|
|
127
|
+
best = app.best_match("partially reinstall the set screws if they fall out")
|
|
128
|
+
|
|
129
|
+
print(best.page)
|
|
130
|
+
print(best.snippet)
|
|
131
|
+
print(best.page_image_path)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Or try the sample corpus from the CLI:
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
mare-demo --query "show me the architecture diagram of transformer"
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Or without installing the package yet:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
PYTHONPATH=src python3 -m mare.demo --query "show me the architecture diagram of transformer"
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## Simplest way to use it
|
|
147
|
+
|
|
148
|
+
Use one command:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
python3 ask.py "MacBook Pro (14-inch, M5 Pro or M5 Max) MagSafe 3 Board - Apple Support.pdf" "partially reinstall the set screws if they fall out"
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
That will:
|
|
155
|
+
|
|
156
|
+
- ingest the PDF if needed
|
|
157
|
+
- retrieve the best matching page
|
|
158
|
+
- print the page number
|
|
159
|
+
- print the exact snippet
|
|
160
|
+
- print the rendered page image path
|
|
161
|
+
|
|
162
|
+
If you want to reuse a previously generated corpus:
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
python3 ask.py --reuse "MacBook Pro (14-inch, M5 Pro or M5 Max) MagSafe 3 Board - Apple Support.pdf" "partially reinstall the set screws if they fall out"
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
If the PDF filename is awkward, rename it first:
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
mv ./*.pdf ./manual.pdf
|
|
172
|
+
PYTHONPATH=src python3 ask.py ./manual.pdf "partially reinstall the set screws if they fall out"
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
## Public Python API
|
|
176
|
+
|
|
177
|
+
The package is meant to be importable, not just runnable from scripts.
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
from mare import MAREApp, load_corpus, load_pdf
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Create an app from a PDF:
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
app = load_pdf("manual.pdf", reuse=True)
|
|
187
|
+
hit = app.best_match("what does MagSafe 3 refer to")
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Create an app from an existing JSON corpus:
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
app = load_corpus("generated/manual.json")
|
|
194
|
+
results = app.retrieve("show me the comparison table", top_k=3)
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Core methods:
|
|
198
|
+
|
|
199
|
+
- `MAREApp.from_pdf(...)`
|
|
200
|
+
- `MAREApp.from_corpus(...)`
|
|
201
|
+
- `MAREApp.from_documents(...)`
|
|
202
|
+
- `app.explain(query)`
|
|
203
|
+
- `app.retrieve(query)`
|
|
204
|
+
- `app.best_match(query)`
|
|
205
|
+
|
|
206
|
+
## Packaging and release
|
|
207
|
+
|
|
208
|
+
MARE is now structured as a regular Python package with:
|
|
209
|
+
|
|
210
|
+
- `pyproject.toml` metadata
|
|
211
|
+
- legacy-friendly `setup.py`
|
|
212
|
+
- console entry points
|
|
213
|
+
- a PyPI publishing workflow
|
|
214
|
+
|
|
215
|
+
Release notes and PyPI steps live in `PUBLISHING.md`.
|
|
216
|
+
|
|
217
|
+
## Visual demo
|
|
218
|
+
|
|
219
|
+
If you want to show this to users visually, run the Streamlit demo:
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
pip install -e ".[ui]"
|
|
223
|
+
PYTHONPATH=src streamlit run src/mare/streamlit_app.py
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
The demo lets a user:
|
|
227
|
+
|
|
228
|
+
- upload a PDF
|
|
229
|
+
- ask a question
|
|
230
|
+
- see the best matching page
|
|
231
|
+
- read the exact evidence snippet
|
|
232
|
+
- view the rendered page image
|
|
233
|
+
|
|
234
|
+
The technical retrieval plan is hidden under a `Debug details` expander so the default experience stays user-facing.
|
|
235
|
+
|
|
236
|
+
## Ingest a real PDF
|
|
237
|
+
|
|
238
|
+
You can convert a PDF into a page-level JSON corpus and then run retrieval on it.
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
python3 -m venv .venv
|
|
242
|
+
source .venv/bin/activate
|
|
243
|
+
pip install -e ".[dev]"
|
|
244
|
+
mare-ingest "MacBook Pro (14-inch, M5 Pro or M5 Max) MagSafe 3 Board - Apple Support.pdf"
|
|
245
|
+
mare-demo --corpus "generated/MacBook Pro (14-inch, M5 Pro or M5 Max) MagSafe 3 Board - Apple Support.json" --query "what does MagSafe 3 refer to"
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Without installing the package first:
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
PYTHONPATH=src python3 -m mare.ingest "MacBook Pro (14-inch, M5 Pro or M5 Max) MagSafe 3 Board - Apple Support.pdf"
|
|
252
|
+
PYTHONPATH=src python3 -m mare.demo --corpus "generated/MacBook Pro (14-inch, M5 Pro or M5 Max) MagSafe 3 Board - Apple Support.json" --query "what does MagSafe 3 refer to"
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
What the ingest step does right now:
|
|
256
|
+
|
|
257
|
+
- reads each PDF page with `pypdf`
|
|
258
|
+
- renders each PDF page to `generated/<pdf-name>/page-N.png`
|
|
259
|
+
- extracts page text
|
|
260
|
+
- creates one retrieval document per page
|
|
261
|
+
- adds lightweight layout hints when terms like `Table` or `Figure` appear
|
|
262
|
+
- writes a JSON corpus that the retriever can search immediately
|
|
263
|
+
|
|
264
|
+
This is still a simple baseline. OCR, figure extraction, and true layout modeling are the next step.
|
|
265
|
+
|
|
266
|
+
## What you get back
|
|
267
|
+
|
|
268
|
+
The retriever now returns:
|
|
269
|
+
|
|
270
|
+
- the matching page number
|
|
271
|
+
- why that page matched
|
|
272
|
+
- a short exact snippet from the page text
|
|
273
|
+
- the rendered page image path
|
|
274
|
+
|
|
275
|
+
That makes it easier to validate whether retrieval found the right instruction and jump to the exact page image.
|
|
276
|
+
|
|
277
|
+
Example output:
|
|
278
|
+
|
|
279
|
+
```json
|
|
280
|
+
{
|
|
281
|
+
"query": "show me the architecture diagram of transformer",
|
|
282
|
+
"intent": "visual_lookup",
|
|
283
|
+
"selected_modalities": ["image"],
|
|
284
|
+
"discarded_modalities": ["text", "layout"],
|
|
285
|
+
"confidence": 0.8,
|
|
286
|
+
"rationale": "Detected modality cues in query tokens. Selected image based on keyword overlap with routing hints.",
|
|
287
|
+
"results": [
|
|
288
|
+
{
|
|
289
|
+
"doc_id": "paper-transformer-p4",
|
|
290
|
+
"title": "Attention Is All You Need",
|
|
291
|
+
"page": 4,
|
|
292
|
+
"score": 0.6,
|
|
293
|
+
"reason": "image:Matched visual cues: architecture, diagram, transformer"
|
|
294
|
+
}
|
|
295
|
+
]
|
|
296
|
+
}
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
## Why the explainability matters
|
|
300
|
+
|
|
301
|
+
The debug surface is a core feature, not an afterthought. For production retrieval systems, we need to answer:
|
|
302
|
+
|
|
303
|
+
- Which modality did the router choose?
|
|
304
|
+
- Which modalities were skipped?
|
|
305
|
+
- Why did a page rank highly?
|
|
306
|
+
- What tradeoff did fusion make?
|
|
307
|
+
|
|
308
|
+
That is the wedge for MARE: make multimodal retrieval inspectable before trying to make it magical.
|
|
309
|
+
|
|
310
|
+
## Local sample data
|
|
311
|
+
|
|
312
|
+
`examples/sample_corpus.json` contains a tiny IR-paper-style corpus so the routing and fusion path is runnable out of the box.
|
|
313
|
+
|
|
314
|
+
There is also a local PDF in this workspace:
|
|
315
|
+
|
|
316
|
+
- `MacBook Pro (14-inch, M5 Pro or M5 Max) MagSafe 3 Board - Apple Support.pdf`
|
|
317
|
+
|
|
318
|
+
That file can now be ingested into a JSON page corpus with `mare-ingest`.
|
|
319
|
+
|
|
320
|
+
## Roadmap
|
|
321
|
+
|
|
322
|
+
### v0.1
|
|
323
|
+
|
|
324
|
+
- text + image + layout routing
|
|
325
|
+
- weighted late fusion
|
|
326
|
+
- explainable retrieval output
|
|
327
|
+
- tests and runnable demo
|
|
328
|
+
|
|
329
|
+
### v0.2
|
|
330
|
+
|
|
331
|
+
- pluggable embedding backends
|
|
332
|
+
- PDF page ingestion
|
|
333
|
+
- OCR and caption extraction adapters
|
|
334
|
+
- score normalization per modality
|
|
335
|
+
|
|
336
|
+
### v0.3
|
|
337
|
+
|
|
338
|
+
- learned router
|
|
339
|
+
- benchmark harness for IRPAPERS-style evaluation
|
|
340
|
+
- cost-aware routing budgets
|
|
341
|
+
- reranking and cross-modal evidence aggregation
|
|
342
|
+
|
|
343
|
+
## Suggested next open-source moves
|
|
344
|
+
|
|
345
|
+
- Add adapters for FAISS, Qdrant, and Weaviate
|
|
346
|
+
- Add page extraction from PDFs
|
|
347
|
+
- Add a benchmark runner that computes Recall@k per modality
|
|
348
|
+
- Add a small web debug UI for route inspection
|
|
349
|
+
|
|
350
|
+
## License
|
|
351
|
+
|
|
352
|
+
MIT
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
mare/__init__.py,sha256=BueI_gdhuXL672sjwVJIyTsUIkKGgUX1fh4i7dzZPKE,530
|
|
2
|
+
mare/api.py,sha256=t0muhmu3HCLP3ts3JT89wYrGrx69pC3p8i2TFc3-YpU,2253
|
|
3
|
+
mare/ask.py,sha256=4I42db73iNDpD892OW8ePGOM44x4WoNRs2gQ7jkEs9s,1884
|
|
4
|
+
mare/demo.py,sha256=5cjQrqGV1dYCRETKio-VySZRRTbnlKbzktjP81LZqo4,1993
|
|
5
|
+
mare/engine.py,sha256=DO-N8jQ-qN31xGDwmZn-5Cjv4lw6O5VKv2fhmAzdAGI,1684
|
|
6
|
+
mare/fusion.py,sha256=2MAYhkGV0EvogDCic49KU5oP8bJIbXAK4c7y8jbM7DI,2303
|
|
7
|
+
mare/ingest.py,sha256=c5N6LQqTxmYZ_TapOYkjz7dp49kHtkXkxr8A2-DOKBs,4503
|
|
8
|
+
mare/router.py,sha256=jxh9mVv4OOSbDjzcBp2dSkUAbKhHP4slMvythKGVrD8,2624
|
|
9
|
+
mare/streamlit_app.py,sha256=-j2FN36H32Es-4fmeTs82lNTYtikmK_K5zk6d-Zt15Q,3452
|
|
10
|
+
mare/types.py,sha256=RRvhTAN98jyECr5jqY7NOt063LGbo4WIq0-fTWtrIAs,1013
|
|
11
|
+
mare/retrievers/__init__.py,sha256=x5J2xUhM_pXaZKTIJkZCYqWY4wrOHoUFMPwXQQvkcro,52
|
|
12
|
+
mare/retrievers/base.py,sha256=ydwBlr5wVf7bg9LgSMTzLiQT4iqK5ehGTpkV5jLPQkY,405
|
|
13
|
+
mare/retrievers/image.py,sha256=06Fdkqk6jQFNy2DKci9DNvqHv1ldCYUfaeMn5v9TMqo,2377
|
|
14
|
+
mare/retrievers/text.py,sha256=wgOWKTK42qvrq13t6b8HX-4TowjiNGzsCyNE7c71zzo,2482
|
|
15
|
+
mare_retrieval-0.2.0.dist-info/licenses/LICENSE,sha256=dQVLagjhAIyGtgRauZrdLI-lcZh4gJUamSnxHuVqfWQ,1057
|
|
16
|
+
mare_retrieval-0.2.0.dist-info/METADATA,sha256=aN6ZqFTDZ4oGBd0NZhXEjcDQCEJrpCt3Muq--FmblCo,9814
|
|
17
|
+
mare_retrieval-0.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
18
|
+
mare_retrieval-0.2.0.dist-info/entry_points.txt,sha256=g6y2nQgt9SeRCMAZfevAfNABwdpDckU_A-en_x2jbHQ,135
|
|
19
|
+
mare_retrieval-0.2.0.dist-info/top_level.txt,sha256=JZzfkMhGhMGqPhMws49gomWf5L4Cc3FSfnpp8EvGJic,5
|
|
20
|
+
mare_retrieval-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mare
|