right-rag 0.1.0__py3-none-any.whl

right_rag/__init__.py

@@ -0,0 +1,2 @@
+"""right-rag package."""
+

right_rag/__main__.py

@@ -0,0 +1,6 @@
+from .cli import main
+
+
+if __name__ == "__main__":
+    main()
+

right_rag/cli.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .pipeline import cli as pipeline_cli
+
+
+SPEC_PACKAGES = [
+    "document_loaders",
+    "unit_parsers",
+    "chunk_parsers",
+    "unit_savers",
+    "chunk_savers",
+    "retrievers",
+    "answer_generators",
+    "evaluators",
+    "reporters",
+]
+
+
+def main(argv: list[str] | None = None) -> None:
+    sys.path.insert(0, str(Path.cwd()))
+    args = list(sys.argv[1:] if argv is None else argv)
+
+    if not args or args[0] in {"-h", "--help"}:
+        print_help()
+        return
+
+    if args[0] == "init":
+        init_cli(args[1:])
+        return
+
+    pipeline_cli(args)
+
+
+def print_help() -> None:
+    print(
+        "usage: right-rag {init,index,query,evaluate} ...\n\n"
+        "commands:\n"
+        "  init [path]       create a right-rag project\n"
+        "  index             build unit and chunk artifacts\n"
+        "  query TEXT        query saved artifacts\n"
+        "  evaluate [cases]  evaluate RAG quality\n"
+    )
+
+
+def init_cli(argv: list[str]) -> None:
+    parser = argparse.ArgumentParser(prog="right-rag init")
+    parser.add_argument("path", nargs="?", default=".")
+    parser.add_argument("--force", action="store_true")
+    args = parser.parse_args(argv)
+
+    init_project(Path(args.path), force=args.force)
+    print(f"Created right-rag project at {Path(args.path).resolve()}")
+
+
+def init_project(root: Path, force: bool = False) -> None:
+    root = root.resolve()
+    if root.exists() and any(root.iterdir()) and not force:
+        raise SystemExit(f"{root} is not empty. Use --force to write into it.")
+
+    for path in [
+        root / "data" / "documents",
+        root / "output",
+        root / "eval",
+        root / "spec",
+    ]:
+        path.mkdir(parents=True, exist_ok=True)
+
+    write_once(root / "right_rag.toml", RIGHT_RAG_TOML, force)
+    write_once(root / ".gitignore", GITIGNORE, force)
+    write_once(root / "data" / "documents" / ".gitkeep", "", force)
+    write_once(root / "eval" / "cases.example.json", EVAL_CASES, force)
+    write_once(root / "spec" / "__init__.py", "", force)
+
+    for package in SPEC_PACKAGES:
+        directory = root / "spec" / package
+        directory.mkdir(parents=True, exist_ok=True)
+        write_once(directory / "__init__.py", "", force)
+        write_once(directory / "default.py", default_reexport(package), force)
+
+
+def write_once(path: Path, text: str, force: bool) -> None:
+    if path.exists() and not force:
+        return
+
+    path.write_text(text, encoding="utf-8")
+
+
+def default_reexport(package: str) -> str:
+    return f"from right_rag.spec.{package}.default import *  # noqa: F401,F403\n"
+
+
+RIGHT_RAG_TOML = """\
+[right-rag]
+spec_root_package = "spec"
+document_root_folder = "data/documents"
+state_path = ".right-rag-state.json"
+"""
+
+
+GITIGNORE = """\
+__pycache__/
+*.py[oc]
+.right-rag-state.json
+output/
+data/documents/*
+!data/documents/.gitkeep
+"""
+
+
+EVAL_CASES = """\
+{
+  "cases": [
+    {
+      "id": "edit-me",
+      "query": "replace this with a real domain question",
+      "expected": {
+        "answer_contains": [],
+        "hit_contains": [],
+        "hit_ids": [],
+        "must_not_contain": []
+      },
+      "tags": ["smoke"]
+    }
+  ]
+}
+"""

right_rag/interfaces.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable, Iterator, Mapping
+from pathlib import Path
+
+from .model import Answer, Chunk, Document, EvalCase, EvalResult, Hit, ParsedUnit, Query, Unit
+
+
+class DocumentLoader(ABC):
+    @property
+    @abstractmethod
+    def loader_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def __iter__(self) -> Iterator[Document]:
+        pass
+
+    def load_documents(self, root_folder: str) -> Iterator[Document]:
+        return iter(self)
+
+
+class UnitParser(ABC):
+    @property
+    @abstractmethod
+    def parser_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def parser_can_handle_document(self, document: Document) -> bool:
+        pass
+
+    @abstractmethod
+    def parse_document(self, document: Document) -> Iterable[ParsedUnit]:
+        pass
+
+
+class ChunkParser(ABC):
+    @property
+    @abstractmethod
+    def parser_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def parse_unit(self, unit: Unit) -> Iterable[Chunk]:
+        pass
+
+
+class UnitArtifact(ABC):
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+    @abstractmethod
+    def retrieve(self, query: Query) -> Iterable[Hit]:
+        pass
+
+
+class ChunkArtifact(ABC):
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        pass
+
+    @abstractmethod
+    def retrieve(self, query: Query) -> Iterable[Hit]:
+        pass
+
+
+class ChunkSaver(ABC):
+    @property
+    @abstractmethod
+    def saver_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def save_chunks(self, chunks: Iterable[Chunk]) -> ChunkArtifact:
+        pass
+
+    @abstractmethod
+    def open_chunk_artifact(self) -> ChunkArtifact:
+        pass
+
+
+class UnitSaver(ABC):
+    @property
+    @abstractmethod
+    def saver_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def save_units(self, units: Iterable[Unit]) -> UnitArtifact:
+        pass
+
+    @abstractmethod
+    def open_unit_artifact(self) -> UnitArtifact:
+        pass
+
+
+class Retriever(ABC):
+    @property
+    @abstractmethod
+    def retriever_name(self) -> str:
+        pass
+
+    def query_options(self) -> Mapping[str, str]:
+        return {}
+
+    @abstractmethod
+    def retrieve(
+        self,
+        query: Query,
+        unit_artifacts: Mapping[str, UnitArtifact],
+        chunk_artifacts: Mapping[str, ChunkArtifact],
+    ) -> list[Hit]:
+        pass
+
+
+class AnswerGenerator(ABC):
+    @property
+    @abstractmethod
+    def generator_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def generate(self, query: Query, hits: Iterable[Hit]) -> Answer:
+        pass
+
+
+class Evaluator(ABC):
+    @property
+    @abstractmethod
+    def evaluator_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def evaluate(self, case: EvalCase, answer: Answer) -> EvalResult:
+        pass
+
+
+class Reporter(ABC):
+    @property
+    @abstractmethod
+    def reporter_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def write_report(
+        self,
+        results: Iterable[EvalResult],
+        output_path: str | Path,
+    ) -> None:
+        pass