hyperforge-remi 1.0.0.post20__py3-none-any.whl

hyperforge_remi/__init__.py

@@ -0,0 +1,3 @@
+from .agent import RemiAgent
+
+__all__ = ["RemiAgent"]

hyperforge_remi/agent.py

@@ -0,0 +1,177 @@
+import asyncio
+
+from hyperforge.agent import Agent
+from hyperforge.configure import agent
+from hyperforge.manager import Manager
+from hyperforge.memory import Chunk, Context, QuestionMemory
+from hyperforge.trace import trace_agent
+from nuclia_models.predict.remi import RemiResponse
+
+from hyperforge import logger
+from hyperforge_remi.config import (
+    ContextGranularity,
+    RemiAgentConfig,
+)
+
+MAX_CONTEXTS_REMI = 60
+
+
+@agent(
+    id="remi",
+    agent_type="postprocess",
+    title="REMI Evaluation",
+    description="Agent that performs REMI evaluation.",
+    config_schema=RemiAgentConfig,
+)
+class RemiAgent(Agent[RemiAgentConfig]):
+    @trace_agent
+    async def __call__(
+        self,
+        memory: QuestionMemory,
+        manager: Manager,
+    ):
+        error = None
+        # For each context in memory add context query, summary and answer onto a text and the initial question
+        if memory.final_answer is None:
+            raise Exception("No final answer")
+        if memory.original_question is None:
+            raise Exception("No original question")
+
+        # Only answer relevance and groundedness for now
+        # If we also did context relevance, we would combine into a single remi call
+        tasks = [
+            manager.remi(
+                question=memory.original_question,
+                answer=memory.final_answer,
+                contexts=None,
+            )
+        ]
+        contexts = (
+            memory.list_contexts_minimal()
+            if self.config.context_granularity == ContextGranularity.PARTIAL_ANSWERS
+            else memory.list_chunks_markdown()
+        )
+        if len(contexts) > MAX_CONTEXTS_REMI:
+            contexts = contexts[:MAX_CONTEXTS_REMI]
+            error = f"Too many contexts for groundedness evaluation, truncated to {MAX_CONTEXTS_REMI}. "
+
+        if memory.is_answered is True:
+            tasks.append(
+                manager.remi(
+                    question=None,
+                    answer=memory.final_answer,
+                    contexts=contexts,
+                )
+            )
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        ev_answer_rel: RemiResponse | BaseException = results[0]
+        if memory.is_answered is True:
+            ev_groundedness: RemiResponse | BaseException | None = results[1]
+        else:
+            logger.info(
+                "Forcing REMi groundedness to 0 on question flagged as not answered"
+            )
+            ev_groundedness = RemiResponse(
+                groundedness=[0] * max(len(contexts), 1), time=0
+            )
+
+        response_str = ""
+        if (
+            isinstance(ev_answer_rel, BaseException)
+            or not ev_answer_rel
+            or not ev_answer_rel.answer_relevance
+        ):
+            msg = "Error evaluating answer relevance. "
+            logger.warning(
+                msg + str(ev_answer_rel)
+                if isinstance(ev_answer_rel, BaseException)
+                else ""
+            )
+            error = error + msg if error else msg
+        else:
+            response_str += (
+                f"Answer relevance: {ev_answer_rel.answer_relevance.score}/5. "
+            )
+
+        # Max aggregation for groundedness, could be configurable
+        if (
+            isinstance(ev_groundedness, BaseException)
+            or not ev_groundedness
+            or not ev_groundedness.groundedness
+        ):
+            msg = "Error evaluating answer groundedness. "
+            logger.warning(
+                msg + str(ev_groundedness)
+                if isinstance(ev_groundedness, BaseException)
+                else ""
+            )
+            error = error + msg if error else msg
+        else:
+            groundedness = max(
+                [g if g is not None else 0 for g in ev_groundedness.groundedness]
+            )
+            response_str += f"Answer groundedness: {groundedness}/5."
+
+            remi_chunks = []
+
+            # Use chunks directly if context granularity is chunk (markdown)
+            if self.config.context_granularity != ContextGranularity.PARTIAL_ANSWERS:
+                chunk_idx = 0
+                for context in memory.contexts:
+                    agent_name = context.agent_id if context.agent_id else context.agent
+                    if context.chunks:
+                        for chunk in context.chunks:
+                            if chunk_idx >= min(
+                                MAX_CONTEXTS_REMI, len(ev_groundedness.groundedness)
+                            ):
+                                break
+                            score = ev_groundedness.groundedness[chunk_idx]
+                            g_score = score if score is not None else 0
+
+                            c = Chunk(
+                                chunk_id=chunk.chunk_id or f"remi_chunk_{chunk_idx}",
+                                title=f"Groundedness {g_score}/5 - [{agent_name}] {chunk.title or 'Untitled'}",
+                                text=f"**Groundedness: {g_score}/5**\n\n{chunk.text}",
+                                origin_agent=self.config.module,
+                            )
+                            remi_chunks.append(c)
+                            chunk_idx += 1
+            else:
+                for i, context in enumerate(memory.contexts[:MAX_CONTEXTS_REMI]):
+                    if i < len(ev_groundedness.groundedness):
+                        score = ev_groundedness.groundedness[i]
+                        g_score = score if score is not None else 0
+                        agent_name = (
+                            context.agent_id if context.agent_id else context.agent
+                        )
+
+                        evaluated_text = (
+                            context.answer_summary_markdown()
+                            if context.summary.strip()
+                            else context.context_markdown()
+                        )
+                        c = Chunk(
+                            chunk_id=context.id or f"remi_{i}",
+                            title=f"Groundedness {g_score}/5 - [{agent_name}] {context.title or 'Untitled'}",
+                            text=f"**Groundedness: {g_score}/5**\n\n{evaluated_text}",
+                            origin_agent=self.config.module,
+                        )
+                        remi_chunks.append(c)
+
+            if remi_chunks:
+                remi_context = Context(
+                    original_question_uuid=memory.original_question_uuid,
+                    actual_question_uuid=memory.actual_question_uuid,
+                    agent="remi",
+                    agent_id="remi",
+                    title="REMi Evaluation Breakdown",
+                    summary=response_str.strip()
+                    if response_str.strip()
+                    else f"Evaluated {len(remi_chunks)} contexts for groundedness.",
+                    question=memory.original_question or "",
+                    source="remi",
+                    chunks=remi_chunks,
+                )
+                if error:
+                    remi_context.summary += f"\nErrors: {error}"
+                await memory.save_context("postprocess", remi_context)

hyperforge_remi/config.py

@@ -0,0 +1,23 @@
+from enum import Enum
+from typing import Literal
+
+from hyperforge.agent import AgentConfig
+from pydantic import Field
+from pydantic.config import ConfigDict
+
+
+class ContextGranularity(str, Enum):
+    FULL = "full"
+    PARTIAL_ANSWERS = "partial_answers"
+
+
+class RemiAgentConfig(AgentConfig):
+    model_config = ConfigDict(title="REMi evaluation")
+    module: Literal["remi"] = "remi"
+    context_granularity: ContextGranularity = Field(
+        default=ContextGranularity.FULL,
+        title="Granularity of the contexts pieces",
+        description="Granularity of the context pieces sent to REMi for groundedness evaluation. "
+        "If 'partial_answers', the evaluation will use agent-level answer attempts to the question when available for a speedier analysis. "
+        "If 'full', the evaluation will use all individual text chunks that each agent generated for a more detailed analysis.",
+    )

hyperforge_remi-1.0.0.post20.dist-info/METADATA

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: hyperforge_remi
+Version: 1.0.0.post20
+Summary: Remi Hyperforge agent
+Author-email: Nuclia <nucliadb@nuclia.com>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://progress.com
+Project-URL: Repository, https://github.com/nuclia/forge
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <4,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: hyperforge
+
+# Remi Hyperforge agents

hyperforge_remi-1.0.0.post20.dist-info/RECORD

@@ -0,0 +1,7 @@
+hyperforge_remi/__init__.py,sha256=eLmHV2JTI1cj_H9Xw7SAK3GnsSLfbIyUzCnbO13lO4E,54
+hyperforge_remi/agent.py,sha256=eIg6y9Sci8dnL6aPZiZp4UmOkNNmkwvv2NleYOlvOAA,7175
+hyperforge_remi/config.py,sha256=sWarPV5_SzGhpVrTBo46XS5TdVq3cbH2h4u9bdMvgLE,905
+hyperforge_remi-1.0.0.post20.dist-info/METADATA,sha256=eeBfA1unaJyE0Lv9bRbC03A-cAbR21G_F8dbg_5dRT0,716
+hyperforge_remi-1.0.0.post20.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+hyperforge_remi-1.0.0.post20.dist-info/top_level.txt,sha256=-SZGKrIg_uD9_QQD_eZwFrnvu2wb26cAEKbC1mIH07o,16
+hyperforge_remi-1.0.0.post20.dist-info/RECORD,,

hyperforge_remi-1.0.0.post20.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

hyperforge_remi-1.0.0.post20.dist-info/top_level.txt

@@ -0,0 +1 @@
+hyperforge_remi