hyperforge-remi 1.0.0.post20__tar.gz

hyperforge_remi-1.0.0.post20/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: hyperforge_remi
+Version: 1.0.0.post20
+Summary: Remi Hyperforge agent
+Author-email: Nuclia <nucliadb@nuclia.com>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://progress.com
+Project-URL: Repository, https://github.com/nuclia/forge
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <4,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: hyperforge
+
+# Remi Hyperforge agents

hyperforge_remi-1.0.0.post20/README.md

@@ -0,0 +1 @@
+# Remi Hyperforge agents

hyperforge_remi-1.0.0.post20/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hyperforge_remi"
+version = "1.0.0.post20"
+license = "Apache-2.0"
+description = "Remi Hyperforge agent"
+authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]
+readme = "README.md"
+classifiers = [
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">=3.10, <4"
+dependencies = ["hyperforge"]
+
+[dependency-groups]
+dev = [
+    "pytest",
+    "pytest-benchmark",
+    "pytest-docker-fixtures>=1.4.2",
+    "pytest-lazy-fixtures",
+    "pytest-recording",
+    "pytest-asyncio",
+    "pytest-cov",
+    "pytest-mock",
+]
+
+[project.urls]
+Homepage = "https://progress.com"
+Repository = "https://github.com/nuclia/forge"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

hyperforge_remi-1.0.0.post20/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

hyperforge_remi-1.0.0.post20/src/hyperforge_remi/__init__.py

@@ -0,0 +1,3 @@
+from .agent import RemiAgent
+
+__all__ = ["RemiAgent"]

hyperforge_remi-1.0.0.post20/src/hyperforge_remi/agent.py

@@ -0,0 +1,177 @@
+import asyncio
+
+from hyperforge.agent import Agent
+from hyperforge.configure import agent
+from hyperforge.manager import Manager
+from hyperforge.memory import Chunk, Context, QuestionMemory
+from hyperforge.trace import trace_agent
+from nuclia_models.predict.remi import RemiResponse
+
+from hyperforge import logger
+from hyperforge_remi.config import (
+    ContextGranularity,
+    RemiAgentConfig,
+)
+
+MAX_CONTEXTS_REMI = 60
+
+
+@agent(
+    id="remi",
+    agent_type="postprocess",
+    title="REMI Evaluation",
+    description="Agent that performs REMI evaluation.",
+    config_schema=RemiAgentConfig,
+)
+class RemiAgent(Agent[RemiAgentConfig]):
+    @trace_agent
+    async def __call__(
+        self,
+        memory: QuestionMemory,
+        manager: Manager,
+    ):
+        error = None
+        # For each context in memory add context query, summary and answer onto a text and the initial question
+        if memory.final_answer is None:
+            raise Exception("No final answer")
+        if memory.original_question is None:
+            raise Exception("No original question")
+
+        # Only answer relevance and groundedness for now
+        # If we also did context relevance, we would combine into a single remi call
+        tasks = [
+            manager.remi(
+                question=memory.original_question,
+                answer=memory.final_answer,
+                contexts=None,
+            )
+        ]
+        contexts = (
+            memory.list_contexts_minimal()
+            if self.config.context_granularity == ContextGranularity.PARTIAL_ANSWERS
+            else memory.list_chunks_markdown()
+        )
+        if len(contexts) > MAX_CONTEXTS_REMI:
+            contexts = contexts[:MAX_CONTEXTS_REMI]
+            error = f"Too many contexts for groundedness evaluation, truncated to {MAX_CONTEXTS_REMI}. "
+
+        if memory.is_answered is True:
+            tasks.append(
+                manager.remi(
+                    question=None,
+                    answer=memory.final_answer,
+                    contexts=contexts,
+                )
+            )
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        ev_answer_rel: RemiResponse | BaseException = results[0]
+        if memory.is_answered is True:
+            ev_groundedness: RemiResponse | BaseException | None = results[1]
+        else:
+            logger.info(
+                "Forcing REMi groundedness to 0 on question flagged as not answered"
+            )
+            ev_groundedness = RemiResponse(
+                groundedness=[0] * max(len(contexts), 1), time=0
+            )
+
+        response_str = ""
+        if (
+            isinstance(ev_answer_rel, BaseException)
+            or not ev_answer_rel
+            or not ev_answer_rel.answer_relevance
+        ):
+            msg = "Error evaluating answer relevance. "
+            logger.warning(
+                msg + str(ev_answer_rel)
+                if isinstance(ev_answer_rel, BaseException)
+                else ""
+            )
+            error = error + msg if error else msg
+        else:
+            response_str += (
+                f"Answer relevance: {ev_answer_rel.answer_relevance.score}/5. "
+            )
+
+        # Max aggregation for groundedness, could be configurable
+        if (
+            isinstance(ev_groundedness, BaseException)
+            or not ev_groundedness
+            or not ev_groundedness.groundedness
+        ):
+            msg = "Error evaluating answer groundedness. "
+            logger.warning(
+                msg + str(ev_groundedness)
+                if isinstance(ev_groundedness, BaseException)
+                else ""
+            )
+            error = error + msg if error else msg
+        else:
+            groundedness = max(
+                [g if g is not None else 0 for g in ev_groundedness.groundedness]
+            )
+            response_str += f"Answer groundedness: {groundedness}/5."
+
+            remi_chunks = []
+
+            # Use chunks directly if context granularity is chunk (markdown)
+            if self.config.context_granularity != ContextGranularity.PARTIAL_ANSWERS:
+                chunk_idx = 0
+                for context in memory.contexts:
+                    agent_name = context.agent_id if context.agent_id else context.agent
+                    if context.chunks:
+                        for chunk in context.chunks:
+                            if chunk_idx >= min(
+                                MAX_CONTEXTS_REMI, len(ev_groundedness.groundedness)
+                            ):
+                                break
+                            score = ev_groundedness.groundedness[chunk_idx]
+                            g_score = score if score is not None else 0
+
+                            c = Chunk(
+                                chunk_id=chunk.chunk_id or f"remi_chunk_{chunk_idx}",
+                                title=f"Groundedness {g_score}/5 - [{agent_name}] {chunk.title or 'Untitled'}",
+                                text=f"**Groundedness: {g_score}/5**\n\n{chunk.text}",
+                                origin_agent=self.config.module,
+                            )
+                            remi_chunks.append(c)
+                            chunk_idx += 1
+            else:
+                for i, context in enumerate(memory.contexts[:MAX_CONTEXTS_REMI]):
+                    if i < len(ev_groundedness.groundedness):
+                        score = ev_groundedness.groundedness[i]
+                        g_score = score if score is not None else 0
+                        agent_name = (
+                            context.agent_id if context.agent_id else context.agent
+                        )
+
+                        evaluated_text = (
+                            context.answer_summary_markdown()
+                            if context.summary.strip()
+                            else context.context_markdown()
+                        )
+                        c = Chunk(
+                            chunk_id=context.id or f"remi_{i}",
+                            title=f"Groundedness {g_score}/5 - [{agent_name}] {context.title or 'Untitled'}",
+                            text=f"**Groundedness: {g_score}/5**\n\n{evaluated_text}",
+                            origin_agent=self.config.module,
+                        )
+                        remi_chunks.append(c)
+
+            if remi_chunks:
+                remi_context = Context(
+                    original_question_uuid=memory.original_question_uuid,
+                    actual_question_uuid=memory.actual_question_uuid,
+                    agent="remi",
+                    agent_id="remi",
+                    title="REMi Evaluation Breakdown",
+                    summary=response_str.strip()
+                    if response_str.strip()
+                    else f"Evaluated {len(remi_chunks)} contexts for groundedness.",
+                    question=memory.original_question or "",
+                    source="remi",
+                    chunks=remi_chunks,
+                )
+                if error:
+                    remi_context.summary += f"\nErrors: {error}"
+                await memory.save_context("postprocess", remi_context)

hyperforge_remi-1.0.0.post20/src/hyperforge_remi/config.py

@@ -0,0 +1,23 @@
+from enum import Enum
+from typing import Literal
+
+from hyperforge.agent import AgentConfig
+from pydantic import Field
+from pydantic.config import ConfigDict
+
+
+class ContextGranularity(str, Enum):
+    FULL = "full"
+    PARTIAL_ANSWERS = "partial_answers"
+
+
+class RemiAgentConfig(AgentConfig):
+    model_config = ConfigDict(title="REMi evaluation")
+    module: Literal["remi"] = "remi"
+    context_granularity: ContextGranularity = Field(
+        default=ContextGranularity.FULL,
+        title="Granularity of the contexts pieces",
+        description="Granularity of the context pieces sent to REMi for groundedness evaluation. "
+        "If 'partial_answers', the evaluation will use agent-level answer attempts to the question when available for a speedier analysis. "
+        "If 'full', the evaluation will use all individual text chunks that each agent generated for a more detailed analysis.",
+    )

hyperforge_remi-1.0.0.post20/src/hyperforge_remi.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: hyperforge_remi
+Version: 1.0.0.post20
+Summary: Remi Hyperforge agent
+Author-email: Nuclia <nucliadb@nuclia.com>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://progress.com
+Project-URL: Repository, https://github.com/nuclia/forge
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <4,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: hyperforge
+
+# Remi Hyperforge agents

hyperforge_remi-1.0.0.post20/src/hyperforge_remi.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+src/hyperforge_remi/__init__.py
+src/hyperforge_remi/agent.py
+src/hyperforge_remi/config.py
+src/hyperforge_remi.egg-info/PKG-INFO
+src/hyperforge_remi.egg-info/SOURCES.txt
+src/hyperforge_remi.egg-info/dependency_links.txt
+src/hyperforge_remi.egg-info/requires.txt
+src/hyperforge_remi.egg-info/top_level.txt
+tests/test_remi.py

hyperforge_remi-1.0.0.post20/src/hyperforge_remi.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

hyperforge_remi-1.0.0.post20/src/hyperforge_remi.egg-info/requires.txt

@@ -0,0 +1 @@
+hyperforge

hyperforge_remi-1.0.0.post20/src/hyperforge_remi.egg-info/top_level.txt

@@ -0,0 +1 @@
+hyperforge_remi

hyperforge_remi-1.0.0.post20/tests/test_remi.py

@@ -0,0 +1,236 @@
+import os
+import re
+
+import pytest
+from hyperforge.engine import main as arag_main
+from hyperforge.minimal_fixtures import cassette_nua_key
+from hyperforge_remi.config import ContextGranularity
+
+NUA_KEY = os.environ.get(
+    "NUA_KEY",
+) or cassette_nua_key("https://europe-1.nuclia.cloud/")
+
+
+pytestmark = [
+    pytest.mark.vcr(ignore_localhost=True, ignore_hosts=["europe-1.nuclia.cloud"]),
+    pytest.mark.asyncio,
+]
+
+ANSWERED_QUESTION = "What is Progress Agentic RAG?"
+UNANSWERED_QUESTION = (
+    "Como usar max_magic y dime como cambiará este parametro en el futuro"
+)
+
+CONFIG = {
+    "drivers": [
+        {
+            "provider": "nucliadb",
+            "identifier": "nuclia-docs",
+            "name": "nuclia-docs",
+            "config": {
+                "url": "https://europe-1.nuclia.cloud/api",
+                "manager": "https://europe-1.nuclia.cloud/api",
+                "kbid": "df8b4c24-2807-4888-ad6c-ae97357a638b",
+                "key": "eyJhbGciOiJSUzI1NiIsImtpZCI6InNhIiwidHlwIjoiSldUIn0.eyJpc3MiOiJodHRwczovL2V1cm9wZS0xLm51Y2xpYS5jbG91ZC8iLCJpYXQiOjE3NTkyNTQ3NjQsInN1YiI6ImUwNGUwMzcyLTYwNDgtNDY5ZC04NWExLWI1ZTM1MjBmMzdlZiIsImp0aSI6IjgzMzRhY2NlLTIwMTUtNGY0MS05M2U5LTczNzk1MTg2NDdiZiIsImV4cCI6MTc5MDc5MDc2MCwia2V5IjoiMmYxMzYyNTItNjNiMy00NzA1LTg1MjQtMDhmYWJjOWUzMjUyIiwia2lkIjoiN2RmMzY2NDctOTdiOC00NzU0LWExNjUtZWZkY2ZlMDRkMzI2In0.kwHAfx9RRTI-G3S64X0iisr0iAyXRKNRhnN4C67MkLSxeu1AOAnVV8EIQuu4jpXW7O4FkSsthFXEv9ZxlRRh_CaS0z_TjPzIzDPeE6eIKskZ70Q7c-pDe949WE9DZiDyy9_dwKsdX5cnvYpKorp0ROm-GvRXrdHaTZKDSYWht3gvEtm6-0j9C1gx2BzKr2coizUAIde_qjSpLOojO4S-k8P8I9dsQFagdcrjxgGWgrAzjhAs_qkqlRmP0QP6S7ToN0nrbHmtKKb0lWmcpVvlAfH95CM20YUs7IAqU_t7-_V6mm43FstRgGeiHkoapo8nPVJtXMBSlaM7GSz0Kxf2TWQwi94mTEQLdA8CblX0skMCfIHFwbcbm1Vf-2C6LywAsSmTYAwsVPpqeQcVZdrfLMhddCjZKUFCNLSurCSb4TuN79GZicPCJDT-VEBMlNH8ayHOyRib5RyqvgXUwGN9zyM-ma7RrVk4eEwSk7923bn_9GTk-s5tYw_exbYsQ1Qa84GA6NzgJ_kNQmgJwb2zW1V5ddCpYd5k6lNEdPRk0JQKlCC2zTmSvnRcLxfDPi4SZFdLLdtG0j2hIl_QNTEC_3VtqJds4FMofy7TkmUObdbEmXjdAsOxkqj2ntGOsaBNiCI_w47BbPvG_V1LsBHDrrIo0Wo1fgAhUbtWV7Dd5J4",
+                "filters": [],
+                "description": "Documentation of the Nuclia API, recipies, reference",
+            },
+        },
+    ],
+    "rules": {
+        "rules": [
+            {
+                "prompt": "Be polite",
+            },
+            {
+                "prompt": "The documentation of Nuclia is hosted at https://docs.nuclia.dev",
+            },
+        ]
+    },
+    "memory": {},
+    "workflow": {
+        "id": "default",
+        "name": "Default workflow",
+        "description": "Default workflow for testing",
+        "parameters": {},
+    },
+    "preprocess": [],
+    "context": [
+        {
+            "module": "basic_ask",
+            "title": "Nuclia Docs Retrieval Agent",
+            "sources": ["nuclia-docs"],
+            "prune_context": False,
+        },
+    ],
+    "generation": [
+        {"module": "summarize", "title": "Summarize agent"},
+    ],
+    "postprocess": [{"module": "remi"}],
+}
+
+
+# Match on body since we send parallel requests and otherwise they get played back in a different order
+@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"])
+@pytest.mark.parametrize(
+    "granularity", (ContextGranularity.PARTIAL_ANSWERS, ContextGranularity.FULL)
+)
+async def test_remi(granularity: ContextGranularity):
+    CONFIG["postprocess"][0] = {"module": "remi", "context_granularity": granularity}  # type: ignore
+    question_memory = await arag_main(
+        agent_id="default",
+        internal_nua=False,
+        external_nua_api_key=NUA_KEY,
+        question=ANSWERED_QUESTION,
+        config=CONFIG,
+        loaded_modules=[
+            "hyperforge_remi",
+            "hyperforge_summarize",
+            "hyperforge_nucliadb",
+        ],
+    )
+
+    assert question_memory.final_answer is not None
+    # verify the step logic is actually validating the context summary instead
+    remi_contexts = [ctx for ctx in question_memory.contexts if ctx.agent == "remi"]
+    assert len(remi_contexts) > 0, "Missing REMi structured context"
+    remi_ctx = remi_contexts[-1]
+    assert remi_ctx.title == "REMi Evaluation Breakdown"
+
+    assert remi_ctx.summary is not None
+    relevance_match = re.search(r"Answer relevance: (\d+)/5\.", remi_ctx.summary)
+    assert relevance_match is not None, remi_ctx.summary
+    relevance_score = int(relevance_match.group(1))
+    assert relevance_score > 1, f"Relevance score {relevance_score} should be > 1"
+
+    groundedness_match = re.search(r"Answer groundedness: (\d+)/5\.", remi_ctx.summary)
+    assert groundedness_match is not None, remi_ctx.summary
+    groundedness_score = int(groundedness_match.group(1))
+    assert groundedness_score > 1, (
+        f"Groundedness score {groundedness_score} should be > 1"
+    )
+
+    assert len(remi_ctx.chunks) > 0, "Missing evaluated chunks in REMi context"
+    assert remi_ctx.chunks[0].title is not None
+    assert "Groundedness" in remi_ctx.chunks[0].title, (
+        f"Missing score in title: {remi_ctx.chunks[0].title}"
+    )
+    assert "**Groundedness:" in remi_ctx.chunks[0].text, (
+        "Missing score text inside chunk text"
+    )
+
+    if granularity == ContextGranularity.FULL:
+        # original text chunks check
+        original_chunks = sum(
+            len(ctx.chunks) for ctx in question_memory.contexts if ctx.agent != "remi"
+        )
+        assert len(remi_ctx.chunks) == original_chunks
+
+        orig_ids = {
+            c.chunk_id
+            for ctx in question_memory.contexts
+            if ctx.agent != "remi"
+            for c in ctx.chunks
+            if c.chunk_id
+        }
+        for chunk in remi_ctx.chunks:
+            assert chunk.chunk_id in orig_ids or "remi_chunk_" in (chunk.chunk_id or "")
+    else:
+        # One chunk per context evaluated
+        ctx_list = [ctx for ctx in question_memory.contexts if ctx.agent != "remi"]
+        original_contexts = min(len(ctx_list), 60)
+        assert len(remi_ctx.chunks) == original_contexts, (
+            f"remi_ctx: {len(remi_ctx.chunks)}, orig: {original_contexts}. ctx_list: {[c.agent for c in ctx_list]}"
+        )
+        orig_ids = {c.id for c in ctx_list if c.id}
+        for chunk in remi_ctx.chunks:
+            assert chunk.chunk_id in orig_ids or (chunk.chunk_id or "").startswith(
+                "remi_"
+            )
+
+    assert "Errors:" not in remi_ctx.summary
+
+
+@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"])
+@pytest.mark.parametrize(
+    "granularity", [ContextGranularity.PARTIAL_ANSWERS, ContextGranularity.FULL]
+)
+async def test_remi_not_enough_data(granularity: ContextGranularity):
+    CONFIG["postprocess"][0] = {"module": "remi", "context_granularity": granularity}  # type: ignore
+    question_memory = await arag_main(
+        agent_id="default",
+        internal_nua=False,
+        external_nua_api_key=NUA_KEY,
+        question=UNANSWERED_QUESTION,
+        config=CONFIG,
+        loaded_modules=[
+            "hyperforge_remi",
+            "hyperforge_summarize",
+            "hyperforge_nucliadb",
+        ],
+    )
+    assert question_memory.final_answer is not None
+    assert "not enough data to answer this" in question_memory.final_answer.lower()
+    assert question_memory.is_answered is False
+
+    remi_contexts = [ctx for ctx in question_memory.contexts if ctx.agent == "remi"]
+    # If there are context chunks, REMi generates a structured context
+    original_chunks = sum(
+        len(ctx.chunks) for ctx in question_memory.contexts if ctx.agent != "remi"
+    )
+    if original_chunks > 0:
+        assert len(remi_contexts) > 0, "Missing REMi structured context"
+        remi_ctx = remi_contexts[-1]
+        assert remi_ctx.title == "REMi Evaluation Breakdown"
+
+        assert remi_ctx.summary is not None
+        relevance_match = re.search(r"Answer relevance: (\d+)/5\.", remi_ctx.summary)
+        assert relevance_match is not None, remi_ctx.summary
+        relevance_score = int(relevance_match.group(1))
+        assert relevance_score < 1, f"Relevance score {relevance_score} should be < 1"
+
+        groundedness_match = re.search(
+            r"Answer groundedness: (\d+)/5\.", remi_ctx.summary
+        )
+        assert groundedness_match is not None, remi_ctx.summary
+        groundedness_score = int(groundedness_match.group(1))
+        assert groundedness_score == 0, (
+            f"Groundedness score {groundedness_score} should be 0 when the question is not answered"
+        )
+
+        assert len(remi_ctx.chunks) > 0, "Missing evaluated chunks in REMi context"
+        assert remi_ctx.chunks[0].title is not None
+        assert "Groundedness" in remi_ctx.chunks[0].title, (
+            f"Missing score in title: {remi_ctx.chunks[0].title}"
+        )
+        assert "**Groundedness:" in remi_ctx.chunks[0].text, (
+            "Missing score text inside chunk text"
+        )
+
+        if granularity == ContextGranularity.FULL:
+            assert len(remi_ctx.chunks) == original_chunks
+            orig_ids = {
+                c.chunk_id
+                for ctx in question_memory.contexts
+                if ctx.agent != "remi"
+                for c in ctx.chunks
+                if c.chunk_id
+            }
+            for chunk in remi_ctx.chunks:
+                assert chunk.chunk_id in orig_ids or "remi_chunk_" in (
+                    chunk.chunk_id or ""
+                )
+        else:
+            # One chunk per context evaluated
+            ctx_list = [ctx for ctx in question_memory.contexts if ctx.agent != "remi"]
+            original_contexts = min(len(ctx_list), 60)
+            assert len(remi_ctx.chunks) == original_contexts, (
+                f"remi_ctx: {len(remi_ctx.chunks)}, orig: {original_contexts}. ctx_list: {[c.agent for c in ctx_list]}"
+            )
+            orig_ids = {c.id for c in ctx_list if c.id}
+            for chunk in remi_ctx.chunks:
+                assert chunk.chunk_id in orig_ids or (chunk.chunk_id or "").startswith(
+                    "remi_"
+                )
+
+        assert "Errors:" not in remi_ctx.summary