hyperforge 1.0.0.post19__py3-none-any.whl

hyperforge/minimal_fixtures.py

@@ -0,0 +1,75 @@
+"""Lightweight pytest fixtures for hyperforge agents.
+
+This module only depends on pytest and the stdlib — no docker, database, or
+nucliadb deps. Safe to use as a pytest plugin in any agent repo:
+
+    # tests/conftest.py
+    pytest_plugins = ["hyperforge.minimal_fixtures"]
+"""
+
+import base64
+import json
+import logging
+
+import pytest
+
+
+def cassette_nua_key(iss: str) -> str:
+    """Return a minimal parseable JWT stub for cassette-replay runs.
+
+    validate_nua() decodes the middle part of the JWT to extract the ``iss``
+    field before making any HTTP call.  When cassettes are present VCR
+    intercepts that HTTP call, so the key doesn't need to be real — it just
+    needs to parse.
+    """
+    payload = base64.b64encode(json.dumps({"iss": iss}).encode()).decode().rstrip("=")
+    return f"cassette.{payload}.stub"
+
+
+class _VCRTaskExceptionFilter(logging.Filter):
+    """Suppress 'Task exception was never retrieved' asyncio errors from vcrpy.
+
+    vcrpy's httpx stub creates a background task (_record_responses) that can
+    fail with an AssertionError due to a vcrpy/httpx version incompatibility.
+    The exception is noisy but harmless in test runs.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return not (
+            record.levelno == logging.ERROR
+            and "Task exception was never retrieved" in record.getMessage()
+            and "_record_responses" in record.getMessage()
+        )
+
+
+@pytest.fixture(scope="module")
+def vcr_config():
+    return {
+        # Replaces the actual token with 'DUMMY' in the recorded YAML
+        "filter_headers": [
+            ("Authorization", "DUMMY"),
+            ("x-nuclia-nuakey", "DUMMY"),
+            ("x-stf-nuakey", "DUMMY"),
+            ("x-goog-api-key", "DUMMY"),
+        ],
+        # Redacts specific query parameters like API keys
+        "filter_query_parameters": ["api_key", "access_token", "key"],
+        # Redacts fields in POST request bodies (e.g., login forms)
+        "filter_post_data_parameters": ["password", "client_secret"],
+        # Decodes compressed responses so they are human-readable in the cassette
+        "decode_compressed_response": True,
+    }
+
+
+@pytest.fixture(autouse=True, scope="session")
+def suppress_test_noise() -> None:
+    """Suppress known-noisy log lines that add no diagnostic value in tests."""
+    logging.getLogger("hyperforge.memory").setLevel(logging.WARNING)
+    logging.getLogger("mcp.server.streamable_http").setLevel(logging.WARNING)
+    logging.getLogger("hyperforge.server").setLevel(logging.WARNING)
+    logging.getLogger("httpx").setLevel(logging.ERROR)
+    logging.getLogger("httpcore.connection").setLevel(logging.ERROR)
+    logging.getLogger("httpcore.http11").setLevel(logging.ERROR)
+    logging.getLogger("asyncio").setLevel(logging.INFO)
+
+    logging.getLogger("asyncio").addFilter(_VCRTaskExceptionFilter())

hyperforge/models.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+import uuid
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Union,
+)
+
+from nuclia.lib.nua_responses import Image, StoredLearningConfiguration
+from nucliadb_models.resource import (
+    ConversationFieldData,
+    FileFieldData,
+    GenericFieldData,
+    LinkFieldData,
+    TextFieldData,
+)
+from nucliadb_models.search import CatalogFacetsResponse
+from pydantic import BaseModel, Field
+
+from hyperforge import PROMPT_ENVIRONMENT, logger
+
+
+class Metadata:
+    pass
+
+
+class KnowledgeGraph:
+    pass
+
+
+class Reason:
+    pass
+
+
+class NucliaDBMemoryConfig(BaseModel):
+    key: Optional[str] = None
+    url: str
+    kbid: str
+    internal: bool = True
+
+
+class MemoryConfig(BaseModel):
+    nucliadb: Optional[NucliaDBMemoryConfig] = None
+
+
+class Rule(BaseModel):
+    prompt: Optional[str] = None
+
+
+class Rules(BaseModel):
+    rules: List[Union[Rule, str]] = Field(
+        default_factory=list,
+        description="List of rules that the workflow should follow. Each rule can be a string or a Rule object with a prompt.",
+    )
+
+
+class Facets(BaseModel):
+    chunks: Dict[str, int]
+    fields: Dict[str, int]
+
+
+class Source(BaseModel):
+    id: str
+    description: str
+    labels: Dict[str, List[str]]
+    facets_native: CatalogFacetsResponse
+    paragraph_facets: Dict[str, int]
+    learning_configuration: StoredLearningConfiguration
+
+
+class CitationMetadata(BaseModel):
+    context_id: str = Field(
+        description="ID of the context this citation refers to",
+    )
+    origin_urls: list[str] = Field(
+        default_factory=list,
+        description="List of origin URLs that this citation refers to",
+    )
+    chunk_index: Optional[int] = Field(
+        default=None,
+        description="Index of the chunk in the context's chunks list. This is only set for chunk-level citations.",
+    )
+
+
+class AnswerCitations(BaseModel):
+    metadata: dict[str, CitationMetadata] = Field(
+        default_factory=dict,
+        description="Map of citation_id to citation metadata. block-AA",
+    )
+
+
+class VegaLiteVisualization(BaseModel):
+    type: Literal["vega_lite"] = "vega_lite"
+    vega_lite_obj: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="The Vega-Lite Object defining the visualization. Previously validated against the Vega-Lite schema.",
+    )
+
+    # If we do server-side rendering in the future, we can add fields like:
+    # svg: Optional[str] = ...
+
+
+# For once we add more visualization types, we can use a Discriminator
+# Visualization = Annotated[Union[VegaLiteVisualization,NewType], Discriminator("type")]
+# For now, we only have one type.
+Visualization = Union[VegaLiteVisualization]
+
+
+class Step(BaseModel):
+    original_question_uuid: Optional[str]
+    actual_question_uuid: Optional[str]
+    module: str
+    title: str
+    value: Optional[str] = None
+    agent_path: str
+    reason: Optional[str] = None
+    timeit: float
+    input_nuclia_tokens: Optional[float]
+    output_nuclia_tokens: Optional[float]
+    error: Optional[str] = None
+
+    def __str__(self):
+        return f"({self.timeit:.2f}s) {self.module}: {self.title} \n {self.value} \n {self.reason} \n NT:({self.input_nuclia_tokens}:{self.output_nuclia_tokens})"
+
+    def markdown(self):
+        return f"""
+## {self.title}
+
+{self.value}
+
+- reason: {self.reason}
+- timeit: {self.timeit}
+- input_tokens: {self.input_nuclia_tokens}
+- output_tokens: {self.output_nuclia_tokens}
+"""
+
+
+class ChunkImages(BaseModel):
+    table: Optional[str]
+    chunk: Optional[str]
+    page: Optional[str]
+
+
+FieldTypes = Union[
+    TextFieldData,
+    ConversationFieldData,
+    FileFieldData,
+    LinkFieldData,
+    GenericFieldData,
+]
+
+
+class Chunk(BaseModel):
+    chunk_id: str
+    title: Optional[str] = None
+    source: Optional[str] = None
+    text: str
+    labels: List[str] = Field(default_factory=list)
+    url: List[str] = Field(default_factory=list)
+    metadata: Optional[Dict[str, Any]] = None
+    action: Optional[str] = Field(
+        default=None,
+        description="agent and function called to get this chunk.",
+    )
+    origin_url: Optional[str] = Field(
+        default=None,
+        description="URL at the origin of the resource from which this chunk was extracted.",
+    )
+    origin_agent: Optional[str] = Field(
+        default=None,
+        description="Agent that originated this chunk. This is useful to keep track of the provenance of the information ",
+    )
+
+    def render(
+        self,
+        citations_id: Optional[str] = None,
+    ) -> str:
+        if citations_id:
+            lines = [f"## Chunk: [{citations_id}] {self.title or self.chunk_id}"]
+        else:
+            lines = [f"## Chunk: {self.title or self.chunk_id}"]
+        if self.action:
+            lines.append(f"Result of running: {self.action}")
+        if self.labels:
+            lines.append(f"Tags: {', '.join(self.labels)}")
+        if self.url:
+            lines.append(f"URLs: {', '.join(self.url)}")
+        lines.append(f"``` {self.text} ```\n")
+        return "\n".join(lines)
+
+
+class Prompt(BaseModel):
+    prompt: str
+    resources: List[str] = Field(default_factory=list)
+    links: List[str] = Field(default_factory=list)
+    description: Optional[str] = None
+
+    def render(self) -> str:
+        lines = ["## Prompt"]
+        if self.description:
+            lines.append(f"Description: {self.description}")
+        if self.resources:
+            lines.append(f"Resources: {', '.join(self.resources)}")
+        if self.links:
+            lines.append(f"Links: {', '.join(self.links)}")
+        lines.append(f"```PROMPT\n{self.prompt}\n```\n")
+        return "\n".join(lines)
+
+
+class Answer(BaseModel):
+    answer: str
+    original_question_uuid: Optional[str]
+    actual_question_uuid: Optional[str]
+    module: str
+    agent_path: str
+    data_visualizations: Optional[list[Visualization]] = None
+    citations: Optional[AnswerCitations] = None
+    chunks: Optional[list[Chunk]] = None
+    structured: Optional[list[str]] = None
+    images: Optional[Dict[str, Image]] = None
+    image_urls: Optional[list[str]] = None
+
+
+CONTEXT_TEMPLATE = """
+
+{% if con.citations_id is not none -%}
+{% for chunk in con.chunks %}
+{{chunk.render(citations_id=con.citations_id ~ "-" ~ loop.index0)}}
+{% endfor -%}
+{% else -%}
+{% for chunk in con.chunks %}
+{{chunk.render()}}
+{% endfor -%}
+{% endif -%}
+
+{% if con.structured | length > 0 -%}
+## Extra structured info:
+{% for structured in con.structured %}
+{{structured}}
+{% endfor -%}
+{% endif -%}
+"""
+
+CONTEXT_PROMPT_TEMPLATE = PROMPT_ENVIRONMENT.from_string(CONTEXT_TEMPLATE)
+
+
+class Context(BaseModel):
+    id: str = Field(
+        default_factory=lambda: uuid.uuid4().hex,
+        description="Unique identifier for this context instance",
+    )
+    original_question_uuid: Optional[str]
+    actual_question_uuid: Optional[str]
+    question: str
+    chunks: List[Chunk] = Field(default_factory=list)
+    images: Dict[str, Image] = Field(default_factory=dict)
+    prompts: List[Prompt] = Field(default_factory=list)
+    structured: List[str] = Field(default_factory=list)
+    source: str
+    agent: str
+    # XXX: This is not actually a summary, but an answer attempt for now!
+    summary: str = Field(
+        default="",
+        description="Partial or full answer to the question, generated by the context validation step inside a context agent.",
+    )
+    agent_id: str = ""
+    title: Optional[str] = None
+    missing: Optional[str] = None
+    citations: list[str] | None = Field(
+        default=None,
+        description="List of chunk IDs that were considered relevant in the context validation step.",
+    )
+    citations_id: Optional[str] = Field(
+        default=None,
+        description="Block ID used for citations in this context.",
+    )
+    image_urls: List[str] = Field(
+        default_factory=list,
+        description="List of image URLs associated with this context.",
+    )
+
+    def answer_summary_markdown(self) -> str:
+        return "# {question}\n\n {summary}".format(
+            question=self.question, summary=self.summary
+        )
+
+    def context_markdown(self) -> str:
+        return CONTEXT_PROMPT_TEMPLATE.render(con=self)
+
+    def stats(self) -> Dict[str, int | str | None]:
+        return {
+            "chunks": len(self.chunks),
+            "images": len(self.images),
+            "structured": len(self.structured),
+            "source": self.source,
+            "question": self.question,
+            "agent": self.agent,
+            "summary": self.summary,
+            "title": self.title,
+            "missing": self.missing,
+        }
+
+    def prune_to_citations(self) -> None:
+        if self.citations is None:
+            logger.warning(
+                "Cannot prune context as no citations are available.",
+                extra={
+                    "agent": self.agent,
+                    "source": self.source,
+                    "agent_id": self.agent_id,
+                },
+            )
+            return
+        self.chunks = [
+            chunk for chunk in self.chunks if chunk.chunk_id in self.citations
+        ]
+        self.structured = [
+            s
+            for i, s in enumerate(self.structured)
+            if f"structured-{i}" in self.citations
+        ]
+
+
+class HistoryQuestionAnswer(BaseModel):
+    question: str
+    answer: str
+
+
+class TrackingInfo(BaseModel):
+    rao_id: str
+    session: str
+    message: str