causaliq-knowledge 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

causaliq_knowledge/graph/response.py

@@ -0,0 +1,392 @@
+"""Response models and parsing for LLM graph generation.
+
+This module provides Pydantic models for representing LLM-generated
+causal graphs and functions for parsing LLM responses in both edge
+list and adjacency matrix formats.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+logger = logging.getLogger(__name__)
+
+
+class ProposedEdge(BaseModel):
+    """A proposed causal edge from LLM graph generation.
+
+    Represents a single directed edge in the proposed causal graph,
+    with confidence score and optional reasoning.
+
+    Attributes:
+        source: The name of the source variable (cause).
+        target: The name of the target variable (effect).
+        confidence: Confidence score from 0.0 to 1.0.
+        reasoning: Optional explanation for this specific edge.
+
+    Example:
+        >>> edge = ProposedEdge(
+        ...     source="smoking",
+        ...     target="lung_cancer",
+        ...     confidence=0.95,
+        ... )
+        >>> print(f"{edge.source} -> {edge.target}: {edge.confidence}")
+        smoking -> lung_cancer: 0.95
+    """
+
+    source: str = Field(..., description="Source variable name (cause)")
+    target: str = Field(..., description="Target variable name (effect)")
+    confidence: float = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="Confidence score from 0.0 to 1.0",
+    )
+    reasoning: Optional[str] = Field(
+        default=None,
+        description="Optional explanation for this edge",
+    )
+
+    @field_validator("confidence", mode="before")
+    @classmethod
+    def clamp_confidence(cls, v: Any) -> float:
+        """Clamp confidence values to [0.0, 1.0] range."""
+        if v is None:
+            return 0.5
+        try:
+            val = float(v)
+            return max(0.0, min(1.0, val))
+        except (TypeError, ValueError):
+            return 0.5
+
+
+@dataclass
+class GenerationMetadata:
+    """Metadata about a graph generation request.
+
+    Attributes:
+        model: The LLM model used for generation.
+        provider: The LLM provider (e.g., "groq", "gemini").
+        timestamp: When the graph was generated.
+        latency_ms: Request latency in milliseconds.
+        input_tokens: Number of input tokens used.
+        output_tokens: Number of output tokens generated.
+        cost_usd: Estimated cost in USD (if available).
+        from_cache: Whether the response was from cache.
+    """
+
+    model: str
+    provider: str = ""
+    timestamp: datetime = field(
+        default_factory=lambda: datetime.now(timezone.utc)
+    )
+    latency_ms: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cost_usd: float = 0.0
+    from_cache: bool = False
+
+
+@dataclass
+class GeneratedGraph:
+    """A complete causal graph generated by an LLM.
+
+    Represents the full output from an LLM graph generation query,
+    including all proposed edges, metadata, and the LLM's reasoning.
+
+    Attributes:
+        edges: List of proposed causal edges.
+        variables: List of variable names in the graph.
+        reasoning: Overall reasoning provided by the LLM.
+        metadata: Generation metadata (model, timing, etc.).
+        raw_response: The original LLM response for debugging.
+
+    Example:
+        >>> edge1 = ProposedEdge(
+        ...     source="age", target="income", confidence=0.7
+        ... )
+        >>> edge2 = ProposedEdge(
+        ...     source="education", target="income", confidence=0.9
+        ... )
+        >>> graph = GeneratedGraph(
+        ...     edges=[edge1, edge2],
+        ...     variables=["age", "education", "income"],
+        ...     reasoning="Age and education both influence income.",
+        ...     metadata=GenerationMetadata(model="llama-3.1-8b-instant"),
+        ... )
+        >>> print(f"Generated {len(graph.edges)} edges")
+        Generated 2 edges
+    """
+
+    edges: List[ProposedEdge]
+    variables: List[str]
+    reasoning: str = ""
+    metadata: Optional[GenerationMetadata] = None
+    raw_response: Optional[Dict[str, Any]] = field(default=None, repr=False)
+
+    def get_adjacency_matrix(self) -> List[List[float]]:
+        """Convert edges to an adjacency matrix.
+
+        Creates a square matrix where entry (i,j) represents the
+        confidence that variable i causes variable j.
+
+        Returns:
+            Square matrix of confidence scores.
+        """
+        n = len(self.variables)
+        var_to_idx = {var: i for i, var in enumerate(self.variables)}
+        matrix = [[0.0] * n for _ in range(n)]
+
+        for edge in self.edges:
+            src_idx = var_to_idx.get(edge.source)
+            tgt_idx = var_to_idx.get(edge.target)
+            if src_idx is not None and tgt_idx is not None:
+                matrix[src_idx][tgt_idx] = edge.confidence
+
+        return matrix
+
+    def get_edge_list(self) -> List[tuple[str, str, float]]:
+        """Get edges as a list of tuples.
+
+        Returns:
+            List of (source, target, confidence) tuples.
+        """
+        return [
+            (edge.source, edge.target, edge.confidence) for edge in self.edges
+        ]
+
+    def filter_by_confidence(self, threshold: float = 0.5) -> "GeneratedGraph":
+        """Return a new graph with only edges above the threshold.
+
+        Args:
+            threshold: Minimum confidence score to include.
+
+        Returns:
+            New GeneratedGraph with filtered edges.
+        """
+        filtered_edges = [e for e in self.edges if e.confidence >= threshold]
+        return GeneratedGraph(
+            edges=filtered_edges,
+            variables=self.variables,
+            reasoning=self.reasoning,
+            metadata=self.metadata,
+            raw_response=self.raw_response,
+        )
+
+
+def parse_edge_list_response(
+    response: Dict[str, Any],
+    variables: List[str],
+) -> GeneratedGraph:
+    """Parse an edge list format response from the LLM.
+
+    Expected JSON format:
+    {
+        "edges": [
+            {"source": "var1", "target": "var2", "confidence": 0.8},
+            ...
+        ],
+        "reasoning": "explanation"
+    }
+
+    Args:
+        response: Parsed JSON response from the LLM.
+        variables: List of valid variable names for validation.
+
+    Returns:
+        GeneratedGraph with parsed edges.
+
+    Raises:
+        ValueError: If the response format is invalid.
+    """
+    if not isinstance(response, dict):
+        raise ValueError(f"Expected dict response, got {type(response)}")
+
+    edges_data = response.get("edges", [])
+    if not isinstance(edges_data, list):
+        raise ValueError(
+            f"Expected 'edges' to be a list, got {type(edges_data)}"
+        )
+
+    reasoning = response.get("reasoning", "")
+    if not isinstance(reasoning, str):
+        reasoning = str(reasoning)
+
+    valid_vars = set(variables)
+    edges = []
+
+    for i, edge_data in enumerate(edges_data):
+        if not isinstance(edge_data, dict):
+            logger.warning(f"Skipping invalid edge at index {i}: not a dict")
+            continue
+
+        source = edge_data.get("source", "")
+        target = edge_data.get("target", "")
+
+        # Validate variable names
+        if source not in valid_vars:
+            logger.warning(f"Unknown source variable: {source}")
+            continue
+        if target not in valid_vars:
+            logger.warning(f"Unknown target variable: {target}")
+            continue
+        if source == target:
+            logger.warning(f"Skipping self-loop: {source} -> {target}")
+            continue
+
+        confidence = edge_data.get("confidence", 0.5)
+        edge_reasoning = edge_data.get("reasoning")
+
+        edges.append(
+            ProposedEdge(
+                source=source,
+                target=target,
+                confidence=confidence,
+                reasoning=edge_reasoning,
+            )
+        )
+
+    return GeneratedGraph(
+        edges=edges,
+        variables=variables,
+        reasoning=reasoning,
+        raw_response=response,
+    )
+
+
+def parse_adjacency_matrix_response(
+    response: Dict[str, Any],
+    expected_variables: List[str],
+) -> GeneratedGraph:
+    """Parse an adjacency matrix format response from the LLM.
+
+    Expected JSON format:
+    {
+        "variables": ["var1", "var2", "var3"],
+        "adjacency_matrix": [
+            [0.0, 0.8, 0.0],
+            [0.0, 0.0, 0.6],
+            [0.0, 0.0, 0.0]
+        ],
+        "reasoning": "explanation"
+    }
+
+    Args:
+        response: Parsed JSON response from the LLM.
+        expected_variables: List of expected variable names for validation.
+
+    Returns:
+        GeneratedGraph with edges extracted from the matrix.
+
+    Raises:
+        ValueError: If the response format is invalid.
+    """
+    if not isinstance(response, dict):
+        raise ValueError(f"Expected dict response, got {type(response)}")
+
+    variables = response.get("variables", [])
+    if not isinstance(variables, list):
+        raise ValueError(
+            f"Expected 'variables' to be a list, got {type(variables)}"
+        )
+
+    matrix = response.get("adjacency_matrix", [])
+    if not isinstance(matrix, list):
+        raise ValueError(
+            f"Expected 'adjacency_matrix' to be a list, got {type(matrix)}"
+        )
+
+    reasoning = response.get("reasoning", "")
+    if not isinstance(reasoning, str):
+        reasoning = str(reasoning)
+
+    # Validate matrix dimensions
+    n = len(variables)
+    if len(matrix) != n:
+        raise ValueError(
+            f"Matrix has {len(matrix)} rows but {n} variables declared"
+        )
+
+    for i, row in enumerate(matrix):
+        if not isinstance(row, list) or len(row) != n:
+            raise ValueError(f"Matrix row {i} has incorrect dimensions")
+
+    # Validate variables against expected
+    valid_vars = set(expected_variables)
+    for var in variables:
+        if var not in valid_vars:
+            logger.warning(f"Unexpected variable in response: {var}")
+
+    # Extract edges from matrix
+    edges = []
+    for i, source in enumerate(variables):
+        for j, target in enumerate(variables):
+            confidence = matrix[i][j]
+            try:
+                confidence = float(confidence)
+            except (TypeError, ValueError):
+                continue
+
+            if confidence > 0.0 and i != j:  # Skip zeros and self-loops
+                edges.append(
+                    ProposedEdge(
+                        source=source,
+                        target=target,
+                        confidence=max(0.0, min(1.0, confidence)),
+                    )
+                )
+
+    return GeneratedGraph(
+        edges=edges,
+        variables=variables,
+        reasoning=reasoning,
+        raw_response=response,
+    )
+
+
+def parse_graph_response(
+    response_text: str,
+    variables: List[str],
+    output_format: str = "edge_list",
+) -> GeneratedGraph:
+    """Parse an LLM response into a GeneratedGraph.
+
+    Handles JSON extraction from markdown code blocks and parses
+    according to the specified output format.
+
+    Args:
+        response_text: Raw text response from the LLM.
+        variables: List of valid variable names.
+        output_format: Expected format ("edge_list" or "adjacency_matrix").
+
+    Returns:
+        GeneratedGraph with parsed edges and metadata.
+
+    Raises:
+        ValueError: If JSON parsing fails or format is invalid.
+    """
+    # Clean up potential markdown code blocks
+    text = response_text.strip()
+    if text.startswith("```json"):
+        text = text[7:]
+    elif text.startswith("```"):
+        text = text[3:]
+    if text.endswith("```"):
+        text = text[:-3]
+    text = text.strip()
+
+    try:
+        response = json.loads(text)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Failed to parse JSON response: {e}")
+
+    if output_format == "adjacency_matrix":
+        return parse_adjacency_matrix_response(response, variables)
+    else:
+        return parse_edge_list_response(response, variables)

causaliq_knowledge/graph/view_filter.py

@@ -0,0 +1,154 @@
+"""Prompt detail filtering for model specifications.
+
+This module provides functionality to extract filtered views
+(minimal, standard, rich) from model specifications for LLM prompts.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from causaliq_knowledge.graph.models import ModelSpec, VariableSpec
+
+
+class PromptDetail(str, Enum):
+    """Level of detail for variable information in prompts."""
+
+    MINIMAL = "minimal"
+    STANDARD = "standard"
+    RICH = "rich"
+
+
+class ViewFilter:
+    """Filter model specifications to extract specific detail levels.
+
+    This class extracts variable information according to the view
+    definitions in the model specification (minimal, standard, rich).
+
+    By default, llm_name is substituted for name in the output to prevent
+    LLM memorisation of benchmark networks. Use use_llm_names=False to
+    output benchmark names directly (for memorisation testing).
+
+    Example:
+        >>> spec = ModelLoader.load("model.json")
+        >>> view_filter = ViewFilter(spec)
+        >>> minimal_vars = view_filter.filter_variables(PromptDetail.MINIMAL)
+        >>> # Returns list of dicts with llm_name as 'name' field
+    """
+
+    def __init__(self, spec: ModelSpec, *, use_llm_names: bool = True) -> None:
+        """Initialise the view filter.
+
+        Args:
+            spec: The model specification to filter.
+            use_llm_names: If True (default), output llm_name as 'name'.
+                If False, output benchmark name as 'name'.
+        """
+        self._spec = spec
+        self._use_llm_names = use_llm_names
+
+    @property
+    def spec(self) -> ModelSpec:
+        """Return the model specification."""
+        return self._spec
+
+    def get_include_fields(self, level: PromptDetail) -> list[str]:
+        """Get the fields to include for a given detail level.
+
+        Args:
+            level: The prompt detail level (minimal, standard, rich).
+
+        Returns:
+            List of field names to include.
+        """
+        if level == PromptDetail.MINIMAL:
+            return self._spec.prompt_details.minimal.include_fields
+        elif level == PromptDetail.STANDARD:
+            return self._spec.prompt_details.standard.include_fields
+        elif level == PromptDetail.RICH:
+            return self._spec.prompt_details.rich.include_fields
+        else:  # pragma: no cover
+            raise ValueError(f"Unknown prompt detail level: {level}")
+
+    def filter_variable(
+        self,
+        variable: VariableSpec,
+        level: PromptDetail,
+    ) -> dict[str, Any]:
+        """Filter a single variable to include only specified fields.
+
+        Args:
+            variable: The variable specification to filter.
+            level: The prompt detail level determining which fields to include.
+
+        Returns:
+            Dictionary with only the fields specified by the detail level.
+            Enum values are converted to their string representations.
+            If use_llm_names is True, the 'name' field contains llm_name.
+        """
+        include_fields = self.get_include_fields(level)
+        # Use mode="json" to convert enums to their string values
+        var_dict = variable.model_dump(mode="json")
+
+        # If using llm_names, substitute llm_name for name in output
+        if self._use_llm_names and "name" in include_fields:
+            var_dict["name"] = var_dict.get("llm_name", var_dict["name"])
+
+        # Never include llm_name in output (it's internal)
+        return {
+            key: value
+            for key, value in var_dict.items()
+            if key in include_fields
+            and key != "llm_name"
+            and value is not None
+        }
+
+    def filter_variables(self, level: PromptDetail) -> list[dict[str, Any]]:
+        """Filter all variables to the specified detail level.
+
+        Args:
+            level: The prompt detail level (minimal, standard, rich).
+
+        Returns:
+            List of filtered variable dictionaries.
+        """
+        return [
+            self.filter_variable(var, level) for var in self._spec.variables
+        ]
+
+    def get_variable_names(self) -> list[str]:
+        """Get all variable names for LLM output.
+
+        Returns benchmark names if use_llm_names is False,
+        otherwise returns llm_names.
+
+        Returns:
+            List of variable names.
+        """
+        if self._use_llm_names:
+            return self._spec.get_llm_names()
+        return self._spec.get_variable_names()
+
+    def get_domain(self) -> str:
+        """Get the domain from the specification.
+
+        Returns:
+            The domain string.
+        """
+        return self._spec.domain
+
+    def get_context_summary(self, level: PromptDetail) -> dict[str, Any]:
+        """Get a complete context summary for LLM prompts.
+
+        Args:
+            level: The prompt detail level for variable filtering.
+
+        Returns:
+            Dictionary with domain and filtered variables.
+        """
+        return {
+            "domain": self._spec.domain,
+            "dataset_id": self._spec.dataset_id,
+            "variables": self.filter_variables(level),
+        }

causaliq_knowledge/llm/base_client.py

@@ -297,6 +297,8 @@ class BaseLLMClient(ABC):
         Args:
             messages: List of message dicts with "role" and "content" keys.
             **kwargs: Provider-specific options (temperature, max_tokens, etc.)
+                Also accepts request_id (str) for identifying requests in
+                exports. Note: request_id is NOT part of the cache key.
 
         Returns:
             LLMResponse with the generated content and metadata.
@@ -306,6 +308,9 @@ class BaseLLMClient(ABC):
         cache = self.cache
         use_cache = self.use_cache
 
+        # Extract request_id (not part of cache key)
+        request_id = kwargs.pop("request_id", "")
+
         # Build cache key
         temperature = kwargs.get("temperature")
         max_tokens = kwargs.get("max_tokens")
@@ -354,6 +359,7 @@ class BaseLLMClient(ABC):
                 input_tokens=response.input_tokens,
                 output_tokens=response.output_tokens,
                 cost_usd=response.cost,
+                request_id=request_id,
             )
             cache.put_data(cache_key, "llm", entry.to_dict())