causaliq-knowledge 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

causaliq_knowledge/graph/response.py

@@ -0,0 +1,392 @@
+"""Response models and parsing for LLM graph generation.
+
+This module provides Pydantic models for representing LLM-generated
+causal graphs and functions for parsing LLM responses in both edge
+list and adjacency matrix formats.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+logger = logging.getLogger(__name__)
+
+
+class ProposedEdge(BaseModel):
+    """A proposed causal edge from LLM graph generation.
+
+    Represents a single directed edge in the proposed causal graph,
+    with confidence score and optional reasoning.
+
+    Attributes:
+        source: The name of the source variable (cause).
+        target: The name of the target variable (effect).
+        confidence: Confidence score from 0.0 to 1.0.
+        reasoning: Optional explanation for this specific edge.
+
+    Example:
+        >>> edge = ProposedEdge(
+        ...     source="smoking",
+        ...     target="lung_cancer",
+        ...     confidence=0.95,
+        ... )
+        >>> print(f"{edge.source} -> {edge.target}: {edge.confidence}")
+        smoking -> lung_cancer: 0.95
+    """
+
+    source: str = Field(..., description="Source variable name (cause)")
+    target: str = Field(..., description="Target variable name (effect)")
+    confidence: float = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="Confidence score from 0.0 to 1.0",
+    )
+    reasoning: Optional[str] = Field(
+        default=None,
+        description="Optional explanation for this edge",
+    )
+
+    @field_validator("confidence", mode="before")
+    @classmethod
+    def clamp_confidence(cls, v: Any) -> float:
+        """Clamp confidence values to [0.0, 1.0] range."""
+        if v is None:
+            return 0.5
+        try:
+            val = float(v)
+            return max(0.0, min(1.0, val))
+        except (TypeError, ValueError):
+            return 0.5
+
+
+@dataclass
+class GenerationMetadata:
+    """Metadata about a graph generation request.
+
+    Attributes:
+        model: The LLM model used for generation.
+        provider: The LLM provider (e.g., "groq", "gemini").
+        timestamp: When the graph was generated.
+        latency_ms: Request latency in milliseconds.
+        input_tokens: Number of input tokens used.
+        output_tokens: Number of output tokens generated.
+        cost_usd: Estimated cost in USD (if available).
+        from_cache: Whether the response was from cache.
+    """
+
+    model: str
+    provider: str = ""
+    timestamp: datetime = field(
+        default_factory=lambda: datetime.now(timezone.utc)
+    )
+    latency_ms: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cost_usd: float = 0.0
+    from_cache: bool = False
+
+
+@dataclass
+class GeneratedGraph:
+    """A complete causal graph generated by an LLM.
+
+    Represents the full output from an LLM graph generation query,
+    including all proposed edges, metadata, and the LLM's reasoning.
+
+    Attributes:
+        edges: List of proposed causal edges.
+        variables: List of variable names in the graph.
+        reasoning: Overall reasoning provided by the LLM.
+        metadata: Generation metadata (model, timing, etc.).
+        raw_response: The original LLM response for debugging.
+
+    Example:
+        >>> edge1 = ProposedEdge(
+        ...     source="age", target="income", confidence=0.7
+        ... )
+        >>> edge2 = ProposedEdge(
+        ...     source="education", target="income", confidence=0.9
+        ... )
+        >>> graph = GeneratedGraph(
+        ...     edges=[edge1, edge2],
+        ...     variables=["age", "education", "income"],
+        ...     reasoning="Age and education both influence income.",
+        ...     metadata=GenerationMetadata(model="llama-3.1-8b-instant"),
+        ... )
+        >>> print(f"Generated {len(graph.edges)} edges")
+        Generated 2 edges
+    """
+
+    edges: List[ProposedEdge]
+    variables: List[str]
+    reasoning: str = ""
+    metadata: Optional[GenerationMetadata] = None
+    raw_response: Optional[Dict[str, Any]] = field(default=None, repr=False)
+
+    def get_adjacency_matrix(self) -> List[List[float]]:
+        """Convert edges to an adjacency matrix.
+
+        Creates a square matrix where entry (i,j) represents the
+        confidence that variable i causes variable j.
+
+        Returns:
+            Square matrix of confidence scores.
+        """
+        n = len(self.variables)
+        var_to_idx = {var: i for i, var in enumerate(self.variables)}
+        matrix = [[0.0] * n for _ in range(n)]
+
+        for edge in self.edges:
+            src_idx = var_to_idx.get(edge.source)
+            tgt_idx = var_to_idx.get(edge.target)
+            if src_idx is not None and tgt_idx is not None:
+                matrix[src_idx][tgt_idx] = edge.confidence
+
+        return matrix
+
+    def get_edge_list(self) -> List[tuple[str, str, float]]:
+        """Get edges as a list of tuples.
+
+        Returns:
+            List of (source, target, confidence) tuples.
+        """
+        return [
+            (edge.source, edge.target, edge.confidence) for edge in self.edges
+        ]
+
+    def filter_by_confidence(self, threshold: float = 0.5) -> "GeneratedGraph":
+        """Return a new graph with only edges above the threshold.
+
+        Args:
+            threshold: Minimum confidence score to include.
+
+        Returns:
+            New GeneratedGraph with filtered edges.
+        """
+        filtered_edges = [e for e in self.edges if e.confidence >= threshold]
+        return GeneratedGraph(
+            edges=filtered_edges,
+            variables=self.variables,
+            reasoning=self.reasoning,
+            metadata=self.metadata,
+            raw_response=self.raw_response,
+        )
+
+
+def parse_edge_list_response(
+    response: Dict[str, Any],
+    variables: List[str],
+) -> GeneratedGraph:
+    """Parse an edge list format response from the LLM.
+
+    Expected JSON format:
+    {
+        "edges": [
+            {"source": "var1", "target": "var2", "confidence": 0.8},
+            ...
+        ],
+        "reasoning": "explanation"
+    }
+
+    Args:
+        response: Parsed JSON response from the LLM.
+        variables: List of valid variable names for validation.
+
+    Returns:
+        GeneratedGraph with parsed edges.
+
+    Raises:
+        ValueError: If the response format is invalid.
+    """
+    if not isinstance(response, dict):
+        raise ValueError(f"Expected dict response, got {type(response)}")
+
+    edges_data = response.get("edges", [])
+    if not isinstance(edges_data, list):
+        raise ValueError(
+            f"Expected 'edges' to be a list, got {type(edges_data)}"
+        )
+
+    reasoning = response.get("reasoning", "")
+    if not isinstance(reasoning, str):
+        reasoning = str(reasoning)
+
+    valid_vars = set(variables)
+    edges = []
+
+    for i, edge_data in enumerate(edges_data):
+        if not isinstance(edge_data, dict):
+            logger.warning(f"Skipping invalid edge at index {i}: not a dict")
+            continue
+
+        source = edge_data.get("source", "")
+        target = edge_data.get("target", "")
+
+        # Validate variable names
+        if source not in valid_vars:
+            logger.warning(f"Unknown source variable: {source}")
+            continue
+        if target not in valid_vars:
+            logger.warning(f"Unknown target variable: {target}")
+            continue
+        if source == target:
+            logger.warning(f"Skipping self-loop: {source} -> {target}")
+            continue
+
+        confidence = edge_data.get("confidence", 0.5)
+        edge_reasoning = edge_data.get("reasoning")
+
+        edges.append(
+            ProposedEdge(
+                source=source,
+                target=target,
+                confidence=confidence,
+                reasoning=edge_reasoning,
+            )
+        )
+
+    return GeneratedGraph(
+        edges=edges,
+        variables=variables,
+        reasoning=reasoning,
+        raw_response=response,
+    )
+
+
+def parse_adjacency_matrix_response(
+    response: Dict[str, Any],
+    expected_variables: List[str],
+) -> GeneratedGraph:
+    """Parse an adjacency matrix format response from the LLM.
+
+    Expected JSON format:
+    {
+        "variables": ["var1", "var2", "var3"],
+        "adjacency_matrix": [
+            [0.0, 0.8, 0.0],
+            [0.0, 0.0, 0.6],
+            [0.0, 0.0, 0.0]
+        ],
+        "reasoning": "explanation"
+    }
+
+    Args:
+        response: Parsed JSON response from the LLM.
+        expected_variables: List of expected variable names for validation.
+
+    Returns:
+        GeneratedGraph with edges extracted from the matrix.
+
+    Raises:
+        ValueError: If the response format is invalid.
+    """
+    if not isinstance(response, dict):
+        raise ValueError(f"Expected dict response, got {type(response)}")
+
+    variables = response.get("variables", [])
+    if not isinstance(variables, list):
+        raise ValueError(
+            f"Expected 'variables' to be a list, got {type(variables)}"
+        )
+
+    matrix = response.get("adjacency_matrix", [])
+    if not isinstance(matrix, list):
+        raise ValueError(
+            f"Expected 'adjacency_matrix' to be a list, got {type(matrix)}"
+        )
+
+    reasoning = response.get("reasoning", "")
+    if not isinstance(reasoning, str):
+        reasoning = str(reasoning)
+
+    # Validate matrix dimensions
+    n = len(variables)
+    if len(matrix) != n:
+        raise ValueError(
+            f"Matrix has {len(matrix)} rows but {n} variables declared"
+        )
+
+    for i, row in enumerate(matrix):
+        if not isinstance(row, list) or len(row) != n:
+            raise ValueError(f"Matrix row {i} has incorrect dimensions")
+
+    # Validate variables against expected
+    valid_vars = set(expected_variables)
+    for var in variables:
+        if var not in valid_vars:
+            logger.warning(f"Unexpected variable in response: {var}")
+
+    # Extract edges from matrix
+    edges = []
+    for i, source in enumerate(variables):
+        for j, target in enumerate(variables):
+            confidence = matrix[i][j]
+            try:
+                confidence = float(confidence)
+            except (TypeError, ValueError):
+                continue
+
+            if confidence > 0.0 and i != j:  # Skip zeros and self-loops
+                edges.append(
+                    ProposedEdge(
+                        source=source,
+                        target=target,
+                        confidence=max(0.0, min(1.0, confidence)),
+                    )
+                )
+
+    return GeneratedGraph(
+        edges=edges,
+        variables=variables,
+        reasoning=reasoning,
+        raw_response=response,
+    )
+
+
+def parse_graph_response(
+    response_text: str,
+    variables: List[str],
+    output_format: str = "edge_list",
+) -> GeneratedGraph:
+    """Parse an LLM response into a GeneratedGraph.
+
+    Handles JSON extraction from markdown code blocks and parses
+    according to the specified output format.
+
+    Args:
+        response_text: Raw text response from the LLM.
+        variables: List of valid variable names.
+        output_format: Expected format ("edge_list" or "adjacency_matrix").
+
+    Returns:
+        GeneratedGraph with parsed edges and metadata.
+
+    Raises:
+        ValueError: If JSON parsing fails or format is invalid.
+    """
+    # Clean up potential markdown code blocks
+    text = response_text.strip()
+    if text.startswith("```json"):
+        text = text[7:]
+    elif text.startswith("```"):
+        text = text[3:]
+    if text.endswith("```"):
+        text = text[:-3]
+    text = text.strip()
+
+    try:
+        response = json.loads(text)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Failed to parse JSON response: {e}")
+
+    if output_format == "adjacency_matrix":
+        return parse_adjacency_matrix_response(response, variables)
+    else:
+        return parse_edge_list_response(response, variables)

causaliq_knowledge/graph/view_filter.py

@@ -0,0 +1,154 @@
+"""Prompt detail filtering for model specifications.
+
+This module provides functionality to extract filtered views
+(minimal, standard, rich) from model specifications for LLM prompts.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from causaliq_knowledge.graph.models import ModelSpec, VariableSpec
+
+
+class PromptDetail(str, Enum):
+    """Level of detail for variable information in prompts."""
+
+    MINIMAL = "minimal"
+    STANDARD = "standard"
+    RICH = "rich"
+
+
+class ViewFilter:
+    """Filter model specifications to extract specific detail levels.
+
+    This class extracts variable information according to the view
+    definitions in the model specification (minimal, standard, rich).
+
+    By default, llm_name is substituted for name in the output to prevent
+    LLM memorisation of benchmark networks. Use use_llm_names=False to
+    output benchmark names directly (for memorisation testing).
+
+    Example:
+        >>> spec = ModelLoader.load("model.json")
+        >>> view_filter = ViewFilter(spec)
+        >>> minimal_vars = view_filter.filter_variables(PromptDetail.MINIMAL)
+        >>> # Returns list of dicts with llm_name as 'name' field
+    """
+
+    def __init__(self, spec: ModelSpec, *, use_llm_names: bool = True) -> None:
+        """Initialise the view filter.
+
+        Args:
+            spec: The model specification to filter.
+            use_llm_names: If True (default), output llm_name as 'name'.
+                If False, output benchmark name as 'name'.
+        """
+        self._spec = spec
+        self._use_llm_names = use_llm_names
+
+    @property
+    def spec(self) -> ModelSpec:
+        """Return the model specification."""
+        return self._spec
+
+    def get_include_fields(self, level: PromptDetail) -> list[str]:
+        """Get the fields to include for a given detail level.
+
+        Args:
+            level: The prompt detail level (minimal, standard, rich).
+
+        Returns:
+            List of field names to include.
+        """
+        if level == PromptDetail.MINIMAL:
+            return self._spec.prompt_details.minimal.include_fields
+        elif level == PromptDetail.STANDARD:
+            return self._spec.prompt_details.standard.include_fields
+        elif level == PromptDetail.RICH:
+            return self._spec.prompt_details.rich.include_fields
+        else:  # pragma: no cover
+            raise ValueError(f"Unknown prompt detail level: {level}")
+
+    def filter_variable(
+        self,
+        variable: VariableSpec,
+        level: PromptDetail,
+    ) -> dict[str, Any]:
+        """Filter a single variable to include only specified fields.
+
+        Args:
+            variable: The variable specification to filter.
+            level: The prompt detail level determining which fields to include.
+
+        Returns:
+            Dictionary with only the fields specified by the detail level.
+            Enum values are converted to their string representations.
+            If use_llm_names is True, the 'name' field contains llm_name.
+        """
+        include_fields = self.get_include_fields(level)
+        # Use mode="json" to convert enums to their string values
+        var_dict = variable.model_dump(mode="json")
+
+        # If using llm_names, substitute llm_name for name in output
+        if self._use_llm_names and "name" in include_fields:
+            var_dict["name"] = var_dict.get("llm_name", var_dict["name"])
+
+        # Never include llm_name in output (it's internal)
+        return {
+            key: value
+            for key, value in var_dict.items()
+            if key in include_fields
+            and key != "llm_name"
+            and value is not None
+        }
+
+    def filter_variables(self, level: PromptDetail) -> list[dict[str, Any]]:
+        """Filter all variables to the specified detail level.
+
+        Args:
+            level: The prompt detail level (minimal, standard, rich).
+
+        Returns:
+            List of filtered variable dictionaries.
+        """
+        return [
+            self.filter_variable(var, level) for var in self._spec.variables
+        ]
+
+    def get_variable_names(self) -> list[str]:
+        """Get all variable names for LLM output.
+
+        Returns benchmark names if use_llm_names is False,
+        otherwise returns llm_names.
+
+        Returns:
+            List of variable names.
+        """
+        if self._use_llm_names:
+            return self._spec.get_llm_names()
+        return self._spec.get_variable_names()
+
+    def get_domain(self) -> str:
+        """Get the domain from the specification.
+
+        Returns:
+            The domain string.
+        """
+        return self._spec.domain
+
+    def get_context_summary(self, level: PromptDetail) -> dict[str, Any]:
+        """Get a complete context summary for LLM prompts.
+
+        Args:
+            level: The prompt detail level for variable filtering.
+
+        Returns:
+            Dictionary with domain and filtered variables.
+        """
+        return {
+            "domain": self._spec.domain,
+            "dataset_id": self._spec.dataset_id,
+            "variables": self.filter_variables(level),
+        }

causaliq_knowledge/llm/base_client.py

@@ -5,11 +5,18 @@ must implement. This provides a consistent API regardless of the
 underlying LLM provider.
 """
 
+from __future__ import annotations
+
+import hashlib
 import json
 import logging
+import time
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+if TYPE_CHECKING:  # pragma: no cover
+    from causaliq_knowledge.cache import TokenCache
 
 logger = logging.getLogger(__name__)
 
@@ -218,3 +225,142 @@ class BaseLLMClient(ABC):
             Model identifier string.
         """
         return getattr(self, "config", LLMConfig(model="unknown")).model
+
+    def _build_cache_key(
+        self,
+        messages: List[Dict[str, str]],
+        temperature: Optional[float] = None,
+        max_tokens: Optional[int] = None,
+    ) -> str:
+        """Build a deterministic cache key for the request.
+
+        Creates a SHA-256 hash from the model, messages, temperature, and
+        max_tokens. The hash is truncated to 16 hex characters (64 bits).
+
+        Args:
+            messages: List of message dicts with "role" and "content" keys.
+            temperature: Sampling temperature (defaults to config value).
+            max_tokens: Maximum tokens (defaults to config value).
+
+        Returns:
+            16-character hex string cache key.
+        """
+        config = getattr(self, "config", LLMConfig(model="unknown"))
+        key_data = {
+            "model": config.model,
+            "messages": messages,
+            "temperature": (
+                temperature if temperature is not None else config.temperature
+            ),
+            "max_tokens": (
+                max_tokens if max_tokens is not None else config.max_tokens
+            ),
+        }
+        key_json = json.dumps(key_data, sort_keys=True, separators=(",", ":"))
+        return hashlib.sha256(key_json.encode()).hexdigest()[:16]
+
+    def set_cache(
+        self,
+        cache: Optional["TokenCache"],
+        use_cache: bool = True,
+    ) -> None:
+        """Configure caching for this client.
+
+        Args:
+            cache: TokenCache instance for caching, or None to disable.
+            use_cache: Whether to use the cache (default True).
+        """
+        self._cache = cache
+        self._use_cache = use_cache
+
+    @property
+    def cache(self) -> Optional["TokenCache"]:
+        """Return the configured cache, if any."""
+        return getattr(self, "_cache", None)
+
+    @property
+    def use_cache(self) -> bool:
+        """Return whether caching is enabled."""
+        return getattr(self, "_use_cache", True)
+
+    def cached_completion(
+        self,
+        messages: List[Dict[str, str]],
+        **kwargs: Any,
+    ) -> LLMResponse:
+        """Make a completion request with caching.
+
+        If caching is enabled and a cached response exists, returns
+        the cached response without making an API call. Otherwise,
+        makes the API call and caches the result.
+
+        Args:
+            messages: List of message dicts with "role" and "content" keys.
+            **kwargs: Provider-specific options (temperature, max_tokens, etc.)
+                Also accepts request_id (str) for identifying requests in
+                exports. Note: request_id is NOT part of the cache key.
+
+        Returns:
+            LLMResponse with the generated content and metadata.
+        """
+        from causaliq_knowledge.llm.cache import LLMCacheEntry, LLMEntryEncoder
+
+        cache = self.cache
+        use_cache = self.use_cache
+
+        # Extract request_id (not part of cache key)
+        request_id = kwargs.pop("request_id", "")
+
+        # Build cache key
+        temperature = kwargs.get("temperature")
+        max_tokens = kwargs.get("max_tokens")
+        cache_key = self._build_cache_key(messages, temperature, max_tokens)
+
+        # Check cache
+        if use_cache and cache is not None:
+            # Ensure encoder is registered
+            if not cache.has_encoder("llm"):
+                cache.register_encoder("llm", LLMEntryEncoder())
+
+            if cache.exists(cache_key, "llm"):
+                cached_data = cache.get_data(cache_key, "llm")
+                if cached_data is not None:
+                    entry = LLMCacheEntry.from_dict(cached_data)
+                    return LLMResponse(
+                        content=entry.response.content,
+                        model=entry.model,
+                        input_tokens=entry.metadata.tokens.input,
+                        output_tokens=entry.metadata.tokens.output,
+                        cost=entry.metadata.cost_usd or 0.0,
+                    )
+
+        # Make API call with timing
+        start_time = time.perf_counter()
+        response = self.completion(messages, **kwargs)
+        latency_ms = int((time.perf_counter() - start_time) * 1000)
+
+        # Store in cache
+        if use_cache and cache is not None:
+            config = getattr(self, "config", LLMConfig(model="unknown"))
+            entry = LLMCacheEntry.create(
+                model=config.model,
+                messages=messages,
+                content=response.content,
+                temperature=(
+                    temperature
+                    if temperature is not None
+                    else config.temperature
+                ),
+                max_tokens=(
+                    max_tokens if max_tokens is not None else config.max_tokens
+                ),
+                provider=self.provider_name,
+                latency_ms=latency_ms,
+                input_tokens=response.input_tokens,
+                output_tokens=response.output_tokens,
+                cost_usd=response.cost,
+                request_id=request_id,
+            )
+            cache.put_data(cache_key, "llm", entry.to_dict())
+
+        return response