graphite-engine 0.3.0__py3-none-any.whl

graphite/__init__.py

@@ -0,0 +1,47 @@
+"""
+Graphite — Open-source claim verification engine for high-stakes decisions.
+
+Core primitives:
+  - Claim: the atomic unit of trust — structured assertion with provenance
+  - ClaimStore: SQLite-backed registry for persisting and querying claims
+  - Provenance: first-class evidence source (document, quote, confidence)
+  - ConfidenceScorer: explainable confidence scoring with named factors
+
+Pipeline:
+  Fetcher → DocumentContext → Extractor → Claim[] → ClaimStore → Verify
+
+Also included:
+  Graph assembly (GraphAssembler) and shock propagation (simulate, scenario)
+"""
+
+# ── Core schemas ──
+from .schemas import ExtractedEdge, NodeRef, Provenance, InferenceBasis, ExtractionError
+from .enums import EdgeType, NodeType, SourceType, ConfidenceLevel, AssertionMode, EvidenceType
+from .evidence import EvidencePacket, EvidenceData
+
+# ── Trust engine primitives ──
+from .claim import Claim, ClaimType, ClaimStatus, ClaimGranularity, ReviewState, ClaimOrigin
+from .claim import ConfidenceFactor, ConfidenceResult
+from .claim_store import ClaimStore
+from .confidence import ConfidenceScorer
+
+# ── Assembly ──
+from .assembler import GraphAssembler
+
+# ── Domain plugin contracts ──
+from .domain import BaseFetcher, BaseExtractor, BasePipeline, DocumentContext, DomainSpec
+from .domain import register_domain, get_domain, list_domains
+
+# ── Rules ──
+from .rules import BaseRuleEngine, RuleResult, ScoreBreakdown
+
+# ── Simulation ──
+from .simulate import top_k_paths_from_source, build_blast_radius, map_to_tier
+
+# ── Scenario ──
+from .scenario import ScenarioShock, ScenarioRunner
+
+# ── I/O ──
+from .io import save_graph, load_graph
+
+__version__ = "0.3.0"

graphite/adapters/__init__.py

@@ -0,0 +1 @@
+"""graphite.adapters — External data adapters for Graphite Terra."""

graphite/adapters/alphaearth.py

@@ -0,0 +1,213 @@
+"""
+graphite/adapters/alphaearth.py — Cache-first AlphaEarth embedding adapter.
+
+Reads 64-dimensional AlphaEarth Foundations embeddings for geographic locations.
+Primary path: local .npy cache files (deterministic, no network needed).
+Optional: GCS COGs at gs://alphaearth_foundations/ (Requester Pays).
+
+AlphaEarth Foundations:
+  - 64-dim embeddings at ~10m/pixel resolution
+  - Annual layers: 2017–2025
+  - Multi-sensor: Sentinel-1/2, Landsat, climate sims, 3D laser
+  - Earth Engine dataset: GOOGLE/SATELLITE_EMBEDDING/V1/ANNUAL
+  - GCS bucket: gs://alphaearth_foundations (Requester Pays)
+"""
+import json
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+
+# ── Constants ──
+EMBEDDING_DIM = 64
+GCS_BUCKET = "gs://alphaearth_foundations"
+EE_DATASET = "GOOGLE/SATELLITE_EMBEDDING/V1/ANNUAL"
+AVAILABLE_YEARS = list(range(2017, 2026))
+
+
+class AlphaEarthAdapter:
+    """Read AlphaEarth embeddings — local cache first, GCS optional.
+
+    Primary path: cache/{year}/{node_id}.npy
+      - Pre-fetched embeddings stored as numpy arrays
+      - Deterministic: same result every run
+      - No network dependency for demos/CI
+
+    Optional path: GCS COGs (Requester Pays)
+      - Requires billing_project to be set
+      - Reads Cloud Optimized GeoTIFFs via rasterio
+      - Results cached locally after first fetch
+
+    Usage:
+        adapter = AlphaEarthAdapter(cache_dir="cache/alphaearth")
+
+        # From cache (fast, deterministic)
+        emb = adapter.get_embedding(29.7355, -95.2690, year=2017)
+
+        # With GCS fallback (requires rasterio + billing project)
+        adapter = AlphaEarthAdapter(
+            cache_dir="cache/alphaearth",
+            billing_project="my-gcp-project",
+        )
+        emb = adapter.get_embedding(29.7355, -95.2690, year=2017)
+    """
+
+    def __init__(
+        self,
+        cache_dir: str = "cache/alphaearth",
+        billing_project: Optional[str] = None,
+    ):
+        self.cache_dir = Path(cache_dir)
+        self.billing_project = billing_project
+
+    def get_embedding(
+        self,
+        lat: float,
+        lon: float,
+        year: int = 2017,
+        node_id: Optional[str] = None,
+    ) -> np.ndarray:
+        """Return 64-dim embedding for a point location.
+
+        Tries cache first (by node_id or lat/lon key), then GCS if configured.
+
+        Args:
+            lat: Latitude
+            lon: Longitude
+            year: Annual embedding year (2017–2025)
+            node_id: Optional cache key (e.g., "PORT_HOUSTON")
+
+        Returns:
+            numpy array of shape (64,)
+        """
+        cache_key = node_id or f"{lat:.4f}_{lon:.4f}"
+
+        # 1. Try cache
+        cached = self._read_cache(cache_key, year)
+        if cached is not None:
+            return cached
+
+        # 2. Try GCS (optional)
+        if self.billing_project:
+            embedding = self._fetch_from_gcs(lat, lon, year)
+            if embedding is not None:
+                self._write_cache(cache_key, year, embedding)
+                return embedding
+
+        # 3. No data available
+        raise FileNotFoundError(
+            f"No AlphaEarth embedding found for {cache_key} (year={year}). "
+            f"Either pre-fetch to {self.cache_dir}/{year}/{cache_key}.npy "
+            f"or set billing_project for GCS access."
+        )
+
+    def get_area_embedding(
+        self,
+        bbox: Tuple[float, float, float, float],
+        year: int = 2017,
+        node_id: Optional[str] = None,
+    ) -> np.ndarray:
+        """Return mean 64-dim embedding for a bounding box area.
+
+        For cache mode, this falls back to get_embedding with the bbox centroid.
+        For GCS mode, this would read and average the full bbox region.
+
+        Args:
+            bbox: (min_lat, min_lon, max_lat, max_lon)
+            year: Annual embedding year
+            node_id: Optional cache key
+
+        Returns:
+            numpy array of shape (64,)
+        """
+        cache_key = node_id or f"bbox_{bbox[0]:.4f}_{bbox[1]:.4f}_{bbox[2]:.4f}_{bbox[3]:.4f}"
+
+        cached = self._read_cache(cache_key, year)
+        if cached is not None:
+            return cached
+
+        # Fallback: centroid point embedding
+        centroid_lat = (bbox[0] + bbox[2]) / 2
+        centroid_lon = (bbox[1] + bbox[3]) / 2
+        return self.get_embedding(centroid_lat, centroid_lon, year, node_id=cache_key)
+
+    def get_embedding_safe(
+        self,
+        lat: float,
+        lon: float,
+        year: int = 2017,
+        node_id: Optional[str] = None,
+    ) -> Optional[np.ndarray]:
+        """Like get_embedding but returns None instead of raising."""
+        try:
+            return self.get_embedding(lat, lon, year, node_id)
+        except FileNotFoundError:
+            return None
+
+    # ── Cache I/O ──
+
+    def _cache_path(self, key: str, year: int) -> Path:
+        return self.cache_dir / str(year) / f"{key}.npy"
+
+    def _read_cache(self, key: str, year: int) -> Optional[np.ndarray]:
+        path = self._cache_path(key, year)
+        if path.exists():
+            arr = np.load(path)
+            if arr.shape == (EMBEDDING_DIM,):
+                return arr
+        return None
+
+    def _write_cache(self, key: str, year: int, embedding: np.ndarray):
+        path = self._cache_path(key, year)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        np.save(path, embedding)
+
+    # ── GCS fetch (optional, requires rasterio) ──
+
+    def _fetch_from_gcs(
+        self, lat: float, lon: float, year: int
+    ) -> Optional[np.ndarray]:
+        """Fetch embedding from GCS Cloud Optimized GeoTIFF.
+
+        Requires:
+          - rasterio package installed
+          - GCP billing project with access to requester-pays buckets
+        """
+        try:
+            import rasterio
+            from rasterio.crs import CRS
+        except ImportError:
+            return None
+
+        # GCS COGs are organized by year and UTM zone
+        # For now, return None — actual implementation would:
+        # 1. Determine UTM zone from lat/lon
+        # 2. Construct GCS path: gs://alphaearth_foundations/{year}/{zone}/...
+        # 3. Open COG with rasterio using GDAL_HTTP_HEADER_AUTH
+        # 4. Sample at lat/lon → 64-dim array
+        #
+        # This is left as a stub because:
+        # - GCS bucket is Requester Pays (needs billing project)
+        # - Demo should work from cache without network
+        # - Full implementation needs GDAL/rasterio + GCP auth setup
+        return None
+
+    # ── Batch operations ──
+
+    def list_cached(self, year: int = 2017) -> List[str]:
+        """List all cached embedding keys for a year."""
+        year_dir = self.cache_dir / str(year)
+        if not year_dir.exists():
+            return []
+        return [p.stem for p in year_dir.glob("*.npy")]
+
+    def cache_stats(self) -> Dict[str, int]:
+        """Return count of cached embeddings per year."""
+        stats = {}
+        for year in AVAILABLE_YEARS:
+            count = len(self.list_cached(year))
+            if count > 0:
+                stats[str(year)] = count
+        return stats

graphite/adapters/weathernext.py

@@ -0,0 +1,93 @@
+"""
+graphite/adapters/weathernext.py — Sample-first WeatherNext 2 forecast adapter.
+
+Reads WeatherNext 2 ensemble forecast data for geographic locations.
+Primary path: local forecast_snapshot.json (deterministic, no network needed).
+Optional (--live): Earth Engine / BigQuery query (requires approved data request form).
+
+WeatherNext 2:
+  - 0.25° resolution, 64-member ensemble
+  - Fields: temperature, wind, precipitation, humidity, pressure
+  - Coverage: 2022-present, 6-hour init times, up to 15-day lead time
+  - Access: EE/BigQuery (requires data request form)
+  - Note: Experimental dataset, not validated for real-world use
+"""
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+class WeatherNextAdapter:
+    """Read WeatherNext 2 forecasts — sample snapshot first, live optional.
+
+    Primary path: forecast_snapshot.json
+      - Deterministic forecast data for demo nodes
+      - No network dependency for demos/CI
+
+    Optional path (live=True): Earth Engine / BigQuery
+      - Requires approved data request form
+      - Not implemented in v1
+
+    Usage:
+        adapter = WeatherNextAdapter(snapshot_path="forecast_snapshot.json")
+        forecast = adapter.get_forecast("asset:PORT_HOUSTON")
+    """
+
+    def __init__(
+        self,
+        snapshot_path: Optional[str] = None,
+        live: bool = False,
+    ):
+        self.live = live
+        self._data = None
+        self._snapshot_path = snapshot_path
+
+        if snapshot_path:
+            self._load_snapshot(snapshot_path)
+
+    def _load_snapshot(self, path: str):
+        """Load forecast data from a snapshot JSON file."""
+        with open(path) as f:
+            raw = json.load(f)
+
+        self._meta = raw.get("meta", {})
+        self._data = {}
+
+        for point in raw.get("forecast_points", []):
+            node_id = point.get("node_id", "")
+            if node_id:
+                self._data[node_id] = point
+
+    @property
+    def meta(self) -> Dict[str, Any]:
+        """Return forecast metadata."""
+        return self._meta if self._meta else {}
+
+    def get_forecast(self, node_id: str) -> Optional[Dict[str, Any]]:
+        """Return forecast fields for a node.
+
+        Returns None if the node is not in the snapshot.
+        """
+        if self._data and node_id in self._data:
+            return self._data[node_id]
+
+        if self.live:
+            return self._fetch_live(node_id)
+
+        return None
+
+    def get_all_forecasts(self) -> Dict[str, Dict[str, Any]]:
+        """Return all forecast points from the snapshot."""
+        return dict(self._data) if self._data else {}
+
+    def list_nodes(self) -> List[str]:
+        """List all node IDs with forecast data."""
+        return list(self._data.keys()) if self._data else []
+
+    def _fetch_live(self, node_id: str) -> Optional[Dict[str, Any]]:
+        """Fetch live forecast from Earth Engine / BigQuery.
+
+        Not implemented in v1 — requires approved data request form.
+        """
+        # Stub: would use ee.ImageCollection or BigQuery SQL
+        return None

graphite/assembler.py

@@ -0,0 +1,299 @@
+"""
+graphite/assembler.py — Graph assembly from extracted edges.
+
+The assembler sits between extractors and writers:
+  Extractor → ExtractedEdge[] → **Assembler** → nx.DiGraph → Writer
+
+Responsibilities:
+  - Deduplicate edges (merge provenance when same relationship from different sources)
+  - Resolve conflicting attributes (merge policy with priority)
+  - Normalize nodes
+  - Collect extraction errors
+  - Stamp graph metadata
+"""
+import json
+import math
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Set
+
+import networkx as nx
+
+from .enums import AssertionMode, ConfidenceLevel, SourceType, EvidenceType
+from .schemas import ExtractedEdge, ExtractionError, Provenance
+from .domain import DomainSpec
+
+
+# ═══════════════════════════════════════
+# Merge Policy
+# ═══════════════════════════════════════
+
+_EVIDENCE_PRIORITY = {
+    EvidenceType.TABLE_CELL: 4,
+    EvidenceType.TEXT_QUOTE: 3,
+    EvidenceType.DERIVED: 2,
+    EvidenceType.MANUAL: 1,
+}
+
+_SOURCE_PRIORITY = {
+    SourceType.USGS_MCS: 4,
+    SourceType.SEC_10K: 3,
+    SourceType.SEC_20F: 3,
+    SourceType.PDF: 2,
+    SourceType.WEB: 1,
+    SourceType.MANUAL: 1,
+}
+
+_ASSERTION_PRIORITY = {
+    AssertionMode.EXTRACTED: 3,
+    AssertionMode.INFERRED: 2,
+    AssertionMode.SEEDED: 1,
+}
+
+_CONFIDENCE_PRIORITY = {
+    ConfidenceLevel.HIGH: 3,
+    ConfidenceLevel.MEDIUM: 2,
+    ConfidenceLevel.LOW: 1,
+}
+
+
+def _provenance_score(p: Provenance) -> int:
+    """Score a provenance for merge priority. Higher = prefer."""
+    return (
+        _EVIDENCE_PRIORITY.get(p.evidence_type, 0) * 100
+        + _SOURCE_PRIORITY.get(p.source_type, 0) * 10
+        + _CONFIDENCE_PRIORITY.get(p.confidence, 0)
+    )
+
+
+class GraphAssembler:
+    """Assemble a NetworkX graph from normalized ExtractedEdge objects.
+
+    Usage:
+        assembler = GraphAssembler()
+        G = assembler.assemble(edges)
+    """
+
+    def __init__(
+        self,
+        pipeline_version: str = "1.0",
+        domain_spec: Optional[DomainSpec] = None,
+        drop_zero_provenance: bool = True,
+        drop_low_inferred: bool = False,
+    ):
+        self.pipeline_version = pipeline_version
+        self.domain_spec = domain_spec
+        self.drop_zero_provenance = drop_zero_provenance
+        self.drop_low_inferred = drop_low_inferred
+        self.errors: List[ExtractionError] = []
+
+    def assemble(
+        self,
+        edges: List[ExtractedEdge],
+        node_labels: Optional[Dict[str, str]] = None,
+    ) -> nx.DiGraph:
+        """Assemble edges into a NetworkX DiGraph.
+
+        Args:
+            edges: Normalized extracted edges
+            node_labels: Optional node_id → display label mapping
+
+        Returns:
+            Assembled and stamped nx.DiGraph
+        """
+        # 1. Validate edge types against domain registry
+        if self.domain_spec:
+            edges = self._validate_edge_types(edges)
+
+        # 2. Quality filters
+        edges = self._quality_filter(edges)
+
+        # 3. Deduplicate
+        deduped = self.dedupe_edges(edges)
+
+        # 4. Build graph
+        G = nx.DiGraph()
+        labels = node_labels or {}
+
+        # Collect all nodes
+        all_nodes: Dict[str, dict] = {}
+        for edge in deduped:
+            for nref in (edge.from_node, edge.to_node):
+                if nref.node_id not in all_nodes:
+                    all_nodes[nref.node_id] = {
+                        "node_type": nref.node_type.value,
+                        "name": nref.label or labels.get(nref.node_id, nref.node_id),
+                    }
+
+        for nid, attrs in all_nodes.items():
+            G.add_node(nid, **attrs)
+
+        # Add edges
+        for edge in deduped:
+            weight = edge.attributes.get("bucket_weight", 0.5)
+            cost = -math.log(max(weight, 0.01))
+
+            edge_attrs = {
+                "edge_type": edge.edge_type,
+                "assertion_mode": edge.assertion_mode.value,
+                "bucket_weight": weight,
+                "cost": round(cost, 6),
+                "confidence": edge.best_confidence.value,
+            }
+
+            # Flatten domain attributes (but not nested structures)
+            for k, v in edge.attributes.items():
+                if k not in edge_attrs and not isinstance(v, (dict, list)):
+                    edge_attrs[k] = v
+
+            # Serialize provenance as JSON string (GraphML compatible)
+            edge_attrs["provenance_json"] = json.dumps(
+                [p.model_dump() for p in edge.provenance], default=str
+            )
+            edge_attrs["provenance_count"] = len(edge.provenance)
+            edge_attrs["evidence"] = edge.provenance[0].evidence_quote if edge.provenance else ""
+            edge_attrs["data_source"] = edge.provenance[0].source_type.value if edge.provenance else ""
+
+            # Inference basis
+            if edge.inference_basis:
+                edge_attrs["inference_method"] = edge.inference_basis.method
+                edge_attrs["inference_reason"] = edge.inference_basis.reason
+
+            # Claim linkage (trust engine v1)
+            if edge.claim_ids:
+                edge_attrs["claim_ids"] = json.dumps(edge.claim_ids)
+
+            G.add_edge(edge.from_node.node_id, edge.to_node.node_id, **edge_attrs)
+
+        # 5. Stamp
+        return self._stamp_graph(G)
+
+    def dedupe_edges(self, edges: List[ExtractedEdge]) -> List[ExtractedEdge]:
+        """Merge edges with same (from, to, type) — combine provenances."""
+        by_key: Dict[str, ExtractedEdge] = {}
+
+        for edge in edges:
+            key = edge.edge_key
+            if key in by_key:
+                existing = by_key[key]
+                merged = self._merge_edge_pair(existing, edge)
+                by_key[key] = merged
+            else:
+                by_key[key] = edge
+
+        return list(by_key.values())
+
+    def _merge_edge_pair(self, a: ExtractedEdge, b: ExtractedEdge) -> ExtractedEdge:
+        """Merge two edges with the same key."""
+        # Merge provenance (dedupe by source_id)
+        seen_sources = {p.source_id for p in a.provenance}
+        merged_prov = list(a.provenance)
+        for p in b.provenance:
+            if p.source_id not in seen_sources:
+                merged_prov.append(p)
+                seen_sources.add(p.source_id)
+
+        # Stronger assertion mode (EXTRACTED > INFERRED > SEEDED)
+        mode = (
+            a.assertion_mode
+            if _ASSERTION_PRIORITY.get(a.assertion_mode, 0)
+            >= _ASSERTION_PRIORITY.get(b.assertion_mode, 0)
+            else b.assertion_mode
+        )
+
+        # Merge attributes with conflict tracking
+        merged_attrs = dict(a.attributes)
+        for k, v in b.attributes.items():
+            if k in merged_attrs and merged_attrs[k] != v:
+                rv_key = f"{k}_reported_values"
+                existing_rv = merged_attrs.get(rv_key, [])
+                if not existing_rv:
+                    a_prov = a.provenance[0] if a.provenance else None
+                    existing_rv.append({
+                        "value": merged_attrs[k],
+                        "source": a_prov.source_type.value if a_prov else "unknown",
+                        "confidence": a_prov.confidence.value if a_prov else "LOW",
+                    })
+                b_prov = b.provenance[0] if b.provenance else None
+                existing_rv.append({
+                    "value": v,
+                    "source": b_prov.source_type.value if b_prov else "unknown",
+                    "confidence": b_prov.confidence.value if b_prov else "LOW",
+                })
+                merged_attrs[rv_key] = existing_rv
+
+                a_score = max((_provenance_score(p) for p in a.provenance), default=0)
+                b_score = max((_provenance_score(p) for p in b.provenance), default=0)
+                if b_score > a_score:
+                    merged_attrs[k] = v
+            else:
+                merged_attrs[k] = v
+
+        basis = a.inference_basis or b.inference_basis
+
+        return ExtractedEdge(
+            from_node=a.from_node,
+            to_node=a.to_node,
+            edge_type=a.edge_type,
+            assertion_mode=mode,
+            attributes=merged_attrs,
+            provenance=merged_prov,
+            inference_basis=basis,
+        )
+
+    def _validate_edge_types(self, edges: List[ExtractedEdge]) -> List[ExtractedEdge]:
+        """Check edge types against domain registry."""
+        allowed = set(self.domain_spec.allowed_edge_types) if self.domain_spec else set()
+        if not allowed:
+            return edges
+
+        valid = []
+        for edge in edges:
+            if edge.edge_type in allowed:
+                valid.append(edge)
+            else:
+                self.errors.append(ExtractionError(
+                    entity_id=edge.from_node.node_id,
+                    source_type=edge.provenance[0].source_type if edge.provenance else SourceType.MANUAL,
+                    error_type="validation_failed",
+                    message=f"Edge type '{edge.edge_type}' not in domain allowed types: {allowed}",
+                ))
+        return valid
+
+    def _quality_filter(self, edges: List[ExtractedEdge]) -> List[ExtractedEdge]:
+        """Apply quality filters."""
+        filtered = []
+        for edge in edges:
+            if self.drop_zero_provenance and not edge.provenance:
+                self.errors.append(ExtractionError(
+                    entity_id=edge.from_node.node_id,
+                    source_type=SourceType.MANUAL,
+                    error_type="no_edges",
+                    message=f"Edge {edge.edge_key} dropped: zero provenance",
+                ))
+                continue
+            if (self.drop_low_inferred
+                    and edge.assertion_mode == AssertionMode.INFERRED
+                    and edge.best_confidence == ConfidenceLevel.LOW):
+                continue
+            filtered.append(edge)
+        return filtered
+
+    def _stamp_graph(self, G: nx.DiGraph) -> nx.DiGraph:
+        """Add metadata to graph."""
+        edge_types = defaultdict(int)
+        assertion_modes = defaultdict(int)
+        for _, _, d in G.edges(data=True):
+            edge_types[d.get("edge_type", "?")] += 1
+            assertion_modes[d.get("assertion_mode", "?")] += 1
+
+        G.graph["built_at"] = datetime.now(timezone.utc).isoformat()
+        G.graph["pipeline_version"] = self.pipeline_version
+        G.graph["node_count"] = G.number_of_nodes()
+        G.graph["edge_count"] = G.number_of_edges()
+        G.graph["edge_types"] = json.dumps(dict(edge_types))
+        G.graph["assertion_modes"] = json.dumps(dict(assertion_modes))
+        if self.domain_spec:
+            G.graph["domain"] = self.domain_spec.name
+
+        return G