sage-governance 1.0.0

package/sage/startup.py

@@ -0,0 +1,311 @@
+"""
+startup.py — SAGE Pre-Loader
+════════════════════════════
+Imported ONCE when mcp_server.py starts. Loads every heavy dependency,
+policy document, and lookup table into module-level globals.
+
+WHY THIS FILE EXISTS
+────────────────────
+MCP stdio servers run as a persistent Python process — they are NOT
+respawned per tool call. However, if heavy imports live inside tool
+functions they still add latency on the FIRST call of each session.
+Preloading here keeps every tool call fast regardless of import order.
+
+Author: SAGE Team / Team SAGE (Hackathon)
+License: MIT
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import pathlib
+import sys
+from datetime import datetime, timezone
+from typing import Any
+
+# ── Resolve project root regardless of CWD ────────────────────────────────────
+_THIS_FILE = pathlib.Path(__file__).resolve()
+PROJECT_ROOT = _THIS_FILE.parent.parent
+
+RULES_DIR      = PROJECT_ROOT / "rules"
+AUDIT_FILE     = PROJECT_ROOT / "audit-trail" / "decisions.jsonl"
+LOGS_FILE      = PROJECT_ROOT / "LOGS.md"
+LOCAL_MEMORY   = PROJECT_ROOT / "local_memory.md"
+REPORTS_DIR    = PROJECT_ROOT / "reports"
+
+# ── Ensure required dirs & files exist ───────────────────────────────────────
+for _p in (AUDIT_FILE.parent, REPORTS_DIR):
+    _p.mkdir(parents=True, exist_ok=True)
+for _f in (AUDIT_FILE, LOGS_FILE, LOCAL_MEMORY):
+    _f.touch(exist_ok=True)
+
+import hashlib
+
+def write_audit_entry(entry: dict) -> str:
+    """
+    Append one JSON line to decisions.jsonl with SHA-256 chain link.
+    Reads the last line to extract the last entry's hash to serve as prev_hash,
+    ensuring chain integrity across separate runs/processes.
+    """
+    prev_hash = ""
+    session_id = entry.get("session_id")
+
+    if AUDIT_FILE.exists() and AUDIT_FILE.stat().st_size > 1:
+        try:
+            with open(AUDIT_FILE, "r", encoding="utf-8") as fh:
+                lines = fh.readlines()
+                if lines:
+                    last_line = lines[-1].strip()
+                    if last_line:
+                        last_entry = json.loads(last_line)
+                        prev_hash = last_entry.get("entry_hash", "")
+                        if not session_id:
+                            session_id = last_entry.get("session_id")
+        except Exception:
+            pass
+
+    if not session_id:
+        session_id = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+
+    entry["session_id"] = session_id
+    entry["timestamp"] = datetime.now(timezone.utc).isoformat()
+
+    payload = json.dumps(entry, sort_keys=True, default=str) + prev_hash
+    current_hash = hashlib.sha256(payload.encode()).hexdigest()
+
+    entry["entry_hash"] = current_hash
+    entry["prev_hash"] = prev_hash
+
+    with open(AUDIT_FILE, "a", encoding="utf-8") as fh:
+        fh.write(json.dumps(entry, default=str) + "\n")
+
+    return current_hash
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# POLICY DOCUMENTS
+# ══════════════════════════════════════════════════════════════════════════════
+
+POLICY_DOCS: dict[str, str] = {}
+_general_dir = RULES_DIR / "general"
+if _general_dir.exists():
+    for _policy_file in _general_dir.glob("*.md"):
+        POLICY_DOCS[_policy_file.stem] = _policy_file.read_text(encoding="utf-8")
+
+POLICY_INDEX: dict[str, Any] = {}
+_index_path = RULES_DIR / "index.json"
+if _index_path.exists():
+    try:
+        POLICY_INDEX = json.loads(_index_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        POLICY_INDEX = {}
+
+# ══════════════════════════════════════════════════════════════════════════════
+# LLM CLIENT  (OpenAI — used only for human-readable reasoning enrichment)
+# Override model with SAGE_LLM_MODEL env var. Defaults to gpt-4o-mini.
+# ══════════════════════════════════════════════════════════════════════════════
+
+LLM_AVAILABLE = False
+LLM_CLIENT: Any = None
+LLM_MODEL   = os.environ.get("SAGE_LLM_MODEL", "gpt-4o-mini")
+
+try:
+    import openai as _openai
+    _api_key = os.environ.get("OPENAI_API_KEY", "")
+    if _api_key:
+        LLM_CLIENT    = _openai.OpenAI(api_key=_api_key)
+        LLM_AVAILABLE = True
+    else:
+        print(
+            "[SAGE startup] WARNING: OPENAI_API_KEY not set. "
+            "Deterministic-only mode active (no LLM enrichment).",
+            file=sys.stderr,
+        )
+except ImportError:
+    print(
+        "[SAGE startup] WARNING: 'openai' package not found. "
+        "Run: pip install openai",
+        file=sys.stderr,
+    )
+
+# ══════════════════════════════════════════════════════════════════════════════
+# OPTIONAL HEAVY DEPS  (fail gracefully if not installed)
+# ══════════════════════════════════════════════════════════════════════════════
+
+FAIRLEARN_AVAILABLE = False
+try:
+    from fairlearn.metrics import (       # noqa: F401
+        demographic_parity_difference,
+        equalized_odds_difference,
+        MetricFrame,
+    )
+    FAIRLEARN_AVAILABLE = True
+except ImportError:
+    pass
+
+DIFFPRIVLIB_AVAILABLE = False
+try:
+    import diffprivlib  # noqa: F401
+    DIFFPRIVLIB_AVAILABLE = True
+except ImportError:
+    pass
+
+# ══════════════════════════════════════════════════════════════════════════════
+# PROTECTED ATTRIBUTES  (direct usage triggers P1 finding)
+# ══════════════════════════════════════════════════════════════════════════════
+
+PROTECTED_ATTRIBUTES: list[str] = [
+    "race", "ethnicity", "color", "sex", "gender", "age",
+    "religion", "national_origin", "nationality", "disability",
+    "pregnancy", "marital_status", "sexual_orientation",
+    "gender_identity", "skin_color",
+]
+
+# ══════════════════════════════════════════════════════════════════════════════
+# PROXY ATTRIBUTE MAP
+# Semantic — not grep. Maps protected characteristics → known proxy variables.
+# Source: ProPublica COMPAS analysis, Ali et al. 2019, Lambrecht & Tucker 2019.
+# ══════════════════════════════════════════════════════════════════════════════
+
+PROXY_ATTRIBUTE_MAP: dict[str, list[str]] = {
+    "race": [
+        "zip_code", "zip code", "zipcode", "postal_code", "postal code",
+        "neighborhood", "census_tract", "school_district", "ward",
+        "surname", "last_name", "family_name", "name",
+        "bank_branch", "grocery_store_distance", "church_attendance",
+        "prior_arrests", "arrest_history",
+    ],
+    "gender": [
+        "browsing_history", "page_likes", "purchase_history",
+        "maternity", "paternity", "childcare", "toy_preferences",
+        "cosmetics", "sports_interest", "car_type",
+        "clothing_category", "grooming_products",
+    ],
+    "age": [
+        "graduation_year", "years_experience", "first_job_year",
+        "account_age", "profile_creation_date", "class_year",
+    ],
+    "socioeconomic_status": [
+        "education_level", "school_name", "employment_gap",
+        "credit_history_length", "bank_type", "car_ownership",
+        "zip_code", "neighborhood", "income_bracket",
+    ],
+    "criminal_history_as_proxy_for_race": [
+        "prior_arrests", "priors_count", "juv_fel_count",
+        "family_history", "neighborhood_crime_rate",
+        "zip_code", "school_district",
+    ],
+}
+
+# ══════════════════════════════════════════════════════════════════════════════
+# EU AI ACT ANNEX III — HIGH-RISK CATEGORIES
+# ══════════════════════════════════════════════════════════════════════════════
+
+EU_AI_ACT_ANNEX_III: dict[str, dict[str, Any]] = {
+    "criminal_justice": {
+        "annex":       "Annex III.6.d",
+        "description": (
+            "AI systems for law enforcement assessing risk of offending "
+            "or re-offending (e.g. COMPAS, recidivism prediction)"
+        ),
+        "keywords": [
+            "recidivism", "criminal", "parole", "sentencing", "compas",
+            "reoffend", "bail", "arrest", "two_year_recid", "risk score",
+            "reoffending", "criminal justice",
+        ],
+    },
+    "employment": {
+        "annex":       "Annex III.4.a",
+        "description": (
+            "AI systems for recruitment/selection, including targeted "
+            "job advertisements, CV filtering, candidate evaluation"
+        ),
+        "keywords": [
+            "job", "recruitment", "hiring", "resume", "cv", "employment",
+            "advertisement", "candidate", "fairjob", "job ad", "click",
+            "click-through", "ctr",
+        ],
+    },
+    "credit_scoring": {
+        "annex":       "Annex III.5.b",
+        "description": (
+            "AI systems for creditworthiness evaluation or "
+            "credit score determination"
+        ),
+        "keywords": [
+            "credit", "loan", "default", "creditworthiness", "mortgage",
+            "financial risk", "lending", "apple card", "credit card",
+            "credit limit", "credit score",
+        ],
+    },
+    "education": {
+        "annex":       "Annex III.3.a",
+        "description": (
+            "AI systems determining access to educational institutions "
+            "or evaluating students"
+        ),
+        "keywords": [
+            "education", "school", "admission", "student", "grade",
+            "academic", "university", "college", "exam",
+        ],
+    },
+    "essential_services": {
+        "annex":       "Annex III.5.a",
+        "description": (
+            "AI systems for essential private and public services "
+            "(healthcare, insurance, housing, social security)"
+        ),
+        "keywords": [
+            "healthcare", "insurance", "benefit", "welfare",
+            "housing", "social security", "medical",
+        ],
+    },
+    "children_safety": {
+        "annex":       "Annex III (context-dependent)",
+        "description": (
+            "AI systems impacting children's safety, privacy, or wellbeing; "
+            "UNICEF 10 Principles apply"
+        ),
+        "keywords": [
+            "child", "minor", "children", "safeguarding", "grooming",
+            "bullying", "moderation", "self-harm", "abuse", "distress",
+            "escalation", "chat safety",
+        ],
+    },
+}
+
+# ══════════════════════════════════════════════════════════════════════════════
+# UDHR ARTICLE MAP  (domain → relevant UDHR articles)
+# ══════════════════════════════════════════════════════════════════════════════
+
+UDHR_ARTICLE_MAP: dict[str, list[str]] = {
+    "criminal_justice":  ["Article 7", "Article 10", "Article 11"],
+    "employment":        ["Article 23"],
+    "credit_scoring":    ["Article 22"],
+    "education":         ["Article 26"],
+    "essential_services": ["Article 22", "Article 25"],
+    "children_safety":   [
+        "UN CRC Article 3 (best interests)",
+        "UN CRC Article 12 (right to be heard)",
+        "UN CRC Article 16 (privacy)",
+        "UN CRC Article 19 (protection from abuse)",
+        "UN CRC Article 34 (sexual exploitation)",
+    ],
+    "general":           ["Article 2", "Article 7"],
+}
+
+# ══════════════════════════════════════════════════════════════════════════════
+# STARTUP REPORT
+# ══════════════════════════════════════════════════════════════════════════════
+
+print(
+    f"[SAGE startup] ✅ Policies loaded: {list(POLICY_DOCS.keys()) or 'none (add to rules/general/)'}",
+    file=sys.stderr,
+)
+print(f"[SAGE startup] ✅ LLM: {'available (' + LLM_MODEL + ')' if LLM_AVAILABLE else 'deterministic-only mode'}",
+      file=sys.stderr)
+print(f"[SAGE startup] ✅ Fairlearn: {FAIRLEARN_AVAILABLE} | diffprivlib: {DIFFPRIVLIB_AVAILABLE}",
+      file=sys.stderr)
+print(f"[SAGE startup] ✅ Session start: {datetime.now(timezone.utc).isoformat()}",
+      file=sys.stderr)