insightforge 0.1.0__py3-none-any.whl

insightforge/__init__.py

@@ -0,0 +1,7 @@
+"""InsightForge public package interface."""
+
+from .analyzer import build_trace
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__", "build_trace"]

insightforge/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

insightforge/analyzer.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+from .models import RiskFlag, TraceNode, TraceRecord
+
+
+HEDGE_PATTERNS = (
+    r"\bmaybe\b",
+    r"\bprobably\b",
+    r"\bI think\b",
+    r"\blikely\b",
+    r"\bappears to\b",
+    r"\bseems\b",
+)
+
+SOURCE_PATTERNS = (
+    r"https?://",
+    r"\bsource\b",
+    r"\bcitation\b",
+    r"\breference\b",
+    r"\bdocumentation\b",
+    r"\bresearch paper\b",
+    r"\bstudy\b",
+)
+
+BIAS_PATTERNS = (
+    r"\balways\b",
+    r"\bnever\b",
+    r"\bobviously\b",
+    r"\beveryone\b",
+    r"\bno one\b",
+)
+
+
+def _count_matches(patterns: Sequence[str], text: str) -> int:
+    return sum(len(re.findall(pattern, text, flags=re.IGNORECASE)) for pattern in patterns)
+
+
+def _build_summary(prompt: str, stdout: str, stderr: str, risk_count: int) -> str:
+    if stderr.strip():
+        return "The wrapped command produced stderr output; review execution details before trusting the answer."
+    if risk_count:
+        return "InsightForge detected language patterns that correlate with weak grounding or overconfident claims."
+    if not stdout.strip():
+        return "The wrapped command returned no stdout, so the trace is mostly execution metadata."
+    if prompt.strip():
+        return "The response completed without obvious risk markers, but the trace should still be reviewed for source quality."
+    return "Execution completed successfully with a low-risk heuristic profile."
+
+
+def build_trace(
+    *,
+    prompt: str,
+    command: Sequence[str],
+    model_hint: str,
+    provider: str = "unknown",
+    system_prompt: str = "",
+    stdout: str,
+    stderr: str,
+    exit_code: int,
+    metadata: dict[str, str] | None = None,
+    provenance_notes: Sequence[str] | None = None,
+) -> TraceRecord:
+    output_blob = "\n".join(part for part in (stdout, stderr) if part)
+    hedge_hits = _count_matches(HEDGE_PATTERNS, output_blob)
+    source_hits = _count_matches(SOURCE_PATTERNS, output_blob)
+    bias_hits = _count_matches(BIAS_PATTERNS, output_blob)
+    stderr_penalty = 0.2 if stderr.strip() else 0.0
+    empty_penalty = 0.15 if not stdout.strip() else 0.0
+
+    confidence = 0.72
+    confidence -= min(0.24, hedge_hits * 0.04)
+    confidence -= min(0.20, bias_hits * 0.05)
+    confidence -= stderr_penalty + empty_penalty
+    confidence += min(0.18, source_hits * 0.06)
+    confidence = max(0.05, min(0.99, round(confidence, 2)))
+
+    bias_flags: list[RiskFlag] = []
+    hallucination_flags: list[RiskFlag] = []
+    provenance: list[str] = list(provenance_notes or [])
+
+    if source_hits:
+        provenance.append("Sources or citations were mentioned in the output.")
+    elif not provenance:
+        provenance.append("No explicit sources or citations were detected.")
+
+    if bias_hits:
+        bias_flags.append(
+            RiskFlag(
+                code="OVERGENERALIZATION",
+                title="Overgeneralized claim pattern",
+                severity="medium",
+                evidence="The output uses absolute language that can hide edge cases or demographic skew.",
+                recommendation="Ask the model to qualify claims, state assumptions, and list known exceptions.",
+            )
+        )
+
+    if hedge_hits and not source_hits:
+        hallucination_flags.append(
+            RiskFlag(
+                code="UNGROUNDED_HEDGING",
+                title="Ungrounded uncertainty",
+                severity="high",
+                evidence="The output contains hedging language without nearby source signals.",
+                recommendation="Request citations, intermediate evidence, or a narrower task boundary.",
+            )
+        )
+
+    if stderr.strip():
+        hallucination_flags.append(
+            RiskFlag(
+                code="EXECUTION_ANOMALY",
+                title="Execution anomaly",
+                severity="medium",
+                evidence="The wrapped command emitted stderr output, which may indicate tool failure or partial completion.",
+                recommendation="Inspect stderr and rerun before relying on the result for audits or downstream actions.",
+            )
+        )
+
+    nodes = [
+        TraceNode(id="prompt", label="Prompt", kind="input", detail=prompt or "No prompt recorded."),
+        TraceNode(
+            id="system",
+            label="System Prompt",
+            kind="input",
+            detail=system_prompt or "No system prompt recorded.",
+        ),
+        TraceNode(
+            id="execution",
+            label="Execution",
+            kind="process",
+            detail=" ".join(command) if command else "No command recorded.",
+            score=1.0 if exit_code == 0 else 0.4,
+        ),
+        TraceNode(
+            id="analysis",
+            label="Heuristic Analysis",
+            kind="analysis",
+            detail=f"Hedges={hedge_hits}, source signals={source_hits}, bias markers={bias_hits}",
+            score=confidence,
+        ),
+        TraceNode(
+            id="output",
+            label="Model Output",
+            kind="output",
+            detail=(stdout or stderr or "No output captured.")[:1200],
+            score=confidence,
+        ),
+    ]
+
+    risk_count = len(bias_flags) + len(hallucination_flags)
+    summary = _build_summary(prompt, stdout, stderr, risk_count)
+
+    return TraceRecord(
+        model_hint=model_hint or "unknown",
+        provider=provider or "unknown",
+        prompt=prompt,
+        system_prompt=system_prompt,
+        command=list(command),
+        exit_code=exit_code,
+        stdout=stdout,
+        stderr=stderr,
+        metadata=dict(metadata or {}),
+        confidence_score=confidence,
+        bias_flags=bias_flags,
+        hallucination_flags=hallucination_flags,
+        provenance=provenance,
+        nodes=nodes,
+        summary=summary,
+    )

insightforge/cli.py

@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+import argparse
+import shlex
+import subprocess
+from pathlib import Path
+
+from . import __version__
+from .analyzer import build_trace
+from .config import load_config
+from .diffing import build_diff, render_diff_text, write_diff_html
+from .migrations import CURRENT_DB_SCHEMA_VERSION, get_schema_version, migrate_storage
+from .policy import evaluate_policies
+from .providers import ProviderError, get_provider
+from .redaction import apply_redaction
+from .renderer import write_html, write_json
+from .store import index_trace, load_trace, load_registry
+from .updater import maybe_get_update_message
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="insightforge",
+        description="Wrap AI interactions and emit audit-friendly traces.",
+    )
+    subparsers = parser.add_subparsers(dest="command_name", required=True)
+
+    wrap = subparsers.add_parser("wrap", help="Run a command and generate JSON + HTML traces.")
+    wrap.add_argument("model", help="Model or tool hint, e.g. claude, grok, local-llm")
+    wrap.add_argument("prompt", help="The prompt or intent associated with the wrapped run")
+    wrap.add_argument(
+        "--cmd",
+        required=True,
+        help="Shell command to execute, quoted as a single string",
+    )
+    wrap.add_argument(
+        "--out",
+        default="traces/latest",
+        help="Output prefix for generated files, without extension",
+    )
+
+    ask = subparsers.add_parser("ask", help="Prompt a supported provider and generate JSON + HTML traces.")
+    ask.add_argument("provider", help="Provider name: mock, openai, anthropic")
+    ask.add_argument("model", help="Model name understood by the provider")
+    ask.add_argument("prompt", help="User prompt to send to the provider")
+    ask.add_argument(
+        "--system",
+        default="",
+        help="Optional system prompt or instruction block",
+    )
+    ask.add_argument(
+        "--out",
+        default="traces/latest",
+        help="Output prefix for generated files, without extension",
+    )
+
+    list_cmd = subparsers.add_parser("list", help="List indexed traces from the local registry.")
+    list_cmd.add_argument(
+        "--limit",
+        type=int,
+        default=10,
+        help="Maximum number of traces to display",
+    )
+
+    diff = subparsers.add_parser("diff", help="Compare two saved traces by path or trace id.")
+    diff.add_argument("before", help="Older trace path or trace id")
+    diff.add_argument("after", help="Newer trace path or trace id")
+    diff.add_argument(
+        "--out",
+        default="traces/diff-latest.html",
+        help="HTML destination for the diff report",
+    )
+
+    schema = subparsers.add_parser("schema-version", help="Show the current storage schema version.")
+    schema.add_argument(
+        "--expected",
+        action="store_true",
+        help="Also print the expected application schema version",
+    )
+
+    subparsers.add_parser("migrate", help="Run storage migrations to the latest schema version.")
+
+    version = subparsers.add_parser("version", help="Show the installed CLI version.")
+    version.add_argument(
+        "--check-updates",
+        action="store_true",
+        help="Also check whether a newer package version is available",
+    )
+    return parser
+
+
+def _cap_output(value: str, limit: int) -> str:
+    if len(value) <= limit:
+        return value
+    return value[:limit] + "\n[InsightForge truncated persisted output]"
+
+
+def _finalize_trace(trace, app_config):
+    trace.policy_results, trace.overall_status = evaluate_policies(trace, app_config.policy)
+    return apply_redaction(trace, app_config.redaction)
+
+
+def _write_trace_artifacts(args: argparse.Namespace, trace, app_config) -> None:
+    out_prefix = Path(args.out)
+    out_prefix.parent.mkdir(parents=True, exist_ok=True)
+    json_path = out_prefix.with_suffix(".json")
+    html_path = out_prefix.with_suffix(".html")
+    write_json(trace, json_path)
+    write_html(trace, html_path)
+    index_trace(trace, json_path, html_path, app_config.storage)
+
+    print(f"InsightForge trace captured for {trace.model_hint}")
+    print(f"Trace ID: {trace.trace_id}")
+    print(f"Provider: {trace.provider}")
+    print(f"Status: {trace.overall_status}")
+    print(f"Confidence score: {trace.confidence_score:.2f}")
+    print(f"JSON: {json_path}")
+    print(f"HTML: {html_path}")
+
+    for flag in trace.bias_flags + trace.hallucination_flags:
+        print(f"[{flag.severity}] {flag.title}: {flag.recommendation}")
+    for result in trace.policy_results:
+        print(f"[policy:{result.status}] {result.policy_id}: {result.message}")
+
+
+def run_wrap(args: argparse.Namespace) -> int:
+    app_config = load_config()
+    command = shlex.split(args.cmd)
+    completed = subprocess.run(command, capture_output=True, text=True, check=False)
+
+    trace = build_trace(
+        prompt=args.prompt,
+        command=command,
+        model_hint=args.model,
+        provider="shell",
+        stdout=_cap_output(completed.stdout, app_config.policy.max_output_chars),
+        stderr=_cap_output(completed.stderr, app_config.policy.max_output_chars),
+        exit_code=completed.returncode,
+        metadata={"mode": "wrapped shell command"},
+    )
+    trace = _finalize_trace(trace, app_config)
+
+    _write_trace_artifacts(args, trace, app_config)
+
+    return completed.returncode
+
+
+def run_ask(args: argparse.Namespace) -> int:
+    app_config = load_config()
+    try:
+        provider = get_provider(args.provider)
+        response = provider.generate(model=args.model, prompt=args.prompt, system_prompt=args.system)
+        stdout = response.output_text
+        stderr = ""
+        exit_code = 0
+        metadata = response.metadata
+        provenance_notes = response.provenance
+    except ProviderError as exc:
+        stdout = ""
+        stderr = str(exc)
+        exit_code = 1
+        metadata = {"transport": "provider call failed"}
+        provenance_notes = []
+
+    trace = build_trace(
+        prompt=args.prompt,
+        system_prompt=args.system,
+        command=[args.provider, args.model],
+        model_hint=args.model,
+        provider=args.provider,
+        stdout=_cap_output(stdout, app_config.policy.max_output_chars),
+        stderr=_cap_output(stderr, app_config.policy.max_output_chars),
+        exit_code=exit_code,
+        metadata=metadata,
+        provenance_notes=provenance_notes,
+    )
+    trace = _finalize_trace(trace, app_config)
+
+    _write_trace_artifacts(args, trace, app_config)
+    print(stdout if stdout else stderr)
+    return exit_code
+
+
+def run_list(args: argparse.Namespace) -> int:
+    app_config = load_config()
+    entries = load_registry(app_config.storage, max(1, args.limit))
+    if not entries:
+        print("No traces indexed yet.")
+        return 0
+
+    for entry in entries:
+        print(
+            f"{entry['trace_id']} | {entry['captured_at']} | {entry['provider']} | "
+            f"{entry['model_hint']} | status={entry['overall_status']} | "
+            f"score={entry['confidence_score']:.2f} | {entry['json_path']}"
+        )
+    return 0
+
+
+def run_diff(args: argparse.Namespace) -> int:
+    app_config = load_config()
+    before = load_trace(args.before, app_config.storage)
+    after = load_trace(args.after, app_config.storage)
+    diff = build_diff(before, after)
+
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    write_diff_html(diff, out_path)
+
+    print(render_diff_text(diff))
+    print(f"HTML: {out_path}")
+    return 0
+
+
+def run_schema_version(args: argparse.Namespace) -> int:
+    app_config = load_config()
+    current = get_schema_version(app_config.storage)
+    print(f"Current schema version: {current}")
+    if args.expected:
+        print(f"Expected schema version: {CURRENT_DB_SCHEMA_VERSION}")
+    return 0
+
+
+def run_migrate(_: argparse.Namespace) -> int:
+    app_config = load_config()
+    before, after = migrate_storage(app_config.storage)
+    print(f"Migrated storage schema: {before} -> {after}")
+    print(f"SQLite: {app_config.storage.sqlite_path}")
+    return 0
+
+
+def run_version(args: argparse.Namespace) -> int:
+    app_config = load_config()
+    print(f"InsightForge version: {__version__}")
+    if args.check_updates:
+        message = maybe_get_update_message(__version__, app_config.updates)
+        print(message if message else "InsightForge is up to date or update check is unavailable.")
+    return 0
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    app_config = load_config()
+    if args.command_name != "version":
+        update_message = maybe_get_update_message(__version__, app_config.updates)
+        if update_message:
+            print(update_message)
+
+    if args.command_name == "wrap":
+        return run_wrap(args)
+    if args.command_name == "ask":
+        return run_ask(args)
+    if args.command_name == "list":
+        return run_list(args)
+    if args.command_name == "diff":
+        return run_diff(args)
+    if args.command_name == "schema-version":
+        return run_schema_version(args)
+    if args.command_name == "migrate":
+        return run_migrate(args)
+    if args.command_name == "version":
+        return run_version(args)
+
+    parser.error("Unknown command")
+    return 2

insightforge/config.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+DEFAULT_CONFIG_PATH = Path(".insightforge.toml")
+
+
+@dataclass(slots=True)
+class PolicyConfig:
+    min_confidence: float = 0.65
+    require_sources: bool = False
+    fail_on_stderr: bool = True
+    block_absolute_language: bool = False
+    max_output_chars: int = 20000
+
+
+@dataclass(slots=True)
+class RedactionConfig:
+    enabled: bool = True
+    mask: str = "[REDACTED]"
+    patterns: list[str] = field(
+        default_factory=lambda: [
+            r"sk-[A-Za-z0-9_\-]+",
+            r"AIza[0-9A-Za-z\-_]+",
+            r"ghp_[A-Za-z0-9]+",
+            r"\b[\w.\-]+@[\w.\-]+\.\w+\b",
+            r"Bearer\s+[A-Za-z0-9\-._~+/]+=*",
+        ]
+    )
+
+
+@dataclass(slots=True)
+class StorageConfig:
+    sqlite_path: str = ".insightforge/traces.db"
+
+
+@dataclass(slots=True)
+class UpdateConfig:
+    enabled: bool = True
+    package_name: str = "insightforge"
+    check_interval_hours: int = 24
+
+
+@dataclass(slots=True)
+class AppConfig:
+    policy: PolicyConfig = field(default_factory=PolicyConfig)
+    redaction: RedactionConfig = field(default_factory=RedactionConfig)
+    storage: StorageConfig = field(default_factory=StorageConfig)
+    updates: UpdateConfig = field(default_factory=UpdateConfig)
+
+
+def load_config(cwd: Path | None = None) -> AppConfig:
+    base = cwd or Path.cwd()
+    path = base / DEFAULT_CONFIG_PATH
+    config = AppConfig()
+    if not path.exists():
+        return config
+
+    payload = tomllib.loads(path.read_text(encoding="utf-8"))
+    policy = payload.get("policy", {})
+    redaction = payload.get("redaction", {})
+    storage = payload.get("storage", {})
+    updates = payload.get("updates", {})
+
+    config.policy = PolicyConfig(
+        min_confidence=float(policy.get("min_confidence", config.policy.min_confidence)),
+        require_sources=bool(policy.get("require_sources", config.policy.require_sources)),
+        fail_on_stderr=bool(policy.get("fail_on_stderr", config.policy.fail_on_stderr)),
+        block_absolute_language=bool(
+            policy.get("block_absolute_language", config.policy.block_absolute_language)
+        ),
+        max_output_chars=int(policy.get("max_output_chars", config.policy.max_output_chars)),
+    )
+    config.redaction = RedactionConfig(
+        enabled=bool(redaction.get("enabled", config.redaction.enabled)),
+        mask=str(redaction.get("mask", config.redaction.mask)),
+        patterns=[str(item) for item in redaction.get("patterns", config.redaction.patterns)],
+    )
+    config.storage = StorageConfig(
+        sqlite_path=str(storage.get("sqlite_path", config.storage.sqlite_path)),
+    )
+    config.updates = UpdateConfig(
+        enabled=bool(updates.get("enabled", config.updates.enabled)),
+        package_name=str(updates.get("package_name", config.updates.package_name)),
+        check_interval_hours=int(
+            updates.get("check_interval_hours", config.updates.check_interval_hours)
+        ),
+    )
+    return config

insightforge/diffing.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import html
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from .models import TraceRecord
+
+
+@dataclass(slots=True)
+class TraceDiff:
+    before: TraceRecord
+    after: TraceRecord
+    summary_lines: list[str]
+    score_delta: float
+    added_flags: list[str]
+    removed_flags: list[str]
+
+
+def build_diff(before: TraceRecord, after: TraceRecord) -> TraceDiff:
+    before_flags = {_flag_key(flag) for flag in before.bias_flags + before.hallucination_flags}
+    after_flags = {_flag_key(flag) for flag in after.bias_flags + after.hallucination_flags}
+    score_delta = round(after.confidence_score - before.confidence_score, 2)
+
+    summary_lines = [
+        f"Provider: {before.provider} -> {after.provider}",
+        f"Model: {before.model_hint} -> {after.model_hint}",
+        f"Status: {before.overall_status} -> {after.overall_status}",
+        f"Confidence: {before.confidence_score:.2f} -> {after.confidence_score:.2f} ({score_delta:+.2f})",
+        f"Exit code: {before.exit_code} -> {after.exit_code}",
+        "Prompt changed." if before.prompt != after.prompt else "Prompt unchanged.",
+        "Output changed." if before.stdout != after.stdout else "Output unchanged.",
+    ]
+
+    return TraceDiff(
+        before=before,
+        after=after,
+        summary_lines=summary_lines,
+        score_delta=score_delta,
+        added_flags=sorted(after_flags - before_flags),
+        removed_flags=sorted(before_flags - after_flags),
+    )
+
+
+def write_diff_html(diff: TraceDiff, destination: Path) -> None:
+    before_json = html.escape(json.dumps(diff.before.to_dict(), indent=2))
+    after_json = html.escape(json.dumps(diff.after.to_dict(), indent=2))
+    document = f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>InsightForge Diff</title>
+  <style>
+    :root {{
+      --bg: #f3f1ea;
+      --panel: #fffdf8;
+      --ink: #1d1d1b;
+      --muted: #666050;
+      --good: #2f6f4f;
+      --bad: #aa3d2a;
+      --line: #d7d0c2;
+    }}
+    body {{
+      margin: 0;
+      font-family: "IBM Plex Sans", "Avenir Next", sans-serif;
+      color: var(--ink);
+      background: linear-gradient(180deg, #ebe4d8 0%, var(--bg) 100%);
+    }}
+    main {{
+      max-width: 1240px;
+      margin: 0 auto;
+      padding: 28px 18px 64px;
+    }}
+    .panel {{
+      background: rgba(255,253,248,0.92);
+      border: 1px solid var(--line);
+      border-radius: 22px;
+      padding: 20px;
+      box-shadow: 0 20px 70px rgba(52, 42, 28, 0.08);
+      margin-bottom: 18px;
+    }}
+    .grid {{
+      display: grid;
+      grid-template-columns: 1fr 1fr;
+      gap: 18px;
+    }}
+    .chips {{
+      display: flex;
+      flex-wrap: wrap;
+      gap: 10px;
+      margin-top: 12px;
+    }}
+    .chip {{
+      border-radius: 999px;
+      padding: 8px 12px;
+      border: 1px solid var(--line);
+      background: var(--panel);
+    }}
+    .good {{ color: var(--good); }}
+    .bad {{ color: var(--bad); }}
+    pre {{
+      overflow: auto;
+      padding: 14px;
+      border-radius: 16px;
+      background: #1d1d1b;
+      color: #f7f0e8;
+      font-size: 13px;
+    }}
+    @media (max-width: 860px) {{
+      .grid {{ grid-template-columns: 1fr; }}
+    }}
+  </style>
+</head>
+<body>
+  <main>
+    <section class="panel">
+      <h1>Trace diff</h1>
+      {''.join(f"<p>{html.escape(line)}</p>" for line in diff.summary_lines)}
+      <div class="chips">
+        {''.join(f"<span class='chip bad'>Added flag: {html.escape(flag)}</span>" for flag in diff.added_flags) or "<span class='chip'>No new flags</span>"}
+        {''.join(f"<span class='chip good'>Removed flag: {html.escape(flag)}</span>" for flag in diff.removed_flags) or "<span class='chip'>No removed flags</span>"}
+      </div>
+    </section>
+    <section class="grid">
+      <section class="panel">
+        <h2>Before</h2>
+        <pre>{before_json}</pre>
+      </section>
+      <section class="panel">
+        <h2>After</h2>
+        <pre>{after_json}</pre>
+      </section>
+    </section>
+  </main>
+</body>
+</html>
+"""
+    destination.write_text(document, encoding="utf-8")
+
+
+def render_diff_text(diff: TraceDiff) -> str:
+    lines = ["InsightForge diff"]
+    lines.extend(diff.summary_lines)
+    if diff.added_flags:
+        lines.append("Added flags: " + ", ".join(diff.added_flags))
+    if diff.removed_flags:
+        lines.append("Removed flags: " + ", ".join(diff.removed_flags))
+    return "\n".join(lines)
+
+
+def _flag_key(flag) -> str:
+    return f"{flag.code}:{flag.severity}:{flag.title}"