scroot 0.2.0__py3-none-any.whl

scroot/cli/__init__.py

@@ -0,0 +1,167 @@
+"""Scroot CLI - `scroot serve` starts the review dashboard."""
+from __future__ import annotations
+
+try:
+    import typer
+    app = typer.Typer(name="scroot", help="Scroot LLM response quality tools.")
+
+    @app.command("download-model")
+    def download_model_cmd(
+        model: str = typer.Option("phi4-mini", help="Model ID to download (phi4-mini, smollm3)"),
+    ):
+        """Download a local LLM model for offline correction."""
+        from scroot.cli.download import download_model
+        try:
+            download_model(model)
+        except Exception as e:
+            typer.echo(f"ERROR: {e}")
+            raise typer.Exit(1)
+
+    @app.command("model-info")
+    def model_info_cmd():
+        """List available local LLM models and their download status."""
+        from scroot.cli.model_info import print_model_info
+        print_model_info()
+
+    @app.command()
+    def score(
+        query: str = typer.Option(..., "--query", "-q", help="The user's query/question."),
+        response: str = typer.Option(..., "--response", "-r", help="The LLM-generated response."),
+        context: list[str] = typer.Option(
+            None, "--context", "-c", help="Grounding context chunk. Repeat for multiple chunks."
+        ),
+        json_output: bool = typer.Option(
+            False, "--json", help="Print the full result as JSON instead of a summary."
+        ),
+    ):
+        """Score a single query/response pair and print the result."""
+        import json as json_module
+        from scroot import score as score_fn
+
+        result = score_fn(query=query, response=response, context=context or None)
+
+        if json_output:
+            typer.echo(json_module.dumps(result.to_dict(), indent=2, default=str))
+            return
+
+        typer.echo(f"IQS:          {result.iqs:.2f}")
+        typer.echo(
+            f"Groundedness: {result.groundedness:.2f}"
+            if result.groundedness is not None
+            else "Groundedness: n/a (no context provided)"
+        )
+        typer.echo(f"Completeness: {result.completeness:.2f}")
+        typer.echo(f"Relevance:    {result.relevance:.2f}")
+        typer.echo(f"Consistency:  {result.consistency:.2f}")
+        typer.echo(f"Confidence:   {result.confidence:.2f}")
+        typer.echo(f"Flags:        {result.flags}")
+
+    @app.command("eval")
+    def eval_cmd(
+        suite: str = typer.Option(..., "--suite", "-s", help="Path to a YAML eval suite."),
+        fail_below: float = typer.Option(
+            None, "--fail-below", help="Override the suite's fail_below_iqs threshold."
+        ),
+        json_output: bool = typer.Option(
+            False, "--json", help="Print a machine-readable JSON summary instead of text."
+        ),
+        output: str = typer.Option(
+            None, "--output", help="Write a JUnit XML report to this path (for CI)."
+        ),
+    ):
+        """Run a YAML-defined quality regression suite (CI/CD quality gate)."""
+        import json as json_module
+        from scroot.cli.eval import format_junit_xml, format_report, load_suite, run_suite
+
+        try:
+            suite_obj = load_suite(suite)
+        except Exception as e:
+            typer.echo(f"ERROR: {e}")
+            raise typer.Exit(1)
+
+        result = run_suite(suite_obj, fail_below=fail_below)
+
+        if output:
+            with open(output, "w", encoding="utf-8") as f:
+                f.write(format_junit_xml(suite_obj, result))
+
+        if json_output:
+            typer.echo(json_module.dumps({
+                "name": suite_obj.name,
+                "passed": result.passed_count,
+                "failed": result.failed_count,
+                "avg_iqs": result.avg_iqs,
+                "results": [
+                    {
+                        "query": r.example.query,
+                        "iqs": r.iqs,
+                        "passed": r.passed,
+                        "gate_reason": r.gate_reason,
+                        "tags": r.example.tags,
+                    }
+                    for r in result.results
+                ],
+            }, indent=2))
+        else:
+            typer.echo(format_report(suite_obj, result))
+
+        if result.failed_count:
+            raise typer.Exit(1)
+
+    @app.command()
+    def serve(
+        port:  int = typer.Option(7432,          help="Port to listen on"),
+        store: str = typer.Option("./scroot_store.jsonl", help="JSONL feedback store path"),
+        host:  str = typer.Option("127.0.0.1",   help="Host to bind to"),
+        token: str = typer.Option(
+            None, "--token",
+            help="Require this shared token on all /api routes (for network "
+                 "binds). Falls back to SCROOT_DASHBOARD_TOKEN.",
+        ),
+        hosted: bool = typer.Option(False,        hidden=True),
+    ):
+        """Start the Scroot Review Console at http://localhost:7432
+
+        The dashboard has no per-user login. The default 127.0.0.1 bind is
+        single-user safe. If you bind to a routable host (e.g. --host 0.0.0.0),
+        set --token (or SCROOT_DASHBOARD_TOKEN) and/or front it with an
+        authenticating reverse proxy - otherwise the correction store and the
+        stored LLM API key are reachable by anyone on the network.
+        """
+        try:
+            import uvicorn
+        except ImportError:
+            typer.echo("ERROR: Install dashboard deps: pip install 'scroot[dashboard]'")
+            raise typer.Exit(1)
+
+        from scroot.dashboard.security import is_loopback_host, resolve_dashboard_token
+        from scroot.dashboard.server import create_app
+
+        fa_app = create_app(store_path=store, hosted=hosted, host=host, auth_token=token)
+
+        typer.echo("\n  * SCROOT Review Console")
+        typer.echo(f"  Store : {store}")
+        typer.echo(f"  URL   : http://{host}:{port}")
+        if resolve_dashboard_token(token) is not None:
+            typer.echo("  Auth  : token required (Authorization: Bearer / X-Scroot-Token)")
+        elif not is_loopback_host(host):
+            typer.echo(
+                "  WARNING: non-loopback bind with NO auth - the store and "
+                "stored API key are exposed to the network. Set --token."
+            )
+        typer.echo("")
+        uvicorn.run(fa_app, host=host, port=port, log_level="info")
+
+except ImportError:
+    # typer not installed - provide a minimal fallback
+    import sys
+
+    class _FakeCLI:
+        def command(self, *a, **kw):
+            def dec(fn): return fn
+            return dec
+        def __call__(self):
+            print("scroot: install typer for CLI support: pip install typer")
+            sys.exit(1)
+
+    app = _FakeCLI()

scroot/cli/download.py

@@ -0,0 +1,49 @@
+"""scroot download-model command."""
+from __future__ import annotations
+
+from scroot.corrector.models import (
+    DEFAULT_MODEL_ID,
+    MODEL_REGISTRY,
+    get_model_dir,
+    get_model_path,
+    is_model_downloaded,
+)
+
+
+def download_model(model_id: str = DEFAULT_MODEL_ID) -> None:
+    if model_id not in MODEL_REGISTRY:
+        ids = ", ".join(MODEL_REGISTRY.keys())
+        raise ValueError(f"Unknown model '{model_id}'. Available: {ids}")
+
+    spec = MODEL_REGISTRY[model_id]
+
+    if is_model_downloaded(model_id):
+        print(f"{spec.name} is already downloaded at {get_model_path(model_id)}")
+        return
+
+    try:
+        from huggingface_hub import hf_hub_download
+    except ImportError:
+        raise RuntimeError(
+            "huggingface-hub is not installed. "
+            "Run: pip install 'scroot[local]'"
+        )
+
+    dest = get_model_dir() / model_id
+    dest.mkdir(parents=True, exist_ok=True)
+
+    print(f"Downloading {spec.name} ({spec.size_gb} GB)...")
+    print(f"Source      : {spec.hf_repo}")
+    print(f"Destination : {dest}")
+    print()
+
+    hf_hub_download(
+        repo_id=spec.hf_repo,
+        filename=spec.hf_filename,
+        local_dir=str(dest),
+        resume_download=True,
+        token=False,
+    )
+
+    print(f"\nOK {spec.name} ready at {dest / spec.hf_filename}")
+    print("Run `scroot serve` to start the dashboard.")

scroot/cli/eval.py

@@ -0,0 +1,230 @@
+"""`scroot eval` - run a YAML-defined quality regression suite.
+
+Loads a suite of (query, response, context) examples with expected IQS/
+groundedness floors, scores each with scroot, and reports pass/fail using
+EntailmentResult.passes_gate() / gate_reason(). Intended as a CI/CD quality
+gate - exits non-zero if any example fails its gate.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from xml.etree import ElementTree as ET
+
+from scroot import score
+
+
+@dataclass
+class EvalExample:
+    """A single (query, response, context) case in an eval suite."""
+
+    query: str
+    response: str
+    context: "str | list[str] | None" = None
+    expected_iqs_min: "float | None" = None
+    tags: list[str] = field(default_factory=list)
+
+
+@dataclass
+class EvalSuite:
+    """A YAML-defined collection of EvalExamples with default gate thresholds."""
+
+    name: str
+    examples: list[EvalExample]
+    fail_below_iqs: "float | None" = None
+    fail_below_groundedness: "float | None" = None
+
+
+@dataclass
+class ExampleResult:
+    """Outcome of scoring a single EvalExample."""
+
+    example: EvalExample
+    iqs: float
+    passed: bool
+    gate_reason: "str | None"
+
+
+@dataclass
+class EvalRunResult:
+    """Aggregate outcome of running an EvalSuite."""
+
+    results: list[ExampleResult]
+
+    @property
+    def passed_count(self) -> int:
+        return sum(1 for r in self.results if r.passed)
+
+    @property
+    def failed_count(self) -> int:
+        return sum(1 for r in self.results if not r.passed)
+
+    @property
+    def avg_iqs(self) -> float:
+        if not self.results:
+            return 0.0
+        return sum(r.iqs for r in self.results) / len(self.results)
+
+
+def _import_yaml():
+    try:
+        import yaml
+    except ImportError as exc:
+        raise RuntimeError(
+            "pyyaml is required for `scroot eval`. "
+            "Install it with: pip install pyyaml"
+        ) from exc
+    return yaml
+
+
+def load_suite(path: str) -> EvalSuite:
+    """Load an EvalSuite from a YAML file.
+
+    Expected shape:
+
+        name: Support regression suite
+        fail_below_iqs: 0.70
+        fail_below_groundedness: 0.80
+        examples:
+          - query: "..."
+            response: "..."
+            context: "..."          # or a list of strings
+            expected_iqs_min: 0.75  # optional, overrides fail_below_iqs
+            tags: [billing]
+
+    Args:
+        path: Path to the YAML suite file.
+
+    Returns:
+        Parsed EvalSuite.
+
+    Raises:
+        RuntimeError: If pyyaml is not installed.
+        OSError: If the file cannot be read.
+        ValueError: If the YAML is malformed or missing required fields.
+    """
+    yaml = _import_yaml()
+
+    with open(path, encoding="utf-8") as f:
+        try:
+            data = yaml.safe_load(f)
+        except yaml.YAMLError as exc:
+            raise ValueError(f"Invalid YAML in {path}: {exc}") from exc
+
+    if not isinstance(data, dict):
+        raise ValueError(f"{path}: top-level YAML must be a mapping")
+
+    raw_examples = data.get("examples") or []
+    if not isinstance(raw_examples, list):
+        raise ValueError(f"{path}: 'examples' must be a list")
+
+    examples = []
+    for i, raw in enumerate(raw_examples):
+        if "query" not in raw or "response" not in raw:
+            raise ValueError(f"{path}: examples[{i}] is missing 'query' or 'response'")
+        examples.append(EvalExample(
+            query=raw["query"],
+            response=raw["response"],
+            context=raw.get("context"),
+            expected_iqs_min=raw.get("expected_iqs_min"),
+            tags=raw.get("tags") or [],
+        ))
+
+    return EvalSuite(
+        name=data.get("name", path),
+        examples=examples,
+        fail_below_iqs=data.get("fail_below_iqs"),
+        fail_below_groundedness=data.get("fail_below_groundedness"),
+    )
+
+
+def run_suite(suite: EvalSuite, fail_below: "float | None" = None) -> EvalRunResult:
+    """Score every example in a suite and evaluate its quality gate.
+
+    Args:
+        suite: The eval suite to run.
+        fail_below: Optional CLI override for the IQS gate threshold,
+            applied to examples that don't set their own
+            ``expected_iqs_min``.
+
+    Returns:
+        EvalRunResult with per-example outcomes and aggregate stats.
+    """
+    results = []
+    for example in suite.examples:
+        result = score(query=example.query, response=example.response, context=example.context)
+        threshold = (
+            example.expected_iqs_min
+            if example.expected_iqs_min is not None
+            else fail_below
+            if fail_below is not None
+            else suite.fail_below_iqs
+            if suite.fail_below_iqs is not None
+            else 0.70
+        )
+        reason = result.gate_reason(
+            threshold=threshold,
+            require_groundedness=suite.fail_below_groundedness,
+        )
+        results.append(ExampleResult(
+            example=example,
+            iqs=result.iqs,
+            passed=reason is None,
+            gate_reason=reason,
+        ))
+
+    return EvalRunResult(results=results)
+
+
+def format_report(suite: EvalSuite, run_result: EvalRunResult) -> str:
+    """Format a plain-text report of an eval run for CLI output."""
+    lines = [f"Eval suite: {suite.name}", ""]
+
+    for i, result in enumerate(run_result.results, start=1):
+        if result.passed:
+            continue
+        tags = f" [{', '.join(result.example.tags)}]" if result.example.tags else ""
+        lines.append(f"FAIL #{i}{tags}")
+        lines.append(f"  Query: {result.example.query}")
+        lines.append(f"  IQS:   {result.iqs:.2f}")
+        lines.append(f"  Reason: {result.gate_reason}")
+        lines.append("")
+
+    lines.append(
+        f"Summary: {run_result.passed_count}/{len(run_result.results)} passed "
+        f"- avg IQS {run_result.avg_iqs:.2f}"
+    )
+    return "\n".join(lines)
+
+
+def format_junit_xml(suite: EvalSuite, run_result: EvalRunResult) -> str:
+    """Format an eval run as a JUnit XML report for CI integration.
+
+    Each example becomes a ``<testcase>``; failing examples (per
+    ``passes_gate()``/``gate_reason()``) get a ``<failure>`` child with the
+    gate reason as the failure message.
+    """
+    testsuite = ET.Element("testsuite", {
+        "name": suite.name,
+        "tests": str(len(run_result.results)),
+        "failures": str(run_result.failed_count),
+    })
+
+    for i, result in enumerate(run_result.results, start=1):
+        tags = ", ".join(result.example.tags) if result.example.tags else ""
+        case_name = f"#{i} {tags}".strip() if tags else f"#{i} {result.example.query[:40]}"
+        testcase = ET.SubElement(testsuite, "testcase", {
+            "classname": suite.name,
+            "name": case_name,
+        })
+        if not result.passed:
+            failure = ET.SubElement(testcase, "failure", {
+                "message": result.gate_reason or "gate failed",
+            })
+            failure.text = (
+                f"Query: {result.example.query}\n"
+                f"IQS: {result.iqs:.2f}\n"
+                f"Reason: {result.gate_reason}"
+            )
+
+    return '<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(testsuite, encoding="unicode")

scroot/cli/model_info.py

@@ -0,0 +1,28 @@
+"""scroot model-info command."""
+from __future__ import annotations
+
+from scroot.corrector.models import (
+    DEFAULT_MODEL_ID,
+    MODEL_REGISTRY,
+    get_model_dir,
+    is_model_downloaded,
+)
+
+
+def print_model_info() -> None:
+    col = "{:<16} {:<16} {:<8} {:<8} {:<12}"
+    print()
+    print("  scroot models")
+    print()
+    print("  " + col.format("Model", "Status", "Size", "RAM", "License"))
+    print("  " + "-" * 60)
+    for model_id, spec in MODEL_REGISTRY.items():
+        default_tag = "  <- default" if model_id == DEFAULT_MODEL_ID else ""
+        status = "ready" if is_model_downloaded(model_id) else "not downloaded"
+        size = f"{spec.size_gb} GB"
+        ram = f"{spec.rec_ram_gb} GB"
+        print(f"  {spec.name:<20} {status:<16} {size:<8} {ram:<8} {spec.license}{default_tag}")
+    print()
+    print(f"  Models stored at: {get_model_dir()}")
+    print("  To download: scroot download-model [--model smollm3]")
+    print()

scroot/composite.py

@@ -0,0 +1,170 @@
+"""Information Quality Score (IQS) - composite metric.
+
+IQS = n / sum(w_i / s_i) -- the weighted harmonic mean of the five
+metric scores, where n = sum(w_i).
+
+Two scoring modes:
+  harmonic (default): weighted harmonic mean.  Any metric near zero drives
+      IQS to zero.  A response with groundedness=0.1 and all others at 0.9
+      scores ~0.31. Zero tolerance: a single quality failure dominates the
+      score, which matches the goal of flagging unreliable responses.
+
+  geometric: weighted geometric mean.  Penalizes low scores but does not
+      collapse to zero unless a metric is literally zero. Reflects partial
+      quality more gently: 9 correct claims + 1 wrong claim -> ~0.8 IQS
+      (not near 0).
+
+Harmonic is the default and the formula documented in the README: any
+metric near zero (e.g. a hallucinated claim) should drive the composite
+score down hard rather than being averaged away.
+
+Default weights:
+    groundedness  0.35  (most important: is it faithful to the source?)
+    completeness  0.25  (did it answer the full question?)
+    relevance     0.20  (is it on topic?)
+    consistency   0.15  (does it contradict itself?)
+    confidence    0.05  (calibration signal, low weight)
+
+When context is not provided, groundedness weight is redistributed
+proportionally across the remaining metrics.
+"""
+
+from __future__ import annotations
+
+import math
+
+
+DEFAULT_WEIGHTS = {
+    "groundedness": 0.35,
+    "completeness": 0.25,
+    "relevance": 0.20,
+    "consistency": 0.15,
+    "confidence": 0.05,
+}
+
+# RAG-optimised preset: boost groundedness, reduce completeness weight.
+# Use when the source context IS the ground truth and faithfulness is
+# the primary concern.
+RAG_WEIGHTS = {
+    "groundedness": 0.50,
+    "completeness": 0.15,
+    "relevance": 0.20,
+    "consistency": 0.10,
+    "confidence": 0.05,
+}
+
+
+def compute_iqs(
+    groundedness: float | None,
+    completeness: float,
+    relevance: float,
+    consistency: float,
+    confidence: float,
+    weights: dict | None = None,
+    mode: str = "harmonic",
+) -> float:
+    """Compute the Information Quality Score.
+
+    IQS = n / sum(w_i / s_i), where n = sum(w_i) and s_i are the metric
+    scores. This is the weighted harmonic mean.
+
+    Args:
+        groundedness: 0-1 or None if no context was provided.
+        completeness: 0-1.
+        relevance: 0-1.
+        consistency: 0-1.
+        confidence: 0-1.
+        weights: Optional custom weight dict. Missing keys default to
+            DEFAULT_WEIGHTS.
+        mode: Scoring formula.
+            "harmonic" (default) - weighted harmonic mean. Zero tolerance:
+                any metric near zero drives IQS toward zero.
+            "geometric" - weighted geometric mean. Gracefully handles
+                partial quality; does not collapse to zero unless a metric
+                is literally zero.
+
+    Returns:
+        IQS score in [0, 1].
+    """
+    scores: dict[str, float] = {
+        "completeness": completeness,
+        "relevance": relevance,
+        "consistency": consistency,
+        "confidence": confidence,
+    }
+    # None groundedness (no context) is excluded so its weight is
+    # redistributed proportionally rather than counted as a zero.
+    if groundedness is not None:
+        scores["groundedness"] = groundedness
+
+    iqs, _ = compute_iqs_detailed(scores, weights=weights, mode=mode)
+    return iqs
+
+
+def compute_iqs_detailed(
+    scores: "dict[str, float | None]",
+    weights: "dict | None" = None,
+    mode: str = "harmonic",
+) -> "tuple[float, dict[str, float]]":
+    """Compute IQS and report the effective weights actually used.
+
+    The dict-based companion to :func:`compute_iqs`. A metric is *active* when
+    it has a non-``None`` score **and** a positive weight; only active metrics
+    contribute to IQS, and their weights are renormalised to sum to 1.0
+    (proportional redistribution). This is how an inapplicable metric -
+    typically ``groundedness`` when no context was provided - is excluded
+    without being treated as a catastrophic zero.
+
+    A metric value of exactly ``0.0`` is a *real* measurement (not missing
+    data) and collapses IQS to ``0.0`` under both means.
+
+    Args:
+        scores: Metric name -> score (float) or ``None`` (inapplicable).
+        weights: Metric name -> weight. Missing keys fall back to
+            ``DEFAULT_WEIGHTS``. Need not sum to 1.0; active weights are
+            renormalised.
+        mode: ``"harmonic"`` (default) or ``"geometric"``.
+
+    Returns:
+        ``(iqs, effective_weights)`` - the IQS in ``[0.0, 1.0]`` and the
+        normalised weights of the active metrics (sums to 1.0).
+
+    Raises:
+        ValueError: if no metric is active (all ``None`` or zero-weighted).
+    """
+    w = dict(DEFAULT_WEIGHTS)
+    if weights:
+        w.update(weights)
+
+    # Active = scored (non-None) AND positively weighted. Excluding
+    # zero-weighted metrics lets a caller opt a metric out explicitly (e.g.
+    # groundedness weight 0.0 when context is never available).
+    active = {
+        k: v for k, v in scores.items()
+        if v is not None and w.get(k, 0.0) > 0.0
+    }
+    if not active:
+        raise ValueError(
+            "All metrics are None (or zero-weighted) - nothing to compute IQS "
+            "from. At least one metric must have a non-None score and a "
+            "positive weight."
+        )
+
+    total_active_weight = sum(w[k] for k in active)
+    effective_weights = {k: w[k] / total_active_weight for k in active}
+
+    # A genuine zero score is a real failure: collapse IQS to 0.0 under both
+    # means (harmonic does this via eps anyway; this makes geometric match).
+    if any(v == 0.0 for v in active.values()):
+        return 0.0, effective_weights
+
+    eps = 1e-6
+    if mode == "geometric":
+        # Weighted geometric mean: exp(sum(w_i * log(s_i))) == prod(s_i ^ w_i)
+        log_iqs = sum(effective_weights[k] * math.log(max(active[k], eps)) for k in active)
+        iqs = math.exp(log_iqs)
+    else:
+        # Weighted harmonic mean: 1 / sum(w_i / s_i), weights summing to 1.
+        iqs = 1.0 / sum(effective_weights[k] / max(active[k], eps) for k in active)
+
+    return round(min(max(iqs, 0.0), 1.0), 4), effective_weights