mobiusguard-eval 0.1.0__py3-none-any.whl

mobiusguard_eval/__init__.py

@@ -0,0 +1,7 @@
+"""Evaluation harness for LLM guardrails.
+
+Held-out attack families, false-positive sets, recall at a fixed false-positive
+rate, and CPU latency.
+"""
+
+__version__ = "0.1.0"

mobiusguard_eval/checkpoint.py

@@ -0,0 +1,72 @@
+"""Checkpoints: save finished work as it happens, so a run resumes instead of restarting.
+
+A checkpoint is an append-only JSONL file. Its first line is a *key*: everything that
+decides the result (detector and model revision, the suite's item IDs, the patterns). The
+other lines are finished units of work. Opening a checkpoint whose key differs sets the
+old file aside (``.stale``) and starts afresh, so saved work is never reused for a
+different question. A last line cut short by a crash is ignored, and its unit redone.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any, cast
+
+
+def fingerprint(value: Any) -> str:
+    """A stable hash of JSON-able data (keys sorted)."""
+    return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest()
+
+
+class Checkpoint:
+    def __init__(self, path: Path, key: dict[str, Any]) -> None:
+        self.path = path
+        self.key = key
+        self.key_hash = fingerprint(key)
+        self.units: dict[str, Any] = {}
+        path.parent.mkdir(parents=True, exist_ok=True)
+        if path.exists() and not self._load():
+            path.replace(path.with_suffix(path.suffix + ".stale"))
+        if not path.exists():
+            with path.open("w", encoding="utf-8") as f:
+                f.write(json.dumps({"checkpoint": self.key_hash, "key": key}) + "\n")
+
+    def _load(self) -> bool:
+        """Read saved units; False when the file belongs to a different key."""
+        with self.path.open(encoding="utf-8") as f:
+            lines = f.read().splitlines()
+        try:
+            header = cast("dict[str, Any]", json.loads(lines[0])) if lines else {}
+        except json.JSONDecodeError:
+            return False
+        if header.get("checkpoint") != self.key_hash:
+            return False
+        for line in lines[1:]:
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue  # cut short by a crash: that unit is done again
+            self.units[row["unit"]] = row["data"]
+        return True
+
+    def __contains__(self, unit: str) -> bool:
+        return unit in self.units
+
+    def add(self, unit: str, data: Any) -> None:
+        """Record a finished unit; it is on disk before this returns."""
+        with self.path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps({"unit": unit, "data": data}, ensure_ascii=False) + "\n")
+            f.flush()
+            os.fsync(f.fileno())
+        self.units[unit] = data
+
+
+def write_atomic(path: Path, text: str) -> None:
+    """Write a file so that readers see the old or the new content, never half of it."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(text, encoding="utf-8")
+    tmp.replace(path)

mobiusguard_eval/cli.py

@@ -0,0 +1,317 @@
+"""Command line: ``mobiusguard-eval <command>``.
+
+    sources                          list the ledger
+    build  --out runs/suite          fetch sources and build a suite
+    score  --suite S --detector D --out runs/D [--device cuda]
+    report --suite S --runs runs/a runs/b --out runs/report
+    latency --detector D [--threads N] --out runs/latency-D.json [--suite S]
+    redaction --suite S --out runs/redaction [--per-pattern 200]
+    pipeline --out runs [--tag v0.1] [--device cuda]   all of the above, in order
+
+Every step saves its work as it goes and skips what is already done for the same inputs,
+so running a command again resumes it. ``--force`` redoes a step from the start.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import sys
+import time
+import traceback
+from collections.abc import Callable, Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from . import detectors
+from .checkpoint import fingerprint, write_atomic
+from .fetch import fetch
+from .items import Item
+from .suite import build, is_built, ledger_sources, load_suite
+
+
+def log(message: str) -> None:
+    print(f"[{datetime.now(UTC):%H:%M:%S}] {message}", file=sys.stderr, flush=True)
+
+
+# ------------------------------------------------------------------------------ steps
+
+
+def step_build(out: Path, source_ids: Sequence[str] | None = None, *, force: bool = False) -> None:
+    sources = ledger_sources(source_ids)
+    if not force and is_built(sources, out):
+        log(f"suite: {out} is up to date, skipping (--force to rebuild)")
+        return
+    loaded: dict[str, list[Item]] = {}
+    for s in sources:
+        loaded[s.id] = list(s.parse(fetch(s)))
+        log(f"suite: {s.id} {len(loaded[s.id]):,} items")
+    manifest = build(sources, loaded, out)
+    log(f"suite: {manifest['items']:,} items, dropped {manifest['dropped']}")
+
+
+def step_score(
+    suite: Path,
+    detector: str,
+    out: Path,
+    *,
+    device: str = "cpu",
+    batch_size: int = 32,
+    limit: int = 0,
+    force: bool = False,
+) -> None:
+    from .run import score
+
+    items = load_suite(suite)
+    if limit:
+        items = items[:limit]
+    score(items, detectors.get(detector, device=device, batch_size=batch_size), out,
+          force=force, log=log)  # fmt: skip
+    log(f"score: wrote {out}")
+
+
+def step_latency(
+    detector: str,
+    out: Path,
+    *,
+    suite: Path | None = None,
+    threads: int = 0,
+    iterations: int = 200,
+    force: bool = False,
+) -> None:
+    from .latency import LENGTHS, measure
+
+    if threads:
+        importlib.import_module("torch").set_num_threads(threads)
+    det = detectors.get(detector)
+    config = {k: v for k, v in det.fingerprint().items() if k != "output"}
+    key: dict[str, Any] = {"config": config, "lengths": list(LENGTHS),
+                           "iterations": iterations, "threads": threads}  # fmt: skip
+    if detector == "tfidf":
+        if suite is None:
+            raise ValueError("tfidf latency needs --suite (it trains first)")
+        from .detectors.tfidf import _pipeline  # pyright: ignore[reportPrivateUsage]
+
+        items = [it for it in load_suite(suite) if not it.family.startswith("obf:")]
+        key["suite"] = fingerprint([it.id for it in items])
+        if not force and _latency_done(out, key):
+            return
+        model = _pipeline().fit([it.text for it in items], [it.label for it in items])
+
+        def one(text: str) -> float:
+            return float(model.predict_proba([text])[0][1])
+    elif detector == "keyword":
+        from .detectors.keyword import KeywordDetector
+
+        one = KeywordDetector().score_text
+    elif detector == "mobiusguard-rules":
+        from .detectors.proxy import RulesDetector
+
+        one = RulesDetector().score_text
+    elif detector.endswith("-norm"):
+        from mobiusguard.normalize import views
+
+        from .detectors.proxy import NormalizedClassifier
+
+        if not isinstance(det, NormalizedClassifier):
+            raise TypeError(detector)
+        nc = det
+
+        def one(text: str) -> float:  # every view is scored, as the detector does
+            return max(nc.score_texts([v.text for v in views(text)]))
+    else:
+        from .detectors.hf import HFClassifier
+
+        if not isinstance(det, HFClassifier):
+            raise TypeError(detector)
+        hf = det
+
+        def one(text: str) -> float:
+            return hf.score_texts([text])[0]
+
+    if detector != "tfidf" and not force and _latency_done(out, key):
+        return
+    result = measure(det, one, iterations=iterations)
+    result["key"] = fingerprint(key)
+    write_atomic(out, json.dumps(result, indent=2) + "\n")
+    log(f"latency: {detector} {json.dumps(result['tokens'])}")
+
+
+def _latency_done(out: Path, key: dict[str, Any]) -> bool:
+    if out.exists() and json.loads(out.read_text()).get("key") == fingerprint(key):
+        log(f"latency: {out} is up to date, skipping (--force to redo)")
+        return True
+    return False
+
+
+def step_redaction(suite: Path, out: Path, *, per_pattern: int = 200, force: bool = False) -> None:
+    from .redaction import evaluate
+
+    summary = evaluate(load_suite(suite), out, per_pattern=per_pattern, force=force, log=log)
+    log(f"redaction: {summary['texts']:,} texts; wrote {out}/redaction_report.md")
+
+
+def step_report(suite: Path, runs: Sequence[Path], out: Path) -> None:
+    from .run import report
+
+    ledger_sources()  # register sources so contamination marks resolve
+    report(load_suite(suite), runs, out)
+    log(f"report: wrote {out / 'report.md'}")
+
+
+# ------------------------------------------------------------------------------ pipeline
+
+
+def pipeline(
+    root: Path,
+    *,
+    tag: str = "v0.1",
+    names: Sequence[str] = detectors.NAMES,
+    device: str = "cpu",
+    batch_size: int = 32,
+    threads: int = 4,
+    per_pattern: int = 200,
+    latency: bool = True,
+    redaction: bool = True,
+    force: bool = False,
+) -> int:
+    """Every step in order. A failed step is reported and the rest still run; running the
+    pipeline again resumes from what was saved."""
+    suite = root / f"suite-{tag}"
+    scores = root / f"scores-{tag}"
+    steps: list[tuple[str, Callable[[], None]]] = [
+        ("build", lambda: step_build(suite, force=force))
+    ]
+    for name in names:
+        dev = device if name in detectors.GPU else "cpu"
+        steps.append((f"score {name}", lambda n=name, d=dev: step_score(
+            suite, n, scores / n, device=d, batch_size=batch_size, force=force)))  # fmt: skip
+    if redaction:
+        steps.append(("redaction", lambda: step_redaction(
+            suite, root / f"redaction-{tag}", per_pattern=per_pattern, force=force)))  # fmt: skip
+    if latency:  # last before the report, so nothing else competes for the CPU
+        for name in names:
+            steps.append((f"latency {name}", lambda n=name: step_latency(
+                n, root / f"latency-{tag}" / f"{n}.json", suite=suite, threads=threads,
+                force=force)))  # fmt: skip
+
+    def report() -> None:
+        done = [scores / n for n in names if (scores / n / "run.json").exists()]
+        if not done:
+            raise RuntimeError("no finished scores to report")
+        step_report(suite, done, root / f"report-{tag}")
+
+    steps.append(("report", report))
+
+    results: list[tuple[str, str, float]] = []
+    for name, run in steps:
+        t0 = time.perf_counter()
+        log(f"== {name}")
+        try:
+            run()
+            results.append((name, "ok", time.perf_counter() - t0))
+        except Exception:
+            traceback.print_exc()
+            results.append((name, "FAILED", time.perf_counter() - t0))
+            if name == "build":
+                break  # nothing else can run without a suite
+    log("summary:")
+    for name, status, seconds in results:
+        log(f"  {status:<6} {name:<24} {seconds:7.1f}s")
+    return 1 if any(status != "ok" for _, status, _ in results) else 0
+
+
+# ------------------------------------------------------------------------------ main
+
+
+def _sources(_: argparse.Namespace) -> int:
+    for s in ledger_sources():
+        trained = ", ".join(s.trained_on_by) or "-"
+        print(f"{s.id:<14} {s.role:<12} {s.licence:<11} trained on by: {trained:<22} {s.title}")
+    return 0
+
+
+def _run(a: argparse.Namespace) -> int:
+    cmd: str = a.command
+    if cmd == "sources":
+        return _sources(a)
+    if cmd == "build":
+        step_build(Path(a.out), a.sources, force=a.force)
+    elif cmd == "score":
+        step_score(Path(a.suite), a.detector, Path(a.out), device=a.device,
+                   batch_size=a.batch_size, limit=a.limit, force=a.force)  # fmt: skip
+    elif cmd == "report":
+        step_report(Path(a.suite), [Path(r) for r in a.runs], Path(a.out))
+    elif cmd == "latency":
+        step_latency(a.detector, Path(a.out), suite=Path(a.suite) if a.suite else None,
+                     threads=a.threads, iterations=a.iterations, force=a.force)  # fmt: skip
+    elif cmd == "redaction":
+        step_redaction(Path(a.suite), Path(a.out), per_pattern=a.per_pattern, force=a.force)
+    elif cmd == "pipeline":
+        return pipeline(Path(a.out), tag=a.tag, names=a.detectors, device=a.device,
+                        batch_size=a.batch_size, threads=a.threads, per_pattern=a.per_pattern,
+                        latency=not a.no_latency, redaction=not a.no_redaction,
+                        force=a.force)  # fmt: skip
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(prog="mobiusguard-eval", description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)  # fmt: skip
+    sub = ap.add_subparsers(dest="command", required=True)
+
+    sub.add_parser("sources")
+
+    p = sub.add_parser("build")
+    p.add_argument("--out", required=True)
+    p.add_argument("--sources", nargs="*", help="source IDs (default: all)")
+    p.add_argument("--force", action="store_true")
+
+    p = sub.add_parser("score")
+    p.add_argument("--suite", required=True)
+    p.add_argument("--detector", required=True, choices=detectors.NAMES)
+    p.add_argument("--out", required=True)
+    p.add_argument("--device", default="cpu")
+    p.add_argument("--batch-size", type=int, default=32)
+    p.add_argument("--limit", type=int, default=0, help="score only the first N items")
+    p.add_argument("--force", action="store_true")
+
+    p = sub.add_parser("report")
+    p.add_argument("--suite", required=True)
+    p.add_argument("--runs", nargs="+", required=True)
+    p.add_argument("--out", required=True)
+
+    p = sub.add_parser("latency")
+    p.add_argument("--detector", required=True, choices=detectors.NAMES)
+    p.add_argument("--out", required=True)
+    p.add_argument("--suite", help="needed for tfidf, which trains first")
+    p.add_argument("--threads", type=int, default=0)
+    p.add_argument("--iterations", type=int, default=200)
+    p.add_argument("--force", action="store_true")
+
+    p = sub.add_parser("redaction")
+    p.add_argument("--suite", required=True)
+    p.add_argument("--out", required=True)
+    p.add_argument("--per-pattern", type=int, default=200)
+    p.add_argument("--force", action="store_true")
+
+    p = sub.add_parser("pipeline", help="every step in order, resuming what was saved")
+    p.add_argument("--out", default="runs")
+    p.add_argument("--tag", default="v0.1")
+    p.add_argument("--detectors", nargs="+", default=list(detectors.NAMES),
+                   choices=detectors.NAMES)  # fmt: skip
+    p.add_argument("--device", default="cpu", help="for the transformer baselines")
+    p.add_argument("--batch-size", type=int, default=32)
+    p.add_argument("--threads", type=int, default=4, help="CPU threads for latency")
+    p.add_argument("--per-pattern", type=int, default=200)
+    p.add_argument("--no-latency", action="store_true")
+    p.add_argument("--no-redaction", action="store_true")
+    p.add_argument("--force", action="store_true", help="redo every step")
+
+    return _run(ap.parse_args(argv))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mobiusguard_eval/dedup.py

@@ -0,0 +1,106 @@
+"""Exact and near-duplicate detection.
+
+Near duplicates are found with MinHash over character 5-gram sets and grouped into
+clusters; a suite puts a whole cluster in one split, so a near copy of a test item can't
+sit in the calibration split (or in a baseline's training data).
+"""
+
+# numpy's stubs leave some overloads partially unknown under strict mode.
+# pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Sequence
+from typing import cast
+
+import numpy as np
+import numpy.typing as npt
+
+from .items import normalize
+
+SHINGLE = 5
+N_PERM = 128
+BANDS = 32  # 32 bands x 4 rows: pairs above ~0.5 Jaccard become candidates
+JACCARD = 0.8  # candidates at or above this estimated Jaccard are duplicates
+
+_U64 = np.uint64
+_rng = np.random.default_rng(20261027)
+_A = _rng.integers(1, 2**63, N_PERM, dtype=np.int64).astype(_U64) | _U64(1)
+_B = _rng.integers(0, 2**63, N_PERM, dtype=np.int64).astype(_U64)
+_P = _U64(1_000_003)
+
+
+def _shingle_hashes(text: str) -> npt.NDArray[np.uint64]:
+    codes = np.frombuffer(normalize(text).encode("utf-32-le"), dtype=np.uint32).astype(_U64)
+    if len(codes) < SHINGLE:
+        codes = np.concatenate([codes, np.zeros(SHINGLE - len(codes), dtype=_U64)])
+    n = len(codes) - SHINGLE + 1
+    h = np.zeros(n, dtype=_U64)
+    with np.errstate(over="ignore"):
+        for j in range(SHINGLE):
+            h = h * _P + codes[j : j + n]
+    return np.unique(h)
+
+
+def signature(text: str) -> npt.NDArray[np.uint64]:
+    """MinHash signature: for each of N_PERM hash functions, the minimum over shingles."""
+    h = _shingle_hashes(text)
+    with np.errstate(over="ignore"):
+        sig = (np.outer(_A, h) + _B[:, None]).min(axis=1)
+    return cast("npt.NDArray[np.uint64]", sig)
+
+
+def similarity(a: npt.NDArray[np.uint64], b: npt.NDArray[np.uint64]) -> float:
+    """Estimated Jaccard similarity of the two shingle sets."""
+    return float(np.mean(a == b))
+
+
+class _UnionFind:
+    def __init__(self, n: int) -> None:
+        self.parent = list(range(n))
+
+    def find(self, i: int) -> int:
+        while self.parent[i] != i:
+            self.parent[i] = self.parent[self.parent[i]]
+            i = self.parent[i]
+        return i
+
+    def union(self, i: int, j: int) -> None:
+        a, b = self.find(i), self.find(j)
+        if a != b:
+            self.parent[max(a, b)] = min(a, b)
+
+
+def clusters(texts: Sequence[str], *, threshold: float = JACCARD) -> list[int]:
+    """Cluster index for each text: exact (normalised) and near duplicates share one.
+
+    The index is the position of the cluster's first member, so it is deterministic for a
+    given input order.
+    """
+    n = len(texts)
+    uf = _UnionFind(n)
+    first: dict[str, int] = {}
+    for i, t in enumerate(texts):
+        key = normalize(t)
+        if key in first:
+            uf.union(first[key], i)
+        else:
+            first[key] = i
+
+    reps = sorted(first.values())
+    sigs = {i: signature(texts[i]) for i in reps}
+    rows = N_PERM // BANDS
+    # Each bucket keeps a few representatives; a new text is compared with those only.
+    buckets: dict[tuple[int, bytes], list[int]] = defaultdict(list)
+    for i in reps:
+        sig = sigs[i]
+        for band in range(BANDS):
+            key = (band, sig[band * rows : (band + 1) * rows].tobytes())
+            members = buckets[key]
+            for j in members:
+                if uf.find(i) != uf.find(j) and similarity(sig, sigs[j]) >= threshold:
+                    uf.union(i, j)
+            if len(members) < 8:
+                members.append(i)
+    return [uf.find(i) for i in range(n)]

mobiusguard_eval/detectors/__init__.py

@@ -0,0 +1,110 @@
+"""Detectors the harness can score: our own and the baselines (decision 0006, rule 6).
+
+A detector gives every item a score where higher means "more likely an attack". It sees the
+whole list of items so that a baseline which needs training (TF-IDF) can train
+leave-one-source-out; detectors that don't train just score the texts.
+
+Work is split into *groups* (chunks of items, or one source for TF-IDF) so that a run can
+save each group as it finishes and resume after a crash (:mod:`..checkpoint`).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import Protocol
+
+from ..items import Item
+
+
+class Detector(Protocol):
+    name: str
+
+    def info(self) -> dict[str, str]:
+        """Model ID, revision, versions, device: whatever identifies what was measured."""
+        ...
+
+    def fingerprint(self) -> dict[str, str]:
+        """What must match for saved scores to be reused (no timings or counts)."""
+        ...
+
+    def groups(self, items: Sequence[Item]) -> list[tuple[str, list[int]]]:
+        """Units of work: a name and the positions of the items it scores."""
+        ...
+
+    def score_group(self, items: Sequence[Item], idx: Sequence[int]) -> list[float]:
+        """Scores for ``items[i]`` for each ``i`` in ``idx``, in that order."""
+        ...
+
+
+CHUNK = 2048
+
+
+def chunks(n: int, size: int = CHUNK) -> list[tuple[str, list[int]]]:
+    """Fixed-size groups by position, for detectors that score items independently."""
+    return [(f"chunk-{i // size:05d}", list(range(i, min(i + size, n)))) for i in range(0, n, size)]
+
+
+def score_items(detector: Detector, items: Sequence[Item]) -> list[float]:
+    """Score everything in one go, without checkpoints."""
+    scores = [0.0] * len(items)
+    for _, idx in detector.groups(items):
+        for i, s in zip(idx, detector.score_group(items, idx), strict=True):
+            scores[i] = s
+    return scores
+
+
+def get(name: str, *, device: str = "cpu", batch_size: int = 32) -> Detector:
+    """A detector by name (see NAMES)."""
+    factories: dict[str, Callable[[], Detector]] = {}
+
+    from .keyword import KeywordDetector
+
+    factories["keyword"] = KeywordDetector
+
+    def tfidf() -> Detector:
+        from .tfidf import TfidfLosoDetector
+
+        return TfidfLosoDetector()
+
+    def hf(key: str) -> Callable[[], Detector]:
+        def make() -> Detector:
+            from .hf import HF_MODELS, HFClassifier
+
+            return HFClassifier(HF_MODELS[key], device=device, batch_size=batch_size)
+
+        return make
+
+    def rules() -> Detector:
+        from .proxy import RulesDetector
+
+        return RulesDetector()
+
+    def norm(key: str) -> Callable[[], Detector]:
+        def make() -> Detector:
+            from .proxy import NormalizedClassifier
+
+            return NormalizedClassifier(key, device=device, batch_size=batch_size)
+
+        return make
+
+    factories["tfidf"] = tfidf
+    factories["protectai-v2"] = hf("protectai-v2")
+    factories["piguard"] = hf("piguard")
+    factories["mobiusguard-rules"] = rules
+    factories["protectai-v2-norm"] = norm("protectai-v2")
+    factories["piguard-norm"] = norm("piguard")
+    if name not in factories:
+        raise KeyError(f"unknown detector {name!r}; choose from {sorted(factories)}")
+    return factories[name]()
+
+
+NAMES = (
+    "keyword",
+    "tfidf",
+    "protectai-v2",
+    "piguard",
+    "mobiusguard-rules",
+    "protectai-v2-norm",
+    "piguard-norm",
+)
+GPU = ("protectai-v2", "piguard", "protectai-v2-norm", "piguard-norm")

mobiusguard_eval/detectors/hf.py

@@ -0,0 +1,38 @@
+"""Pretrained classifiers for the harness: the proxy's own ``mobiusguard.classifier``,
+with the methods the harness needs (fingerprint, groups). Needs the ``hf`` extra.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from mobiusguard.classifier import HF_MODELS, HFModel
+from mobiusguard.classifier import HFClassifier as _ProxyClassifier
+
+from ..items import Item
+from . import chunks
+
+__all__ = ["HF_MODELS", "HFClassifier", "HFModel"]
+
+
+class HFClassifier(_ProxyClassifier):
+    def fingerprint(self) -> dict[str, str]:
+        return {
+            "repo": self.spec.repo,
+            "revision": self.spec.revision,
+            "max_length": str(self.spec.max_length),
+            "device": self.device.split(":")[0],
+            "output": "log-odds",
+        }
+
+    def groups(self, items: Sequence[Item]) -> list[tuple[str, list[int]]]:
+        return chunks(len(items))
+
+    def score_group(self, items: Sequence[Item], idx: Sequence[int]) -> list[float]:
+        # Sort by length so batches pad less; return in the given order.
+        order = sorted(range(len(idx)), key=lambda k: len(items[idx[k]].text))
+        scores = self.score_texts([items[idx[k]].text for k in order])
+        result = [0.0] * len(idx)
+        for k, s in zip(order, scores, strict=True):
+            result[k] = s
+        return result