foldkeep 0.9.2__py3-none-any.whl

foldkeep/__init__.py

@@ -0,0 +1,16 @@
+"""FoldKeep -- Agent Context Infrastructure.
+
+Layers:
+  compressor : FastLingua engine (entity-preserving heuristic compression)
+  estimate   : token & cost estimation across popular models
+  store      : named compressed-context snapshots for agent workflows
+
+Layers: core.compressor | estimate | store(SQLite memory bank) | packing | mcp_server.
+"""
+__version__ = "0.6.0"
+
+from .core.compressor import FastLingua  # noqa: F401
+from .benchmark.suite import run_bench, evolve  # noqa: F401
+from .estimate import estimate_tokens, estimate_cost, savings  # noqa: F401
+from .memory import store  # noqa: F401
+from .packing import pack, render  # noqa: F401

foldkeep/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

foldkeep/cli.py

@@ -0,0 +1,246 @@
+"""FoldKeep unified CLI: oc compress / estimate / store / bench / evolve."""
+import argparse
+import json
+import os
+import sys
+
+from . import __version__
+from .core.compressor import FastLingua, GENES
+from .benchmark.suite import run_bench, evolve, EVOLVE_FILE
+from .memory import store
+from .estimate import estimate_cost, savings
+from .packing import pack, render
+from .diff import commit as ctx_commit, log as ctx_log, show as ctx_show
+from . import session as sess
+
+def _read(src):
+    if src == "-":
+        raw = sys.stdin.buffer.read()
+        try:
+            return raw.decode("utf-8")
+        except UnicodeDecodeError:
+            return raw.decode("gbk", "replace")
+    with open(src, encoding="utf-8-sig") as f:
+        return f.read().lstrip("\ufeff")
+
+def _adapt(fl, stats, target):
+    try:
+        cur = {}
+        if os.path.exists(EVOLVE_FILE):
+            with open(EVOLVE_FILE, encoding="utf-8") as f:
+                cur = json.load(f)
+        err = stats["ratio"] - target
+        if abs(err) > 0.05:
+            lo, hi = GENES["dup_th"]
+            d = float(cur.get("dup_th", 0.72)) - (0.01 if err > 0 else -0.008)
+            cur["dup_th"] = round(min(hi, max(lo, d)), 4)
+            lo, hi = GENES["ent_w"]
+            d = float(cur.get("ent_w", 3.0)) - (0.08 if err > 0 else -0.10)
+            cur["ent_w"] = round(min(hi, max(lo, d)), 4)
+            lo, hi = GENES["cont_w"]
+            d = float(cur.get("cont_w", 1.0)) - (0.05 if err > 0 else -0.04)
+            cur["cont_w"] = round(min(hi, max(lo, d)), 4)
+            with open(EVOLVE_FILE, "w", encoding="utf-8") as f:
+                json.dump(cur, f, indent=1)
+    except Exception:
+        pass
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(
+        prog="foldkeep",
+        description="FoldKeep: Agent Context Infrastructure (compress | estimate | store | bench | evolve)")
+    ap.add_argument("--version", action="version", version="foldkeep " + __version__)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    c = sub.add_parser("compress", help="compress text via FastLingua engine")
+    c.add_argument("input", help="file path or '-' for stdin")
+    c.add_argument("--ratio", type=float, default=0.5)
+    c.add_argument("--mode", choices=FastLingua.MODES, default="balanced")
+    c.add_argument("--anchors", action="store_true")
+    c.add_argument("--collapse", action="store_true")
+    c.add_argument("--adapt", action="store_true")
+    c.add_argument("--no-auto", action="store_true")
+    c.add_argument("--save", metavar="NAME", help="also store snapshot for later reuse")
+    c.add_argument("-o", "--output")
+
+    e = sub.add_parser("estimate", help="token & cost estimate")
+    e.add_argument("input")
+    e.add_argument("--model", default="claude-sonnet")
+    e.add_argument("--compress", action="store_true", help="also show post-compression savings")
+
+    st = sub.add_parser("store", help="snapshot management")
+    st.add_argument("op", choices=["list", "get", "delete"])
+    st.add_argument("name", nargs="?")
+
+    b = sub.add_parser("bench", help="run benchmark")
+    b.add_argument("--full", action="store_true", help="1000-sample synthetic suite")
+
+    pk = sub.add_parser("pack", help="fit multiple files into a token budget")
+    pk.add_argument("inputs", nargs="+")
+    pk.add_argument("--budget", type=int, default=8000)
+    pk.add_argument("--ratio", type=float, default=0.5)
+    pk.add_argument("--model", default=None, help="also show cost for this model")
+    pk.add_argument("--save", metavar="NAME")
+    pk.add_argument("-o", "--output")
+    sub.add_parser("selftest", help="run the 16-case edge battery")
+
+    ss = sub.add_parser("session", help="long-conversation rolling compression")
+    ss.add_argument("op", choices=["new", "push", "show", "stat"])
+    ss.add_argument("name")
+    ss.add_argument("-r", "--role", default="user", choices=["user", "assistant", "system"])
+    ss.add_argument("-t", "--text")
+    ss.add_argument("-f", "--file")
+    ss.add_argument("--budget", type=int, default=None)
+    ss.add_argument("--roll-threshold", type=int, default=None,
+                    help="override auto-rollup token threshold")
+    cx = sub.add_parser("ctx", help="context version chain: commit/log/show/restore")
+    cx.add_argument("op", choices=["commit", "log", "show", "restore"])
+    cx.add_argument("target", nargs="?", help="file path (commit) or version id")
+    cx.add_argument("-p", "--project", default="default")
+    cx.add_argument("-l", "--label", default="")
+    cx.add_argument("-o", "--output")
+    ev = sub.add_parser("evolve", help="self-evolve compressor genes")
+    ev.add_argument("--gens", type=int, default=8)
+    ev.add_argument("--pop", type=int, default=8)
+    ev.add_argument("--seed", type=int, default=42)
+
+    a = ap.parse_args(argv)
+
+    if a.cmd == "compress":
+        text = _read(a.input)
+        fl = FastLingua(a.mode, a.ratio, a.anchors,
+                        auto=not a.no_auto, collapse=a.collapse)
+        comp, stats = fl.compress(text)
+        if a.adapt:
+            _adapt(fl, stats, a.ratio)
+        out = comp  # engine already renders [F#] anchors when enabled
+        if a.output:
+            with open(a.output, "w", encoding="utf-8") as f:
+                f.write(out)
+        else:
+            print(out)
+        if a.save:
+            p = store.save(a.save, text, comp,
+                           {"ratio": stats["ratio"], "mode": a.mode})
+            print("[stored] %s" % p, file=sys.stderr)
+
+    elif a.cmd == "estimate":
+        text = _read(a.input)
+        info = estimate_cost(text, a.model)
+        print("input : %6d tokens | $%.6f  (%s)" % (info["tokens"], info["usd"], a.model))
+        if a.compress:
+            comp, _st = FastLingua("balanced", 0.5).compress(text)
+            after = estimate_cost(comp, a.model)
+            sv = savings(text, comp, a.model)
+            print("after : %6d tokens | $%.6f" % (after["tokens"], after["usd"]))
+            print("saved : %6d tokens | $%.6f  | compressed to %.1f%%"
+                  % (sv["tokens_saved"], sv["usd_saved"], sv["ratio"] * 100))
+
+    elif a.cmd == "store":
+        if a.op == "list":
+            rows = store.list_all()
+            if not rows:
+                print("(empty)")
+            for r in rows:
+                print("%-24s %7d -> %7d chars" % (r["name"], r["orig"], r["comp"]))
+        elif a.op == "get":
+            r = store.get(a.name or "")
+            print(r["text"] if r else "(not found)")
+        else:
+            print("deleted" if store.delete(a.name or "") else "(not found)")
+
+    elif a.cmd == "bench":
+        run_bench(getattr(a, "full", False))
+
+    elif a.cmd == "pack":
+        items = []
+        for src in a.inputs:
+            try:
+                items.append((os.path.basename(src), _read(src)))
+            except Exception as ex:
+                print("[skip] %s: %s" % (src, ex), file=sys.stderr)
+        if not items:
+            print("(no readable inputs)", file=sys.stderr)
+            return
+        res = pack(items, a.budget, a.ratio)
+        out = render(res)
+        if a.model:
+            from .estimate import estimate_cost
+            info = estimate_cost(out, a.model)
+            out += "\n=== cost: %d tokens | $%.6f (%s) ===" % (
+                info["tokens"], info["usd"], a.model)
+        if a.output:
+            with open(a.output, "w", encoding="utf-8") as f:
+                f.write(out)
+        else:
+            print(out)
+        if a.save:
+            pth = store.save(a.save, "".join(t for _, t in items), out,
+                             {"budget": a.budget, "used": res["used"]})
+            print("[stored] %s" % pth, file=sys.stderr)
+
+    elif a.cmd == "selftest":
+        from .selftest import run
+        R, fails = run()
+        print("---")
+        print("TOTAL %d | PASS %d | FAIL %d" % (len(R), len(R) - len(fails), len(fails)))
+        if fails:
+            raise SystemExit(1)
+
+    elif a.cmd == "session":
+        if a.op == "new":
+            sess.new(a.name)
+            print("session '%s' ready" % a.name)
+        elif a.op == "push":
+            text = a.text if a.text is not None else (_read(a.file) if a.file else None)
+            if not text:
+                print("need -t TEXT or -f FILE", file=sys.stderr)
+                return
+            kw = {} if a.roll_threshold is None else {"roll_threshold": a.roll_threshold}
+            r = sess.push(a.name, a.role, text, **kw)
+            st = sess.stat(a.name)
+            print("pushed #%d (%s) | rolled %d | session: %d turns, ~%d tok"
+                  % (r["seq"], a.role, r["rolled"], st["turns"], st["est_tokens"]))
+        elif a.op == "show":
+            out, used, dropped = sess.render(a.name, a.budget)
+            print(out)
+        else:
+            print(sess.stat(a.name))
+
+    elif a.cmd == "ctx":
+        if a.op == "commit":
+            if not a.target:
+                print("need file path", file=sys.stderr)
+                return
+            text = _read(a.target)
+            r = ctx_commit(a.project, text, a.label)
+            print("committed v%d [%s] %d -> %d chars (%.1f%%), stored %s"
+                  % (r["id"], a.project, r["orig"], r["comp"],
+                     r["ratio"] * 100, r["stored"]))
+        elif a.op == "log":
+            rows = ctx_log(a.project)
+            if not rows:
+                print("(empty)")
+            for r in rows:
+                print("v%-4d %-16s parent=%-4s %6d->%6d %s"
+                      % (r["id"], r["label"] or "-", r["parent"] or "-",
+                         r["orig"], r["comp"], "FULL" if r["full"] else "delta"))
+        else:
+            if not a.target or not a.target.isdigit():
+                print("need version id", file=sys.stderr)
+                return
+            text = ctx_show(int(a.target))
+            if text is None:
+                print("(version not found)", file=sys.stderr)
+                return
+            if a.output:
+                with open(a.output, "w", encoding="utf-8") as f:
+                    f.write(text)
+            else:
+                print(text)
+
+    elif a.cmd == "evolve":
+        evolve(a.gens, a.pop, a.seed)
+
+if __name__ == "__main__":
+    main()

foldkeep/core/compressor.py

@@ -0,0 +1,346 @@
+# -*- coding: utf-8 -*-
+"""FastLingua v2 - model-free context compressor.
+Guarantees: entity retention (hard rules), determinism, no model needed."""
+import json
+import argparse, json, os, random, re, sys, time
+from collections import Counter
+
+ENTITY_PATTERNS = [
+    re.compile(r'https?://[A-Za-z0-9\-._~:/?#@!$&*+,;=%\[\]]+|www\.[A-Za-z0-9\-._~/%]+', re.I),
+    re.compile(r'\b[\w.+-]+@[\w-]+\.[\w.]+\b'),
+    re.compile(r'[A-Za-z]:\\[^\s，。；：！？、）（]+'),
+    re.compile(r'(?:/[\w.\-]+){2,}'),
+    re.compile(r'\b[\w\-]+(?:[/\\][\w\-]+)+\.(?:py|js|ts|json|md|txt|html|css|java|c|cpp|go|rs|sql|yaml|yml|bat|sh)\b', re.I),
+    re.compile(r'(?<![\w.\-])[\w\-]+\.(?:py|js|ts|json|md|txt|log|db|exe|bat|sh)(?![\w\-])', re.I),
+    re.compile(r'\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:[ T]\d{1,2}:\d{2}(?::\d{2})?)?'),
+    re.compile(r'\b\d{1,2}:\d{2}(?::\d{2})?\b'),
+    re.compile(r'\b[vV]?\d+(?:\.\d+)+[a-z]?\b'),
+    re.compile(r'\d+(?:\.\d+)?\s*(?:%|px|ms|[kKmMgGtT]?B|kb|mb|gb|tb|hz|万|亿|元|秒|毫秒|分钟|小时|天|周|月|年|个|人|次|倍)', re.I),
+    re.compile(r'(?<![A-Za-z0-9_])[A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*)+\(\)'),
+    re.compile(r'(?<![A-Za-z0-9_])[A-Za-z_][\w]*(?:\.[A-Za-z_][\w]*){1,5}(?![A-Za-z0-9_])'),
+    re.compile(r'(?<![A-Za-z0-9_])[a-z_][a-z0-9]*(?:_[a-z0-9]+)+'),
+    re.compile(r'(?<![A-Za-z0-9_])[a-z]+(?:[A-Z][a-z0-9]+)+'),
+    re.compile(r'(?<![A-Za-z0-9_])[A-Z][A-Z0-9_]{2,}'),
+    re.compile(r'"[^"\n]{2,80}"'),
+]
+# single-pass merged entity scanner (order = priority)
+COMBINED_ENT_PAT = re.compile("|".join("(?:%s)" % pp.pattern for pp in ENTITY_PATTERNS))
+
+
+KEEP_KEYWORDS = ['must', 'need', 'error', 'Error', 'ERROR', 'fail', 'crash',
+                 'TODO', 'FIXME', 'critical', 'root cause', 'fix',
+                 '必须', '错误', '报错', '重要', '关键', '原因', '决定', '结论',
+                 '总之', '综上', '总结', '建议', '注意', 'summary', 'recommend', 'conclusion']
+
+CAUSAL_TERMS = ['because', 'therefore', 'thus', 'hence', 'root cause', 'due to',
+                'as a result', '因为', '所以', '因此', '导致', '造成', '原因', '结果',
+                '由于', '从而', '进而', '以至于']
+CODE_TERMS = ['error', 'exception', 'traceback', 'bug', 'fix', 'patch', 'deploy',
+              'build', 'test', 'crash', 'stack', 'assert', 'timeout', 'rollback',
+              'compile', 'runtime', '报错', '异常', '崩溃', '部署', '编译', '构建',
+              '测试', '回滚', '补丁', '上线']
+FILLERS = ['basically', 'actually', 'literally', 'obviously', 'essentially',
+           'you know', 'i mean', 'i think', 'i believe', 'in my opinion',
+           'sort of', 'kind of', 'to be honest', 'needless to say',
+           '我觉得', '我认为', '说实话', '坦白说', '众所周知', '毫无疑问',
+           '基本上', '一般来说', '就是说', '也就是说',
+           '这样的话', '其实吧', '你知道吗', '我们可以看到']
+
+STOPWORDS_EN = set('''a an the and or but nor so yet for of to in on at by with
+from as is are was were be been being am do does did doing have has had having
+will would shall should can could may might this that these those i you he she
+it we they me him her us them my your his its our their there here when where
+why how what which who whom whose if then than too very just also only own same
+such no not now out up down off over under again further once about into through
+during before after above below between both each few more most other some any
+all'''.split())
+
+STOPCHARS_ZH = set('的吗呢吧啊呀哦喔嘛么呗啦喽哇嘿哟欸诶之乎者矣焉哉')
+
+ENT_RE = re.compile('|'.join('(?:%s)' % pp.pattern for pp in ENTITY_PATTERNS))
+PH_RE = re.compile(r'\ue000\d+\ue001')  # placeholder scanner (protected text)
+
+
+GENES = {
+    'dup_th':     (0.55, 0.90),
+    'ent_w':      (1.0, 6.0),
+    'cont_w':     (0.4, 3.0),
+    'keep_bonus': (2.0, 20.0),
+    'causal_w':   (0.5, 4.0),
+    'code_w':     (0.5, 4.0),
+}
+
+_EVOLVING = False
+
+def _evolved_path():
+    return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                        'evolved.json')
+
+
+class FastLingua:
+    MODES = ('light', 'balanced', 'aggressive')
+
+    def __init__(self, mode='balanced', target_ratio=0.5, anchors=False, auto=True, collapse=False):
+        self.mode = mode
+        self.target_ratio = target_ratio
+        self.anchors = anchors
+        self.auto = auto          # statistical auto-discovery of fillers
+        self.collapse = collapse  # exact-repeat phrase collapsing (opt-in)
+        # ---- evolvable genes ----
+        self.dup_th = 0.72
+        self.ent_w = 3.0
+        self.cont_w = 1.0
+        self.keep_bonus = 10.0
+        self.causal_w = 1.5
+        self.code_w = 1.5
+        self.evolved = self._load_genome()
+        self.last_auto = set()    # words auto-discovered in last run
+        self._store = []
+
+    def _load_genome(self):
+        try:
+            with open(_evolved_path(), encoding='utf-8') as f:
+                evo = json.load(f)
+            n = 0
+            for k in GENES:
+                if k in evo:
+                    setattr(self, k, float(evo[k]))
+                    n += 1
+            return n > 0
+        except Exception:
+            return False
+
+    def _protect(self, text):
+        self._store = []
+        def sub(m):
+            self._store.append(m.group(0))
+            return '\uE000%d\uE001' % (len(self._store) - 1)
+        text = COMBINED_ENT_PAT.sub(sub, text)
+        return text
+
+    def _validate(self, orig, comp):
+        return _instance_validate(self, orig, comp)
+
+    def _restore(self, text):
+        def sub(m):
+            i = int(m.group(1))
+            return self._store[i] if i < len(self._store) else m.group(0)
+        return re.sub('\uE000(\\d+)\uE001', sub, text)
+
+    @staticmethod
+    def _split_sentences(text):
+        parts = re.split(r'(?<=[。！？!?；;])|\n'
+                         r'|(?<=[A-Za-z0-9)\]”])\.\s+(?=[A-Z\u4e00-\u9fff])', text)
+        parts = [p.strip() for p in parts if p and p.strip()]
+        abbrev = {'com', 'net', 'org', 'gov', 'edu', 'io', 'inc', 'ltd', 'co',
+                  'vs', 'etc', 'fig', 'no', 'dr', 'mr', 'mrs', 'ms', 'st',
+                  'jr', 'sr', 'eg', 'ie'}
+        merged = []
+        for p in parts:
+            if merged:
+                words = merged[-1].split()
+                w = re.sub(r'[^A-Za-z]', '', words[-1]) if words else ''
+                if w in abbrev or (len(w) <= 2 and w.isalpha() and w.islower()):
+                    merged[-1] = merged[-1] + ' ' + p
+                    continue
+            merged.append(p)
+        # long-line chunking: punctuation-free mega-lines (raw dumps) can
+        # never be pruned as one giant sentence; split into ~500-char pieces
+        chunked = []
+        for p in merged:
+            if len(p) > 2000:
+                pieces = []
+                for w in p.split(' '):
+                    while len(w) > 500:          # spaceless mega-word: hard slice
+                        pieces.append(w[:500])
+                        w = w[500:]
+                    pieces.append(w)
+                buf = ''
+                for w in pieces:
+                    if buf and len(buf) + len(w) + 1 > 500:
+                        chunked.append(buf)
+                        buf = w
+                    else:
+                        buf = (buf + ' ' + w) if buf else w
+                if buf:
+                    chunked.append(buf)
+            else:
+                chunked.append(p)
+        return chunked
+
+    @staticmethod
+    def _sig(s):
+        s = PH_RE.sub('', s)
+        words = set(w.lower() for w in re.findall(r'[A-Za-z]{2,}|\d+', s))
+        if words:
+            return words
+        chars = [c for c in s if '\u4e00' <= c <= '\u9fff']
+        return set('%s%s' % b for b in zip(chars, chars[1:]))
+
+    def _dedupe(self, sentences):
+        kept, sigs = [], []
+        for s in sentences:
+            sig = self._sig(s)
+            if not sig or PH_RE.search(s):
+                kept.append(s)
+                continue
+            dup = any(len(sig & p) / max(len(sig | p), 1) > self.dup_th for p in sigs[-12:])
+            if not dup:
+                kept.append(s)
+                sigs.append(sig)
+        return kept
+
+    def _density(self, s):
+        if not s:
+            return 0.0
+        ent = len(PH_RE.findall(s))
+        words = len(re.findall(r'[A-Za-z]{2,}|[\u4e00-\u9fff]', s))
+        keep = any(k in s for k in KEEP_KEYWORDS)
+        causal = sum(1 for k in CAUSAL_TERMS if k in s)
+        code = sum(1 for k in CODE_TERMS if k in s)
+        base = (ent * self.ent_w + words * self.cont_w
+                + causal * self.causal_w + code * self.code_w)
+        return base / max(len(s), 1) * (self.keep_bonus if keep else 1.0)
+
+    def _auto_filler_words(self, text):
+        """Statistical discovery: frequent short lowercase words = likely filler.
+        Guards: never touch Capitalized words (proper nouns), len<=7, freq>=4."""
+        tokens = re.findall(r'(?<![\w\-])[A-Za-z]+(?![\w\-])', PH_RE.sub(' ', text))
+        total = len(tokens)
+        if total < 60:
+            return set()
+        cnt = Counter(w.lower() for w in tokens)
+        need = max(4, total // 100)
+        found = set()
+        for w, c in cnt.items():
+            if c < need or len(w) > 7 or w in STOPWORDS_EN:
+                continue
+            # proper-noun guard: capitalized occurrence anywhere -> skip
+            if re.search(r'(?<![\w\-])' + re.escape(w.capitalize()) + r'(?![\w\-])', text):
+                continue
+            found.add(w)
+        return found
+
+    @staticmethod
+    def _collapse_repeats(s, min_count=2, long_len=16):
+        """Collapse EXACT repeated phrases (zero info loss).
+        Eligible: appears >=3 times, or >=2 times when phrase is long (>=16 chars)."""
+        changed = True
+        while changed:
+            changed = False
+            L = len(s)
+            max_n = min(40, L // max(min_count, 2))
+            for n in range(max_n, 5, -1):
+                seen = {}
+                for i in range(L - n + 1):
+                    if i > 0 and s[i - 1] not in ' \n':
+                        continue
+                    j = i + n
+                    if j < L and s[j] not in ' \n.,;:!?)':
+                        continue
+                    seen.setdefault(s[i:i + n], []).append(i)
+                hit = None
+                for g, idxs in seen.items():
+                    if '\n' in g or '\ue000' in g or '\ue001' in g:
+                        continue
+                    ok = len(idxs) >= 3 or (len(idxs) >= min_count and n >= long_len)
+                    if ok:
+                        hit = (g, idxs)
+                        break
+                if hit:
+                    g, idxs = hit
+                    for i in reversed(idxs[1:]):
+                        s = s[:i] + s[i + n:]
+                    changed = True
+                    break
+        return s
+
+    def compress(self, text):
+        t0 = time.perf_counter()
+        orig_len = len(text)
+        has_zh = bool(re.search('[\u4e00-\u9fff]', text))
+        orig_text = text
+        text = self._protect(text)
+        n_ent = len(self._store)
+        for f in FILLERS:
+            text = re.sub(re.escape(f), '', text, flags=re.I)
+        sents = self._dedupe(self._split_sentences(text))
+        if self.mode in ('balanced', 'aggressive'):
+            # hyphen-aware: never split compounds like in-memory / state-of-the-art
+            def prune(m):
+                w = m.group(0)
+                return '' if w.lower() in STOPWORDS_EN else w
+            sents = [re.sub(r'(?<![\w\-])[A-Za-z]+(?![\w\-])', prune, s) for s in sents]
+        if self.mode == 'aggressive':
+            def prune_zh(m):
+                return ''.join(c for c in m.group(0) if c not in STOPCHARS_ZH)
+            sents = [re.sub(r'[\u4e00-\u9fff]+', prune_zh, s) for s in sents]
+        if self.auto and self.mode != 'light':
+            extra = self._auto_filler_words(text)
+            self.last_auto = extra
+            if extra:
+                alt = '|'.join(re.escape(w) for w in sorted(extra, key=len, reverse=True))
+                apat = re.compile(r'(?<![\w\-])(?:%s)(?![\w\-])' % alt, re.I)
+                sents = [apat.sub('', x) for x in sents]
+            if self.collapse:
+                joined = self._collapse_repeats('\n'.join(sents))
+                sents = [x for x in joined.split('\n') if x.strip()]
+        cur = sum(len(s) for s in sents)
+        budget = max(int(orig_len * self.target_ratio), 1)
+        if cur > budget and self.mode != 'light':
+            order = sorted(range(len(sents)), key=lambda i: self._density(sents[i]))
+            alive = [True] * len(sents)
+            remaining = cur
+            for i in order:
+                if remaining <= budget:
+                    break
+                if i == 0 or i == len(sents) - 1 or '?' in sents[i] or '？' in sents[i]:
+                    continue
+                # HARD GUARD: never drop a sentence carrying protected entities
+                if PH_RE.search(sents[i]):
+                    continue
+                if any(k in sents[i] for k in KEEP_KEYWORDS):
+                    continue
+                remaining -= len(sents[i])
+                alive[i] = False
+            sents = [s for s, a in zip(sents, alive) if a]
+        result = ''.join(sents) if has_zh else ' '.join(sents)
+        result = self._restore(result)
+        result = re.sub(r'[ \t]{2,}', ' ', result)
+        result = re.sub(r'\s+([.,;:!?)\]])', r'\1', result)
+        result = result.strip()
+        # ---- self-validation with graceful degradation ----
+        problems = self._validate(orig_text, result)
+        if problems and self.mode != 'light':
+            fb = FastLingua('light', min(self.target_ratio * 1.6, 0.95),
+                            auto=self.auto, collapse=False)
+            alt, _st2 = fb.compress(text)
+            if not fb._validate(orig_text, alt):
+                result = alt
+                n_ent = len(fb._store)
+        elapsed = time.perf_counter() - t0
+        stats = {'orig': orig_len, 'comp': len(result),
+                 'ratio': round(len(result) / max(orig_len, 1), 3),
+                 'entities': n_ent, 'ms': round(elapsed * 1000, 2)}
+        if not result.strip() and orig_text.strip():
+            keep = [sn for sn in self._split_sentences(orig_text) if sn.strip()][:2]
+            result = ' '.join(keep) if keep else orig_text.strip()[:200]
+        if self.anchors:
+            result = '\n'.join('[F%d] %s' % (i + 1, f)
+                               for i, f in enumerate(self._split_sentences(result)))
+        return result, stats
+
+
+def _instance_validate(fl, orig, comp):
+    """Post-compression validation: entities + keep-keywords retention."""
+    problems = []
+    raw = sorted(set(ENT_RE.findall(orig)))
+    raw = [e for e in raw if not any(e != o and e in o for o in raw)]
+    miss = [e for e in raw if e not in comp]
+    if miss:
+        problems.append('entities:' + ','.join(miss[:3]))
+    ko = sum(1 for k in KEEP_KEYWORDS if k in orig)
+    kc = sum(1 for k in KEEP_KEYWORDS if k in comp)
+    if ko and kc < ko:
+        problems.append('keywords %d/%d' % (kc, ko))
+    return problems