simurg 1.0.0__py3-none-any.whl

simurg/__init__.py

@@ -0,0 +1,51 @@
+# ═══════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+#
+# Online (in-stream, token-time) detection of LLM decoding corruption —
+# repetition collapse, cross-lingual drift, training-data regurgitation, structural
+# breakdown, semantic discontinuity — with onset localization and a zero-leak
+# hold/release/abort protocol. A pluggable five-detector ensemble (rules + n-gram
+# surprise + Count-Min repetition + SimHash drift + entropy) fused under conformal
+# calibration, plus an online-learnable tier.
+#
+# Host integration is five lines:
+#     from simurg import Simurg, OnlineLogReg
+#     s = Simurg(model=OnlineLogReg.load("weights/simurg_model.json"))   # model optional
+#     for token in llm_stream:
+#         v = s.feed(token)
+#         if v.state == "corrupt":
+#             abort_and_retry(reason=v.reasons, onset=v.onset_char); break
+#         ui.write(v.released)
+#     final = s.finish(); ui.write(final.released)
+# ═══════════════════════════════════════════════════════════════════════════════
+from .core import (DRIFT, REGISTRY, REGURGITATION, REPETITION, SEMANTIC,
+                   STRUCTURAL, TAXONOMY, Detector, DetectorScore, stable_hash)
+from .features import StreamFeatures
+from .signals.calibrate import RobustEWMA
+from .signals.ngram_lm import OnlineCharNGram
+from .signals.sketch import CountMinSketch, RepetitionTracker
+from .signals.simhash import RollingSimHash
+from .detection.rules import rule_verdict
+from .detection.fusion import ConformalEnsemble
+from .detection.sentinel import CLEAN, CORRUPT, SUSPECT, Simurg, Verdict
+from .learning.model import OnlineLogReg
+from .learning.custom import (CustomLearnedDetector, DetectabilityReport,
+                              LexiconDetector, fit_custom_detector)
+from .integrations.openai_guard import GuardedLLM
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "Simurg", "Verdict", "StreamFeatures", "OnlineLogReg", "ConformalEnsemble",
+    "GuardedLLM",
+    "fit_custom_detector", "DetectabilityReport", "CustomLearnedDetector",
+    "LexiconDetector",
+    "Detector", "DetectorScore", "REGISTRY", "rule_verdict", "stable_hash",
+    "OnlineCharNGram", "CountMinSketch", "RepetitionTracker", "RollingSimHash",
+    "RobustEWMA",
+    "CLEAN", "SUSPECT", "CORRUPT",
+    "REPETITION", "DRIFT", "REGURGITATION", "STRUCTURAL", "SEMANTIC", "TAXONOMY",
+]

simurg/core.py

@@ -0,0 +1,83 @@
+# ═══════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+#
+# core — shared contracts of the package: the corruption taxonomy, the
+# `DetectorScore` / `Signal` value objects, the `Detector` protocol every
+# detector implements, a `DetectorRegistry` for plug-in extensibility, and a
+# process-stable hash used by the sketch/SimHash estimators (so results are
+# reproducible across runs — important for the benchmark and the paper).
+# ═══════════════════════════════════════════════════════════════════════════════
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Protocol, runtime_checkable
+
+# ── corruption taxonomy ──────────────────────────────────────────────────────
+REPETITION = "repetition_collapse"
+DRIFT = "cross_lingual_drift"
+REGURGITATION = "regurgitation"
+STRUCTURAL = "structural_breakdown"
+SEMANTIC = "semantic_discontinuity"     # new axis surfaced by the SimHash detector
+TAXONOMY = (REPETITION, DRIFT, REGURGITATION, STRUCTURAL, SEMANTIC)
+
+
+def stable_hash(s: str, seed: int = 0) -> int:
+    """Deterministic 64-bit hash (blake2b). Unlike Python's built-in ``hash`` it
+    is stable across processes/runs, so sketch buckets and SimHash bits are
+    reproducible — a hard requirement for a benchmark others can replicate."""
+    key = seed.to_bytes(8, "little") if seed else b""
+    return int.from_bytes(
+        hashlib.blake2b(s.encode("utf-8", "ignore"), key=key, digest_size=8).digest(),
+        "little")
+
+
+@dataclass
+class DetectorScore:
+    """One detector's read on the current stream state."""
+    name: str
+    p: float                              # corruption probability contributed [0,1]
+    reasons: List[str] = field(default_factory=list)
+    classes: List[str] = field(default_factory=list)   # taxonomy labels
+    features: Dict[str, float] = field(default_factory=dict)   # exposed numeric signals
+
+
+@runtime_checkable
+class Detector(Protocol):
+    """A detector is a cheap read over the shared incremental `StreamState`.
+    It never re-scans text — all O(1)-amortized work happens in StreamState.feed;
+    `evaluate` only interprets the already-computed signals. This is what keeps
+    a five-detector ensemble as cheap as a single pass."""
+    name: str
+
+    def evaluate(self, state) -> DetectorScore:  # noqa: D401
+        ...
+
+
+class DetectorRegistry:
+    """Plug-in registry so open-source contributors can add a detector without
+    touching the sentinel: `@registry.register` a factory, and it joins the
+    ensemble. The fusion layer weights whatever is registered."""
+
+    def __init__(self):
+        self._factories: Dict[str, Callable[[], Detector]] = {}
+
+    def register(self, name: str):
+        def deco(factory: Callable[[], Detector]):
+            self._factories[name] = factory
+            return factory
+        return deco
+
+    def build(self, names=None) -> List[Detector]:
+        names = names or list(self._factories)
+        return [self._factories[n]() for n in names if n in self._factories]
+
+    @property
+    def names(self):
+        return tuple(self._factories)
+
+
+REGISTRY = DetectorRegistry()

simurg/data/__init__.py

@@ -0,0 +1,6 @@
+# ═════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+# ═════════════════════════════════════════════════════════════════════════════

simurg/data/dataset.py

@@ -0,0 +1,92 @@
+# ═══════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+#
+# dataset — benchmark dataset builder. Clean corpus = REAL production texts (AI
+# answers + desk reports read from the deployment DB — AZ/EN/RU macro-economics
+# prose with legitimate figures, the hardest possible negatives), with bundled
+# fallback seeds when the DB is offline. Positives = synthetic CorruptBench
+# corruptions of held-out clean texts, with exact onset ground truth. This module
+# only READS the DB; it never imports or mutates backend runtime code.
+# ═══════════════════════════════════════════════════════════════════════════════
+from __future__ import annotations
+
+import os
+import random
+
+from .synth import CLASSES, corrupt
+
+_FALLBACK = [
+    ("Büdcə xərclərinin 20% artımı qeyri-neft buraxılış kəsirini -1.64%-dən -0.44%-ə "
+     "yaxşılaşdırır, ÜDM-in illik artımı 2.94%-dən 3.78%-ə yüksəlir. İnflyasiya 5.45%-dən "
+     "5.71%-ə qalxsa da 2-6% dəhlizində qalır və Mərkəzi Bank faiz dərəcəsini sabit saxlayır. "
+     "Tikinti və nəqliyyat sektorları fiskal impulsdan ən çox faydalanan sahələrdir. ") * 6,
+    ("The non-oil output gap improves from -1.6% in the baseline to -0.4%, a +1.2pp effect "
+     "of the shock. GDP grows 1.93% in 2026 versus 1.45% baseline, while inflation stays at "
+     "5.71%, inside the 2-6% corridor. The current account remains near 7.1% of GDP and the "
+     "policy rate rises only 15 basis points, so monetary conditions stay accommodative. ") * 6,
+    ("Повышение бюджетных расходов на 20% ускоряет рост ВВП до 1.93% против 1.45% в базовом "
+     "сценарии. Инфляция остаётся в коридоре 2-6%, а ненефтяной разрыв выпуска сокращается "
+     "с -1.64% до -0.44%. Строительство и транспорт получают наибольший фискальный импульс. ") * 6,
+]
+
+
+def load_clean_corpus(min_len: int = 700, limit: int = 400) -> list[str]:
+    """Clean-corpus loader, three tiers:
+    1. ``SIMURG_CORPUS_JSONL`` env var → path to a .jsonl with a ``text`` field
+       per line (bring your own corpus — e.g. the CorruptBench-HF clean rows);
+    2. optional deployment hook: the Black Swan production DB when this package
+       sits inside that repo (silently skipped anywhere else);
+    3. bundled fallback seed texts."""
+    texts: list[str] = []
+    corpus_path = os.environ.get("SIMURG_CORPUS_JSONL")
+    if corpus_path and os.path.exists(corpus_path):
+        import json
+        with open(corpus_path, encoding="utf-8") as f:
+            for line in f:
+                try:
+                    t = json.loads(line).get("text") or ""
+                except Exception:
+                    continue
+                if len(t) >= min_len:
+                    texts.append(t)
+                if len(texts) >= limit:
+                    break
+    if not texts:
+        try:  # deployment hook — absent in the open-source layout, and that's fine
+            from backend.db import engine
+            from sqlalchemy import text as _t
+            with engine.connect() as c:
+                rows = c.execute(_t(
+                    "SELECT content, payload->>'report' FROM chat_messages "
+                    "WHERE sender='AI' ORDER BY created_at DESC LIMIT :lim"),
+                    {"lim": limit}).fetchall()
+            for content, report in rows:
+                for t in (content, report):
+                    if t and len(t) >= min_len:
+                        texts.append(t)
+        except Exception:
+            pass
+    if len(texts) < 10:
+        texts += _FALLBACK
+    return texts
+
+
+def build(seed: int = 7, per_class: int = 60):
+    """Returns (streams, meta): streams = list of (text, label, onset, cls);
+    clean streams have label 0, onset None."""
+    rng = random.Random(seed)
+    corpus = [t for t in load_clean_corpus() if len(t) >= 700]
+    rng.shuffle(corpus)
+    n_clean_pool = max(8, len(corpus) // 2)
+    clean_pool, corrupt_pool = corpus[:n_clean_pool], corpus[n_clean_pool:] or corpus
+    streams = [(t, 0, None, "clean") for t in clean_pool]
+    for cls in CLASSES:
+        for _ in range(per_class):
+            base = rng.choice(corrupt_pool)
+            text, onset, c = corrupt(base, cls, rng)
+            streams.append((text, 1, onset, c))
+    rng.shuffle(streams)
+    return streams

simurg/data/evaluate.py

@@ -0,0 +1,159 @@
+# ═══════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+#
+# evaluate — the end-to-end benchmark & trainer. Builds CorruptBench, trains the
+# learned tier on StreamState checkpoints, derives conformal thresholds from clean
+# streams, and reports the paper's results table: stream TPR/FPR, AUROC, per-class
+# recall, detection latency past onset, onset-localization error, zero-leak rate
+# and throughput. Persists `simurg_model.json` (weights) + `simurg_calib.json`
+# (conformal thresholds).  Run:  python3 -m simurg.evaluate
+# ═══════════════════════════════════════════════════════════════════════════════
+from __future__ import annotations
+
+import os
+import random
+import time
+
+import numpy as np
+
+from .dataset import build
+from ..features import StreamFeatures
+from ..detection.fusion import ConformalEnsemble
+from ..learning.model import OnlineLogReg
+from ..detection.sentinel import CORRUPT, Simurg
+from .synth import CLASSES
+
+_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "weights")
+_MODEL_PATH = os.path.join(_DIR, "simurg_model.json")
+_CALIB_PATH = os.path.join(_DIR, "simurg_calib.json")
+
+
+def _checkpoints(text: str, every: int = 300):
+    f = StreamFeatures()
+    out = []
+    for i in range(0, len(text), every):
+        f.feed(text[i:i + every])
+        if f.total_len >= 350:
+            f.freeze_baseline()
+        out.append((f.total_len, f.vector()))
+    return out
+
+
+def _train_model(streams):
+    X, y = [], []
+    for text, label, onset, _cls in streams:
+        for pos, vec in _checkpoints(text):
+            if label == 0:
+                X.append(vec); y.append(0)
+            elif pos >= (onset or 0) + 250:
+                X.append(vec); y.append(1)
+            elif pos <= (onset or 0):
+                X.append(vec); y.append(0)
+    model = OnlineLogReg(len(StreamFeatures.VECTOR))
+    model.partial_fit(X, y, epochs=6)
+    return model, len(X), int(sum(y))
+
+
+def _run_stream(text, model, fusion):
+    s = Simurg(model=model, fusion=fusion)
+    forwarded = 0
+    for i in range(0, len(text), 40):
+        v = s.feed(text[i:i + 40])
+        if v.state != CORRUPT:
+            forwarded += len(v.released)
+        else:
+            return v, s, forwarded
+    return s.finish(), s, forwarded
+
+
+def _auroc(scores, labels) -> float:
+    order = sorted(zip(scores, labels))
+    pos = sum(labels); neg = len(labels) - pos
+    if not pos or not neg:
+        return float("nan")
+    rank_sum, rank = 0.0, 1
+    for _s, l in order:
+        if l == 1:
+            rank_sum += rank
+        rank += 1
+    return (rank_sum - pos * (pos + 1) / 2) / (pos * neg)
+
+
+def run(seed: int = 7, save: bool = False):
+    rng = random.Random(seed)
+    streams = build(seed=seed)
+    rng.shuffle(streams)
+    n_test = max(20, len(streams) // 3)
+    test, train = streams[:n_test], streams[n_test:]
+
+    model, n_vec, n_pos = _train_model(train)
+
+    # conformal calibration: fused scores of CLEAN training streams → thresholds
+    fusion = ConformalEnsemble()
+    clean_scores = []
+    for text, label, _o, _c in train:
+        if label == 0:
+            v, _s, _f = _run_stream(text, model, fusion)
+            clean_scores.append(v.p_corrupt)
+    fusion.calibrate(clean_scores, alpha=0.02, suspect_alpha=0.1)
+
+    print(f"dataset: {len(streams)} streams ({sum(1 for s in streams if s[1]==1)} corrupt) | "
+          f"train vectors: {n_vec} ({n_pos} pos) | features: {len(StreamFeatures.VECTOR)} | "
+          f"test: {len(test)}")
+    print("conformal thresholds:", {k: round(v, 3) for k, v in fusion.to_dict().items()})
+    print("learned weights:", {k: round(float(w), 2)
+                               for k, w in zip(StreamFeatures.VECTOR, model.w)})
+
+    scores, labels, latencies, onset_errs = [], [], [], []
+    flagged = total_corrupt = false_alarms = clean_n = leaks_blocked = early = 0
+    cls_hits = {c: [0, 0] for c in CLASSES}
+    t0 = time.monotonic(); chars = 0
+
+    for text, label, onset, cls in test:
+        v, s, forwarded = _run_stream(text, model, fusion)
+        chars += len(text)
+        scores.append(v.p_corrupt); labels.append(label)
+        if label == 1:
+            total_corrupt += 1; cls_hits[cls][1] += 1
+            if (onset or 0) < 350:
+                early += 1
+            if v.state == CORRUPT:
+                flagged += 1; cls_hits[cls][0] += 1
+                latencies.append(max(0, s.f.total_len - (onset or 0)))
+                if v.onset_char is not None and onset is not None:
+                    onset_errs.append(abs(v.onset_char - onset))
+                if (onset or 0) < 350 and forwarded == 0:
+                    leaks_blocked += 1
+        else:
+            clean_n += 1
+            if v.state == CORRUPT:
+                false_alarms += 1
+    dt = time.monotonic() - t0
+
+    print("\n── results ─────────────────────────────────────────────")
+    print(f"stream TPR: {flagged}/{total_corrupt} = {flagged/max(1,total_corrupt):.3f}")
+    print(f"stream FPR: {false_alarms}/{clean_n} = {false_alarms/max(1,clean_n):.3f}")
+    print(f"AUROC: {_auroc(scores, labels):.3f}")
+    for c, (hit, tot) in cls_hits.items():
+        if tot:
+            print(f"  recall[{c}]: {hit}/{tot} = {hit/tot:.2f}")
+    if latencies:
+        print(f"detection latency past onset: median {int(np.median(latencies))}, "
+              f"p90 {int(np.percentile(latencies, 90))} chars")
+    if onset_errs:
+        print(f"onset localization |err|: median {int(np.median(onset_errs))} chars")
+    print(f"zero-leak (onset<hold): {leaks_blocked}/{early}")
+    print(f"throughput: {chars/max(dt,1e-6):,.0f} chars/sec")
+
+    if save:
+        model.save(_MODEL_PATH); fusion.save(_CALIB_PATH)
+        print(f"\nsaved → {os.path.basename(_MODEL_PATH)}, {os.path.basename(_CALIB_PATH)}")
+    return model, fusion
+
+
+if __name__ == "__main__":
+    import sys
+    run(save="--save" in sys.argv)

simurg/data/generate_dataset.py

@@ -0,0 +1,167 @@
+# ═══════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+#
+# hf_dataset/generate_dataset.py — builds the open-source CorruptBench release
+# for HuggingFace. Clean seed texts are GENERATED (not taken from production
+# chats — no user data leaves the deployment) by HAL-X's wahoo-1.5-preview 12B
+# model across three languages and a spread of analytic topics; a stratified
+# subset is corrupted by the simurg.synth injectors with exact onset ground
+# truth. Output: corruptbench_train.jsonl / corruptbench_test.jsonl (+ stats).
+# Split is BY SEED TEXT, so a corrupt variant never shares its clean source with
+# the other split. Run:  python3 -m simurg.data.generate_dataset
+# ═══════════════════════════════════════════════════════════════════════════════
+from __future__ import annotations
+
+import json
+import os
+import random
+import threading
+import time
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from simurg.data.synth import CLASSES, corrupt
+
+# Any OpenAI-compatible /v1/chat/completions endpoint can generate the clean
+# seed corpus — point these env vars at yours.
+URL = os.environ.get("SIMURG_GEN_URL", "http://localhost:8000/v1/chat/completions")
+MODEL = os.environ.get("SIMURG_GEN_MODEL", "wahoo-1.5-preview")
+OUT = os.path.dirname(os.path.abspath(__file__))
+N_SEEDS = 200
+CORRUPT_PER_SEED = 3
+TEST_FRAC = 0.2
+SEED = 20260718
+
+LANGS = [("en", "English"), ("az", "Azerbaijani"), ("ru", "Russian")]
+TOPICS = [
+    "inflation dynamics and monetary policy in a small open economy",
+    "the fiscal transmission of a budget expenditure shock to non-oil sectors",
+    "how oil price movements propagate to the current account and the exchange rate",
+    "credit market conditions, overdue loans and financial stability",
+    "tourism and transport sectors under geopolitical risk",
+    "a quarterly macroeconomic outlook for an oil-exporting economy",
+    "the labor market implications of a construction boom",
+    "central bank policy corridors and interest rate decisions",
+    "diversification strategies for a resource-dependent economy",
+    "exchange rate pass-through to consumer prices",
+    "public investment multipliers and infrastructure spending",
+    "banking sector liquidity and deposit rate dynamics",
+    "agricultural output volatility and food price inflation",
+    "trade balance dynamics under changing global demand",
+    "sovereign wealth fund transfers and fiscal sustainability",
+]
+STYLES = [
+    "an analytical desk report with a few concrete figures",
+    "a policy briefing for a ministry, with sector detail",
+    "an explanatory note for a general audience",
+    "a quarterly bulletin section, moderately technical",
+]
+
+_print_lock = threading.Lock()
+
+
+def _gen_one(idx: int, rng_seed: int) -> dict | None:
+    rng = random.Random(rng_seed)
+    lang_code, lang_name = LANGS[idx % len(LANGS)]
+    topic = rng.choice(TOPICS)
+    style = rng.choice(STYLES)
+    words = rng.randint(350, 750)
+    prompt = (f"Write {style} about {topic}. Length: roughly {words} words. "
+              f"Write ENTIRELY in {lang_name}. Use a professional economist's voice; "
+              f"you may include a handful of plausible illustrative figures "
+              f"(percentages, growth rates). Do not add any preamble or title markers "
+              f"beyond normal markdown headings.")
+    body = {"model": MODEL, "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": 1600, "temperature": 0.9, "top_p": 0.95}
+    for attempt in range(3):
+        try:
+            req = urllib.request.Request(
+                URL, data=json.dumps(body).encode(),
+                headers={"Content-Type": "application/json"}, method="POST")
+            with urllib.request.urlopen(req, timeout=180) as r:
+                d = json.loads(r.read())
+            text = (d["choices"][0]["message"].get("content") or "").strip()
+            if len(text) >= 900:
+                return {"seed_id": idx, "language": lang_code, "topic": topic,
+                        "style": style, "text": text}
+            time.sleep(1)
+        except Exception as e:
+            with _print_lock:
+                print(f"  seed {idx} attempt {attempt+1} failed: {e}")
+            time.sleep(2 + attempt * 3)
+    return None
+
+
+def main():
+    rng = random.Random(SEED)
+    print(f"generating {N_SEEDS} clean seed texts with {MODEL} …")
+    seeds = []
+    t0 = time.monotonic()
+    with ThreadPoolExecutor(max_workers=8) as ex:
+        futs = {ex.submit(_gen_one, i, SEED + i): i for i in range(N_SEEDS)}
+        for n, fut in enumerate(as_completed(futs), 1):
+            res = fut.result()
+            if res:
+                seeds.append(res)
+            if n % 20 == 0:
+                print(f"  {n}/{N_SEEDS} done ({time.monotonic()-t0:.0f}s), "
+                      f"ok={len(seeds)}")
+    print(f"clean seeds: {len(seeds)} in {time.monotonic()-t0:.0f}s")
+
+    # split BY SEED (no leakage), stratified by language
+    rng.shuffle(seeds)
+    n_test = int(len(seeds) * TEST_FRAC)
+    for i, s in enumerate(seeds):
+        s["split"] = "test" if i < n_test else "train"
+
+    # build examples: 1 clean + CORRUPT_PER_SEED corrupt per seed
+    examples = []
+    counter = 0
+    for s in seeds:
+        counter += 1
+        examples.append({
+            "id": f"cb-{counter:05d}", "split": s["split"],
+            "text": s["text"], "label": 0, "corruption_class": "clean",
+            "onset_char": None, "language": s["language"], "topic": s["topic"],
+            "n_chars": len(s["text"]), "seed_model": MODEL,
+        })
+        classes = rng.sample(list(CLASSES), k=min(CORRUPT_PER_SEED, len(CLASSES)))
+        for cls in classes:
+            counter += 1
+            crng = random.Random(hash((s["seed_id"], cls)) & 0xFFFFFFFF)
+            text_c, onset, _ = corrupt(s["text"], cls, crng, onset_frac=(0.05, 0.75))
+            examples.append({
+                "id": f"cb-{counter:05d}", "split": s["split"],
+                "text": text_c, "label": 1, "corruption_class": cls,
+                "onset_char": onset, "language": s["language"], "topic": s["topic"],
+                "n_chars": len(text_c), "seed_model": MODEL,
+            })
+
+    rng.shuffle(examples)
+    stats = {"total": len(examples)}
+    for split in ("train", "test"):
+        rows = [e for e in examples if e["split"] == split]
+        path = os.path.join(OUT, f"corruptbench_{split}.jsonl")
+        with open(path, "w", encoding="utf-8") as f:
+            for e in rows:
+                f.write(json.dumps(e, ensure_ascii=False) + "\n")
+        stats[split] = {
+            "n": len(rows),
+            "clean": sum(1 for e in rows if e["label"] == 0),
+            "corrupt": sum(1 for e in rows if e["label"] == 1),
+            "by_class": {c: sum(1 for e in rows if e["corruption_class"] == c)
+                         for c in ("clean",) + tuple(CLASSES)},
+            "by_lang": {l: sum(1 for e in rows if e["language"] == l)
+                        for l, _ in LANGS},
+        }
+        print(f"wrote {path} ({len(rows)} rows)")
+    with open(os.path.join(OUT, "stats.json"), "w", encoding="utf-8") as f:
+        json.dump(stats, f, indent=2, ensure_ascii=False)
+    print(json.dumps(stats, indent=2))
+
+
+if __name__ == "__main__":
+    main()

simurg/data/synth.py

@@ -0,0 +1,101 @@
+# ═══════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+#
+# synth — the "CorruptBench" synthetic corruption generator. Injects a corruption
+# of a chosen taxonomy class into CLEAN text at a controlled onset position,
+# yielding (corrupt_text, onset_char, class) triples with EXACT ground truth for
+# onset-localization and detection-latency benchmarks. Payloads are modeled on
+# REAL production derails (table echo with '#REF!', Chinese-news drift, English
+# README regurgitation, Persian name-list loops) — not toy noise.
+# ═══════════════════════════════════════════════════════════════════════════════
+from __future__ import annotations
+
+import random
+
+from ..core import DRIFT, REGURGITATION, REPETITION, STRUCTURAL
+
+# ── payload corpora (modeled on observed derails) ────────────────────────────
+_CJK = ("由国立大学金融科技研发团队主导开发的智能合约审计系统今日正式宣布完成测试并开放企业试用"
+        "这套系统结合了人工智能与区块链技术能够自动侦测合约中的潜在漏洞大幅提升审计效率"
+        "根据团队表示传统的合约审计需要大量的人力而智能审计系统可以在短时间内完成分析")
+_ARABIC = "بهره‌های محمد رضایی سید علیزاده در بخصوص په دې نه سره بادي مورد بررسی قرار گرفت "
+_README = ("## How to use\n1. Install Python packages\n```bash\npip install -r requirements.txt\n```\n"
+           "2. Set the API Key\nCreate a .env file in the root directory:\n```\nAPI_KEY=your_key_here\n```\n"
+           "3. Run the application\n```bash\npython main.py\n```\nFeatures\n- Multi-threat detection\n"
+           "- Reentrancy attacks\n- Integer overflow/underflow\n- Detailed report\n")
+_NEWS = ("2024年01月25日 記者報導 台灣的金融科技領域迎來了一項重大突破 "
+         "The system uses deep learning to analyze code, flagging potential vulnerabilities "
+         "and suggesting fixes. This is particularly relevant given the increasing complexity. ")
+
+_SERIES = ["MCI", "Foreign Demand", "Geopolitical Risk", "Consumer Credit",
+           "Tax Gap (lag)", "Budget Exp.", "Lag / Expectation / Residual", "Actual"]
+
+
+def _table_echo(rng: random.Random, n_chars: int) -> str:
+    out = ["#REF! "]
+    while sum(len(p) for p in out) < n_chars:
+        name = rng.choice(_SERIES)
+        vals = " ".join(f"{rng.uniform(-3, 3):.2f}" if rng.random() < 0.4 else "0.00"
+                        for _ in range(rng.randint(12, 30)))
+        out.append(f"{name} {vals} ")
+    return "".join(out)[:n_chars]
+
+
+def _repetition_loop(rng: random.Random, base: str, n_chars: int) -> str:
+    words = [w for w in base.split() if len(w) > 3]
+    phrase = " ".join(rng.sample(words, min(4, len(words)))) if len(words) >= 4 \
+        else "the same phrase again"
+    sep = rng.choice(["\n", " ", " — ", "\n• "])
+    unit = phrase + sep
+    return (unit * (n_chars // len(unit) + 1))[:n_chars]
+
+
+def _drift(rng: random.Random, n_chars: int) -> str:
+    src = _CJK if rng.random() < 0.6 else _ARABIC * 6
+    start = rng.randrange(0, max(1, len(src) - 50))
+    body = (src[start:] + src)[: n_chars]
+    return body
+
+
+def _regurgitation(rng: random.Random, n_chars: int) -> str:
+    src = _README if rng.random() < 0.6 else _NEWS * 3
+    return (src * (n_chars // len(src) + 1))[:n_chars]
+
+
+def _structural(rng: random.Random, n_chars: int) -> str:
+    if rng.random() < 0.5:
+        return _table_echo(rng, n_chars)
+    frags = ["#### ", "``` ", "===== ", "________ ", "J) Comb $## = 2 2Community ",
+             "original}.original/m4/. # , 1 ", "{ # ,) $## 8P8 pack # { . mM # 1 ", "─" * 20 + " "]
+    out = []
+    while sum(len(p) for p in out) < n_chars:
+        out.append(rng.choice(frags))
+    return "".join(out)[:n_chars]
+
+
+_INJECTORS = {
+    REPETITION: _repetition_loop,
+    DRIFT: _drift,
+    REGURGITATION: _regurgitation,
+    STRUCTURAL: _structural,
+}
+CLASSES = tuple(_INJECTORS)
+
+
+def corrupt(clean: str, cls: str, rng: random.Random,
+            onset_frac: tuple[float, float] = (0.0, 0.7)) -> tuple[str, int, str]:
+    """Inject a `cls` corruption into `clean`. Returns (text, onset_char, cls).
+    onset position is uniform in onset_frac of the clean length; the corruption
+    replaces the remainder (mirrors production: after collapse the model never
+    returns to the topic)."""
+    lo, hi = onset_frac
+    onset = int(len(clean) * rng.uniform(lo, hi))
+    tail_len = max(400, len(clean) - onset)
+    if cls == REPETITION:
+        payload = _INJECTORS[cls](rng, clean[:max(onset, 200)], tail_len)
+    else:
+        payload = _INJECTORS[cls](rng, tail_len)
+    return clean[:onset] + payload, onset, cls

simurg/detection/__init__.py

@@ -0,0 +1,6 @@
+# ═════════════════════════════════════════════════════════════════════════════
+# SIMURG · Streaming Integrity Monitor & Universal Regeneration Guard
+#
+# Developed by doofZ (a.k.a Farid Aghayev from HAL-X AI)
+# Co-Founder & Head of AI at HAL-X AI.
+# ═════════════════════════════════════════════════════════════════════════════