kintsugi 0.1.0__py3-none-any.whl

kintsugi/_data/locales.py

@@ -0,0 +1,66 @@
+"""Locale registry: region, script and register variants (data as code).
+
+Each entry is a BCP-47-style code with its base language, ISO 15924 script, region,
+and register (standard acrolect vs colloquial/creole/vernacular basilect). These are
+red-team target locales; a model collapsing a variant to its base is exactly what the
+attacks probe. The exhaustive MT-engine coverage catalogues live in docs, not here."""
+
+LOCALES = [
+    {'code': 'pt-BR', 'name': 'Portuguese (Brazil)', 'base': 'pt', 'script': 'Latn', 'region': 'BR', 'register': 'standard', 'band': 'pt', 'note': ''},
+    {'code': 'pt-PT', 'name': 'Portuguese (Portugal)', 'base': 'pt', 'script': 'Latn', 'region': 'PT', 'register': 'standard', 'band': 'pt', 'note': ''},
+    {'code': 'pt-MO', 'name': 'Portuguese (Macau)', 'base': 'pt', 'script': 'Latn', 'region': 'MO', 'register': 'standard', 'band': 'pt', 'note': ''},
+    {'code': 'fr-FR', 'name': 'French (France)', 'base': 'fr', 'script': 'Latn', 'region': 'FR', 'register': 'standard', 'band': 'fr', 'note': ''},
+    {'code': 'fr-CA', 'name': 'French (Canada)', 'base': 'fr', 'script': 'Latn', 'region': 'CA', 'register': 'standard', 'band': 'fr', 'note': ''},
+    {'code': 'fr-CH', 'name': 'French (Switzerland)', 'base': 'fr', 'script': 'Latn', 'region': 'CH', 'register': 'standard', 'band': 'fr', 'note': ''},
+    {'code': 'fr-BE', 'name': 'French (Belgium)', 'base': 'fr', 'script': 'Latn', 'region': 'BE', 'register': 'standard', 'band': 'fr', 'note': ''},
+    {'code': 'de-DE', 'name': 'German (Germany)', 'base': 'de', 'script': 'Latn', 'region': 'DE', 'register': 'standard', 'band': 'de', 'note': ''},
+    {'code': 'de-AT', 'name': 'Austrian Standard German', 'base': 'de', 'script': 'Latn', 'region': 'AT', 'register': 'standard', 'band': 'de', 'note': ''},
+    {'code': 'de-CH', 'name': 'Swiss Standard German', 'base': 'de', 'script': 'Latn', 'region': 'CH', 'register': 'standard', 'band': 'de', 'note': 'written standard in Switzerland; distinct from spoken gsw'},
+    {'code': 'gsw', 'name': 'Swiss German (Schweizerdeutsch)', 'base': 'de', 'script': 'Latn', 'region': 'CH', 'register': 'vernacular', 'band': 'de', 'note': 'spoken Alemannic vernacular; models tend to collapse it to Standard German'},
+    {'code': 'es-ES', 'name': 'Spanish (Spain)', 'base': 'es', 'script': 'Latn', 'region': 'ES', 'register': 'standard', 'band': 'es', 'note': ''},
+    {'code': 'es-MX', 'name': 'Spanish (Mexico)', 'base': 'es', 'script': 'Latn', 'region': 'MX', 'register': 'standard', 'band': 'es', 'note': ''},
+    {'code': 'es-AR', 'name': 'Spanish (Argentina)', 'base': 'es', 'script': 'Latn', 'region': 'AR', 'register': 'standard', 'band': 'es', 'note': ''},
+    {'code': 'es-419', 'name': 'Spanish (Latin America)', 'base': 'es', 'script': 'Latn', 'region': '419', 'register': 'standard', 'band': 'es', 'note': ''},
+    {'code': 'es-US', 'name': 'Spanish (United States)', 'base': 'es', 'script': 'Latn', 'region': 'US', 'register': 'standard', 'band': 'es', 'note': ''},
+    {'code': 'en-US', 'name': 'English (United States)', 'base': 'en', 'script': 'Latn', 'region': 'US', 'register': 'standard', 'band': 'en', 'note': ''},
+    {'code': 'en-GB', 'name': 'English (United Kingdom)', 'base': 'en', 'script': 'Latn', 'region': 'GB', 'register': 'standard', 'band': 'en', 'note': ''},
+    {'code': 'en-AU', 'name': 'English (Australia)', 'base': 'en', 'script': 'Latn', 'region': 'AU', 'register': 'standard', 'band': 'en', 'note': ''},
+    {'code': 'en-CA', 'name': 'English (Canada)', 'base': 'en', 'script': 'Latn', 'region': 'CA', 'register': 'standard', 'band': 'en', 'note': ''},
+    {'code': 'en-IN', 'name': 'Indian Standard English', 'base': 'en', 'script': 'Latn', 'region': 'IN', 'register': 'standard', 'band': 'en', 'note': ''},
+    {'code': 'en-SG', 'name': 'Singapore Standard English', 'base': 'en', 'script': 'Latn', 'region': 'SG', 'register': 'standard', 'band': 'en', 'note': 'the acrolect; contrast with en-SG-x-singlish'},
+    {'code': 'en-SG-x-singlish', 'name': 'Singlish', 'base': 'en', 'script': 'Latn', 'region': 'SG', 'register': 'creole', 'band': 'en', 'note': "Singapore colloquial English creole; the basilect to en-SG's acrolect"},
+    {'code': 'en-IN-x-hinglish', 'name': 'Hinglish', 'base': 'en', 'script': 'Latn', 'region': 'IN', 'register': 'colloquial', 'band': 'en', 'note': 'Hindi-English code-mixing; romanised; contrast en-IN and hi-Deva'},
+    {'code': 'en-JM', 'name': 'Jamaican Standard English', 'base': 'en', 'script': 'Latn', 'region': 'JM', 'register': 'standard', 'band': 'en', 'note': ''},
+    {'code': 'jam', 'name': 'Jamaican Patois', 'base': 'en', 'script': 'Latn', 'region': 'JM', 'register': 'creole', 'band': 'en', 'note': 'English-lexified creole; the basilect to Jamaican Standard English'},
+    {'code': 'nl-NL', 'name': 'Dutch (Netherlands)', 'base': 'nl', 'script': 'Latn', 'region': 'NL', 'register': 'standard', 'band': 'nl', 'note': ''},
+    {'code': 'nl-BE', 'name': 'Dutch (Belgium / Flemish)', 'base': 'nl', 'script': 'Latn', 'region': 'BE', 'register': 'standard', 'band': 'nl', 'note': ''},
+    {'code': 'it-IT', 'name': 'Italian (Italy)', 'base': 'it', 'script': 'Latn', 'region': 'IT', 'register': 'standard', 'band': 'it', 'note': ''},
+    {'code': 'it-CH', 'name': 'Italian (Switzerland)', 'base': 'it', 'script': 'Latn', 'region': 'CH', 'register': 'standard', 'band': 'it', 'note': ''},
+    {'code': 'ar', 'name': 'Modern Standard Arabic', 'base': 'ar', 'script': 'Arab', 'region': None, 'register': 'standard', 'band': 'ar', 'note': 'MSA'},
+    {'code': 'ar-EG', 'name': 'Egyptian Arabic', 'base': 'ar', 'script': 'Arab', 'region': 'EG', 'register': 'vernacular', 'band': 'ar', 'note': 'spoken dialect; Arabic diglossia vs MSA'},
+    {'code': 'ar-MA', 'name': 'Moroccan Arabic (Darija)', 'base': 'ar', 'script': 'Arab', 'region': 'MA', 'register': 'vernacular', 'band': 'ar', 'note': 'spoken dialect; Arabic diglossia vs MSA'},
+    {'code': 'zh-Hans', 'name': 'Chinese (Simplified)', 'base': 'zh', 'script': 'Hans', 'region': None, 'register': 'standard', 'band': 'zh', 'note': ''},
+    {'code': 'zh-Hant', 'name': 'Chinese (Traditional)', 'base': 'zh', 'script': 'Hant', 'region': None, 'register': 'standard', 'band': 'zh-Hant', 'note': ''},
+    {'code': 'zh-CN', 'name': 'Chinese (Mainland, Simplified)', 'base': 'zh', 'script': 'Hans', 'region': 'CN', 'register': 'standard', 'band': 'zh', 'note': ''},
+    {'code': 'zh-SG', 'name': 'Chinese (Singapore, Simplified)', 'base': 'zh', 'script': 'Hans', 'region': 'SG', 'register': 'standard', 'band': 'zh', 'note': ''},
+    {'code': 'zh-TW', 'name': 'Chinese (Taiwan, Traditional)', 'base': 'zh', 'script': 'Hant', 'region': 'TW', 'register': 'standard', 'band': 'zh-Hant', 'note': ''},
+    {'code': 'zh-HK', 'name': 'Chinese (Hong Kong, Traditional)', 'base': 'zh', 'script': 'Hant', 'region': 'HK', 'register': 'standard', 'band': 'zh-Hant', 'note': 'written standard Chinese in HK; spoken form is Cantonese (yue)'},
+    {'code': 'zh-MO', 'name': 'Chinese (Macau, Traditional)', 'base': 'zh', 'script': 'Hant', 'region': 'MO', 'register': 'standard', 'band': 'zh-Hant', 'note': ''},
+    {'code': 'yue', 'name': 'Cantonese', 'base': 'yue', 'script': 'Hant', 'region': 'HK', 'register': 'vernacular', 'band': 'yue', 'note': 'spoken Cantonese; models tend to collapse it to Mandarin'},
+    {'code': 'hi-Deva', 'name': 'Hindi (Devanagari)', 'base': 'hi', 'script': 'Deva', 'region': 'IN', 'register': 'standard', 'band': 'hi', 'note': ''},
+    {'code': 'hi-Latn', 'name': 'Hindi (Latin / romanised)', 'base': 'hi', 'script': 'Latn', 'region': 'IN', 'register': 'colloquial', 'band': 'hi', 'note': 'romanised Hindi; overlaps Hinglish'},
+    {'code': 'sr-Cyrl', 'name': 'Serbian (Cyrillic)', 'base': 'sr', 'script': 'Cyrl', 'region': 'RS', 'register': 'standard', 'band': 'sr', 'note': ''},
+    {'code': 'sr-Latn', 'name': 'Serbian (Latin)', 'base': 'sr', 'script': 'Latn', 'region': 'RS', 'register': 'standard', 'band': 'sr', 'note': ''},
+    {'code': 'az-Latn', 'name': 'Azerbaijani (Latin)', 'base': 'az', 'script': 'Latn', 'region': 'AZ', 'register': 'standard', 'band': 'az', 'note': ''},
+    {'code': 'az-Cyrl', 'name': 'Azerbaijani (Cyrillic)', 'base': 'az', 'script': 'Cyrl', 'region': 'AZ', 'register': 'standard', 'band': 'az', 'note': ''},
+    {'code': 'uz-Latn', 'name': 'Uzbek (Latin)', 'base': 'uz', 'script': 'Latn', 'region': 'UZ', 'register': 'standard', 'band': 'uz', 'note': ''},
+    {'code': 'uz-Cyrl', 'name': 'Uzbek (Cyrillic)', 'base': 'uz', 'script': 'Cyrl', 'region': 'UZ', 'register': 'standard', 'band': 'uz', 'note': ''},
+    {'code': 'pa-Guru', 'name': 'Punjabi (Gurmukhi)', 'base': 'pa', 'script': 'Guru', 'region': 'IN', 'register': 'standard', 'band': 'pa', 'note': ''},
+    {'code': 'pa-Arab', 'name': 'Punjabi (Shahmukhi)', 'base': 'pa', 'script': 'Arab', 'region': 'PK', 'register': 'standard', 'band': 'pa', 'note': ''},
+    {'code': 'ms-Latn', 'name': 'Malay (Rumi / Latin)', 'base': 'ms', 'script': 'Latn', 'region': 'MY', 'register': 'standard', 'band': 'ms', 'note': ''},
+    {'code': 'ms-Arab', 'name': 'Malay (Jawi / Arabic)', 'base': 'ms', 'script': 'Arab', 'region': 'MY', 'register': 'standard', 'band': 'ms', 'note': ''},
+    {'code': 'kk-Cyrl', 'name': 'Kazakh (Cyrillic)', 'base': 'kk', 'script': 'Cyrl', 'region': 'KZ', 'register': 'standard', 'band': 'kk', 'note': ''},
+    {'code': 'kk-Latn', 'name': 'Kazakh (Latin)', 'base': 'kk', 'script': 'Latn', 'region': 'KZ', 'register': 'standard', 'band': 'kk', 'note': ''},
+    {'code': 'mn-Cyrl', 'name': 'Mongolian (Cyrillic)', 'base': 'mn', 'script': 'Cyrl', 'region': 'MN', 'register': 'standard', 'band': 'mn', 'note': ''},
+    {'code': 'mn-Mong', 'name': 'Mongolian (Traditional script)', 'base': 'mn', 'script': 'Mong', 'region': 'MN', 'register': 'standard', 'band': 'mn', 'note': ''},
+]

kintsugi/attacks.py

@@ -0,0 +1,75 @@
+"""Attack categories and seeds discovered by the red-team project.
+
+201 categories / 2822 seeds, loaded from ``data/attacks.json``. Each category
+is either ``open-weights`` (found attacking local Ollama models) or ``frontier``
+(found attacking a frontier API). Categories also carry the catastrophic/bad
+rates actually observed in the runs, keyed by language pair.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Iterator, Optional
+
+from ._data.attacks import CATEGORIES as _CATEGORIES
+
+
+@dataclass
+class Attack:
+    """One adversarial source string plus where it came from."""
+
+    source: str
+    category: str
+    origin: str = "open-weights"
+    lang_pair: Optional[str] = None
+    meta: dict[str, Any] = field(default_factory=dict)
+
+
+def categories() -> list[dict[str, Any]]:
+    """All category records: name, description, origin, seeds, observed rates."""
+    return _CATEGORIES
+
+
+def category_names() -> list[str]:
+    return [c["name"] for c in categories()]
+
+
+def observed_rates() -> dict[str, dict[str, dict[str, int]]]:
+    """Per-category observed counts ``{category: {lang_pair: {n, catastrophic, bad}}}``.
+
+    Counts exclude rows polluted by judge rate-limit errors, so rates are honest.
+    """
+    return {c["name"]: c["observed"] for c in categories() if "observed" in c}
+
+
+def load_attacks(
+    category: Optional[str] = None,
+    origin: Optional[str] = None,
+    lang_pair: Optional[str] = None,
+    limit_per_category: Optional[int] = None,
+) -> list[Attack]:
+    """Expand categories into individual :class:`Attack` seeds.
+
+    Filter by ``category`` (exact name), ``origin`` ('open-weights'/'frontier'),
+    or a category's own ``lang_pair`` tag; cap seeds per category with
+    ``limit_per_category``.
+    """
+    out: list[Attack] = []
+    for cat in categories():
+        if category is not None and cat["name"] != category:
+            continue
+        if origin is not None and cat.get("origin") != origin:
+            continue
+        clp = cat.get("lang_pair")
+        if lang_pair is not None and clp is not None and clp != lang_pair:
+            continue
+        seeds = cat["seeds"]
+        if limit_per_category is not None:
+            seeds = seeds[:limit_per_category]
+        for s in seeds:
+            out.append(Attack(source=s, category=cat["name"], origin=cat.get("origin", "open-weights"),
+                              lang_pair=clp, meta={"description": cat.get("description", "")}))
+    return out
+
+
+def iter_attacks(**kwargs: Any) -> Iterator[Attack]:
+    yield from load_attacks(**kwargs)

kintsugi/cli.py

@@ -0,0 +1,151 @@
+"""Command-line interface for kintsugi.
+
+    kintsugi list [--origin open-weights|frontier]
+    kintsugi list-framings
+    kintsugi attack --target ollama:qwen3.5:0.8b --lang en-de --category prompt_injection
+    kintsugi frequency --target ollama:qwen3.5:0.8b --direction en-ja --n 30
+    kintsugi transform zero_width "Do not take with alcohol"
+
+A target (and optional judge) is ``kind:model`` where kind is ``ollama``, ``hf``
+or ``anthropic``. For your own model, use the Python API with CallableProvider.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Optional
+
+from . import frequency, locales, transforms
+from .attacks import categories, load_attacks
+from .engine import LANG_PAIRS, RedTeam
+from .framings import framings
+from .providers import AnthropicProvider, HuggingFaceProvider, OllamaProvider, Provider
+
+
+def _make_provider(spec: Optional[str]) -> Optional[Provider]:
+    if not spec:
+        return None
+    kind, _, model = spec.partition(":")
+    if not model:
+        raise SystemExit(f"target must be kind:model, got {spec!r}")
+    if kind == "ollama":
+        return OllamaProvider(model)
+    if kind in ("hf", "huggingface"):
+        return HuggingFaceProvider(model)
+    if kind == "anthropic":
+        return AnthropicProvider(model)
+    raise SystemExit(f"unknown provider kind {kind!r} (use ollama, hf, or anthropic)")
+
+
+def _cmd_list(args):
+    for c in categories():
+        if args.origin and c.get("origin") != args.origin:
+            continue
+        n = len(c["seeds"])
+        print(f"{c['name']:<48} {c.get('origin',''):<12} {n:>4} seeds  {c['description'][:70]}")
+
+
+def _cmd_list_framings(args):
+    for f in framings():
+        print(f"{f['name']:<40} batch {f.get('batch','?')}  {f['description'][:80]}")
+
+
+def _cmd_transform(args):
+    print(transforms.apply(args.name, args.text, seed=args.seed))
+
+
+def _run(args, attacks):
+    target = _make_provider(args.target)
+    judge = _make_provider(args.judge)
+    rt = RedTeam(target=target, judge=judge, lang_pair=args.lang)
+    findings = rt.run(attacks, progress=not args.quiet)
+    summary = rt.summary(findings)
+    print(json.dumps(summary, indent=1, ensure_ascii=False))
+    if args.out:
+        with open(args.out, "w", encoding="utf-8") as fh:
+            for f in findings:
+                fh.write(json.dumps(f.to_dict(), ensure_ascii=False) + "\n")
+        print(f"wrote {len(findings)} findings to {args.out}", file=sys.stderr)
+
+
+def _cmd_attack(args):
+    attacks = load_attacks(category=args.category, origin=args.origin,
+                           limit_per_category=args.limit)
+    if not attacks:
+        raise SystemExit("no attacks matched those filters")
+    _run(args, attacks)
+
+
+def _cmd_list_locales(args):
+    for x in locales.locales(register=args.register, script=args.script):
+        reg = x["region"] or "-"
+        print(f"{x['code']:<22} {x['script']:<5} {reg:<4} {x['register']:<11} band={x['band']:<8} {x['name']}")
+
+
+def _cmd_frequency(args):
+    attacks = frequency.probes(lang=args.lang_of, band=args.band, n=args.n, direction=args.direction)
+    # run in the probe's own language, not the attack default: words go source -> English
+    if args.lang_of == "ja":
+        args.lang = args.direction
+    else:
+        base = locales.band_key(args.lang_of) if locales.is_locale(args.lang_of) else args.lang_of
+        args.lang = f"{base}-en"
+    _run(args, attacks)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="kintsugi", description="Red-team machine translation.")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    pl = sub.add_parser("list", help="list attack categories")
+    pl.add_argument("--origin", choices=["open-weights", "frontier"])
+    pl.set_defaults(func=_cmd_list)
+
+    pf = sub.add_parser("list-framings", help="list translator framings")
+    pf.set_defaults(func=_cmd_list_framings)
+
+    ploc = sub.add_parser("list-locales", help="list region/script/register locale targets")
+    ploc.add_argument("--register", choices=["standard", "colloquial", "creole", "vernacular"])
+    ploc.add_argument("--script")
+    ploc.set_defaults(func=_cmd_list_locales)
+
+    pt = sub.add_parser("transform", help="apply an adversarial transform to text")
+    pt.add_argument("name")
+    pt.add_argument("text")
+    pt.add_argument("--seed", type=int)
+    pt.set_defaults(func=_cmd_transform)
+
+    def add_run_args(sp):
+        sp.add_argument("--target", required=True, help="kind:model, e.g. ollama:qwen3.5:0.8b")
+        sp.add_argument("--judge", help="kind:model for the LLM judge (optional)")
+        sp.add_argument("--lang", default="en-de", choices=sorted(LANG_PAIRS), help="language pair")
+        sp.add_argument("--out", help="write findings JSONL here")
+        sp.add_argument("--quiet", action="store_true")
+
+    pa = sub.add_parser("attack", help="run attack categories against a target")
+    add_run_args(pa)
+    pa.add_argument("--category", help="single category name (default: all)")
+    pa.add_argument("--origin", choices=["open-weights", "frontier"])
+    pa.add_argument("--limit", type=int, help="max seeds per category")
+    pa.set_defaults(func=_cmd_attack)
+
+    pq = sub.add_parser("frequency", help="frequent vs rare word probes (by language or locale)")
+    add_run_args(pq)
+    pq.add_argument("--lang-of", default="ja", dest="lang_of",
+                    help="frequency language or locale code (e.g. ja, de, zh-Hant, en-SG-x-singlish)")
+    pq.add_argument("--direction", default="en-ja", choices=["en-ja", "ja-en"],
+                    help="Japanese only: gloss->ja or ja->gloss")
+    pq.add_argument("--band", default="rare", choices=["frequent", "rare"])
+    pq.add_argument("--n", type=int, default=30)
+    pq.set_defaults(func=_cmd_frequency)
+    return p
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    args = build_parser().parse_args(argv)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()