ftgate 0.1.0__py3-none-any.whl

ftgate/__init__.py

@@ -0,0 +1,2 @@
+"""ftgate — do runtimes send the bytes the model was trained on?"""
+__version__ = "0.1.0"

ftgate/assets/assert_tools.js

@@ -0,0 +1,77 @@
+// ftgate: judge one tool-calling response against the expected calls.
+//
+// Written once by ftgate, run by promptfoo for every case. It reads the
+// model's output in whatever shape the provider produced — OpenAI
+// tool_calls JSON, a list of function_call items, or raw text with
+// <tool_call> blocks — and reports named scores rather than one pass/fail,
+// so the table can separate "wrong tool", "wrong arguments" and "the
+// call was there but the provider's parser lost it".
+module.exports = (output, context) => {
+  const expected = context.vars.__expected_calls;   // [{name, arguments}]
+  const norm = (v) => {
+    if (Array.isArray(v)) return v.map(norm);
+    if (v && typeof v === "object") { const o = {}; for (const k of Object.keys(v).sort()) o[k] = norm(v[k]); return o; }
+    if (typeof v === "number") return Number(v);
+    if (typeof v === "boolean") return v;
+    return String(v).trim().toLowerCase();
+  };
+  // Expected arguments must be present and equal; extra (optional) arguments
+  // are allowed and reported separately, so a model that adds unit="celsius"
+  // to a weather call is not marked wrong for being thorough.
+  const subsetKey = (c, exp) => {
+    const got = c.arguments || {}, want = exp.arguments || {};
+    const picked = {}; for (const k of Object.keys(want)) picked[k] = got[k];
+    return JSON.stringify([c.name, norm(picked)]);
+  };
+  const wantKey = (e) => JSON.stringify([e.name, norm(e.arguments || {})]);
+
+  let structured = [];
+  let text = typeof output === "string" ? output : JSON.stringify(output);
+  const takeCall = (c) => {
+    const fn = c.function || c;
+    let args = fn.arguments ?? fn.input ?? {};
+    if (typeof args === "string") { try { args = JSON.parse(args); } catch (e) { return { name: fn.name, arguments: null, malformed: true }; } }
+    return { name: fn.name, arguments: args };
+  };
+  try {
+    const o = typeof output === "string" ? JSON.parse(output) : output;
+    const list = Array.isArray(o) ? o : (o && o.tool_calls) ? o.tool_calls : (o && o.name) ? [o] : [];
+    structured = list.filter(c => c && (c.function || c.name)).map(takeCall);
+  } catch (e) { /* not JSON: raw text */ }
+
+  let fromText = [];
+  const re = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g; let m;
+  while ((m = re.exec(text)) !== null) {
+    try { const d = JSON.parse(m[1]); fromText.push({ name: d.name, arguments: d.arguments || {} }); }
+    catch (e) { fromText.push({ name: null, arguments: null, malformed: true }); }
+  }
+
+  const calls = structured.length ? structured : fromText;
+  const parserLost = structured.length === 0 && fromText.length > 0;   // the model called; the provider did not surface it
+  const malformed = calls.some(c => c.malformed);
+  const names = (a) => a.map(c => c.name).sort().join("|");
+  const namesOk = names(calls) === names(expected);
+  // Pair each expected call with a produced call of the same name (greedy).
+  let argsOk = namesOk && !malformed, extra = false;
+  if (argsOk) {
+    const pool = calls.slice();
+    for (const e of expected) {
+      const i = pool.findIndex(c => c.name === e.name && subsetKey(c, e) === wantKey(e));
+      if (i < 0) { argsOk = false; break; }
+      const got = pool[i].arguments || {};
+      if (Object.keys(got).some(k => !(k in (e.arguments || {})))) extra = true;
+      pool.splice(i, 1);
+    }
+  }
+
+  return {
+    pass: argsOk,
+    score: argsOk ? 1 : namesOk ? 0.5 : 0,
+    reason: argsOk ? (extra ? "ok, extra optional arguments" : "exact")
+      : !calls.length ? "no call"
+      : malformed ? "malformed arguments: " + text.slice(0, 160)
+      : !namesOk ? `tools ${names(calls) || "-"} vs expected ${names(expected)}`
+      : "arguments differ: got " + JSON.stringify(calls.map(c => c.arguments)) + " expected " + JSON.stringify(expected.map(e => e.arguments)),
+    namedScores: { called: calls.length ? 1 : 0, names: namesOk ? 1 : 0, args: argsOk ? 1 : 0, extra_args: extra ? 1 : 0, parser_lost: parserLost ? 1 : 0, malformed: malformed ? 1 : 0 },
+  };
+};

ftgate/cli.py

@@ -0,0 +1,125 @@
+"""ftgate command line."""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from ftgate import __version__, reference, render
+from ftgate.compare import compare
+from ftgate.conversations import BUILTIN, load_dataset
+from ftgate.runtimes import llama as llama_rt
+from ftgate.runtimes import ollama as ollama_rt
+from ftgate import data as data_mod
+import json as _json
+from ftgate import tools_eval
+
+
+def cmd_tools(args) -> int:
+    tools = _json.loads(Path(args.tools).read_text(encoding="utf-8")) if args.tools else []
+    if args.from_dataset:
+        cases, ds_tools = tools_eval.cases_from_dataset(Path(args.from_dataset), args.holdout)
+        tools = tools or ds_tools
+    else:
+        cases = tools_eval.load_cases(Path(args.cases))
+    if not cases:
+        print("no cases", file=sys.stderr); return 2
+    if not tools:
+        print("no tools: pass --tools tools.json (or a dataset whose rows carry `tools`)", file=sys.stderr); return 2
+    arms = tools_eval.run(cases, tools, args.provider, keep=Path(args.keep) if args.keep else None, concurrency=args.concurrency)
+    if args.format == "json":
+        print(_json.dumps([a.__dict__ for a in arms], indent=2, ensure_ascii=False))
+    else:
+        print(tools_eval.to_text(arms), end="")
+    if args.min_exact is not None:
+        worst = min((a.args / a.n) for a in arms if a.n) if arms else 0
+        return 1 if worst < args.min_exact else 0
+    return 0
+
+
+def cmd_data(args) -> int:
+    reps = [data_mod.lint(Path(p), model=args.model, max_seq_len=args.max_seq_len,
+                          eval_path=Path(args.eval) if args.eval else None, limit=args.limit) for p in args.path]
+    if args.format == "json":
+        print(_json.dumps([r.as_dict() for r in reps] if len(reps) > 1 else reps[0].as_dict(), indent=2, ensure_ascii=False))
+    else:
+        print("".join(data_mod.to_text(r) for r in reps), end="")
+    errors = sum(r.count("error") for r in reps); warnings = sum(r.count("warning") for r in reps)
+    if args.fail_on == "error":
+        return 1 if errors else 0
+    if args.fail_on == "warning":
+        return 1 if errors or warnings else 0
+    return 0
+
+
+def cmd_template(args) -> int:
+    convs = load_dataset(Path(args.dataset), args.sample) if args.dataset else BUILTIN
+    if not args.llama and not args.ollama:
+        print("nothing to compare against: pass --llama URL and/or --ollama MODEL", file=sys.stderr); return 2
+
+    results = []
+    for conv in convs:
+        msgs = conv.inference_messages
+        ref = reference.render(args.model, msgs, conv.tools, add_generation_prompt=True)
+        if args.llama:
+            rt = llama_rt.render(args.llama.rstrip("/"), msgs, conv.tools)
+            results.append(compare(conv.name, ref, rt, reference.tokenize(args.model, rt.text)))
+        if args.ollama:
+            rt = ollama_rt.render(args.ollama_url.rstrip("/"), args.ollama, msgs, conv.tools, verify=not args.no_verify)
+            results.append(compare(conv.name, ref, rt, reference.tokenize(args.model, rt.text)))
+
+    print(render.to_json(results, args.model) if args.format == "json" else render.to_text(results, args.model, args.diff), end="")
+    if args.fail_on == "mismatch":
+        return 1 if any(r.verdict == "mismatch" for r in results) else 0
+    if args.fail_on == "drift":
+        return 1 if any(r.verdict != "identical" for r in results) else 0
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(prog="ftgate", description="Do runtimes send the bytes the model was trained on?")
+    ap.add_argument("--version", action="version", version=__version__)
+    sub = ap.add_subparsers(dest="command", required=True)
+
+    t = sub.add_parser("template", help="compare runtime prompts and tokens against the training-side rendering")
+    t.add_argument("--model", required=True, help="HF model id or path whose tokenizer and chat template were used for training")
+    t.add_argument("--llama", metavar="URL", help="llama-server base URL, e.g. http://localhost:8080")
+    t.add_argument("--ollama", metavar="MODEL", help="Ollama model name, e.g. qwen2.5:0.5b-instruct")
+    t.add_argument("--ollama-url", default="http://localhost:11434")
+    t.add_argument("--no-verify", action="store_true", help="skip the live cross-check of Ollama's prompt token count")
+    t.add_argument("--dataset", metavar="JSONL", help="your own conversations (messages[, tools] per line) instead of the built-in set")
+    t.add_argument("--sample", type=int, default=20, help="rows to take from --dataset (default 20)")
+    t.add_argument("--diff", action="store_true", help="print unified diffs for findings")
+    t.add_argument("--format", choices=["text", "json"], default="text")
+    t.add_argument("--fail-on", choices=["never", "drift", "mismatch"], default="never")
+    t.set_defaults(func=cmd_template)
+
+    d = sub.add_parser("data", help="lint a fine-tuning dataset with the target model's tokenizer and template")
+    d.add_argument("path", nargs="+", help="JSONL file(s): messages / ShareGPT / Alpaca / preference rows")
+    d.add_argument("--model", help="HF model id; enables tokenizer and template checks")
+    d.add_argument("--max-seq-len", type=int, help="the trainer's cutoff; rows beyond it are reported")
+    d.add_argument("--eval", metavar="JSONL", help="eval set to check for leakage (near-duplicates)")
+    d.add_argument("--limit", type=int)
+    d.add_argument("--format", choices=["text", "json"], default="text")
+    d.add_argument("--fail-on", choices=["never", "warning", "error"], default="never")
+    d.set_defaults(func=cmd_data)
+
+    tl = sub.add_parser("tools", help="evaluate tool calling on your tools and cases, base vs tuned vs quantised, via promptfoo")
+    tl.add_argument("--tools", metavar="JSON", help="OpenAI-style tool list")
+    tl.add_argument("--cases", metavar="FILE", help="cases: {user, calls[, system]} per row (json/jsonl/yaml)")
+    tl.add_argument("--from-dataset", metavar="JSONL", help="hold out cases from your training rows instead")
+    tl.add_argument("--holdout", type=int, default=50)
+    tl.add_argument("--provider", action="append", required=True,
+                    help="repeatable: ollama:MODEL | openai:MODEL@http://host:port/v1 | any promptfoo provider id")
+    tl.add_argument("--keep", metavar="DIR", help="keep the generated promptfoo config and results here")
+    tl.add_argument("--concurrency", type=int, default=1, help="requests in flight (default 1: local servers answer one at a time)")
+    tl.add_argument("--min-exact", type=float, help="exit 1 if any arm's exact-args rate is below this")
+    tl.add_argument("--format", choices=["text", "json"], default="text")
+    tl.set_defaults(func=cmd_tools)
+
+    args = ap.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ftgate/compare.py

@@ -0,0 +1,119 @@
+"""Compare a runtime's bytes with the reference and say what kind of difference it is.
+
+Findings are ordered by how much they matter to a model that was trained
+on the reference:
+
+  schema_not_json     a tool definition reached the model as something other
+                      than JSON — the schema itself is gone
+  injected_system     the runtime added a system prompt the training bytes
+                      did not have (Ollama's Modelfile SYSTEM)
+  special_prefix      the runtime prepends a special token the training
+                      bytes did not have (the BOS case)
+  structural          text differs beyond whitespace/JSON spacing
+  spacing             same content, different whitespace or JSON separators —
+                      different tokens, same meaning
+  tokenizer           identical text, different ids — the runtime's
+                      tokenizer disagrees with the training tokenizer
+  identical           byte-for-byte and id-for-id
+"""
+from __future__ import annotations
+
+import difflib
+import json
+import re
+from dataclasses import dataclass, field
+
+from ftgate.reference import Rendering
+from ftgate.runtimes import RuntimeRendering
+
+_TOOLS_BLOCK = re.compile(r"<tools>(.*?)</tools>", re.S)
+
+
+@dataclass
+class Finding:
+    kind: str
+    detail: str
+    diff: str = ""
+
+
+@dataclass
+class Comparison:
+    conversation: str
+    runtime: str
+    reference_tokens: int
+    runtime_tokens: int | None
+    findings: list[Finding] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def verdict(self) -> str:
+        kinds = {f.kind for f in self.findings}
+        if kinds & {"schema_not_json", "structural", "special_prefix", "injected_system"}:
+            return "mismatch"
+        if kinds & {"spacing", "tokenizer"}:
+            return "drift"
+        return "identical"
+
+
+def _canon(text: str) -> str:
+    """Collapse whitespace and JSON separators so spacing-only differences vanish."""
+    t = re.sub(r"[ \t]+", " ", text)
+    t = re.sub(r"\s*([,:{}\[\]])\s*", r"\1", t)
+    return t.strip()
+
+
+def _tool_schemas_valid(text: str) -> list[str]:
+    """Every line inside <tools> must parse as JSON with a function name."""
+    bad = []
+    for block in _TOOLS_BLOCK.findall(text):
+        for line in block.strip().splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+                fn = obj.get("function", obj)
+                if not isinstance(fn, dict) or "name" not in fn:
+                    bad.append(line[:120])
+            except json.JSONDecodeError:
+                bad.append(line[:120])
+    return bad
+
+
+def compare(name: str, ref: Rendering, rt: RuntimeRendering, ref_ids_of_runtime_text: list[int]) -> Comparison:
+    c = Comparison(name, rt.runtime, len(ref.ids), len(rt.ids) if rt.ids is not None else None, notes=list(rt.notes))
+
+    bad = _tool_schemas_valid(rt.text)
+    if bad and not _tool_schemas_valid(ref.text):
+        c.findings.append(Finding("schema_not_json",
+            f"{len(bad)} tool definition(s) are not JSON in the runtime prompt", "\n".join(bad)))
+
+    if any(n.startswith("Modelfile SYSTEM injected") for n in rt.notes):
+        c.findings.append(Finding("injected_system", "runtime prepends a system prompt the training conversation does not have"))
+
+    if rt.special_prefix and not ref.ids[:len(rt.special_prefix)] == rt.special_prefix:
+        c.findings.append(Finding("special_prefix",
+            f"runtime prepends special token(s) {rt.special_prefix} that the training bytes do not start with"))
+
+    if rt.text != ref.text:
+        diff = "\n".join(difflib.unified_diff(ref.text.splitlines(), rt.text.splitlines(), "training", rt.runtime, lineterm="", n=1))
+        if _canon(rt.text) == _canon(ref.text):
+            c.findings.append(Finding("spacing", "same content, different whitespace or JSON spacing — different tokens", diff))
+        elif not bad and not any(f.kind == "injected_system" for f in c.findings):
+            c.findings.append(Finding("structural", "prompt text differs from the training bytes", diff))
+        else:
+            c.findings[0].diff = diff
+    elif rt.ids is not None and rt.ids != ref.ids:
+        c.findings.append(Finding("tokenizer", f"identical text, different ids ({len(rt.ids)} vs {len(ref.ids)})"))
+
+    if rt.ids is None:
+        c.runtime_tokens = len(ref_ids_of_runtime_text)
+        c.notes.append("runtime cannot tokenize; ids computed with the training tokenizer on the runtime's text")
+        for n in rt.notes:
+            if n.startswith("__prompt_eval_count="):
+                reported = int(n.split("=", 1)[1])
+                c.notes = [x for x in c.notes if not x.startswith("__")]
+                c.notes.append("cross-check: matches the runtime's reported prompt token count" if reported == c.runtime_tokens
+                               else f"cross-check FAILED: runtime reported {reported} prompt tokens, render has {c.runtime_tokens}")
+    c.notes = [x for x in c.notes if not x.startswith("__")]
+    return c

ftgate/conversations.py

@@ -0,0 +1,84 @@
+"""Conversations to compare: built-in fixtures, or rows from the user's own dataset.
+
+The built-in set covers the shapes a chat template has to get right —
+plain chat, a system prompt, a tool call with a response, several tools.
+The user's dataset is the one that matters: those are the rows the model
+was actually trained on, so a mismatch there is a mismatch that counts.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+WEATHER = {"type": "function", "function": {"name": "get_weather", "description": "Get the current weather for a city.",
+    "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "City name"},
+    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, "required": ["city"]}}}
+CONVERT = {"type": "function", "function": {"name": "convert_currency", "description": "Convert an amount between currencies.",
+    "parameters": {"type": "object", "properties": {"amount": {"type": "number"}, "from_currency": {"type": "string"},
+    "to_currency": {"type": "string"}}, "required": ["amount", "from_currency", "to_currency"]}}}
+
+
+@dataclass
+class Conversation:
+    name: str
+    messages: list[dict]
+    tools: list[dict] = field(default_factory=list)
+
+    @property
+    def inference_messages(self) -> list[dict]:
+        """Everything up to the final assistant turn — what a runtime is sent."""
+        msgs = self.messages
+        if msgs and msgs[-1].get("role") == "assistant":
+            msgs = msgs[:-1]
+        return msgs
+
+
+BUILTIN = [
+    Conversation("plain-chat", [
+        {"role": "user", "content": "Name three primary colours."},
+        {"role": "assistant", "content": "Red, blue and yellow."},
+    ]),
+    Conversation("system-prompt", [
+        {"role": "system", "content": "You are a terse assistant. Answer in one sentence."},
+        {"role": "user", "content": "Why is the sky blue?"},
+        {"role": "assistant", "content": "Air scatters short wavelengths more than long ones."},
+    ]),
+    Conversation("tool-call-and-response", [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "What's the weather in Tbilisi?"},
+        {"role": "assistant", "content": "", "tool_calls": [{"type": "function", "function": {"name": "get_weather", "arguments": {"city": "Tbilisi", "unit": "celsius"}}}]},
+        {"role": "tool", "content": "{\"temp_c\": 21, \"sky\": \"clear\"}"},
+        {"role": "assistant", "content": "It's 21°C and clear in Tbilisi."},
+    ], tools=[WEATHER]),
+    Conversation("two-tools-three-params", [
+        {"role": "user", "content": "Convert 100 USD to EUR."},
+        {"role": "assistant", "content": "", "tool_calls": [{"type": "function", "function": {"name": "convert_currency", "arguments": {"amount": 100, "from_currency": "USD", "to_currency": "EUR"}}}]},
+    ], tools=[WEATHER, CONVERT]),
+]
+
+
+def load_dataset(path: Path, limit: int | None = None) -> list[Conversation]:
+    """Rows from a JSONL file in the common `messages` (+ optional `tools`) shape.
+
+    ShareGPT-style `conversations` with `from`/`value` is mapped onto it; any
+    other shape is reported rather than guessed at.
+    """
+    out: list[Conversation] = []
+    with path.open(encoding="utf-8") as fh:
+        for n, line in enumerate(fh):
+            line = line.strip()
+            if not line:
+                continue
+            row = json.loads(line)
+            if "messages" in row:
+                msgs = row["messages"]
+            elif "conversations" in row:
+                role = {"human": "user", "gpt": "assistant", "system": "system", "tool": "tool"}
+                msgs = [{"role": role.get(m.get("from"), m.get("from")), "content": m.get("value", "")} for m in row["conversations"]]
+            else:
+                raise ValueError(f"{path}:{n+1}: no `messages` or `conversations` key; keys are {sorted(row)}")
+            out.append(Conversation(f"{path.name}:{n+1}", msgs, row.get("tools") or []))
+            if limit and len(out) >= limit:
+                break
+    return out