poolprobe 1.0.0__py3-none-any.whl

poolprobe/chains.py

@@ -0,0 +1,414 @@
+"""poolprobe.chains — a fallback chain that looks configured and is not.
+
+A chain is the last thing standing between an agent and a dead upstream, and
+every way it breaks is silent. The runtime does not warn about a rung it
+dropped, a duplicate it collapsed, or a key nobody reads; the agent just falls
+through to a worse model and the operator finds out a week later.
+
+Two lanes exist and they do not share rules:
+
+* **main** — top-level ``fallback_providers`` (plus the legacy single-dict
+  ``fallback_model``). A rung needs *both* ``provider`` and ``model``; an
+  incomplete one is skipped without a word. Identical rungs are deduped away.
+* **aux** — ``auxiliary.<task>.fallback_chain``. A rung needs *only*
+  ``provider``; ``model`` is optional and resolves later. A chain that is not a
+  list is ignored in its entirety. No dedup, so a repeated rung really does
+  burn a retry.
+
+And in both lanes ``priority`` is decoration: nothing sorts by it, the order in
+the file is the order they are tried.
+
+``validate`` returns plain dicts — ``{"label", "rungs", "issues"}`` — which is
+exactly what :class:`poolprobe.report.Report` renders. No logic lives in the
+report layer.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from .adapter import ConfigError, _load_yaml
+
+__all__ = [
+    "ConfigError",
+    "load_config",
+    "validate",
+    "validate_file",
+    "selfcheck",
+]
+
+#: keys that look like chain config but are read by nothing
+DEAD_CHAIN_KEYS: tuple[tuple[str, str], ...] = (
+    ("model", "fallback_chain"),
+)
+
+
+def _norm(value: Any) -> str:
+    """Config values arrive as whatever YAML produced; chains compare strings."""
+    if value is None:
+        return ""
+    return str(value).strip()
+
+
+def _short(value: Any, limit: int = 48) -> str:
+    text = value if isinstance(value, str) else repr(value)
+    return text if len(text) <= limit else text[: limit - 1] + "\u2026"
+
+
+def load_config(path: str | Path) -> dict:
+    """Read a gateway/agent YAML config into a dict (empty dict if not a mapping)."""
+    data = _load_yaml(Path(path).expanduser())
+    return data if isinstance(data, dict) else {}
+
+
+def _same_backend(
+    rung_provider: str,
+    rung_model: str,
+    primary_provider: str,
+    primary_model: str,
+) -> bool:
+    """A rung pointing at the backend that just failed is not a fallback."""
+    if not rung_model or not primary_model:
+        return False
+    if rung_model.lower() != primary_model.lower():
+        return False
+    if not rung_provider or not primary_provider:
+        return True
+    return rung_provider.lower() == primary_provider.lower()
+
+
+def _entries_of(raw: Any) -> list[Any] | None:
+    """Normalise a chain value the way the runtime does.
+
+    ``None`` means "nothing to check". A non-list non-dict value yields ``[]``
+    with the caller reporting it, because that is precisely the silent drop.
+    """
+    if raw is None:
+        return None
+    if isinstance(raw, dict):
+        return [raw]
+    if isinstance(raw, list):
+        return raw
+    return []
+
+
+def _priority_issue(raw: Any) -> str:
+    """``priority`` never reorders anything — flag it only when it misleads."""
+    if not isinstance(raw, list):
+        return ""
+    declared: list[tuple[int, Any]] = []
+    for index, entry in enumerate(raw, start=1):
+        if isinstance(entry, dict) and entry.get("priority") is not None:
+            declared.append((index, entry.get("priority")))
+    if len(declared) < 2:
+        return ""
+    for index, value in declared:
+        try:
+            number = int(value)
+        except (TypeError, ValueError):
+            return (
+                f"rung {index} declares priority {_short(value)!r} \u2014 priority is not "
+                "read at all; file order is the only order"
+            )
+        if number != index:
+            return (
+                f"rung {index} declares priority {number} \u2014 nothing sorts by priority, "
+                "rungs are tried in file order; the numbers are decoration"
+            )
+    return ""
+
+
+def check_chain(
+    label: str,
+    raw: Any,
+    *,
+    primary: tuple[str, str] = ("", ""),
+    require_model: bool = True,
+    dedup: bool = True,
+    live_models: set[str] | None = None,
+) -> dict:
+    """Validate one chain against the lane rules it will actually be read by.
+
+    ``require_model`` and ``dedup`` are the two places the main and aux lanes
+    genuinely differ; everything else is shared so the wording stays consistent.
+    """
+    rungs: list[str] = []
+    issues: list[str] = []
+    entries = _entries_of(raw)
+    if entries is None:
+        return {"label": label, "rungs": rungs, "issues": issues}
+
+    if raw is not None and not isinstance(raw, (list, dict)):
+        issues.append(
+            f"declared as {_short(raw)} ({type(raw).__name__}), not a list \u2014 the whole "
+            "chain is ignored silently"
+        )
+        return {"label": label, "rungs": rungs, "issues": issues}
+
+    if isinstance(raw, list) and not raw:
+        issues.append("declared as an empty list \u2014 nothing to fall back to")
+        return {"label": label, "rungs": rungs, "issues": issues}
+
+    priority_issue = _priority_issue(raw)
+    if priority_issue:
+        issues.append(priority_issue)
+
+    primary_provider, primary_model = primary
+    seen: dict[tuple[str, str, str], int] = {}
+    usable = 0
+
+    for index, entry in enumerate(entries, start=1):
+        if not isinstance(entry, dict):
+            text = _short(entry)
+            issues.append(
+                f"rung {index} is a bare {type(entry).__name__} ({text}) \u2014 a rung needs "
+                "a mapping with at least a provider; dropped silently"
+            )
+            rungs.append(text)
+            continue
+
+        provider = _norm(entry.get("provider"))
+        model = _norm(entry.get("model"))
+        base_url = _norm(entry.get("base_url")).rstrip("/")
+        shown = f"{provider}/{model}" if provider and model else (provider or model or _short(entry))
+        rungs.append(shown)
+
+        if not provider:
+            issues.append(f"rung {index} ({shown}) has no provider \u2014 dropped silently")
+            continue
+        if require_model and not model:
+            issues.append(
+                f"rung {index} ({provider}) has no model \u2014 this lane requires provider "
+                "AND model; dropped silently"
+            )
+            continue
+
+        usable += 1
+        identity = (provider.lower(), model.lower(), base_url.lower())
+        if identity in seen:
+            first = seen[identity]
+            tail = (
+                "the runtime dedupes it away, it never runs"
+                if dedup
+                else "no dedup here, it just burns a retry on the same backend"
+            )
+            issues.append(f"rung {index} ({shown}) duplicates rung {first} \u2014 {tail}")
+            continue
+        seen[identity] = index
+
+        if _same_backend(provider, model, primary_provider, primary_model):
+            issues.append(
+                f"rung {index} ({shown}) is the primary itself \u2014 falls back onto the "
+                "backend that just failed"
+            )
+        if live_models is not None and model and model not in live_models:
+            issues.append(
+                f"rung {index} model '{model}' is not in the live gateway catalog \u2014 this "
+                "rung can only ever fail"
+            )
+
+    if entries and usable == 0:
+        issues.append(
+            "every rung is dropped or duplicated \u2014 the lane falls straight through to "
+            "the safety net"
+        )
+    return {"label": label, "rungs": rungs, "issues": issues}
+
+
+def _dead_key_chains(config: dict) -> list[dict]:
+    """Keys an operator edits in good faith that no code path reads."""
+    rungs: list[str] = []
+    issues: list[str] = []
+    for parent, key in DEAD_CHAIN_KEYS:
+        section = config.get(parent)
+        if isinstance(section, dict) and key in section:
+            rungs.append(f"{parent}.{key}")
+            issues.append(
+                f"{parent}.{key} is set but nothing reads it \u2014 the main lane reads "
+                "top-level fallback_providers only; edits here are lost"
+            )
+    if not rungs:
+        return []
+    return [{"label": "dead keys", "rungs": rungs, "issues": issues}]
+
+
+def validate(
+    config: dict | None,
+    *,
+    live_models: set[str] | None = None,
+) -> list[dict]:
+    """Validate every fallback chain in an agent config.
+
+    ``live_models`` is optional: pass the gateway catalog to also catch rungs
+    that name a model the upstream no longer serves.
+    """
+    cfg = config if isinstance(config, dict) else {}
+    out: list[dict] = []
+
+    model_cfg = cfg.get("model")
+    model_cfg = model_cfg if isinstance(model_cfg, dict) else {}
+    main_primary = (_norm(model_cfg.get("provider")), _norm(model_cfg.get("default")))
+
+    main = check_chain(
+        "main",
+        cfg.get("fallback_providers"),
+        primary=main_primary,
+        require_model=True,
+        dedup=True,
+        live_models=live_models,
+    )
+    if main["rungs"] or main["issues"]:
+        out.append(main)
+
+    legacy = cfg.get("fallback_model")
+    if legacy is not None:
+        out.append(
+            check_chain(
+                "main (legacy fallback_model)",
+                legacy,
+                primary=main_primary,
+                require_model=True,
+                dedup=True,
+                live_models=live_models,
+            )
+        )
+
+    aux_root = cfg.get("auxiliary")
+    if isinstance(aux_root, dict):
+        for task in sorted(aux_root):
+            task_cfg = aux_root[task]
+            if not isinstance(task_cfg, dict) or "fallback_chain" not in task_cfg:
+                continue
+            out.append(
+                check_chain(
+                    f"aux:{task}",
+                    task_cfg.get("fallback_chain"),
+                    primary=(_norm(task_cfg.get("provider")), _norm(task_cfg.get("model"))),
+                    require_model=False,
+                    dedup=False,
+                    live_models=live_models,
+                )
+            )
+
+    out.extend(_dead_key_chains(cfg))
+    return out
+
+
+def validate_file(
+    path: str | Path,
+    *,
+    live_models: set[str] | None = None,
+) -> list[dict]:
+    """``load_config`` + ``validate`` in one call."""
+    return validate(load_config(path), live_models=live_models)
+
+
+def selfcheck() -> None:
+    """Offline asserts over the lane rules. Run: ``python -m poolprobe.chains``."""
+    # main lane: incomplete rung dropped, duplicate deduped, priority is a lie
+    broken = {
+        "model": {"provider": "custom:cliproxy", "default": "qwen3.8-max"},
+        "fallback_providers": [
+            {"provider": "custom:cliproxy", "model": "glm-5.3-flash", "priority": 1},
+            {"provider": "custom:cliproxy", "model": "glm-5.3-flash", "priority": 2},
+            {"provider": "custom:cliproxy", "priority": 3},
+            "deepseek-v4-flash",
+        ],
+    }
+    (main,) = validate(broken)
+    assert main["label"] == "main"
+    assert main["rungs"] == [
+        "custom:cliproxy/glm-5.3-flash",
+        "custom:cliproxy/glm-5.3-flash",
+        "custom:cliproxy",
+        "deepseek-v4-flash",
+    ], main["rungs"]
+    joined = "\n".join(main["issues"])
+    assert "duplicates rung 1" in joined
+    assert "dedupes it away" in joined
+    assert "has no model" in joined
+    assert "bare str" in joined
+    # priority 1,2,3 sitting on rungs 1,2,3 misleads nobody: no issue by design
+    assert "priority" not in joined
+
+    # priority that disagrees with position is the trap worth naming
+    shuffled = {
+        "fallback_providers": [
+            {"provider": "p", "model": "a", "priority": 9},
+            {"provider": "p", "model": "b", "priority": 1},
+        ]
+    }
+    (misleading,) = validate(shuffled)
+    assert "nothing sorts by priority" in "\n".join(misleading["issues"])
+
+    # a rung that is the primary is not a fallback
+    loop = {
+        "model": {"provider": "custom:cliproxy", "default": "gpt-5.4"},
+        "fallback_providers": [
+            {"provider": "custom:cliproxy", "model": "gpt-5.4"},
+            {"provider": "custom:cliproxy", "model": "glm-5.3-flash"},
+        ],
+    }
+    (chain,) = validate(loop)
+    assert "is the primary itself" in "\n".join(chain["issues"])
+
+    # aux lane: model is optional, no dedup, non-list chain is ignored whole
+    aux = {
+        "auxiliary": {
+            "vision": {
+                "provider": "custom:cliproxy",
+                "model": "gemini-3.8-flash-high",
+                "fallback_chain": [
+                    {"provider": "custom:cliproxy"},
+                    {"provider": "custom:cliproxy"},
+                ],
+            },
+            "compression": {"fallback_chain": "glm-5.3-flash"},
+            "curator": {"fallback_chain": []},
+        }
+    }
+    by_label = {item["label"]: item for item in validate(aux)}
+    assert by_label["aux:vision"]["rungs"] == ["custom:cliproxy", "custom:cliproxy"]
+    assert "no model" not in "\n".join(by_label["aux:vision"]["issues"])
+    assert "burns a retry" in "\n".join(by_label["aux:vision"]["issues"])
+    assert "not a list" in "\n".join(by_label["aux:compression"]["issues"])
+    assert "empty list" in "\n".join(by_label["aux:curator"]["issues"])
+
+    # dead key + live-catalog cross-check
+    dead = {"model": {"fallback_chain": [{"provider": "x", "model": "y"}]},
+            "fallback_providers": [{"provider": "p", "model": "gone-model"}]}
+    labels = {item["label"] for item in validate(dead, live_models={"live-model"})}
+    assert "dead keys" in labels
+    dead_report = validate(dead, live_models={"live-model"})
+    assert "model.fallback_chain is set but nothing reads it" in "\n".join(
+        issue for item in dead_report for issue in item["issues"]
+    )
+    main_issues = "\n".join(
+        issue for item in dead_report if item["label"] == "main" for issue in item["issues"]
+    )
+    assert "not in the live gateway catalog" in main_issues
+    # same chain, catalog says the model exists -> that issue disappears
+    assert "not in the live gateway catalog" not in "\n".join(
+        issue
+        for item in validate(dead, live_models={"gone-model"})
+        if item["label"] == "main"
+        for issue in item["issues"]
+    )
+
+    # a clean config produces no chains at all, not fake-green ones
+    assert validate({}) == []
+    clean = {
+        "model": {"provider": "custom:cliproxy", "default": "gpt-5.4"},
+        "fallback_providers": [
+            {"provider": "custom:cliproxy", "model": "glm-5.3-flash"},
+            {"provider": "custom:cliproxy", "model": "deepseek-v4-flash"},
+        ],
+    }
+    (only,) = validate(clean)
+    assert only["issues"] == [], only["issues"]
+
+
+if __name__ == "__main__":
+    selfcheck()
+    print("chains selfcheck ok")