knowledge-gateway 0.7.4__py3-none-any.whl

gateway/__init__.py

@@ -0,0 +1,7 @@
+"""knowledge-gateway: filesystem/git-native FastMCP knowledge gateway (vault + code-graph + convert)."""
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("knowledge-gateway")
+except PackageNotFoundError:  # running from a source tree that is not installed
+    __version__ = "0.2.0"

gateway/__main__.py

@@ -0,0 +1,3 @@
+from .server import main
+
+main()

gateway/acl.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+class AccessDenied(Exception):
+    """Raised when a token may not touch a vault. Message is deliberately opaque
+    so a caller cannot tell a forbidden vault apart from a non-existent one."""
+
+
+@dataclass(frozen=True)
+class TokenInfo:
+    sub: str
+    vaults: frozenset[str]
+    write: bool
+    email: str = ""
+
+
+def build_registry(tokens_cfg: dict) -> dict[str, TokenInfo]:
+    reg: dict[str, TokenInfo] = {}
+    for token, meta in (tokens_cfg or {}).items():
+        if not isinstance(token, str) or not token:
+            raise ValueError("token key must be a non-empty string")
+        sub = meta.get("sub", "unknown")
+        vaults = meta.get("vaults", [])
+        write = meta.get("write", False)
+        # Validate types explicitly: `vaults: myvault` (a str) would otherwise
+        # become a frozenset of characters, and `write: "false"` (a str) would be
+        # truthy via bool() and silently grant write access.
+        if not isinstance(vaults, list) or not all(isinstance(x, str) for x in vaults):
+            raise ValueError(f"token '{sub}': vaults must be a list of strings")
+        if not isinstance(write, bool):
+            raise ValueError(f"token '{sub}': write must be a boolean (true/false)")
+        reg[token] = TokenInfo(
+            sub=str(sub),
+            vaults=frozenset(vaults),
+            write=write,
+            email=str(meta.get("email", "")),
+        )
+    return reg
+
+
+def scopes_for(info: TokenInfo) -> list[str]:
+    scopes = [f"vault:{v}" for v in sorted(info.vaults)]
+    if info.write:
+        scopes.append("write")
+    return scopes
+
+
+def allowed_vaults(scopes) -> set[str]:
+    return {s.split(":", 1)[1] for s in scopes if s.startswith("vault:")}
+
+
+def can_write(scopes) -> bool:
+    return "write" in set(scopes)
+
+
+def authorize(scopes, vault: str, *, write: bool) -> None:
+    if vault not in allowed_vaults(scopes):
+        raise AccessDenied(f"vault_forbidden: {vault}")
+    if write and not can_write(scopes):
+        raise AccessDenied(f"write_forbidden: {vault}")

gateway/codegraph/__init__.py

@@ -0,0 +1,15 @@
+"""codegraph - build a queryable code/Ansible knowledge graph from a source tree.
+
+Our own extractor (no third-party graph tool): stdlib `ast` for Python, PyYAML for
+Ansible, and an optional generic tree-sitter pass for more languages (JS/TS/Go/
+Terraform/bash/PowerShell/... via the [graph-all] extra). Output is a NetworkX
+node-link `graph.json` the gateway serves read-only. AST-only: no LLM, no network.
+
+Node:  {id, label, type, file_type, source_file, source_location?, community?}
+Edge:  {source, target, relation, confidence(EXTRACTED|INFERRED|AMBIGUOUS), confidence_score?}
+"""
+from __future__ import annotations
+
+from .build import build_graph, SCHEMA_VERSION
+
+__all__ = ["build_graph", "SCHEMA_VERSION"]

gateway/codegraph/build.py

@@ -0,0 +1,89 @@
+"""Build a NetworkX code/Ansible graph from a source tree and return node-link JSON.
+
+Passes: Ansible (PyYAML, repo-level) + Python (ast, per file) + optional broad
+languages (tree-sitter, per file). Communities (greedy modularity) are baked onto
+nodes. AST-only, read-only on the source tree. networkx is imported lazily so this
+module is importable without the [graph] extra.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from . import extract_ansible, extract_python, treesitter
+
+SCHEMA_VERSION = 1
+_PRUNE = extract_ansible.PRUNE  # one prune set shared by the Ansible pass and the file walk
+
+
+def _iter_files(root: Path):
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in _PRUNE]  # prune in place: do not descend
+        for fn in filenames:
+            yield Path(dirpath) / fn
+
+
+def build_graph(root, languages: list[str] | None = None) -> dict:
+    """Build the graph for `root`. Returns NetworkX node-link data (with a `graph`
+    metadata block). `languages` optionally restricts the broad tree-sitter pass."""
+    try:
+        import networkx as nx
+        from networkx.algorithms.community import greedy_modularity_communities
+    except ImportError:
+        raise ValueError("graph_unavailable: install the [graph] extra (networkx) to build graphs")
+
+    root = Path(root).resolve()
+    if not root.is_dir():
+        raise FileNotFoundError(f"not_found: {root}")
+
+    fragments: list[dict] = [extract_ansible.extract(root)]
+    n_py = n_ts = 0
+    for p in _iter_files(root):
+        rel = p.relative_to(root).as_posix()
+        if p.suffix == ".py":
+            fragments.append(extract_python.extract(p, rel))
+            n_py += 1
+        elif p.suffix.lower() in treesitter.LANG_EXTS:
+            if languages and treesitter.EXT_LANG.get(p.suffix.lower()) not in languages:
+                continue
+            frag = treesitter.extract(p, rel)
+            if frag["nodes"]:
+                n_ts += 1
+            fragments.append(frag)
+
+    G = nx.DiGraph()
+    for frag in fragments:
+        for n in frag["nodes"]:
+            nid = n["id"]
+            if nid in G:
+                G.nodes[nid].update({k: v for k, v in n.items() if v is not None and k != "id"})
+            else:
+                G.add_node(nid, **{k: v for k, v in n.items() if k != "id"})
+        for e in frag["edges"]:
+            for end in (e["source"], e["target"]):
+                if end not in G:
+                    G.add_node(end)
+            G.add_edge(e["source"], e["target"],
+                       **{k: v for k, v in e.items() if k not in ("source", "target")})
+
+    communities: list = []
+    if G.number_of_nodes():
+        try:
+            communities = list(greedy_modularity_communities(G.to_undirected()))
+        except Exception:
+            communities = []
+        for cid, members in enumerate(communities):
+            for nid in members:
+                G.nodes[nid]["community"] = cid
+
+    G.graph.update({
+        "schema_version": SCHEMA_VERSION,
+        "root": root.name,
+        "files_python": n_py,
+        "files_treesitter": n_ts,
+        "treesitter_available": treesitter.AVAILABLE,
+        "communities": len(communities),
+        "node_count": G.number_of_nodes(),
+        "edge_count": G.number_of_edges(),
+    })
+    return nx.node_link_data(G, edges="links")

gateway/codegraph/cli.py

@@ -0,0 +1,37 @@
+"""CLI: build a code graph from a source tree into a graph.json.
+
+    knowledge-gateway-graph <source> -o <vault>/.graph/<name>.json [--languages js ts ...]
+
+Runs where the code is (the source tree may be outside any vault); writes a node-link
+graph.json the gateway then serves read-only. AST-only, no network, no LLM.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(prog="knowledge-gateway-graph",
+                                 description="Build a code/Ansible knowledge graph (graph.json).")
+    ap.add_argument("source", help="source tree (code repo) to graph")
+    ap.add_argument("-o", "--out", required=True, help="output graph.json path")
+    ap.add_argument("--languages", nargs="*", default=None,
+                    help="restrict the tree-sitter pass to these languages (default: all available)")
+    args = ap.parse_args(argv)
+
+    from .build import build_graph  # imports networkx; needs the [graph] extra
+    data = build_graph(args.source, languages=args.languages)
+    out = Path(args.out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+    g = data.get("graph", {})
+    print(f"built: {g.get('node_count')} nodes, {g.get('edge_count')} edges, "
+          f"{g.get('communities')} communities (tree-sitter: {g.get('treesitter_available')}) -> {out}")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

gateway/codegraph/extract_ansible.py

@@ -0,0 +1,183 @@
+"""Ansible/YAML extractor - the relationships a generic AST tool cannot see.
+
+Repo-level pass: walks playbooks, roles (tasks/handlers/meta), and filter_plugins,
+emitting nodes for playbooks/plays/roles/tasksfiles/tasks/handlers/filters and edges
+for uses_role, role_depends_on, include_role, include_tasks, notifies, has_tasks,
+calls_filter (Ansible task -> Python filter plugin), and implemented_by (filter -> fn).
+"""
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+import yaml
+
+INCLUDE_TASKS = {"include_tasks", "import_tasks",
+                 "ansible.builtin.include_tasks", "ansible.builtin.import_tasks"}
+INCLUDE_ROLE = {"include_role", "import_role",
+                "ansible.builtin.include_role", "ansible.builtin.import_role"}
+_FILTER_RE = re.compile(r"\|\s*([a-zA-Z_]\w*)")
+PRUNE = {".git", "node_modules", ".venv", "venv", "__pycache__", ".graph",
+         "dist", "build", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".tox"}
+
+
+def _pruned(p, root) -> bool:
+    return any(part in PRUNE for part in p.relative_to(root).parts)
+
+
+def _safe_load(path: Path):
+    try:
+        return yaml.safe_load(path.read_text(encoding="utf-8", errors="replace"))
+    except Exception:
+        return None
+
+
+def _iter_strings(obj):
+    if isinstance(obj, str):
+        yield obj
+    elif isinstance(obj, dict):
+        for v in obj.values():
+            yield from _iter_strings(v)
+    elif isinstance(obj, list):
+        for v in obj:
+            yield from _iter_strings(v)
+
+
+def extract(root: Path) -> dict:
+    root = Path(root)
+    nodes: dict[str, dict] = {}
+    edges: list[dict] = []
+
+    def node(nid, **attrs):
+        cur = nodes.setdefault(nid, {"id": nid})
+        cur.update({k: v for k, v in attrs.items() if v is not None})
+
+    def edge(s, t, relation, confidence="EXTRACTED", score=None):
+        node(s)
+        node(t)
+        e = {"source": s, "target": t, "relation": relation, "confidence": confidence}
+        if score is not None:
+            e["confidence_score"] = score
+        edges.append(e)
+
+    # --- filter plugins: name -> function (what `| name` resolves to) ---
+    filter_names: set[str] = set()
+    for py in root.rglob("filter_plugins/*.py"):
+        if _pruned(py, root):
+            continue
+        rel = py.relative_to(root).as_posix()
+        try:
+            tree = ast.parse(py.read_text(encoding="utf-8", errors="replace"))
+        except SyntaxError:
+            continue
+        for f in tree.body:  # module-level functions only - avoid bogus method ids
+            if isinstance(f, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                node(f"pyfunc:{rel}:{f.name}", label=f.name, type="function",
+                     file_type="python", source_file=rel)
+        for n in ast.walk(tree):
+            if isinstance(n, ast.FunctionDef) and n.name == "filters":
+                for d in ast.walk(n):
+                    if isinstance(d, ast.Dict):
+                        for k, v in zip(d.keys, d.values):
+                            if isinstance(k, ast.Constant) and isinstance(k.value, str):
+                                fname = k.value
+                                filter_names.add(fname)
+                                node(f"filter:{fname}", label=fname, type="ansible_filter",
+                                     file_type="python", source_file=rel)
+                                if isinstance(v, ast.Name):
+                                    edge(f"filter:{fname}", f"pyfunc:{rel}:{v.id}", "implemented_by")
+
+    def walk_tasks(tasks, owner, rel):
+        if not isinstance(tasks, list):
+            return
+        for i, t in enumerate(tasks):
+            if not isinstance(t, dict):
+                continue
+            for blk in ("block", "rescue", "always"):
+                if blk in t:
+                    walk_tasks(t[blk], owner, rel)
+            name = t.get("name")
+            tid = f"task:{rel}:{i}:" + (name or "unnamed")[:40]
+            interesting = False
+            for k in t:
+                if k in INCLUDE_TASKS:
+                    spec = t[k]
+                    f = spec.get("file") if isinstance(spec, dict) else spec
+                    if isinstance(f, str):
+                        edge(owner, f"tasksfile:{(Path(rel).parent / f).as_posix()}", "include_tasks")
+                        interesting = True
+                if k in INCLUDE_ROLE:
+                    spec = t[k]
+                    rn = spec.get("name") if isinstance(spec, dict) else spec
+                    if isinstance(rn, str):
+                        edge(owner, f"role:{rn}", "include_role")
+                        interesting = True
+            nt = t.get("notify")
+            for h in ([nt] if isinstance(nt, str) else nt or []):
+                if isinstance(h, str):
+                    node(tid, label=name or "unnamed", type="task", file_type="ansible",
+                         source_file=rel, source_location=f"#{i}")
+                    edge(tid, f"handler:{h}", "notifies")
+                    interesting = True
+            used = {m for s in _iter_strings(t) for m in _FILTER_RE.findall(s) if m in filter_names}
+            for fn in used:
+                node(tid, label=name or "unnamed", type="task", file_type="ansible",
+                     source_file=rel, source_location=f"#{i}")
+                edge(tid, f"filter:{fn}", "calls_filter")
+            if (interesting or used) and tid in nodes:
+                edge(owner, tid, "has_task")
+
+    for meta in root.rglob("roles/*/meta/main.yml"):
+        if _pruned(meta, root):
+            continue
+        rname = meta.parts[-3]
+        node(f"role:{rname}", label=rname, type="role", file_type="ansible",
+             source_file=meta.relative_to(root).as_posix())
+        data = _safe_load(meta) or {}
+        if isinstance(data, dict):
+            for dep in (data.get("dependencies") or []):
+                dn = (dep.get("role") or dep.get("name")) if isinstance(dep, dict) else dep
+                if isinstance(dn, str):
+                    edge(f"role:{rname}", f"role:{dn}", "role_depends_on")
+
+    for tf in root.rglob("roles/*/tasks/*.yml"):
+        if _pruned(tf, root):
+            continue
+        rel = tf.relative_to(root).as_posix()
+        rname = tf.parts[-3]
+        node(f"tasksfile:{rel}", label=Path(rel).name, type="tasksfile",
+             file_type="ansible", source_file=rel)
+        node(f"role:{rname}", label=rname, type="role", file_type="ansible")
+        edge(f"role:{rname}", f"tasksfile:{rel}", "has_tasks")
+        walk_tasks(_safe_load(tf), f"tasksfile:{rel}", rel)
+
+    for hf in root.rglob("roles/*/handlers/main.yml"):
+        if _pruned(hf, root):
+            continue
+        hs = _safe_load(hf) or []
+        rel = hf.relative_to(root).as_posix()
+        for h in hs if isinstance(hs, list) else []:
+            if isinstance(h, dict) and h.get("name"):
+                node(f"handler:{h['name']}", label=h["name"], type="handler",
+                     file_type="ansible", source_file=rel)
+
+    for pb in list(root.glob("playbook_*.yml")) + list(root.glob("*.yml")):
+        plays = _safe_load(pb)
+        if not isinstance(plays, list) or not any(
+            isinstance(p, dict) and ("hosts" in p or "roles" in p or "tasks" in p) for p in plays
+        ):
+            continue  # not a playbook
+        rel = pb.relative_to(root).as_posix()
+        node(f"playbook:{rel}", label=pb.name, type="playbook", file_type="ansible", source_file=rel)
+        for play in plays:
+            if not isinstance(play, dict):
+                continue
+            for r in (play.get("roles") or []):
+                rn = (r.get("role") or r.get("name")) if isinstance(r, dict) else r
+                if isinstance(rn, str):
+                    edge(f"playbook:{rel}", f"role:{rn}", "uses_role")
+            for sec in ("tasks", "pre_tasks", "post_tasks", "handlers"):
+                walk_tasks(play.get(sec) or [], f"playbook:{rel}", rel)
+
+    return {"nodes": list(nodes.values()), "edges": edges}

gateway/codegraph/extract_python.py

@@ -0,0 +1,64 @@
+"""Python extractor (stdlib `ast`, no dependency): module/function/class/method nodes,
+import edges, and within-file call edges (INFERRED).
+
+Top-level functions get stable `pyfunc:<rel>:<name>` ids (these are what Ansible filter
+plugins and call edges reference); methods are scoped as `pymethod:<rel>:<Class>.<name>`
+so same-named functions/methods do not collide.
+"""
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+_FUNC = (ast.FunctionDef, ast.AsyncFunctionDef)
+
+
+def extract(path: Path, rel: str) -> dict:
+    try:
+        tree = ast.parse(Path(path).read_text(encoding="utf-8", errors="replace"))
+    except SyntaxError:
+        return {"nodes": [], "edges": []}
+
+    nodes: list[dict] = []
+    edges: list[dict] = []
+    mod = f"module:{rel}"
+    nodes.append({"id": mod, "label": rel, "type": "module", "file_type": "python", "source_file": rel})
+
+    top_funcs: set[str] = set()
+    for node in tree.body:  # module-level only -> stable, collision-free ids
+        if isinstance(node, _FUNC):
+            top_funcs.add(node.name)
+            nid = f"pyfunc:{rel}:{node.name}"
+            nodes.append({"id": nid, "label": node.name, "type": "function",
+                          "file_type": "python", "source_file": rel, "source_location": f"L{node.lineno}"})
+            edges.append({"source": mod, "target": nid, "relation": "defines", "confidence": "EXTRACTED"})
+        elif isinstance(node, ast.ClassDef):
+            cid = f"pyclass:{rel}:{node.name}"
+            nodes.append({"id": cid, "label": node.name, "type": "class",
+                          "file_type": "python", "source_file": rel, "source_location": f"L{node.lineno}"})
+            edges.append({"source": mod, "target": cid, "relation": "defines", "confidence": "EXTRACTED"})
+            for m in node.body:
+                if isinstance(m, _FUNC):
+                    mid = f"pymethod:{rel}:{node.name}.{m.name}"
+                    nodes.append({"id": mid, "label": f"{node.name}.{m.name}", "type": "method",
+                                  "file_type": "python", "source_file": rel, "source_location": f"L{m.lineno}"})
+                    edges.append({"source": cid, "target": mid, "relation": "defines", "confidence": "EXTRACTED"})
+
+    for n in ast.walk(tree):
+        if isinstance(n, ast.Import):
+            for a in n.names:
+                edges.append({"source": mod, "target": f"extmodule:{a.name.split('.')[0]}",
+                              "relation": "imports", "confidence": "EXTRACTED"})
+        elif isinstance(n, ast.ImportFrom) and n.module:
+            edges.append({"source": mod, "target": f"extmodule:{n.module.split('.')[0]}",
+                          "relation": "imports", "confidence": "EXTRACTED"})
+
+    # within-file calls between top-level functions (INFERRED)
+    for fn in [n for n in tree.body if isinstance(n, _FUNC)]:
+        caller = f"pyfunc:{rel}:{fn.name}"
+        for c in ast.walk(fn):
+            if (isinstance(c, ast.Call) and isinstance(c.func, ast.Name)
+                    and c.func.id in top_funcs and c.func.id != fn.name):
+                edges.append({"source": caller, "target": f"pyfunc:{rel}:{c.func.id}",
+                              "relation": "calls", "confidence": "INFERRED", "confidence_score": 0.85})
+    return {"nodes": nodes, "edges": edges}

gateway/codegraph/treesitter.py

@@ -0,0 +1,84 @@
+"""Optional broad-language pass via tree-sitter-language-pack (the [graph-all] extra).
+
+One dependency covers many languages (JS/TS/Go/Rust/Java/C#/PHP/Ruby/bash/PowerShell/
+Terraform/...). If the package is not installed, AVAILABLE is False and the build simply
+skips these languages - the Python (ast) and Ansible (yaml) passes need no tree-sitter.
+
+Adding a language later = one line in EXT_LANG (data, not a rewrite).
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+try:  # the broad pass is opt-in; core install stays light
+    from tree_sitter_language_pack import get_parser
+    AVAILABLE = True
+except Exception:  # pragma: no cover - exercised only with the [graph-all] extra
+    AVAILABLE = False
+
+# extension -> tree-sitter language name. Extend freely; coverage is data, not code.
+EXT_LANG = {
+    ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
+    ".ts": "typescript", ".tsx": "tsx",
+    ".go": "go", ".rs": "rust", ".java": "java", ".cs": "c_sharp",
+    ".rb": "ruby", ".php": "php", ".sh": "bash", ".bash": "bash",
+    ".ps1": "powershell", ".psm1": "powershell",
+    ".tf": "hcl", ".tfvars": "hcl", ".hcl": "hcl",
+    ".c": "c", ".h": "c", ".cpp": "cpp", ".cc": "cpp", ".hpp": "cpp",
+    ".lua": "lua", ".kt": "kotlin", ".swift": "swift", ".scala": "scala",
+}
+LANG_EXTS = set(EXT_LANG)
+
+# tree-sitter node types that denote a definition, across grammars (heuristic, language-agnostic).
+_DEF_TYPES = {
+    "function_declaration", "function_definition", "function_item", "method_definition",
+    "method_declaration", "function", "method", "arrow_function", "func_literal",
+    "class_declaration", "class_definition", "class_specifier", "class", "struct_item",
+    "struct_specifier", "interface_declaration", "type_declaration", "module",
+    "block",  # terraform/hcl resource/module blocks
+}
+_NAME_TYPES = ("identifier", "name", "type_identifier", "field_identifier",
+               "constant", "word", "string_literal")
+
+
+def _name(node, src: bytes) -> str | None:
+    for ch in node.children:
+        if ch.type in _NAME_TYPES:
+            return src[ch.start_byte:ch.end_byte].decode("utf-8", "replace").strip('"').strip()
+    return None
+
+
+def extract(path: Path, rel: str) -> dict:
+    if not AVAILABLE:
+        return {"nodes": [], "edges": []}
+    lang = EXT_LANG.get(path.suffix.lower())
+    if not lang:
+        return {"nodes": [], "edges": []}
+    try:
+        parser = get_parser(lang)
+        src = path.read_bytes()
+        tree = parser.parse(src)
+    except Exception:
+        return {"nodes": [], "edges": []}
+
+    nodes: list[dict] = []
+    edges: list[dict] = []
+    mod = f"module:{rel}"
+    nodes.append({"id": mod, "label": rel, "type": "module", "file_type": lang, "source_file": rel})
+
+    stack = [tree.root_node]
+    seen: set[str] = set()
+    while stack:
+        n = stack.pop()
+        if n.type in _DEF_TYPES:
+            nm = _name(n, src)
+            if nm:
+                nid = f"{lang}:{rel}:{nm}:L{n.start_point[0] + 1}"
+                if nid not in seen:
+                    seen.add(nid)
+                    kind = "class" if "class" in n.type or "struct" in n.type or "interface" in n.type else "function"
+                    nodes.append({"id": nid, "label": nm, "type": kind, "file_type": lang,
+                                  "source_file": rel, "source_location": f"L{n.start_point[0] + 1}"})
+                    edges.append({"source": mod, "target": nid, "relation": "defines", "confidence": "EXTRACTED"})
+        stack.extend(n.children)
+    return {"nodes": nodes, "edges": edges}

gateway/config.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import yaml
+
+from .vaults import Vault
+
+DEFAULT_VAULTS_FILE = "vaults.yaml"
+DEFAULT_TOKENS_FILE = "tokens.yaml"
+
+
+def _base_dir() -> Path:
+    return Path(__file__).resolve().parent.parent
+
+
+def _resolve(env_var: str, default_name: str) -> Path:
+    override = os.environ.get(env_var)
+    if override:
+        return Path(override).expanduser().resolve()
+    return _base_dir() / default_name
+
+
+def load_vaults(path: Path | None = None) -> dict[str, Vault]:
+    cfg = path or _resolve("KNOWLEDGE_GATEWAY_VAULTS", DEFAULT_VAULTS_FILE)
+    if not cfg.exists():
+        raise FileNotFoundError(
+            f"vaults config not found: {cfg} — copy vaults.example.yaml to vaults.yaml"
+        )
+    data = yaml.safe_load(cfg.read_text()) or {}
+    vaults = {
+        name: Vault.from_spec(name, spec)
+        for name, spec in (data.get("vaults") or {}).items()
+    }
+    if not vaults:
+        raise ValueError(f"no vaults defined in {cfg}")
+
+    items = list(vaults.values())
+    for v in items:
+        if v.subdir == "." and v.repo_root != v.path:
+            raise ValueError(
+                f"vault '{v.name}': subdir '.' with repo_root != path would commit the whole repo"
+            )
+        if v.subdir.startswith("/") or ".." in Path(v.subdir).parts:
+            raise ValueError(f"vault '{v.name}': subdir must be a relative path without '..'")
+        # subdir must actually locate path under repo_root, else a commit scoped
+        # to the subdir could touch a sibling tree (e.g. backend/) the vault
+        # token was never granted.
+        if (v.repo_root / v.subdir).resolve() != v.path:
+            raise ValueError(
+                f"vault '{v.name}': repo_root/subdir does not resolve to path"
+            )
+    for i, a in enumerate(items):
+        for b in items[i + 1:]:
+            if a.path == b.path or a.path.is_relative_to(b.path) or b.path.is_relative_to(a.path):
+                raise ValueError(
+                    f"vault paths overlap ('{a.name}', '{b.name}') — grants would leak across them"
+                )
+    return vaults
+
+
+def load_tokens(path: Path | None = None) -> dict:
+    cfg = path or _resolve("KNOWLEDGE_GATEWAY_TOKENS", DEFAULT_TOKENS_FILE)
+    if not cfg.exists():
+        raise FileNotFoundError(
+            f"tokens config not found: {cfg} — copy tokens.example.yaml to tokens.yaml"
+        )
+    # Secrets file: refuse to load if it is group/world-accessible. The docs tell
+    # admins to `chmod 0600`, but nothing enforced it — a 0644 tokens.yaml would
+    # silently expose every bearer token to other local users.
+    if os.name == "posix":
+        mode = cfg.stat().st_mode & 0o777
+        if mode & 0o077:
+            raise PermissionError(
+                f"{cfg} is group/world-accessible (mode {mode:#o}); run: chmod 0600 {cfg}"
+            )
+    data = yaml.safe_load(cfg.read_text()) or {}
+    return data.get("tokens") or {}

gateway/convert.py

@@ -0,0 +1,25 @@
+"""Convert attachments (PDF / Office / images / HTML / ...) to Markdown via markitdown.
+
+markitdown is an optional dependency (the [convert] extra); it is imported lazily so the
+gateway core never requires it. Returns Markdown text; writes nothing.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+MAX_CONVERT_BYTES = 50 * 1024 * 1024
+
+
+def to_markdown(path: Path) -> str:
+    try:
+        from markitdown import MarkItDown
+    except ImportError:
+        raise ValueError("convert_unavailable: install the [convert] extra (markitdown) to convert documents")
+    p = Path(path)
+    if p.stat().st_size > MAX_CONVERT_BYTES:
+        raise ValueError(f"too_large: {p.name} is over {MAX_CONVERT_BYTES // (1024 * 1024)} MiB")
+    try:
+        result = MarkItDown().convert(str(p))
+    except Exception as e:
+        raise ValueError(f"convert_failed: {p.name}: {e}")
+    return result.text_content or ""