index-graph 1.0.0__py3-none-any.whl

index_graph/__init__.py

@@ -0,0 +1,16 @@
+"""Compact JSON repository inventory maps for multi-repo workspaces."""
+
+from __future__ import annotations
+
+from .classify import classify
+from .config import Config, Rule, default_config, load_config
+from .model import SCHEMA_VERSION, Map, RepoRow
+from .scan import build_map, discover_repos, write_map
+
+__version__ = "1.0.0"
+__all__ = [
+    "build_map", "write_map", "discover_repos",
+    "Map", "RepoRow", "SCHEMA_VERSION",
+    "Config", "Rule", "load_config", "default_config",
+    "classify", "__version__",
+]

index_graph/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

index_graph/classify.py

@@ -0,0 +1,29 @@
+"""Pure classification: ordered glob rules, then a remote-host fallback."""
+
+from __future__ import annotations
+
+from urllib.parse import urlsplit
+
+from .config import PUBLIC_HOSTS, Config
+
+
+def _remote_host(origin: str) -> str | None:
+    if not origin:
+        return None
+    if "://" not in origin and "@" in origin and ":" in origin:
+        # scp-like SSH form: git@github.com:owner/repo.git
+        return origin.split("@", 1)[1].split(":", 1)[0] or None
+    return urlsplit(origin).hostname
+
+
+def classify(path: str, is_repo: bool, origin: str, config: Config) -> str:
+    for rule in config.rules:
+        if rule.regex.match(path):
+            return rule.class_
+    if is_repo:
+        host = _remote_host(origin)
+        if host is None:
+            return "local"
+        return "public" if host in PUBLIC_HOSTS else "private"
+    name = path.rsplit("/", 1)[-1]
+    return "hidden" if name.startswith(".") else "entry"

index_graph/cli.py

@@ -0,0 +1,194 @@
+"""Command-line entry point: map (default) + graph + context subcommands."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import replace
+from pathlib import Path
+
+from . import __version__
+from .config import load_config
+from .context.pack import closure, focus_subgraph, render_text, to_json
+from .graph.build import build_graph
+from .scan import build_map, discover_repos, write_map
+
+_SUBCOMMANDS = {"map", "graph", "context", "viz"}
+
+
+def _add_map_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--root", type=Path, default=Path.cwd())
+    p.add_argument("--output", type=Path, default=None)
+    p.add_argument("--json", action="store_true")
+    p.add_argument("--config", type=Path, default=None)
+    p.add_argument("--jobs", type=int, default=None)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="index",
+        description="Repository inventory maps + dependency graph + context packs.")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    sub = parser.add_subparsers(dest="cmd")
+
+    _add_map_args(sub.add_parser("map", help="Write the repository inventory map (default)."))
+
+    g = sub.add_parser("graph", help="Derive the repo-level dependency graph.")
+    g.add_argument("--root", type=Path, default=Path.cwd())
+    g.add_argument("--json", action="store_true")
+
+    c = sub.add_parser("context", help="Render the synthesis context pack.")
+    c.add_argument("--root", type=Path, default=Path.cwd())
+    c.add_argument("--json", action="store_true")
+    c.add_argument("--focus", default=None)
+    c.add_argument("--audit", action="store_true")
+
+    v = sub.add_parser("viz", help="Render the dependency graph (html/svg/mermaid).")
+    v.add_argument("--root", type=Path, default=Path.cwd())
+    v.add_argument("--format", choices=["html", "svg", "mermaid", "all"], default="html")
+    v.add_argument("--focus", default=None)
+    v.add_argument("--no-external", action="store_true")
+    v.add_argument("--out", default=None)
+    v.add_argument("--out-dir", default=None)
+    return parser
+
+
+def _repo_paths(root: Path) -> dict[str, Path]:
+    # discover_repos requires a Config; use neutral defaults for graph/context.
+    config = load_config(None, root)
+    return {p.name: p for p in discover_repos(root, config)}
+
+
+def _cmd_map(args) -> int:
+    root = args.root.resolve()
+    if not root.is_dir():
+        raise SystemExit(f"root not found: {root}")
+    config = load_config(args.config, root)
+    if args.jobs is not None:
+        if args.jobs < 1:
+            raise SystemExit("--jobs must be a positive integer")
+        config = replace(config, jobs=args.jobs)
+    if args.json:
+        print(json.dumps(build_map(root, config, __version__).to_json(), indent=2))
+    else:
+        output = args.output.resolve() if args.output else root / "INDEX.json"
+        data = write_map(root, config, __version__, output)
+        print(f"wrote {output}")
+        print(f"repos={data.repo_count} dirty={data.dirty_count}")
+    return 0
+
+
+def _cmd_graph(args) -> int:
+    graph = build_graph(_repo_paths(args.root.resolve()))
+    if args.json:
+        print(json.dumps(to_json(graph), indent=2))
+    else:
+        print(render_text(graph, "dependency graph"))
+    return 0
+
+
+def _cmd_context(args) -> int:
+    graph = build_graph(_repo_paths(args.root.resolve()))
+    names = {n.name for n in graph.repos}
+    if args.audit:
+        data = to_json(graph)
+        print(f"salience-faithfulness warnings: {len(data['salience_audit'])}")
+        for w in data["salience_audit"]:
+            print(f"  [{w['kind']}] {w['node']} (in={w['in_degree']}) — {w['note']}")
+        return 0
+    if args.focus:
+        if args.focus not in names:
+            near = [n for n in names if args.focus.lower() in n.lower()]
+            print(f"unknown project: {args.focus!r}"
+                  + (f" — did you mean: {', '.join(sorted(near))}?" if near else ""))
+            return 2
+        graph = focus_subgraph(graph, closure(list(graph.edges), args.focus))
+        title = f"focus={args.focus}"
+    else:
+        title = "workstation context"
+    print(json.dumps(to_json(graph), indent=2) if args.json else render_text(graph, title))
+    return 0
+
+
+def _head_commit(root) -> str | None:
+    import subprocess
+    try:
+        out = subprocess.run(
+            ["git", "-C", str(root), "rev-parse", "HEAD"],
+            capture_output=True, text=True, timeout=5,
+        )
+        return out.stdout.strip() or None
+    except Exception:
+        return None
+
+
+def _cmd_viz(args) -> int:
+    from . import viz
+
+    graph = build_graph(_repo_paths(args.root.resolve()))
+    names = {n.name for n in graph.repos}
+    if args.focus:
+        if args.focus not in names:
+            near = [n for n in names if args.focus.lower() in n.lower()]
+            print(f"unknown project: {args.focus!r}"
+                  + (f" — did you mean: {', '.join(sorted(near))}?" if near else ""))
+            return 2
+        graph = focus_subgraph(graph, closure(list(graph.edges), args.focus))
+    pack = to_json(graph)
+    include_external = not args.no_external
+
+    def _svg() -> str:
+        return viz.render_svg(viz.build_layout(pack, include_external=include_external))
+
+    def _html() -> str:
+        return viz.render_html(pack, svg=_svg(), charts=viz.render_charts(pack, include_external=include_external))
+
+    if args.format == "all":
+        out_dir = Path(args.out_dir or ".")
+        out_dir.mkdir(parents=True, exist_ok=True)
+        files = {
+            "graph.mmd": viz.render_mermaid(pack, include_external=include_external).encode("utf-8"),
+            "graph.svg": _svg().encode("utf-8"),
+            "graph.html": _html().encode("utf-8"),
+            "context.json": json.dumps(pack, indent=2).encode("utf-8"),
+        }
+        for name, data in files.items():
+            (out_dir / name).write_bytes(data)
+        artifacts = {
+            "mermaid": ("graph.mmd", files["graph.mmd"]),
+            "svg": ("graph.svg", files["graph.svg"]),
+            "html": ("graph.html", files["graph.html"]),
+            "context": ("context.json", files["context.json"]),
+        }
+        meta = {"version": __version__, "commit": _head_commit(args.root.resolve()), "root": str(args.root)}
+        manifest = viz.render_manifest(pack, artifacts=artifacts, meta=meta)
+        (out_dir / "context-manifest.json").write_text(
+            json.dumps(manifest, indent=2), encoding="utf-8"
+        )
+        return 0
+
+    text = {"svg": _svg, "mermaid": lambda: viz.render_mermaid(pack, include_external=include_external), "html": _html}[args.format]()
+    if args.out:
+        Path(args.out).write_text(text, encoding="utf-8")
+    else:
+        print(text)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    raw = list(sys.argv[1:] if argv is None else argv)
+    # No leading subcommand: route top-level --version/--help to the root
+    # parser; otherwise treat the invocation as the implicit `map` command
+    # (preserves v0.2.0 behavior).
+    if not raw or raw[0] not in _SUBCOMMANDS:
+        if raw and raw[0] in ("--version", "-h", "--help"):
+            build_parser().parse_args(raw[:1])  # prints and exits
+        raw = ["map", *raw]
+    args = build_parser().parse_args(raw)
+    if args.cmd == "graph":
+        return _cmd_graph(args)
+    if args.cmd == "context":
+        return _cmd_context(args)
+    if args.cmd == "viz":
+        return _cmd_viz(args)
+    return _cmd_map(args)

index_graph/config.py

@@ -0,0 +1,124 @@
+"""Configuration: .index.toml parsing, neutral defaults, glob translation."""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+DEFAULT_PRUNE_DIRS = frozenset({
+    ".git", ".mypy_cache", ".pytest_cache", ".ruff_cache",
+    "__pycache__", ".venv", "venv", "node_modules",
+})
+DEFAULT_MARKERS = (
+    "README.md", "AGENTS.md", "CLAUDE.md", "pyproject.toml", "package.json",
+    "Cargo.toml", "CMakeLists.txt", "Makefile", "requirements.txt",
+)
+PUBLIC_HOSTS = frozenset({
+    "github.com", "gitlab.com", "bitbucket.org", "codeberg.org", "git.sr.ht",
+})
+_KNOWN_TOP = frozenset({"rule", "scan", "privacy", "output"})
+
+
+def _default_jobs() -> int:
+    return min(32, (os.cpu_count() or 4) * 5)
+
+
+def glob_to_regex(pattern: str) -> str:
+    """Translate a path glob to an anchored regex.
+
+    `*` matches within a segment, `**` across segments, `/**` makes the
+    separator optional so `public/**` also matches `public`.
+    """
+    out: list[str] = []
+    i, n = 0, len(pattern)
+    while i < n:
+        if pattern.startswith("/**", i):
+            out.append("(/.*)?")
+            i += 3
+        elif pattern.startswith("**", i):
+            out.append(".*")
+            i += 2
+        elif pattern[i] == "*":
+            out.append("[^/]*")
+            i += 1
+        else:
+            out.append(re.escape(pattern[i]))
+            i += 1
+    return "^" + "".join(out) + "$"
+
+
+@dataclass(frozen=True)
+class Rule:
+    pattern: str
+    class_: str
+    regex: re.Pattern = field(init=False, compare=False, repr=False)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "regex", re.compile(glob_to_regex(self.pattern)))
+
+
+@dataclass(frozen=True)
+class Config:
+    rules: tuple[Rule, ...] = ()
+    extra_prune: frozenset[str] = frozenset()
+    markers: tuple[str, ...] = DEFAULT_MARKERS
+    jobs: int = field(default_factory=_default_jobs)
+    omit_origin_classes: frozenset[str] = frozenset()
+    portable: bool = True
+    annotations: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def prune(self) -> frozenset[str]:
+        return DEFAULT_PRUNE_DIRS | self.extra_prune
+
+
+def default_config() -> Config:
+    return Config()
+
+
+def load_config(path: Path | None, root: Path) -> Config:
+    if path is None:
+        candidate = root / ".index.toml"
+        if not candidate.exists():
+            return default_config()
+        path = candidate
+    elif not path.exists():
+        raise SystemExit(f"config not found: {path}")
+    with path.open("rb") as handle:
+        try:
+            data = tomllib.load(handle)
+        except tomllib.TOMLDecodeError as exc:
+            raise SystemExit(f"{path}: invalid TOML: {exc}") from exc
+    return _build_config(data, path)
+
+
+def _build_config(data: dict[str, Any], path: Path) -> Config:
+    rules: list[Rule] = []
+    for idx, item in enumerate(data.get("rule", [])):
+        if "pattern" not in item or "class" not in item:
+            raise SystemExit(f"{path}: rule[{idx}] requires 'pattern' and 'class'")
+        rules.append(Rule(str(item["pattern"]), str(item["class"])))
+
+    scan = data.get("scan", {})
+    jobs = scan.get("jobs", _default_jobs())
+    if not isinstance(jobs, int) or jobs < 1:
+        raise SystemExit(f"{path}: [scan] jobs must be a positive integer")
+    extra_prune = frozenset(str(d) for d in scan.get("prune", []))
+    markers = tuple(scan["markers"]) if "markers" in scan else DEFAULT_MARKERS
+
+    omit = frozenset(str(c) for c in data.get("privacy", {}).get("omit_origin_classes", []))
+
+    output = data.get("output", {})
+    portable = bool(output.get("portable", True))
+    annotations = dict(output.get("annotations", {}))
+
+    for key in data:
+        if key not in _KNOWN_TOP:
+            print(f"{path}: warning: unknown config key '{key}'", file=sys.stderr)
+
+    return Config(tuple(rules), extra_prune, markers, jobs, omit, portable, annotations)

index_graph/context/__init__.py

@@ -0,0 +1 @@
+"""Synthesis context-pack renderer."""

index_graph/context/pack.py

@@ -0,0 +1,98 @@
+"""Render a DependencyGraph as the synthesis context pack (relations+roles+prose).
+
+No editorializing: every line traces to a data field or an evidence record.
+"""
+from __future__ import annotations
+
+from ..graph.build import DependencyGraph, RepoNode
+from ..graph.edges import Edge
+from ..graph.roles import salience_audit, structural_salience
+
+
+def _marker_list(node: RepoNode) -> list[str]:
+    out = []
+    if "entry" in node.markers:
+        out.append("entry")
+    if "published" in node.markers:
+        out.append("published")
+    return out
+
+
+def render_text(graph: DependencyGraph, title: str) -> str:
+    L = [f"# Context pack: {title}", ""]
+    L.append("## Roles (project: roles — in/out degree)")
+    sal = structural_salience(list(graph.edges))
+    for node in sorted(graph.repos, key=lambda n: n.name):
+        rs = ", ".join(graph.roles.get(node.name, ())) or "(none)"
+        s = sal.get(node.name, {"in_degree": 0, "out_degree": 0})
+        L.append(f"- {node.name}: {rs} — in={s['in_degree']} out={s['out_degree']}")
+    L.append("")
+    L.append("## Relations (A -> B: signals [confidence])")
+    for e in graph.edges:
+        if e.external:
+            continue
+        kinds = "+".join(sorted({s.kind for s in e.signals}))
+        L.append(f"- {e.from_repo} -> {e.to_repo}: {kinds} [{e.confidence}]")
+    L.append("")
+    L.append("## External dependencies (A -> name)")
+    for e in graph.edges:
+        if e.external:
+            L.append(f"- {e.from_repo} -> {e.target_name}")
+    L.append("")
+    L.append("## Inventory (all projects — extracted description)")
+    for node in sorted(graph.repos, key=lambda n: n.name):
+        eco = "/".join(node.ecosystems) or "none"
+        L.append(f"- {node.name} [{eco}]: {node.description}")
+    L.append("")
+    if graph.warnings:
+        L.append(f"## Warnings ({len(graph.warnings)})")
+        for w in graph.warnings:
+            L.append(f"- {w}")
+    return "\n".join(L)
+
+
+def to_json(graph: DependencyGraph) -> dict:
+    sal = structural_salience(list(graph.edges))
+    marked = {n.name: _marker_list(n) for n in graph.repos if _marker_list(n)}
+    relations = [{
+        "from": e.from_repo, "to": e.to_repo, "target_name": e.target_name,
+        "external": e.external, "confidence": e.confidence,
+        "signals": [{"kind": s.kind, "file": s.evidence_file, "line": s.evidence_line,
+                     "raw": s.raw_spec} for s in e.signals],
+    } for e in graph.edges]
+    return {
+        "roles": {n.name: list(graph.roles.get(n.name, ())) for n in graph.repos},
+        "relations": relations,
+        "salience": sal,
+        "salience_audit": salience_audit(sal, marked),
+        "repos": [{"name": n.name, "ecosystems": list(n.ecosystems),
+                   "description": n.description, "markers": sorted(n.markers)}
+                  for n in graph.repos],
+        "warnings": list(graph.warnings),
+    }
+
+
+def closure(edges: list[Edge], focus: str) -> set[str]:
+    adj: dict[str, set[str]] = {}
+    for e in edges:
+        if e.external or e.to_repo is None:
+            continue
+        adj.setdefault(e.from_repo, set()).add(e.to_repo)
+        adj.setdefault(e.to_repo, set()).add(e.from_repo)
+    seen = {focus}
+    stack = [focus]
+    while stack:
+        n = stack.pop()
+        for m in adj.get(n, ()):
+            if m not in seen:
+                seen.add(m)
+                stack.append(m)
+    return seen
+
+
+def focus_subgraph(graph: DependencyGraph, keep: set[str]) -> DependencyGraph:
+    repos = tuple(n for n in graph.repos if n.name in keep)
+    edges = tuple(e for e in graph.edges
+                  if e.from_repo in keep and (e.external or e.to_repo in keep))
+    roles = {k: v for k, v in graph.roles.items() if k in keep}
+    return DependencyGraph(repos, edges, roles, graph.warnings)

index_graph/gitmeta.py

@@ -0,0 +1,47 @@
+"""Git subprocess access and always-on credential redaction."""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+from typing import Any
+
+_USERINFO = re.compile(r"(?i)(https?://)[^/@]+@")
+_SECRET_QUERY = re.compile(r"(?i)\b(token|password|secret|api[_-]?key)=([^@\s]+)")
+
+
+def sanitize_credentials(origin: str) -> str:
+    clean = _USERINFO.sub(r"\1<redacted>@", origin)
+    return _SECRET_QUERY.sub(r"\1=<redacted>", clean)
+
+
+def run_git(repo: Path, args: list[str]) -> str:
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(repo), *args],
+            text=True, capture_output=True, timeout=20, check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return ""
+    return result.stdout.strip() if result.returncode == 0 else ""
+
+
+def repo_metadata(repo: Path) -> dict[str, Any]:
+    status = run_git(repo, ["status", "--porcelain=v1"]).splitlines()
+    untracked = sum(1 for line in status if line.startswith("??"))
+    dirty = sum(1 for line in status if line and not line.startswith("??"))
+    branch = (
+        run_git(repo, ["branch", "--show-current"])
+        or run_git(repo, ["rev-parse", "--abbrev-ref", "HEAD"])
+        or "unknown"
+    )
+    head = run_git(repo, ["rev-parse", "--short=7", "HEAD"]) or "unknown"
+    origin = sanitize_credentials(run_git(repo, ["config", "--get", "remote.origin.url"]) or "")
+    return {
+        "branch": branch,
+        "head": head,
+        "origin": origin,
+        "dirty_count": dirty,
+        "untracked_count": untracked,
+    }

index_graph/graph/__init__.py

@@ -0,0 +1,4 @@
+"""Repo-level dependency inference engine."""
+from .build import DependencyGraph, RepoNode, build_graph
+
+__all__ = ["DependencyGraph", "RepoNode", "build_graph"]

index_graph/graph/build.py

@@ -0,0 +1,127 @@
+"""Assemble repo trees + resolvers into a DependencyGraph."""
+from __future__ import annotations
+
+import configparser
+import json
+import re
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+from .edges import Edge, build_index, resolve_edges
+from .walk import walk_files
+from .resolvers import ALL_RESOLVERS
+from .resolvers.base import RawEdge
+from .roles import derive_roles
+
+_PARA = re.compile(r"\n\s*\n")
+
+
+@dataclass(frozen=True)
+class RepoNode:
+    name: str
+    path: str
+    ecosystems: tuple[str, ...]
+    exposed_names: frozenset[str]
+    description: str
+    markers: frozenset[str]
+
+
+@dataclass(frozen=True)
+class DependencyGraph:
+    repos: tuple[RepoNode, ...]
+    edges: tuple[Edge, ...]
+    roles: dict[str, tuple[str, ...]]
+    warnings: tuple[str, ...]
+
+
+def _description(repo_root: Path) -> str:
+    for readme in ("README.md", "README.rst", "README.txt", "readme.md"):
+        p = repo_root / readme
+        if p.is_file():
+            try:
+                text = p.read_text(encoding="utf-8").strip()
+            except OSError:
+                continue
+            for block in _PARA.split(text):
+                b = block.strip()
+                if b and not b.startswith("#") and not b.startswith("!["):
+                    return " ".join(b.split())[:300]
+    pp = repo_root / "pyproject.toml"
+    if pp.is_file():
+        try:
+            d = tomllib.loads(pp.read_text(encoding="utf-8")).get("project", {})
+            if d.get("description"):
+                return str(d["description"])
+        except (tomllib.TOMLDecodeError, OSError):
+            pass
+    pj = repo_root / "package.json"
+    if pj.is_file():
+        try:
+            d = json.loads(pj.read_text(encoding="utf-8"))
+            if d.get("description"):
+                return str(d["description"])
+        except (json.JSONDecodeError, OSError):
+            pass
+    return "(no description)"
+
+
+def detect_markers(repo_root: Path, exposed: set[str]) -> set[str]:
+    mk: set[str] = set()
+    if exposed:
+        mk.add("published")
+    pp = repo_root / "pyproject.toml"
+    if pp.is_file():
+        try:
+            data = tomllib.loads(pp.read_text(encoding="utf-8"))
+            if data.get("project", {}).get("scripts") or \
+               data.get("project", {}).get("entry-points"):
+                mk.add("entry")
+        except (tomllib.TOMLDecodeError, OSError):
+            pass
+    cfg = repo_root / "setup.cfg"
+    if cfg.is_file():
+        try:
+            cp = configparser.ConfigParser()
+            cp.read(cfg, encoding="utf-8")
+            if cp.has_option("options.entry_points", "console_scripts"):
+                mk.add("entry")
+        except (configparser.Error, OSError):
+            pass
+    pj = repo_root / "package.json"
+    if pj.is_file():
+        try:
+            if json.loads(pj.read_text(encoding="utf-8")).get("bin"):
+                mk.add("entry")
+        except (json.JSONDecodeError, OSError):
+            pass
+    if any(walk_files(repo_root, names=("__main__.py",))):
+        mk.add("entry")
+    return mk
+
+
+def build_graph(repo_paths: dict[str, Path], resolvers=ALL_RESOLVERS) -> DependencyGraph:
+    nodes: list[RepoNode] = []
+    exposed: dict[str, set[str]] = {}
+    repo_raw: dict[str, list[RawEdge]] = {}
+    markers: dict[str, set[str]] = {}
+    for name, root in sorted(repo_paths.items()):
+        ecos: list[str] = []
+        names: set[str] = set()
+        raws: list[RawEdge] = []
+        for r in resolvers:
+            if r.matches(root):
+                ecos.append(r.name)
+                names |= r.exposed_names(root)
+                raws += r.raw_edges(root)
+        exposed[name] = names
+        repo_raw[name] = raws
+        mk = detect_markers(root, names)
+        markers[name] = mk
+        nodes.append(RepoNode(name, str(root), tuple(ecos), frozenset(names),
+                              _description(root), frozenset(mk)))
+
+    index = build_index(exposed)
+    edges, warnings = resolve_edges(repo_raw, index)
+    roles = derive_roles(set(repo_paths), edges, markers)
+    return DependencyGraph(tuple(nodes), tuple(edges), roles, tuple(warnings))