grapholog 0.5.1__py3-none-any.whl

grapholog/__init__.py

@@ -0,0 +1,3 @@
+"""Read-only project maps and comparisons."""
+
+__version__ = "0.5.1"

grapholog/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

grapholog/cache.py

@@ -0,0 +1,199 @@
+"""Optional, disposable parse cache. Source bytes are always hashed again.
+
+Entries contain a JSON header followed by a compact JSON payload. Only successful
+path-independent parse data is stored; dependency resolution is never cached.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict, fields, is_dataclass
+from functools import lru_cache
+from hashlib import sha256
+from importlib import metadata, resources
+import json
+import os
+from pathlib import Path
+import stat
+import sys
+import tempfile
+from typing import get_args, get_origin, get_type_hints
+
+from . import __version__
+from .model import Call, File, Import, Logic, Symbol
+
+FORMAT = "grapholog.parse-cache.v1"
+MAX_ENTRY = 64 * 1024 * 1024
+PARSE_FIELDS = ("lines", "parser", "status", "syntax_hash", "shape_hash", "fingerprints",
+                "nodes", "symbols", "imports", "calls", "logic", "notes")
+RECORDS = {"symbols": Symbol, "imports": Import, "calls": Call, "logic": Logic}
+
+
+@lru_cache(maxsize=1)
+def identity() -> dict:
+    versions = {}
+    for package in ("tree-sitter", "tree-sitter-javascript", "tree-sitter-typescript",
+                    "tree-sitter-php", "tree-sitter-bash"):
+        try:
+            versions[package] = metadata.version(package)
+        except metadata.PackageNotFoundError:
+            versions[package] = "missing"
+    source = resources.files("grapholog")
+    return {"format": FORMAT, "engine": "python", "version": __version__,
+            "interpreter": [sys.implementation.name, *sys.version_info[:3]],
+            "packages": versions,
+            "code": {name: sha256(source.joinpath(name).read_bytes()).hexdigest()
+                     for name in ("cache.py", "model.py", "python_parser.py", "tree_parser.py")}}
+
+
+def _encoded(value) -> bytes:
+    return json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode()
+
+
+@lru_cache(maxsize=None)
+def _hints(cls):
+    return get_type_hints(cls)
+
+
+def _valid(value, kind) -> bool:
+    origin, args = get_origin(kind), get_args(kind)
+    if kind is str:
+        return isinstance(value, str)
+    if kind is int:
+        return type(value) is int and value >= 0
+    if kind is bool:
+        return type(value) is bool
+    if origin is list:
+        return isinstance(value, list) and all(_valid(v, args[0]) for v in value)
+    if origin is dict:
+        return isinstance(value, dict) and all(_valid(k, args[0]) and _valid(v, args[1]) for k, v in value.items())
+    if is_dataclass(kind):
+        hints = _hints(kind)
+        return isinstance(value, dict) and value.keys() == hints.keys() and all(_valid(value[k], v) for k, v in hints.items())
+    return False
+
+
+def _hex(value: str, size: int) -> bool:
+    return len(value) == size and all(c in "0123456789abcdef" for c in value)
+
+
+class ParseCache:
+    def __init__(self, directory: str | Path | None):
+        self.root = None
+        self.namespace = ""
+        self.stats = {"enabled": False}
+        if directory is None:
+            return
+        path = Path(directory).expanduser().absolute()
+        self.stats = {"enabled": False, "directory": str(path), "hits": 0, "misses": 0,
+                      "writes": 0, "invalid": 0, "errors": 0, "skipped_large": 0, "issues": []}
+        try:
+            if path.is_symlink():
+                raise OSError("Cache directory must not be a symlink")
+            path.mkdir(parents=True, exist_ok=True, mode=0o700)
+            self.root = path.resolve()
+            self.namespace = sha256(_encoded(identity())).hexdigest()
+            self.stats.update(enabled=True, directory=str(self.root), namespace=self.namespace)
+        except (OSError, ValueError) as exc:
+            self.root = None
+            self._error(exc)
+
+    def _error(self, error):
+        self.stats["errors"] += 1
+        if len(self.stats["issues"]) < 3:
+            self.stats["issues"].append(str(error))
+
+    def _entry(self, file: File) -> Path:
+        key = sha256((file.language + "\0" + file.sha256).encode()).hexdigest()
+        return self.root / "python-v1" / self.namespace / key[:2] / (key + ".jsonl")
+
+    def _safe_parents(self, path: Path, create: bool = False):
+        # Cache names are generated hashes, never source paths. Reject symlinked shards.
+        for folder in reversed(list(path.parents)):
+            if folder == self.root or self.root in folder.parents:
+                if create:
+                    folder.mkdir(exist_ok=True, mode=0o700)
+                info = folder.lstat()
+                if not stat.S_ISDIR(info.st_mode):
+                    raise OSError("Cache parent is not a regular directory")
+
+    def load(self, file: File) -> bool:
+        if self.root is None:
+            return False
+        try:
+            path = self._entry(file)
+            self._safe_parents(path)
+            flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
+            with os.fdopen(os.open(path, flags), "rb") as stream:
+                if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
+                    raise ValueError("Cache entry is not a regular file")
+                data = stream.read(MAX_ENTRY + 1)
+            if len(data) > MAX_ENTRY:
+                raise ValueError("Cache entry exceeds size limit")
+            head, payload = data.split(b"\n", 1)
+            if len(head) > 4096:
+                raise ValueError("Invalid cache header")
+            header = json.loads(head)
+            expected = {"schema": FORMAT, "namespace": self.namespace, "language": file.language,
+                        "source_sha256": file.sha256, "payload_sha256": sha256(payload).hexdigest()}
+            if header != expected:
+                raise ValueError("Cache header or checksum mismatch")
+            result = json.loads(payload)
+            hints = _hints(File)
+            if (not isinstance(result, dict) or set(result) != set(PARSE_FIELDS)
+                    or not all(_valid(result[k], hints[k]) for k in PARSE_FIELDS)):
+                raise ValueError("Invalid parse data")
+            expected_parser = "python.ast" if file.language == "python" else "tree-sitter:" + file.language
+            if (result["status"] != "parsed" or result["parser"] != expected_parser
+                    or not all(_hex(result[k], 64) for k in ("syntax_hash", "shape_hash"))
+                    or any(not _hex(s, 16) for s in result["fingerprints"])
+                    or any(a >= b for a, b in zip(result["fingerprints"], result["fingerprints"][1:]))):
+                raise ValueError("Incompatible parse data")
+            if (any(i["targets"] or i["status"] != "unresolved" for i in result["imports"])
+                    or any(c["target"] or c["resolution"] != "unresolved" for c in result["calls"])):
+                raise ValueError("Cache contains resolved dependencies")
+            # Apply only after validating the entire entry; no partially restored state.
+            for name, cls in RECORDS.items():
+                result[name] = [cls(**item) for item in result[name]]
+            for name in PARSE_FIELDS:
+                setattr(file, name, result[name])
+            self.stats["hits"] += 1
+            return True
+        except FileNotFoundError:
+            pass
+        except (ValueError, TypeError, KeyError, RecursionError, UnicodeError):
+            self.stats["invalid"] += 1
+        except OSError as exc:
+            self._error(exc)
+        self.stats["misses"] += 1
+        return False
+
+    def store(self, file: File):
+        if self.root is None or file.status != "parsed":
+            return
+        temporary = None
+        try:
+            data = {name: getattr(file, name) for name in PARSE_FIELDS}
+            for name in RECORDS:
+                data[name] = [asdict(item) for item in data[name]]
+            payload = _encoded(data)
+            header = _encoded({"schema": FORMAT, "namespace": self.namespace, "language": file.language,
+                               "source_sha256": file.sha256, "payload_sha256": sha256(payload).hexdigest()})
+            if len(payload) + len(header) + 1 > MAX_ENTRY:
+                self.stats["skipped_large"] += 1
+                return
+            path = self._entry(file)
+            self._safe_parents(path, create=True)
+            fd, temporary = tempfile.mkstemp(prefix=".tmp-", dir=path.parent)
+            with os.fdopen(fd, "wb") as stream:
+                stream.write(header + b"\n")
+                stream.write(payload)
+            os.replace(temporary, path)
+            temporary = None
+            self.stats["writes"] += 1
+        except (OSError, ValueError, RecursionError) as exc:
+            self._error(exc)
+        finally:
+            if temporary is not None:
+                try:
+                    os.unlink(temporary)
+                except OSError:
+                    pass

grapholog/cli.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+from . import __version__
+from .compare import compare, duplicates
+from .render import render_compare, render_project, safe, to_dot
+from .scan import scan
+
+
+def nonnegative(value):
+    number = int(value)
+    if number < 0:
+        raise argparse.ArgumentTypeError("must be >= 0")
+    return number
+
+
+def similarity(value):
+    number = float(value)
+    if not 0 <= number <= 1:
+        raise argparse.ArgumentTypeError("must be between 0 and 1")
+    return number
+
+
+def parser():
+    p = argparse.ArgumentParser(prog="grapholog", description="Jednorazowa mapa plikow, skladni i zaleznosci; bez wykonywania kodu projektu.")
+    p.add_argument("--version", action="version", version=__version__)
+    subs = p.add_subparsers(dest="command", required=True)
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--all", dest="all_files", action="store_true", help="wylacz domyslne wykluczenia i .graphologignore")
+    common.add_argument("--exclude", action="append", default=[], metavar="GLOB", help="wlasne wykluczenie; mozna powtarzac")
+    common.add_argument("--source-root", action="append", default=[], metavar="DIR", help="korzen importow Python wzgledem projektu; domyslnie . oraz src")
+    common.add_argument("--max-bytes", type=nonnegative, default=2 * 1024 * 1024, help="limit parsowania pliku; SHA-256 liczony zawsze; 0=bez limitu")
+    common.add_argument("--min-nodes", type=nonnegative, default=12, help="minimalna liczba wezlow dla grup podobienstwa skladni/funkcji")
+    common.add_argument("--cache-dir", type=Path, help="opcjonalny katalog cache parsera; pliki zrodlowe nadal sa odczytywane i haszowane")
+    common.add_argument("--profile", choices=("inventory", "logic"), default="inventory", help="logic pomija dane i pliki generowane przed odczytem; zachowuje manifesty i licencje")
+    common.add_argument("--ascii", action="store_true", help="widok zgodny z terminalem ASCII")
+    common.add_argument("--color", choices=("auto", "always", "never"), default="auto")
+    common.add_argument("--output", "-o", type=Path, help="zapis raportu do wskazanego pliku; bez nadpisania plikow zrodlowych")
+    common.add_argument("--strict", action="store_true", help="kod 2 przy niepelnej analizie obslugiwanych jezykow/bledach odczytu")
+    show = subs.add_parser("scan", parents=[common], help="pokaz mape jednego projektu")
+    show.add_argument("path", nargs="?", default=".")
+    show.add_argument("--format", choices=("terminal", "json", "dot"), default="terminal")
+    show.add_argument("--detail", choices=("files", "compact", "full"), default="compact")
+    diff = subs.add_parser("diff", parents=[common], help="porownaj dwa drzewa katalogow")
+    diff.add_argument("left")
+    diff.add_argument("right")
+    diff.add_argument("--format", choices=("terminal", "json"), default="terminal")
+    diff.add_argument("--threshold", type=similarity, default=0.70)
+    diff.add_argument("--max-pairs", type=nonnegative, default=20000, help="maks. porownan przy szukaniu podobnych niedopasowanych plikow")
+    diff.add_argument("--check", action="store_true", help="kod 1 jesli projekty roznia sie; przydatne w CI")
+    return p
+
+
+def incomplete(project) -> bool:
+    return bool(project.warnings) or any(f.status in {"parse-error", "read-error", "changed-during-scan", "parser-missing"}
+        or (f.language != "other" and f.status in {"size-limit", "non-utf8", "binary"}) for f in project.files)
+
+
+def main(argv=None):
+    args_list = list(sys.argv[1:] if argv is None else argv)
+    if args_list and args_list[0] == "tickets":
+        from .tickets_cli import main as tickets_main
+        return tickets_main(args_list[1:])
+    if args_list and args_list[0] in {"workspace", "contracts", "serve"}:
+        from .workspace_cli import main as workspace_main
+        return workspace_main(args_list)
+    # `grapholog ./project` is shorthand for `grapholog scan ./project`.
+    if not args_list or (args_list[0] not in {"scan", "diff", "-h", "--help", "--version"}):
+        args_list.insert(0, "scan")
+    args = parser().parse_args(args_list)
+    try:
+        options = dict(all_files=args.all_files, exclude=args.exclude, max_bytes=args.max_bytes,
+                       source_roots=args.source_root or None, cache_dir=args.cache_dir, profile=args.profile)
+        for root in args.source_root:
+            if Path(root).is_absolute() or ".." in Path(root).parts:
+                raise ValueError("--source-root must be a relative directory inside the project")
+        output = args.output.expanduser().absolute() if args.output else None
+        if output and output.exists():
+            raise ValueError(f"Output exists: {output}. Choose a new report path.")
+        if output and output.is_symlink():
+            raise ValueError("Output cannot be a symbolic link")
+        color = args.color == "always" or (args.color == "auto" and sys.stdout.isatty()
+                                          and "NO_COLOR" not in os.environ and output is None)
+        projects = []
+        def run(path):
+            opts = {**options, "exclude": list(options["exclude"])}
+            if output:
+                try:
+                    opts["exclude"].append(output.resolve().relative_to(Path(path).expanduser().resolve()).as_posix())
+                except ValueError:
+                    pass
+            project = scan(path, **opts)
+            projects.append(project)
+            return project
+        if args.command == "scan":
+            project = run(args.path)
+            if args.format == "json":
+                data = project.to_dict()
+                data["duplicates"] = duplicates([("", project)], args.min_nodes)
+                result = None
+            elif args.format == "dot":
+                result = to_dot(project)
+            else:
+                result = render_project(project, detail=args.detail, ascii_only=args.ascii,
+                                        color=color, min_nodes=args.min_nodes)
+            code = 0
+        else:
+            left, right = run(args.left), run(args.right)
+            data = compare(left, right, threshold=args.threshold, max_pairs=args.max_pairs, min_nodes=args.min_nodes)
+            result = None if args.format == "json" else render_compare(data, ascii_only=args.ascii, color=color)
+            code = 1 if args.check and data["has_differences"] else 0
+        def write_result(handle):
+            if result is None:
+                json.dump(data, handle, ensure_ascii=True, indent=2)
+                handle.write("\n")
+            else:
+                handle.write(result)
+        if output:
+            with output.open("x", encoding="utf-8") as handle:
+                write_result(handle)
+            print(f"Saved: {safe(output)}", file=sys.stderr)
+        else:
+            try:
+                write_result(sys.stdout)
+            except UnicodeEncodeError:
+                sys.stdout.write(result.encode(sys.stdout.encoding or "ascii", "backslashreplace").decode(sys.stdout.encoding or "ascii"))
+        if args.strict and any(incomplete(project) for project in projects):
+            return 2
+        return code
+    except BrokenPipeError:
+        # Avoid a second flush failure when piping a long report to head.
+        sys.stdout = open(os.devnull, "w")
+        return 0
+    except (OSError, ValueError) as exc:
+        print("grapholog: " + safe(exc), file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

grapholog/compare.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from dataclasses import asdict
+from .model import File, Project, symbol_ref
+from .similarity import search
+
+
+def _groups(items, key, minimum=2):
+    groups = defaultdict(list)
+    for item in items:
+        value = key(item)
+        if value:
+            groups[value].append(item)
+    return [group for _, group in sorted(groups.items(), key=lambda kv: str(kv[0])) if len(group) >= minimum]
+
+
+def duplicates(projects: list[tuple[str, Project]], min_nodes: int = 12) -> dict:
+    records = [(prefix, file) for prefix, p in projects for file in p.files]
+    ref = lambda r: r[0] + r[1].path
+    result = {"exact_files": [], "syntax_files": [], "shape_candidates": [], "function_clones": [],
+              "function_shape_candidates": []}
+    for group in _groups(records, lambda r: (r[1].kind, r[1].sha256) if r[1].sha256 else None):
+        result["exact_files"].append(sorted(ref(r) for r in group))
+    parsed = [r for r in records if r[1].status == "parsed" and r[1].nodes >= min_nodes]
+    for group in _groups(parsed, lambda r: (r[1].parser, r[1].syntax_hash)):
+        if len({r[1].sha256 for r in group}) > 1:
+            result["syntax_files"].append(sorted(ref(r) for r in group))
+    for group in _groups(parsed, lambda r: (r[1].parser, r[1].shape_hash)):
+        if len({r[1].syntax_hash for r in group}) > 1:
+            result["shape_candidates"].append(sorted(ref(r) for r in group))
+    symbols = [(prefix, f, s) for prefix, f in records for s in f.symbols
+               if s.kind not in {"class", "interface", "type", "enum", "trait"}
+               and s.nodes >= min_nodes and s.body_hash and f.status == "parsed"]
+    sref = lambda r: r[0] + symbol_ref(r[1], r[2])
+    for group in _groups(symbols, lambda r: (r[1].parser, r[2].body_hash)):
+        result["function_clones"].append(sorted(sref(r) for r in group))
+    for group in _groups(symbols, lambda r: (r[1].parser, r[2].shape_hash)):
+        if len({r[2].body_hash for r in group}) > 1:
+            result["function_shape_candidates"].append(sorted(sref(r) for r in group))
+    return {k: sorted(v) for k, v in result.items()}
+
+
+def compatible(a: File, b: File) -> bool:
+    return a.parser == b.parser and a.language == b.language and a.status == b.status == "parsed"
+
+
+def file_status(a: File, b: File) -> str:
+    if a.kind != b.kind:
+        return "type-changed"
+    if not a.sha256 or not b.sha256:
+        return "unreadable"
+    if a.sha256 == b.sha256:
+        return "unchanged" if a.executable == b.executable else "mode-changed"
+    if compatible(a, b) and a.syntax_hash == b.syntax_hash:
+        return "syntax-equal"
+    if compatible(a, b) and a.shape_hash == b.shape_hash:
+        return "identifier-change-candidate"
+    return "changed"
+
+
+def _symbol_delta(a: File, b: File) -> dict:
+    # Repeated overloads/redefinitions are compared as ordered sequences per qualified name.
+    old, new = defaultdict(list), defaultdict(list)
+    for symbol in a.symbols:
+        old[symbol.qualname].append(symbol.syntax_hash)
+    for symbol in b.symbols:
+        new[symbol.qualname].append(symbol.syntax_hash)
+    return {"added": sorted(new.keys() - old.keys()), "removed": sorted(old.keys() - new.keys()),
+            "changed": sorted(k for k in old.keys() & new.keys() if old[k] != new[k])}
+
+
+def _edge_set(project: Project, rename: dict | None = None) -> set[tuple]:
+    rename = rename or {}
+    result = set()
+    for f in project.files:
+        for item in f.imports:
+            if item.targets:
+                for target in item.targets:
+                    result.add((rename.get(f.path, f.path), item.kind, rename.get(target, target), "local"))
+            else:
+                result.add((rename.get(f.path, f.path), item.kind, item.module, item.status))
+    return result
+
+
+def compare(left: Project, right: Project, *, threshold: float = 0.70,
+            max_pairs: int = 20000, min_nodes: int = 12) -> dict:
+    old, new = {f.path: f for f in left.files}, {f.path: f for f in right.files}
+    pairs = [{"left": p, "right": p, "status": file_status(old[p], new[p])}
+             for p in sorted(old.keys() & new.keys())]
+    removed, added = set(old.keys() - new.keys()), set(new.keys() - old.keys())
+    # Pair only unambiguous 1:1 candidates. Many-to-many copies are duplicate groups.
+    for mode in ("bytes", "syntax"):
+        def key(f):
+            if mode == "bytes":
+                return (f.kind, f.sha256) if f.sha256 else None
+            return (f.parser, f.syntax_hash) if f.status == "parsed" and f.syntax_hash else None
+        a, b = defaultdict(list), defaultdict(list)
+        for path in removed:
+            if value := key(old[path]):
+                a[value].append(path)
+        for path in added:
+            if value := key(new[path]):
+                b[value].append(path)
+        for value in sorted(a.keys() & b.keys()):
+            if len(a[value]) == len(b[value]) == 1:
+                src, dst = a[value][0], b[value][0]
+                pairs.append({"left": src, "right": dst,
+                              "status": "relocated-identical" if mode == "bytes" else "relocated-syntax-equal"})
+                removed.remove(src)
+                added.remove(dst)
+    for pair in pairs:
+        a, b = old[pair["left"]], new[pair["right"]]
+        pair["symbol_delta"] = _symbol_delta(a, b)
+        pair["executable_changed"] = a.executable != b.executable
+    for path in sorted(removed):
+        pairs.append({"left": path, "right": None, "status": "removed"})
+    for path in sorted(added):
+        pairs.append({"left": None, "right": path, "status": "added"})
+    pairs.sort(key=lambda p: (p["right"] or p["left"], p["status"]))
+
+    left_candidates = [old[p] for p in sorted(removed) if old[p].nodes >= min_nodes and old[p].fingerprints]
+    right_candidates = [new[p] for p in sorted(added) if new[p].nodes >= min_nodes and new[p].fingerprints]
+    similarities, search_stats = search(left_candidates, right_candidates, threshold, max_pairs)
+    mapping = {p["left"]: p["right"] for p in pairs if p["left"] and p["right"]}
+    old_edges, new_edges = _edge_set(left, mapping), _edge_set(right)
+    directory_added = sorted(set(right.directories) - set(left.directories))
+    directory_removed = sorted(set(left.directories) - set(right.directories))
+    result = {"schema": "grapholog.compare.v1", "left": left.to_dict(), "right": right.to_dict(),
+        "pairs": pairs, "counts": dict(sorted(Counter(p["status"] for p in pairs).items())),
+        "directories_added": directory_added, "directories_removed": directory_removed,
+        "dependencies_added": [list(e) for e in sorted(new_edges - old_edges)],
+        "dependencies_removed": [list(e) for e in sorted(old_edges - new_edges)],
+        "similarities": sorted(similarities, key=lambda p: (-p["score"], p["left"], p["right"])),
+        "similarity_search": search_stats,
+        "duplicates": duplicates([("A:", left), ("B:", right)], min_nodes),
+        "has_differences": any(p["status"] != "unchanged" for p in pairs) or bool(directory_added or directory_removed),
+        "limitations": ["Static candidates, not runtime or semantic equivalence.",
+                        "Similarity search applies to unmatched removed/added files only.",
+                        "Relocation means unambiguous equal content at different paths; it is not VCS history."]}
+    return result