graphitect 0.2.0__py3-none-any.whl

graphify/extractors/json_config.py

@@ -0,0 +1,240 @@
+"""Json_config extractor. Moved verbatim from graphify/extract.py."""
+from __future__ import annotations
+
+
+from pathlib import Path
+from graphify.extractors.base import _file_stem, _make_id, _read_text
+from graphify.ids import normalize_id
+
+
+_CONFIG_JSON_NAMES = frozenset({
+    "package.json", "tsconfig.json", "jsconfig.json", "composer.json",
+    "deno.json", "deno.jsonc", "bower.json", "manifest.json",
+    "app.json", "now.json", "vercel.json", "angular.json", "nest-cli.json",
+    "biome.json", "biome.jsonc", "renovate.json", ".babelrc", ".babelrc.json",
+    ".eslintrc.json", ".prettierrc.json", ".prettierrc", "babel.config.json",
+})
+
+_CONFIG_JSON_KEYS = frozenset({
+    "dependencies", "devDependencies", "peerDependencies",
+    "optionalDependencies", "bundleDependencies", "bundledDependencies",
+    "extends", "$ref", "$schema", "compilerOptions",
+})
+
+def _is_config_json(path: Path, obj_node, source: bytes) -> bool:
+    """True if a .json file is a recognized config/manifest worth AST-extracting.
+
+    Matches by filename first (cheap), then falls back to a top-level key probe
+    so arbitrarily-named config files (e.g. ``api.tsconfig.json``,
+    ``foo.eslintrc.json``) are still picked up. Returns False for data JSON so it
+    is skipped by the structural pass (#1224)."""
+    name = path.name.casefold()
+    if name in _CONFIG_JSON_NAMES:
+        return True
+    # Common compound config names: *.eslintrc.json, *.prettierrc.json, etc.
+    if name.endswith((".eslintrc.json", ".prettierrc.json", ".babelrc.json",
+                      "tsconfig.json", "jsconfig.json")):
+        return True
+    # Top-level key probe: scan the root object's immediate keys (no deep walk).
+    for top_key in obj_node.children:
+        if top_key.type != "pair":
+            continue
+        key_node = top_key.child_by_field_name("key")
+        if key_node is None:
+            continue
+        kc = key_node.child_by_field_name("string_content")
+        text = _read_text(kc, source) if kc else _read_text(key_node, source).strip('"\'')
+        if text in _CONFIG_JSON_KEYS:
+            return True
+    return False
+
+def extract_json(path: Path) -> dict:
+    """Extract structure and dependency edges from a *config/manifest* .json file.
+
+    Data-shaped JSON (eval fixtures, datasets, GeoJSON, API response dumps) is
+    deliberately skipped — AST-walking it produced hundreds of orphan key-nodes
+    and duplicate communities that swamped real structure (#1224). Recognition
+    is by filename (package.json, tsconfig.json, …) or a top-level key probe
+    (dependencies / extends / $ref / $schema / compilerOptions)."""
+    _JSON_MAX_BYTES = 1_048_576  # 1 MiB — skip large fixture dumps / GeoJSON blobs
+
+    try:
+        import tree_sitter_json as tsjson
+        from tree_sitter import Language, Parser
+    except ImportError:
+        return {"nodes": [], "edges": [], "error": "tree-sitter-json not installed"}
+
+    try:
+        # Bounded read instead of stat()+read() to eliminate TOCTOU (J-1):
+        # read one byte beyond the limit so we can detect oversized files even
+        # if the file grows between stat and read.
+        with path.open("rb") as _f:
+            source = _f.read(_JSON_MAX_BYTES + 1)
+        if len(source) > _JSON_MAX_BYTES:
+            return {"nodes": [], "edges": [], "error": "json file too large to index"}
+        language = Language(tsjson.language())
+        parser = Parser(language)
+        tree = parser.parse(source)
+        root = tree.root_node
+    except Exception as e:
+        return {"nodes": [], "edges": [], "error": str(e)}
+
+    stem = _file_stem(path)
+    str_path = str(path)
+    nodes: list[dict] = []
+    edges: list[dict] = []
+    seen_ids: set[str] = set()
+
+    # Keys whose string values become imports (package.json dep blocks)
+    _DEP_KEYS = frozenset({
+        "dependencies", "devDependencies", "peerDependencies",
+        "optionalDependencies", "bundleDependencies", "bundledDependencies",
+    })
+
+    def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None:
+        if nid and nid not in seen_ids:
+            seen_ids.add(nid)
+            nodes.append({"id": nid, "label": label, "file_type": file_type,
+                          "source_file": str_path, "source_location": f"L{line}"})
+
+    def add_edge(src: str, tgt: str, relation: str, line: int,
+                 context: str | None = None) -> None:
+        if not src or not tgt or src == tgt:
+            return
+        edge = {"source": src, "target": tgt, "relation": relation,
+                "confidence": "EXTRACTED", "source_file": str_path,
+                "source_location": f"L{line}", "weight": 1.0}
+        if context:
+            edge["context"] = context
+        edges.append(edge)
+
+    file_nid = _make_id(str(path))
+    add_node(file_nid, path.name, 1)
+
+    def _key_text(pair_node) -> str | None:
+        """Extract the string content of a pair's key."""
+        key_node = pair_node.child_by_field_name("key")
+        if key_node is None:
+            return None
+        if key_node.type == "string":
+            content = key_node.child_by_field_name("string_content")
+            if content:
+                return _read_text(content, source)
+            # fallback: strip surrounding quotes
+            raw = _read_text(key_node, source)
+            return raw.strip('"\'')
+        return _read_text(key_node, source)
+
+    def _val_node(pair_node):
+        return pair_node.child_by_field_name("value")
+
+    def walk_object(obj_node, parent_nid: str, parent_key: str | None,
+                    depth: int, pair_count: list) -> None:
+        if depth > 6:
+            return
+        for child in obj_node.children:
+            if child.type != "pair":
+                continue
+            if pair_count[0] >= 500:  # check per-pair so the cap is honoured exactly (J-3)
+                return
+            pair_count[0] += 1
+            key = _key_text(child)
+            if not key:
+                continue
+            # A key that normalizes to nothing (a JSONC `"//"` comment key, say)
+            # would collapse `_make_id(stem, key)` down to the bare file-stem id,
+            # which is absolute-path-derived and leaks the scan path (#1899). Such
+            # keys carry no graph signal, so drop them.
+            if not normalize_id(key):
+                continue
+            key_nid = _make_id(stem, *(([parent_key] if parent_key else []) + [key]))
+            if not key_nid:
+                continue
+            line = child.start_point[0] + 1
+            add_node(key_nid, key, line)
+            add_edge(parent_nid, key_nid, "contains", line)
+
+            val = _val_node(child)
+            if val is None:
+                continue
+
+            if val.type == "object":
+                walk_object(val, key_nid, key, depth + 1, pair_count)
+
+            elif val.type == "array" and key == "extends":
+                # Only a genuine `extends` array (tsconfig 5.0+, eslint) is
+                # inheritance. This branch used to fire for *every* array under
+                # a tracked key, so `compilerOptions.lib: ["dom","esnext"]` and
+                # `exclude: ["node_modules",".next"]` each emitted an `extends`
+                # edge — array membership relabelled as inheritance. The string
+                # branch below already gates on `key == "extends"`; this matches
+                # it. Non-`extends` arrays keep their `contains` edge only.
+                # Prefix with "ref_" so external refs don't collide with real
+                # code/file node IDs that share the same collapsed _make_id (J-4).
+                for item in val.children:
+                    if item.type == "string":
+                        content = item.child_by_field_name("string_content")
+                        ref = _read_text(content, source) if content else _read_text(item, source).strip('"\'')
+                        if ref:
+                            ref_nid = _make_id("ref", ref)
+                            if ref_nid:
+                                add_node(ref_nid, ref, line, file_type="concept")
+                                add_edge(key_nid, ref_nid, "extends", line, context="import")
+
+            elif val.type == "string":
+                content = val.child_by_field_name("string_content")
+                val_text = _read_text(content, source) if content else _read_text(val, source).strip('"\'')
+
+                if key == "extends" and val_text:
+                    # Namespace external refs to avoid ID collision with file nodes (J-4)
+                    ref_nid = _make_id("ref", val_text)
+                    if ref_nid:
+                        add_node(ref_nid, val_text, line, file_type="concept")
+                        add_edge(file_nid, ref_nid, "extends", line, context="import")
+
+                elif key == "$ref" and val_text:
+                    # Namespace $ref values to prevent edge hijacking into code nodes (J-4)
+                    ref_nid = _make_id("ref", val_text)
+                    if ref_nid:
+                        add_edge(parent_nid, ref_nid, "references", line)
+
+                elif parent_key in _DEP_KEYS and val_text:
+                    # Two defects fixed here, both from the #1764 shape:
+                    #
+                    # 1. The edge ran `key_nid -> dep_nid`, and both nodes carry
+                    #    the *same label* (the dependency name), so every
+                    #    dependency produced a visually self-referential
+                    #    `react --imports--> react` (22 of them on a mid-size
+                    #    Next.js repo). The self-loop guard in
+                    #    test_extract_json_import_and_extends_targets_are_real_nodes
+                    #    compares node ids, and these two ids differ, so the
+                    #    duplicate slipped past it. Sourcing the edge at the
+                    #    manifest yields `package.json --imports--> react`.
+                    #
+                    # 2. `_make_id(key)` minted a *bare* id, skipping the "ref"
+                    #    namespacing every other external reference in this file
+                    #    applies (J-4). A dependency named `utils`, `colors` or
+                    #    `types` could then be collapsed onto a same-named local
+                    #    module by build.py's alias index -- the #1638 failure
+                    #    mode reached through a different door.
+                    dep_nid = _make_id("ref", key)
+                    if dep_nid:
+                        add_node(dep_nid, key, line, file_type="concept")
+                        add_edge(file_nid, dep_nid, "imports", line, context="import")
+
+    # Entry: find root document → object
+    doc = root
+    if doc.type == "document" and doc.child_count > 0:
+        doc = doc.children[0]
+    if doc.type == "object":
+        # Only AST-extract recognized config/manifest JSON. Data JSON (fixtures,
+        # datasets, GeoJSON, API dumps) is skipped so it doesn't explode into
+        # orphan key-nodes (#1224); it's left to the LLM semantic pass.
+        if not _is_config_json(path, doc, source):
+            return {"nodes": [], "edges": [], "skipped": "data json (not a config/manifest)"}
+        walk_object(doc, file_nid, None, 0, [0])
+    else:
+        # Top-level array or scalar => data JSON, never a config/manifest.
+        return {"nodes": [], "edges": [], "skipped": "data json (non-object root)"}
+
+    return {"nodes": nodes, "edges": edges}

graphify/extractors/julia.py

@@ -0,0 +1,289 @@
+"""julia — moved verbatim from graphify/extract.py."""
+from __future__ import annotations
+
+from graphify.extractors.base import _file_stem, _make_id, _read_text
+from graphify.extractors.engine import _semantic_reference_edge
+from pathlib import Path
+
+
+def extract_julia(path: Path) -> dict:
+    """Extract modules, structs, functions, imports, and calls from a .jl file."""
+    try:
+        import tree_sitter_julia as tsjulia
+        from tree_sitter import Language, Parser
+    except ImportError:
+        return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"}
+
+    try:
+        language = Language(tsjulia.language())
+        parser = Parser(language)
+        source = path.read_bytes()
+        tree = parser.parse(source)
+        root = tree.root_node
+    except Exception as e:
+        return {"nodes": [], "edges": [], "error": str(e)}
+
+    stem = _file_stem(path)
+    str_path = str(path)
+    nodes: list[dict] = []
+    edges: list[dict] = []
+    seen_ids: set[str] = set()
+    function_bodies: list[tuple[str, object]] = []
+
+    def add_node(nid: str, label: str, line: int) -> None:
+        if nid not in seen_ids:
+            seen_ids.add(nid)
+            nodes.append({
+                "id": nid,
+                "label": label,
+                "file_type": "code",
+                "source_file": str_path,
+                "source_location": f"L{line}",
+            })
+
+    def add_edge(src: str, tgt: str, relation: str, line: int,
+                 confidence: str = "EXTRACTED", weight: float = 1.0,
+                 context: str | None = None) -> None:
+        edge = {
+            "source": src,
+            "target": tgt,
+            "relation": relation,
+            "confidence": confidence,
+            "source_file": str_path,
+            "source_location": f"L{line}",
+            "weight": weight,
+        }
+        if context:
+            edge["context"] = context
+        edges.append(edge)
+
+    file_nid = _make_id(str(path))
+    add_node(file_nid, path.name, 1)
+
+    def ensure_named_node(name: str, line: int) -> str:
+        nid = _make_id(stem, name)
+        if nid in seen_ids:
+            return nid
+        nid = _make_id(name)
+        if nid not in seen_ids:
+            # The name isn't defined in this file, so this is a cross-file reference
+            # (e.g. a `Thing` type annotation imported from another module). Emit a
+            # SOURCELESS stub — like the inheritance-base path below — so the
+            # corpus-level rewire can collapse it onto the real definition. A sourced
+            # stub here makes _disambiguate_colliding_node_ids bake the referencing
+            # file's path (with extension) into the id and blocks the rewire, which is
+            # the phantom-duplicate-node bug (#1402).
+            seen_ids.add(nid)
+            nodes.append({
+                "id": nid,
+                "label": name,
+                "file_type": "code",
+                "source_file": "",
+                "source_location": "",
+                "origin_file": str_path,
+            })
+        return nid
+
+    def _type_head_names(type_head) -> tuple[str | None, str | None]:
+        """Return (type_name, supertype_name) from a Julia `type_head`.
+
+        A bare declaration (`Foo`) exposes an `identifier`; a subtyping
+        declaration (`Foo <: Bar`) wraps both names in a `binary_expression`.
+        Both the struct and abstract-type paths need this, so parse it once.
+        """
+        bin_expr = next(
+            (c for c in type_head.children if c.type == "binary_expression"), None
+        )
+        if bin_expr:
+            identifiers = [c for c in bin_expr.children if c.type == "identifier"]
+            if identifiers:
+                name = _read_text(identifiers[0], source)
+                super_name = _read_text(identifiers[-1], source) if len(identifiers) >= 2 else None
+                return name, super_name
+            return None, None
+        name_node = next((c for c in type_head.children if c.type == "identifier"), None)
+        return (_read_text(name_node, source) if name_node else None), None
+
+    def _func_name_from_signature(sig_node) -> str | None:
+        """Extract function name from a Julia signature node (call_expression > identifier)."""
+        for child in sig_node.children:
+            if child.type == "call_expression":
+                callee = child.children[0] if child.children else None
+                if callee and callee.type == "identifier":
+                    return _read_text(callee, source)
+        return None
+
+    def walk_calls(body_node, func_nid: str) -> None:
+        if body_node is None:
+            return
+        t = body_node.type
+        if t in ("function_definition", "short_function_definition"):
+            return
+        if t == "call_expression" and body_node.children:
+            callee = body_node.children[0]
+            # Direct call: foo(...)
+            if callee.type == "identifier":
+                callee_name = _read_text(callee, source)
+                target_nid = _make_id(stem, callee_name)
+                add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
+                         confidence="EXTRACTED", context="call")
+            # Method call: obj.method(...)
+            elif callee.type == "field_expression" and len(callee.children) >= 3:
+                method_node = callee.children[-1]
+                method_name = _read_text(method_node, source)
+                target_nid = _make_id(stem, method_name)
+                add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1,
+                         confidence="EXTRACTED", context="call")
+        for child in body_node.children:
+            walk_calls(child, func_nid)
+
+    def walk(node, scope_nid: str) -> None:
+        t = node.type
+
+        # Module
+        if t == "module_definition":
+            name_node = next((c for c in node.children if c.type == "identifier"), None)
+            if name_node:
+                mod_name = _read_text(name_node, source)
+                mod_nid = _make_id(stem, mod_name)
+                line = node.start_point[0] + 1
+                add_node(mod_nid, mod_name, line)
+                add_edge(file_nid, mod_nid, "defines", line)
+                for child in node.children:
+                    walk(child, mod_nid)
+            return
+
+        # Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia)
+        if t == "struct_definition":
+            # type_head may contain: identifier (simple) or binary_expression (Foo <: Bar)
+            type_head = next((c for c in node.children if c.type == "type_head"), None)
+            if not type_head:
+                return
+            struct_name, super_name = _type_head_names(type_head)
+            if not struct_name:
+                return
+            struct_nid = _make_id(stem, struct_name)
+            line = node.start_point[0] + 1
+            add_node(struct_nid, struct_name, line)
+            add_edge(scope_nid, struct_nid, "defines", line)
+            if super_name:
+                add_edge(struct_nid, ensure_named_node(super_name, line),
+                         "inherits", line, confidence="EXTRACTED")
+            # Field types: each `name::Type` lowers to a typed_expression child of struct_definition
+            for child in node.children:
+                if child.type == "typed_expression":
+                    type_ids = [c for c in child.children if c.type == "identifier"]
+                    if len(type_ids) >= 2:
+                        field_line = child.start_point[0] + 1
+                        type_name = _read_text(type_ids[-1], source)
+                        type_nid = ensure_named_node(type_name, field_line)
+                        edges.append(_semantic_reference_edge(
+                            struct_nid, type_nid, "field", str_path, field_line))
+            return
+
+        # Abstract type
+        if t == "abstract_definition":
+            # type_head is a bare `identifier` (`abstract type Foo end`) or a
+            # `binary_expression` for the subtyping form (`abstract type Foo <: Bar end`).
+            # The latter was dropped entirely — abstract types are the backbone of
+            # Julia's dispatch hierarchies, so an intermediate `Foo <: Bar` vanishing
+            # broke the inheritance chain and lost the type node itself.
+            type_head = next((c for c in node.children if c.type == "type_head"), None)
+            if type_head:
+                abs_name, super_name = _type_head_names(type_head)
+                if abs_name:
+                    abs_nid = _make_id(stem, abs_name)
+                    line = node.start_point[0] + 1
+                    add_node(abs_nid, abs_name, line)
+                    add_edge(scope_nid, abs_nid, "defines", line)
+                    if super_name:
+                        add_edge(abs_nid, ensure_named_node(super_name, line),
+                                 "inherits", line, confidence="EXTRACTED")
+            return
+
+        # Function: function foo(...) ... end
+        if t == "function_definition":
+            sig_node = next((c for c in node.children if c.type == "signature"), None)
+            if sig_node:
+                func_name = _func_name_from_signature(sig_node)
+                if func_name:
+                    func_nid = _make_id(stem, func_name)
+                    line = node.start_point[0] + 1
+                    add_node(func_nid, f"{func_name}()", line)
+                    add_edge(scope_nid, func_nid, "defines", line)
+                    function_bodies.append((func_nid, node))
+            return
+
+        # Short function: foo(x) = expr
+        if t == "assignment":
+            lhs = node.children[0] if node.children else None
+            if lhs and lhs.type == "call_expression" and lhs.children:
+                callee = lhs.children[0]
+                if callee.type == "identifier":
+                    func_name = _read_text(callee, source)
+                    func_nid = _make_id(stem, func_name)
+                    line = node.start_point[0] + 1
+                    add_node(func_nid, f"{func_name}()", line)
+                    add_edge(scope_nid, func_nid, "defines", line)
+                    # Only walk the RHS (index 2 after lhs and operator) to avoid self-loops
+                    rhs = node.children[-1] if len(node.children) >= 3 else None
+                    if rhs:
+                        function_bodies.append((func_nid, rhs))
+            return
+
+        # Using / Import
+        if t in ("using_statement", "import_statement"):
+            line = node.start_point[0] + 1
+
+            def _julia_mod_name(n):
+                # identifier (`Foo`), scoped_identifier (`Base.Threads`), or
+                # import_path (relative `..Sibling`) -> the module name. Only bare
+                # identifiers were handled, so qualified/relative imports — and the
+                # scoped package of a `selected_import` — were silently dropped.
+                if n.type == "import_path":
+                    ids = [c for c in n.children if c.type == "identifier"]
+                    return _read_text(ids[-1], source) if ids else None
+                if n.type in ("identifier", "scoped_identifier"):
+                    return _read_text(n, source)
+                return None
+
+            def _emit_import(name):
+                if not name:
+                    return
+                imp_nid = _make_id(name)
+                add_node(imp_nid, name, line)
+                add_edge(scope_nid, imp_nid, "imports", line, context="import")
+
+            for child in node.children:
+                if child.type in ("identifier", "scoped_identifier", "import_path"):
+                    _emit_import(_julia_mod_name(child))
+                elif child.type == "selected_import":
+                    # `import Base.Threads: nthreads` — the package (first named
+                    # child) may itself be a scoped_identifier/import_path.
+                    pkg = next(
+                        (c for c in child.children
+                         if c.type in ("identifier", "scoped_identifier", "import_path")),
+                        None,
+                    )
+                    if pkg is not None:
+                        _emit_import(_julia_mod_name(pkg))
+            return
+
+        for child in node.children:
+            walk(child, scope_nid)
+
+    walk(root, file_nid)
+
+    for func_nid, body_node in function_bodies:
+        # For function_definition nodes, walk children directly to avoid
+        # the boundary check returning early on the top-level node itself.
+        # Skip the "signature" child — it contains the function's own call_expression
+        # which would create a self-loop.
+        if body_node.type == "function_definition":
+            for child in body_node.children:
+                if child.type != "signature":
+                    walk_calls(child, func_nid)
+        else:
+            walk_calls(body_node, func_nid)
+
+    return {"nodes": nodes, "edges": edges}