graphifyy 0.7.2__tar.gz → 0.7.4__tar.gz

{graphifyy-0.7.2/graphifyy.egg-info → graphifyy-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: graphifyy
-Version: 0.7.2
+Version: 0.7.4
 Summary: AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph
 License: MIT License
         
@@ -319,7 +319,7 @@ The MCP server gives your assistant structured access: `query_graph`, `get_node`
 
 - **Code files** — processed locally via tree-sitter. Nothing leaves your machine.
 - **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine.
-- **Docs, PDFs, images** — sent to your AI assistant's model API (Anthropic, OpenAI, etc.) using your own API key.
+- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `ANTHROPIC_API_KEY` or `MOONSHOT_API_KEY`.
 - No telemetry, no usage tracking, no analytics.
 
 ---

{graphifyy-0.7.2 → graphifyy-0.7.4}/README.md

@@ -221,7 +221,7 @@ The MCP server gives your assistant structured access: `query_graph`, `get_node`
 
 - **Code files** — processed locally via tree-sitter. Nothing leaves your machine.
 - **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine.
-- **Docs, PDFs, images** — sent to your AI assistant's model API (Anthropic, OpenAI, etc.) using your own API key.
+- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `ANTHROPIC_API_KEY` or `MOONSHOT_API_KEY`.
 - No telemetry, no usage tracking, no analytics.
 
 ---

{graphifyy-0.7.2 → graphifyy-0.7.4}/graphify/__main__.py

@@ -1094,6 +1094,10 @@ def main() -> None:
         print("    --max-children N        cap children per node (default 200)")
         print("    --top-k-edges N         per-symbol outbound edges in inspector (default 12)")
         print("    --label NAME            project label in header")
+        print("  extract <path>          headless full extraction (AST + semantic LLM) for CI/scripts")
+        print("    --backend B             kimi|claude (default: whichever API key is set)")
+        print("    --out DIR               output dir (default: <path>); writes <DIR>/graphify-out/")
+        print("    --no-cluster            skip clustering, write raw extraction only")
         print("  benchmark [graph.json]  measure token reduction vs naive full-corpus approach")
         print("  hook install            install post-commit/post-checkout git hooks (all platforms)")
         print("  hook uninstall          remove git hooks")
@@ -1882,6 +1886,235 @@ def main() -> None:
                 pass
         result = run_benchmark(graph_path, corpus_words=corpus_words)
         print_benchmark(result)
+
+    elif cmd == "extract":
+        # Headless full-pipeline extraction for CI / scripts (#698).
+        # Runs detect -> AST extraction on code -> semantic LLM extraction on
+        # docs/papers/images -> merge -> build -> cluster -> write outputs.
+        # Unlike the skill.md path (which runs through Claude Code subagents),
+        # this calls extract_corpus_parallel directly using whichever backend
+        # has an API key set.
+        if len(sys.argv) < 3:
+            print(
+                "Usage: graphify extract <path> [--backend kimi|claude] "
+                "[--out DIR] [--no-cluster]",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        target = Path(sys.argv[2]).resolve()
+        if not target.exists():
+            print(f"error: path not found: {target}", file=sys.stderr)
+            sys.exit(1)
+
+        backend: str | None = None
+        out_dir: Path | None = None
+        no_cluster = False
+        args = sys.argv[3:]
+        i = 0
+        while i < len(args):
+            a = args[i]
+            if a == "--backend" and i + 1 < len(args):
+                backend = args[i + 1]; i += 2
+            elif a.startswith("--backend="):
+                backend = a.split("=", 1)[1]; i += 1
+            elif a == "--out" and i + 1 < len(args):
+                out_dir = Path(args[i + 1]); i += 2
+            elif a.startswith("--out="):
+                out_dir = Path(a.split("=", 1)[1]); i += 1
+            elif a == "--no-cluster":
+                no_cluster = True; i += 1
+            else:
+                i += 1
+
+        # Backend resolution. If user did not pass --backend, sniff env.
+        # If backend was explicitly requested, validate its key is present
+        # and surface a clear error early — don't let extract_corpus_parallel
+        # raise mid-run after we've spent time on AST extraction.
+        from graphify.llm import (
+            BACKENDS as _BACKENDS,
+            detect_backend as _detect_backend,
+            estimate_cost as _estimate_cost,
+            extract_corpus_parallel as _extract_corpus_parallel,
+        )
+        if backend is None:
+            backend = _detect_backend()
+            if backend is None:
+                print(
+                    "error: no LLM API key found. Set MOONSHOT_API_KEY (kimi) "
+                    "or ANTHROPIC_API_KEY (claude), or pass --backend.",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+        if backend not in _BACKENDS:
+            print(
+                f"error: unknown backend '{backend}'. "
+                f"Available: {', '.join(sorted(_BACKENDS))}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        env_key = _BACKENDS[backend]["env_key"]
+        if not os.environ.get(env_key):
+            print(
+                f"error: backend '{backend}' requires {env_key} to be set.",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        # Resolve output dir. The user-facing contract is "<out>/graphify-out/"
+        # so a fresh checkout writes graphify-out/ at the project root, matching
+        # the skill.md pipeline.
+        out_root = (out_dir.resolve() if out_dir else target)
+        graphify_out = out_root / "graphify-out"
+        graphify_out.mkdir(parents=True, exist_ok=True)
+
+        from graphify.detect import detect as _detect
+        print(f"[graphify extract] scanning {target}")
+        detection = _detect(target)
+        files_by_type = detection.get("files", {})
+        code_files = [Path(p) for p in files_by_type.get("code", [])]
+        doc_files = [Path(p) for p in files_by_type.get("document", [])]
+        paper_files = [Path(p) for p in files_by_type.get("paper", [])]
+        image_files = [Path(p) for p in files_by_type.get("image", [])]
+        semantic_files = doc_files + paper_files + image_files
+        print(
+            f"[graphify extract] found {len(code_files)} code, "
+            f"{len(doc_files)} docs, {len(paper_files)} papers, "
+            f"{len(image_files)} images"
+        )
+
+        # AST extraction on code files. Empty code list (docs-only corpus) is
+        # the issue #698 case — skip cleanly instead of crashing inside extract().
+        ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0}
+        if code_files:
+            from graphify.extract import extract as _ast_extract
+            print(f"[graphify extract] AST extraction on {len(code_files)} code files...")
+            try:
+                ast_result = _ast_extract(code_files, cache_root=target)
+            except Exception as exc:
+                print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr)
+                ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0}
+
+        # Semantic extraction on docs/papers/images. Skip if there's nothing
+        # for the LLM — saves an empty API call and prevents BACKENDS validation
+        # from firing on a no-op.
+        sem_result: dict = {
+            "nodes": [], "edges": [], "hyperedges": [],
+            "input_tokens": 0, "output_tokens": 0,
+        }
+        if semantic_files:
+            print(
+                f"[graphify extract] semantic extraction on "
+                f"{len(semantic_files)} files via {backend}..."
+            )
+            try:
+                sem_result = _extract_corpus_parallel(
+                    semantic_files,
+                    backend=backend,
+                    root=target,
+                )
+            except ImportError as exc:
+                print(f"error: {exc}", file=sys.stderr)
+                sys.exit(1)
+            except Exception as exc:
+                print(
+                    f"[graphify extract] semantic extraction failed: {exc}",
+                    file=sys.stderr,
+                )
+
+        # Merge AST + semantic. Order matters for deduplication: passing AST
+        # first means semantic node attributes win on collision (richer labels
+        # for symbols also referenced in docs). Hyperedges only come from the
+        # semantic side.
+        merged: dict = {
+            "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])),
+            "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])),
+            "hyperedges": list(sem_result.get("hyperedges", [])),
+            "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0),
+            "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0),
+        }
+
+        graph_json_path = graphify_out / "graph.json"
+        analysis_path = graphify_out / ".graphify_analysis.json"
+
+        if no_cluster:
+            # --no-cluster: dump the raw merged extraction as graph.json.
+            # No NetworkX, no community detection, no analysis sidecar.
+            graph_json_path.write_text(
+                json.dumps(merged, indent=2), encoding="utf-8"
+            )
+            cost = _estimate_cost(
+                backend, merged["input_tokens"], merged["output_tokens"]
+            )
+            print(
+                f"[graphify extract] wrote {graph_json_path} — "
+                f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges "
+                f"(no clustering)"
+            )
+            if merged["input_tokens"] or merged["output_tokens"]:
+                print(
+                    f"[graphify extract] tokens: "
+                    f"{merged['input_tokens']:,} in / "
+                    f"{merged['output_tokens']:,} out, "
+                    f"est. cost: ${cost:.4f}"
+                )
+            sys.exit(0)
+
+        # Build graph + cluster + score + write.
+        from graphify.build import build_from_json as _build_from_json
+        from graphify.cluster import cluster as _cluster, score_all as _score_all
+        from graphify.export import to_json as _to_json
+        from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising
+
+        G = _build_from_json(merged)
+        if G.number_of_nodes() == 0:
+            print(
+                "[graphify extract] graph is empty — extraction produced no nodes. "
+                "Possible causes: all files skipped, binary-only corpus, or LLM "
+                "returned no edges.",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        communities = _cluster(G)
+        cohesion = _score_all(G, communities)
+        try:
+            gods = _god_nodes(G)
+        except Exception:
+            gods = []
+        try:
+            surprises = _surprising(G, communities)
+        except Exception:
+            surprises = []
+
+        _to_json(G, communities, str(graph_json_path), force=True)
+        analysis = {
+            "communities": {str(k): v for k, v in communities.items()},
+            "cohesion": {str(k): v for k, v in cohesion.items()},
+            "gods": gods,
+            "surprises": surprises,
+            "tokens": {
+                "input": merged["input_tokens"],
+                "output": merged["output_tokens"],
+            },
+        }
+        analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8")
+
+        cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"])
+        print(
+            f"[graphify extract] wrote {graph_json_path}: "
+            f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, "
+            f"{len(communities)} communities"
+        )
+        print(f"[graphify extract] wrote {analysis_path}")
+        if merged["input_tokens"] or merged["output_tokens"]:
+            print(
+                f"[graphify extract] tokens: "
+                f"{merged['input_tokens']:,} in / "
+                f"{merged['output_tokens']:,} out, "
+                f"est. cost (~{backend}): ${cost:.4f}"
+            )
+
     else:
         print(f"error: unknown command '{cmd}'", file=sys.stderr)
         print("Run 'graphify --help' for usage.", file=sys.stderr)

{graphifyy-0.7.2 → graphifyy-0.7.4}/graphify/extract.py

@@ -48,18 +48,59 @@ def _file_stem(path: Path) -> str:
 _TSCONFIG_ALIAS_CACHE: dict[str, dict[str, str]] = {}
 
 
+def _strip_jsonc(text: str) -> str:
+    """Strip // line comments, /* */ block comments, and trailing commas from JSONC.
+
+    Preserves string contents (including // and /* inside strings) by skipping over
+    quoted spans first. Required for tsconfig.json files generated by SvelteKit,
+    NestJS, Vite, T3, Astro, etc., which use JSONC by default (#700).
+    """
+    # Remove block and line comments while leaving string literals untouched.
+    pattern = re.compile(
+        r'"(?:\\.|[^"\\])*"'    # double-quoted string (with escapes)
+        r"|/\*.*?\*/"           # /* block comment */
+        r"|//[^\n]*",           # // line comment
+        re.DOTALL,
+    )
+
+    def _replace(match: re.Match) -> str:
+        token = match.group(0)
+        if token.startswith('"'):
+            return token
+        return ""
+
+    stripped = pattern.sub(_replace, text)
+    # Remove trailing commas before } or ] (allowing whitespace between).
+    stripped = re.sub(r",(\s*[}\]])", r"\1", stripped)
+    return stripped
+
+
 def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[str, str]:
     """Recursively read path aliases from a tsconfig, following extends chains.
 
     Child config paths override parent. Circular extends are detected via seen set.
     npm package configs (e.g. @tsconfig/svelte) are skipped since they're not on disk.
+    Handles JSONC (comments + trailing commas) which is the default tsconfig format
+    for SvelteKit, NestJS, Vite, T3, Astro, etc. (#700).
     """
     if str(tsconfig) in seen:
         return {}
     seen.add(str(tsconfig))
     try:
-        data = json.loads(tsconfig.read_text(encoding="utf-8"))
-    except Exception:
+        raw = tsconfig.read_text(encoding="utf-8")
+    except Exception as e:
+        print(f"  warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
+        return {}
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        try:
+            data = json.loads(_strip_jsonc(raw))
+        except json.JSONDecodeError as e:
+            print(f"  warning: failed to parse {tsconfig} as JSON/JSONC ({e.msg} at line {e.lineno} col {e.colno})", file=sys.stderr, flush=True)
+            return {}
+    except Exception as e:
+        print(f"  warning: failed to parse {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True)
         return {}
 
     aliases: dict[str, str] = {}
@@ -1708,13 +1749,50 @@ def extract_svelte(path: Path) -> dict:
         import re as _re
         src = path.read_text(encoding="utf-8", errors="replace")
         existing_ids = {n["id"] for n in result.get("nodes", [])}
-        file_node_id = _make_id(path.stem, str(path))
+        # Source file node ID must match the one _extract_generic creates:
+        # _make_id(str(path)) - single arg, no stem prefix. Otherwise the source
+        # endpoint is a phantom node and build_from_json drops the edge (#701).
+        file_node_id = _make_id(str(path))
+        aliases = _load_tsconfig_aliases(path.parent)
         for m in _re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src):
             raw = m.group(1)
-            if not raw.startswith("."):
+            if not raw:
                 continue
-            node_id = _make_id(raw, str(path))
+            if raw.startswith("."):
+                # Relative import - resolve to full path so IDs match file node IDs.
+                resolved = Path(os.path.normpath(path.parent / raw))
+                # TypeScript ESM: imports written as .js but actual file is .ts/.tsx
+                if resolved.suffix == ".js":
+                    resolved = resolved.with_suffix(".ts")
+                elif resolved.suffix == ".jsx":
+                    resolved = resolved.with_suffix(".tsx")
+                node_id = _make_id(str(resolved))
+            else:
+                # Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", "@/" -> "src/")
+                # before treating as external. Mirrors _import_js logic so SvelteKit alias
+                # imports resolve to the same file node IDs the extractor creates (#701).
+                resolved_alias = None
+                for alias_prefix, alias_base in aliases.items():
+                    if raw == alias_prefix or raw.startswith(alias_prefix + "/"):
+                        rest = raw[len(alias_prefix):].lstrip("/")
+                        resolved_alias = Path(os.path.normpath(Path(alias_base) / rest))
+                        break
+                if resolved_alias is not None:
+                    node_id = _make_id(str(resolved_alias))
+                else:
+                    # Bare/scoped import (node_modules) - use last segment;
+                    # build_from_json drops as external if no matching node exists.
+                    module_name = raw.split("/")[-1]
+                    if not module_name:
+                        continue
+                    node_id = _make_id(module_name)
             if node_id in existing_ids:
+                # Edge target already a real node - just add the edge, don't add a node.
+                result.setdefault("edges", []).append({
+                    "source": file_node_id, "target": node_id,
+                    "relation": "dynamic_import", "confidence": "EXTRACTED",
+                    "source_file": str(path),
+                })
                 continue
             result.setdefault("nodes", []).append({
                 "id": node_id, "label": raw,

{graphifyy-0.7.2 → graphifyy-0.7.4/graphifyy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: graphifyy
-Version: 0.7.2
+Version: 0.7.4
 Summary: AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph
 License: MIT License
         
@@ -319,7 +319,7 @@ The MCP server gives your assistant structured access: `query_graph`, `get_node`
 
 - **Code files** — processed locally via tree-sitter. Nothing leaves your machine.
 - **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine.
-- **Docs, PDFs, images** — sent to your AI assistant's model API (Anthropic, OpenAI, etc.) using your own API key.
+- **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `ANTHROPIC_API_KEY` or `MOONSHOT_API_KEY`.
 - No telemetry, no usage tracking, no analytics.
 
 ---

{graphifyy-0.7.2 → graphifyy-0.7.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "graphifyy"
-version = "0.7.2"
+version = "0.7.4"
 description = "AI coding assistant skill (Claude Code, Codex, OpenCode, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph"
 readme = "README.md"
 license = { file = "LICENSE" }