graphify-vault 0.5.8a0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {graphify_vault-0.5.8a0/graphify_vault.egg-info → graphify_vault-0.6.0}/PKG-INFO +1 -1
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/__main__.py +1 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/build_cmd.py +20 -8
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/query_log.py +32 -3
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/serve.py +64 -17
- graphify_vault-0.6.0/graphify/simplify.py +289 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0/graphify_vault.egg-info}/PKG-INFO +1 -1
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify_vault.egg-info/SOURCES.txt +2 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/pyproject.toml +1 -1
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_query_log.py +85 -0
- graphify_vault-0.6.0/tests/test_simplify.py +366 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/LICENSE +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/README.md +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/__init__.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/_minhash.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/affected.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/analyze.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/audit.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/benchmark.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/bin_cache.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/build.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/cache.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/cluster.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/compat.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/compile.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/dedup.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/detect.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/doctor.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/export.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/extract.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/feedback.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/hooks.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/ingest.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/llm.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/manifest.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/mcp_cmd.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/multigraph_compat.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/report.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/security.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/skill.md +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/transcribe.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/validate.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/watch.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify/wiki.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify_vault.egg-info/dependency_links.txt +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify_vault.egg-info/entry_points.txt +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify_vault.egg-info/requires.txt +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/graphify_vault.egg-info/top_level.txt +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/setup.cfg +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_affected.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_analyze.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_audit.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_benchmark.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_bin_cache.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_build.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_build_multigraph.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_cache.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_claude_md.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_cluster.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_cluster_multigraph.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_compile.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_confidence.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_dedup.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_detect.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_export.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_export_multigraph.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_extract.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_extract_headingless.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_feedback.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_graph_build_perf.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_heading_cluster.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_heading_compat.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_heading_extract.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_heading_query.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_hooks.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_hypergraph.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_ingest.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_install.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_languages.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_llm.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_main_multigraph.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_mcp_cmd.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_multigraph_compat.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_multigraph_edge_compat.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_multilang.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_pipeline.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_query_determinism.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_rationale.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_report.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_rerank.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_security.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_semantic_similarity.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_serve.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_serve_multigraph.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_serve_multigraph_load.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_sort_by.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_spatial_craft_build.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_spatialsdk_query_regression.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_transcribe.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_validate.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_watch.py +0 -0
- {graphify_vault-0.5.8a0 → graphify_vault-0.6.0}/tests/test_wiki.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: graphify_vault
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Fork of github.com/safishamsi/graphify with GRAPHIFY_WORKSPACE path resolution for distributed graph artifacts, same-label node merging, incremental build fixes, and local knowledge vault capabilities
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -1246,6 +1246,7 @@ def _apply_enhance_result(gp: Path, candidates_path: Path, enhance_data: dict) -
|
|
|
1246
1246
|
|
|
1247
1247
|
_raw_out["nodes"] = list(node_map.values())
|
|
1248
1248
|
_raw_out["edges"] = list(edge_set.values())
|
|
1249
|
+
_raw_out.pop("links", None) # remove stale "links" key to avoid duplication
|
|
1249
1250
|
|
|
1250
1251
|
gp.write_text(json.dumps(_raw_out, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
1251
1252
|
print(f"Graph updated: {len(_raw_out['nodes'])} nodes, {len(_raw_out['edges'])} edges → {gp}")
|
|
@@ -15,6 +15,7 @@ resolves to graphify_vault — no sys.path hacks needed.
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
17
|
import json
|
|
18
|
+
import os
|
|
18
19
|
import re
|
|
19
20
|
import sys
|
|
20
21
|
import zipfile
|
|
@@ -569,6 +570,9 @@ def run_build(args: list[str] | None = None) -> None:
|
|
|
569
570
|
"--depth", type=int, default=2, choices=[1, 2, 3], help="Semantic extraction depth (default: 2)"
|
|
570
571
|
)
|
|
571
572
|
parser.add_argument("--no-dedup", action="store_true", help="Skip entity deduplication step")
|
|
573
|
+
parser.add_argument(
|
|
574
|
+
"--no-simplify", action="store_true", help="Skip graph simplification (prune empty/low-info nodes)"
|
|
575
|
+
)
|
|
572
576
|
parser.add_argument("--llm", action="store_true", help="Enable LLM enhancement for Topic layer after build")
|
|
573
577
|
parser.add_argument(
|
|
574
578
|
"--provider",
|
|
@@ -580,8 +584,8 @@ def run_build(args: list[str] | None = None) -> None:
|
|
|
580
584
|
parser.add_argument("--model", default=None, help="LLM model name (overrides env var)")
|
|
581
585
|
opts = parser.parse_args(args)
|
|
582
586
|
|
|
583
|
-
# --workspace takes precedence over positional path;
|
|
584
|
-
workspace_arg = opts.workspace or opts.path or "."
|
|
587
|
+
# --workspace takes precedence over positional path; fall back to GRAPHIFY_WORKSPACE env var, then "."
|
|
588
|
+
workspace_arg = opts.workspace or opts.path or os.environ.get("GRAPHIFY_WORKSPACE", ".")
|
|
585
589
|
|
|
586
590
|
from graphify.analyze import god_nodes, suggest_questions, surprising_connections
|
|
587
591
|
from graphify.build import build_from_json
|
|
@@ -609,15 +613,19 @@ def run_build(args: list[str] | None = None) -> None:
|
|
|
609
613
|
print(f"Extract produced {len(result['nodes'])} nodes / {len(result['edges'])} edges")
|
|
610
614
|
|
|
611
615
|
G = build_from_json(result)
|
|
612
|
-
communities = cluster(G)
|
|
613
616
|
|
|
614
617
|
if not opts.no_dedup:
|
|
615
618
|
from graphify.dedup import deduplicate_graph
|
|
616
619
|
|
|
617
620
|
pre_count = G.number_of_nodes()
|
|
618
621
|
G = deduplicate_graph(G)
|
|
619
|
-
|
|
620
|
-
|
|
622
|
+
|
|
623
|
+
if not opts.no_simplify:
|
|
624
|
+
from graphify.simplify import simplify_graph
|
|
625
|
+
|
|
626
|
+
simplify_graph(G)
|
|
627
|
+
|
|
628
|
+
communities = cluster(G)
|
|
621
629
|
|
|
622
630
|
cohesion = score_all(G, communities)
|
|
623
631
|
gods = god_nodes(G)
|
|
@@ -841,15 +849,19 @@ def run_build(args: list[str] | None = None) -> None:
|
|
|
841
849
|
print(f"After merge: {len(merged_nodes)} nodes, {len(merged_edges)} edges")
|
|
842
850
|
|
|
843
851
|
G = build_from_json(merged_result)
|
|
844
|
-
communities = cluster(G)
|
|
845
852
|
|
|
846
853
|
if not opts.no_dedup:
|
|
847
854
|
from graphify.dedup import deduplicate_graph
|
|
848
855
|
|
|
849
856
|
pre_count = G.number_of_nodes()
|
|
850
857
|
G = deduplicate_graph(G)
|
|
851
|
-
|
|
852
|
-
|
|
858
|
+
|
|
859
|
+
if not opts.no_simplify:
|
|
860
|
+
from graphify.simplify import simplify_graph
|
|
861
|
+
|
|
862
|
+
simplify_graph(G)
|
|
863
|
+
|
|
864
|
+
communities = cluster(G)
|
|
853
865
|
|
|
854
866
|
cohesion = score_all(G, communities)
|
|
855
867
|
gods = god_nodes(G)
|
|
@@ -43,8 +43,15 @@ def write_entry(
|
|
|
43
43
|
workspace: str,
|
|
44
44
|
seed_ids: list[str] | None = None,
|
|
45
45
|
result_node_ids: list[str] | None = None,
|
|
46
|
+
*,
|
|
47
|
+
auto_rotate: bool = True,
|
|
46
48
|
) -> None:
|
|
47
|
-
"""Append one JSONL line to today's log file.
|
|
49
|
+
"""Append one JSONL line to today's log file.
|
|
50
|
+
|
|
51
|
+
auto_rotate: if True (default), spawn a background thread to compress
|
|
52
|
+
old logs and clean up. Set to False when the caller manages rotation
|
|
53
|
+
ordering (e.g. MCP server runs feedback analysis before rotation).
|
|
54
|
+
"""
|
|
48
55
|
if not is_enabled():
|
|
49
56
|
return
|
|
50
57
|
log_dir = _LOG_DIR
|
|
@@ -54,7 +61,8 @@ def write_entry(
|
|
|
54
61
|
print(f"[query_log] cannot create log dir {log_dir}: {e}", file=sys.stderr)
|
|
55
62
|
return
|
|
56
63
|
|
|
57
|
-
|
|
64
|
+
if auto_rotate:
|
|
65
|
+
threading.Thread(target=_rotate_and_cleanup, args=(log_dir,), daemon=True).start()
|
|
58
66
|
|
|
59
67
|
today_str = datetime.now().strftime("%Y-%m-%d")
|
|
60
68
|
log_file = log_dir / f"query_{today_str}.jsonl"
|
|
@@ -83,12 +91,27 @@ def write_entry(
|
|
|
83
91
|
def _rotate_and_cleanup(log_dir: Path) -> None:
|
|
84
92
|
"""Compress old jsonl files to zip and delete zips older than 7 days.
|
|
85
93
|
|
|
86
|
-
Keeps yesterday's
|
|
94
|
+
Keeps today's and yesterday's logs uncompressed. For older logs,
|
|
95
|
+
checks the feedback marker before zipping — if GRAPHIFY_WORKSPACE
|
|
96
|
+
is set and the marker directory exists but the marker file is missing,
|
|
97
|
+
the log is kept uncompressed for feedback analysis. If the workspace
|
|
98
|
+
is unset or the graphify-out directory doesn't exist, zips normally
|
|
99
|
+
to avoid unbounded disk growth.
|
|
87
100
|
"""
|
|
88
101
|
today = datetime.now()
|
|
89
102
|
today_str = today.strftime("%Y-%m-%d")
|
|
90
103
|
yesterday_str = (today - timedelta(days=1)).strftime("%Y-%m-%d")
|
|
91
104
|
|
|
105
|
+
# Resolve feedback marker directory (if available)
|
|
106
|
+
import os
|
|
107
|
+
|
|
108
|
+
marker_dir = None
|
|
109
|
+
ws = os.environ.get("GRAPHIFY_WORKSPACE", "")
|
|
110
|
+
if ws:
|
|
111
|
+
candidate = Path(ws) / "graphify-out"
|
|
112
|
+
if candidate.is_dir():
|
|
113
|
+
marker_dir = candidate
|
|
114
|
+
|
|
92
115
|
# Compress old jsonl files (skip today and yesterday)
|
|
93
116
|
for jsonl_file in list(log_dir.glob("query_*.jsonl")):
|
|
94
117
|
# Extract date from filename: query_YYYY-MM-DD.jsonl
|
|
@@ -97,6 +120,12 @@ def _rotate_and_cleanup(log_dir: Path) -> None:
|
|
|
97
120
|
if file_date_str in (today_str, yesterday_str):
|
|
98
121
|
continue
|
|
99
122
|
|
|
123
|
+
# Don't zip if feedback analysis hasn't processed this log yet.
|
|
124
|
+
if marker_dir is not None:
|
|
125
|
+
marker = marker_dir / f".feedback_analyzed_{stem}"
|
|
126
|
+
if not marker.exists():
|
|
127
|
+
continue
|
|
128
|
+
|
|
100
129
|
zip_path = log_dir / f"{jsonl_file.name}.zip"
|
|
101
130
|
try:
|
|
102
131
|
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
@@ -3166,11 +3166,10 @@ def _task_results_to_text(
|
|
|
3166
3166
|
return "\n".join(lines)
|
|
3167
3167
|
|
|
3168
3168
|
|
|
3169
|
-
def _log_query(question: str, payload: dict) -> None:
|
|
3169
|
+
def _log_query(question: str, payload: dict, *, auto_rotate: bool = True) -> None:
|
|
3170
3170
|
"""Write query_log entry from _query_graph results. Works for both MCP and direct calls."""
|
|
3171
3171
|
try:
|
|
3172
3172
|
import os
|
|
3173
|
-
import time as _time
|
|
3174
3173
|
|
|
3175
3174
|
from graphify.query_log import write_entry
|
|
3176
3175
|
|
|
@@ -3185,6 +3184,7 @@ def _log_query(question: str, payload: dict) -> None:
|
|
|
3185
3184
|
workspace=os.environ.get("GRAPHIFY_WORKSPACE", ""),
|
|
3186
3185
|
seed_ids=seed_ids or None,
|
|
3187
3186
|
result_node_ids=result_ids or None,
|
|
3187
|
+
auto_rotate=auto_rotate,
|
|
3188
3188
|
)
|
|
3189
3189
|
except Exception:
|
|
3190
3190
|
pass
|
|
@@ -3204,10 +3204,10 @@ def _query_graph(
|
|
|
3204
3204
|
god_labels: set[str] | None = None,
|
|
3205
3205
|
rerank: bool = True,
|
|
3206
3206
|
feedback: dict[str, float] | None = None,
|
|
3207
|
+
_auto_rotate_log: bool = True,
|
|
3207
3208
|
) -> dict[str, object]:
|
|
3208
3209
|
# Clamp token_budget to safe range for direct callers bypassing MCP schema
|
|
3209
|
-
|
|
3210
|
-
token_budget = 2000
|
|
3210
|
+
token_budget = max(1, min(token_budget, 100000))
|
|
3211
3211
|
if feedback is None:
|
|
3212
3212
|
feedback = G.graph.get("_feedback")
|
|
3213
3213
|
if feedback is None:
|
|
@@ -3377,7 +3377,7 @@ def _query_graph(
|
|
|
3377
3377
|
payload["text"] = _task_results_to_text(
|
|
3378
3378
|
payload, token_budget=token_budget, traversal_mode=traversal_mode, depth=depth
|
|
3379
3379
|
)
|
|
3380
|
-
_log_query(question, payload)
|
|
3380
|
+
_log_query(question, payload, auto_rotate=_auto_rotate_log)
|
|
3381
3381
|
return payload
|
|
3382
3382
|
|
|
3383
3383
|
scored = _score_nodes(G, terms)
|
|
@@ -3436,7 +3436,7 @@ def _query_graph(
|
|
|
3436
3436
|
"results": results,
|
|
3437
3437
|
"text": text if not json_output else text,
|
|
3438
3438
|
}
|
|
3439
|
-
_log_query(question, payload)
|
|
3439
|
+
_log_query(question, payload, auto_rotate=_auto_rotate_log)
|
|
3440
3440
|
return payload
|
|
3441
3441
|
|
|
3442
3442
|
|
|
@@ -3712,6 +3712,12 @@ def serve(graph_path: str = "graphify-out/graph.json", workspace: str | None = N
|
|
|
3712
3712
|
types.Tool(
|
|
3713
3713
|
name="query_graph",
|
|
3714
3714
|
description="Search the knowledge graph using BFS or DFS. Returns relevant nodes and edges as text context.",
|
|
3715
|
+
annotations=types.ToolAnnotations(
|
|
3716
|
+
readOnlyHint=True,
|
|
3717
|
+
destructiveHint=False,
|
|
3718
|
+
idempotentHint=True,
|
|
3719
|
+
openWorldHint=False,
|
|
3720
|
+
),
|
|
3715
3721
|
inputSchema={
|
|
3716
3722
|
"type": "object",
|
|
3717
3723
|
"properties": {
|
|
@@ -3762,11 +3768,23 @@ def serve(graph_path: str = "graphify-out/graph.json", workspace: str | None = N
|
|
|
3762
3768
|
types.Tool(
|
|
3763
3769
|
name="graph_stats",
|
|
3764
3770
|
description="Return summary statistics: node count, edge count, communities, confidence breakdown.",
|
|
3771
|
+
annotations=types.ToolAnnotations(
|
|
3772
|
+
readOnlyHint=True,
|
|
3773
|
+
destructiveHint=False,
|
|
3774
|
+
idempotentHint=True,
|
|
3775
|
+
openWorldHint=False,
|
|
3776
|
+
),
|
|
3765
3777
|
inputSchema={"type": "object", "properties": {}},
|
|
3766
3778
|
),
|
|
3767
3779
|
types.Tool(
|
|
3768
3780
|
name="switch_workspace",
|
|
3769
3781
|
description="Switch to a different graphify-out workspace (e.g. different version). Hot-reloads the graph without restarting the MCP server.",
|
|
3782
|
+
annotations=types.ToolAnnotations(
|
|
3783
|
+
readOnlyHint=False,
|
|
3784
|
+
destructiveHint=False,
|
|
3785
|
+
idempotentHint=True,
|
|
3786
|
+
openWorldHint=False,
|
|
3787
|
+
),
|
|
3770
3788
|
inputSchema={
|
|
3771
3789
|
"type": "object",
|
|
3772
3790
|
"properties": {
|
|
@@ -3793,8 +3811,7 @@ def serve(graph_path: str = "graphify-out/graph.json", workspace: str | None = N
|
|
|
3793
3811
|
return f"Input validation error: depth={raw_depth} is out of range (1-6)"
|
|
3794
3812
|
depth = raw_depth
|
|
3795
3813
|
budget = int(arguments.get("token_budget", 2000))
|
|
3796
|
-
|
|
3797
|
-
budget = 2000
|
|
3814
|
+
budget = max(1, min(budget, 100000))
|
|
3798
3815
|
max_degree = int(arguments.get("max_degree", 0))
|
|
3799
3816
|
if max_degree < 0:
|
|
3800
3817
|
max_degree = 0
|
|
@@ -3829,18 +3846,48 @@ def serve(graph_path: str = "graphify-out/graph.json", workspace: str | None = N
|
|
|
3829
3846
|
header = f"Traversal: {mode.upper()} depth={depth} max_degree={max_degree or '∞'} | Start: {start_nodes} | {len(results)} ranked matches\n\n"
|
|
3830
3847
|
result_text = header + str(payload.get("text", ""))
|
|
3831
3848
|
try:
|
|
3832
|
-
from graphify.
|
|
3833
|
-
from graphify.query_log import _LOG_DIR
|
|
3849
|
+
from graphify.query_log import _LOG_DIR, _rotate_and_cleanup
|
|
3834
3850
|
|
|
3835
3851
|
_fb_dir = Path(_state["G"].graph.get("_graph_dir", ""))
|
|
3836
|
-
if _fb_dir.is_dir():
|
|
3837
|
-
|
|
3838
|
-
def _refresh_feedback():
|
|
3839
|
-
from graphify.feedback import load_feedback
|
|
3840
|
-
|
|
3841
|
-
_state["feedback"] = load_feedback(_fb_dir)
|
|
3842
3852
|
|
|
3843
|
-
|
|
3853
|
+
def _post_query_maintenance():
|
|
3854
|
+
"""Run feedback analysis BEFORE log rotation in one thread.
|
|
3855
|
+
|
|
3856
|
+
This guarantees analyze_daily_log can read yesterday's
|
|
3857
|
+
uncompressed .jsonl before _rotate_and_cleanup zips it.
|
|
3858
|
+
"""
|
|
3859
|
+
try:
|
|
3860
|
+
if _fb_dir.is_dir():
|
|
3861
|
+
from graphify.feedback import analyze_daily_log, load_feedback
|
|
3862
|
+
|
|
3863
|
+
def _refresh():
|
|
3864
|
+
_state["feedback"] = load_feedback(_fb_dir)
|
|
3865
|
+
|
|
3866
|
+
from datetime import datetime, timedelta
|
|
3867
|
+
|
|
3868
|
+
yesterday_str = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
|
|
3869
|
+
log_path = _LOG_DIR / f"query_{yesterday_str}.jsonl"
|
|
3870
|
+
marker = _fb_dir / f".feedback_analyzed_query_{yesterday_str}"
|
|
3871
|
+
if not marker.exists():
|
|
3872
|
+
n = analyze_daily_log(log_path, _fb_dir)
|
|
3873
|
+
if n > 0:
|
|
3874
|
+
print(
|
|
3875
|
+
f"[feedback] analyzed {yesterday_str} log, updated {n} node(s)",
|
|
3876
|
+
file=sys.stderr,
|
|
3877
|
+
)
|
|
3878
|
+
_refresh()
|
|
3879
|
+
except Exception as exc:
|
|
3880
|
+
print(f"[feedback] maintenance error: {exc}", file=sys.stderr)
|
|
3881
|
+
|
|
3882
|
+
# Now safe to rotate (zip old logs)
|
|
3883
|
+
try:
|
|
3884
|
+
_rotate_and_cleanup(_LOG_DIR)
|
|
3885
|
+
except Exception:
|
|
3886
|
+
pass
|
|
3887
|
+
|
|
3888
|
+
import threading
|
|
3889
|
+
|
|
3890
|
+
threading.Thread(target=_post_query_maintenance, daemon=True).start()
|
|
3844
3891
|
except Exception:
|
|
3845
3892
|
pass
|
|
3846
3893
|
return result_text
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
# simplify.py — prune low-information-density nodes from a knowledge graph
|
|
2
|
+
#
|
|
3
|
+
# Designed to run in the build pipeline between dedup and cluster:
|
|
4
|
+
# extract → assemble → dedup → **simplify** → cluster → export
|
|
5
|
+
#
|
|
6
|
+
# All pruning decisions are based on structural statistics (raws length,
|
|
7
|
+
# degree, line range), never on hardcoded labels or project-specific names.
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
import networkx as nx
|
|
13
|
+
|
|
14
|
+
# ── Thresholds (tunable, based on empirical analysis) ────────────────────
|
|
15
|
+
|
|
16
|
+
_MIN_CONTENT_CHARS = 20 # raws join < this → "no content"
|
|
17
|
+
_LEAF_MAX_DEGREE = 5 # empty + degree ≤ this → prunable leaf
|
|
18
|
+
_HUB_MIN_DEGREE = 6 # empty + degree ≥ this → hub candidate
|
|
19
|
+
_MAX_BRIDGE_NEIGHBORS = 50 # limit bridging fan-out to avoid edge explosion
|
|
20
|
+
_HEADING_MIN_CHARS = 20 # single-line node with raws < this → heading-only
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _has_content(ndata: dict) -> bool:
|
|
24
|
+
"""Return True if a node carries meaningful textual content."""
|
|
25
|
+
raws = ndata.get("raws", [])
|
|
26
|
+
raws_text = " ".join(raws) if isinstance(raws, list) else str(raws or "")
|
|
27
|
+
return len(raws_text.strip()) >= _MIN_CONTENT_CHARS
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _neighbor_count(G: nx.Graph, nid: str) -> int:
|
|
31
|
+
"""Return the number of unique neighbors (not counting parallel edges)."""
|
|
32
|
+
return len(set(G.neighbors(nid)))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _is_single_line_heading(ndata: dict) -> bool:
|
|
36
|
+
"""Return True if a node is a single-line heading with no real content."""
|
|
37
|
+
sl = ndata.get("start_line")
|
|
38
|
+
el = ndata.get("end_line")
|
|
39
|
+
if sl is None or el is None or sl != el:
|
|
40
|
+
return False
|
|
41
|
+
raws = ndata.get("raws", [])
|
|
42
|
+
raws_text = " ".join(raws) if isinstance(raws, list) else str(raws or "")
|
|
43
|
+
return len(raws_text.strip()) < _HEADING_MIN_CHARS
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# ── Strategy 1: prune low-degree empty nodes ─────────────────────────────
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def prune_empty_leaves(G: nx.Graph, *, max_degree: int = _LEAF_MAX_DEGREE) -> int:
|
|
50
|
+
"""Remove nodes with no content and degree ≤ max_degree.
|
|
51
|
+
|
|
52
|
+
These are leaf or near-leaf nodes that contribute nothing to retrieval.
|
|
53
|
+
Returns the number of nodes removed.
|
|
54
|
+
"""
|
|
55
|
+
to_remove = [
|
|
56
|
+
nid for nid in list(G.nodes()) if not _has_content(G.nodes[nid]) and _neighbor_count(G, nid) <= max_degree
|
|
57
|
+
]
|
|
58
|
+
G.remove_nodes_from(to_remove)
|
|
59
|
+
return len(to_remove)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ── Strategy 2: collapse high-degree empty hubs ──────────────────────────
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def collapse_empty_hubs(
|
|
66
|
+
G: nx.Graph,
|
|
67
|
+
*,
|
|
68
|
+
min_degree: int = _HUB_MIN_DEGREE,
|
|
69
|
+
max_bridge: int = _MAX_BRIDGE_NEIGHBORS,
|
|
70
|
+
bridge_degree_cap: int = 30,
|
|
71
|
+
) -> int:
|
|
72
|
+
"""Remove empty hub nodes, optionally bridging their neighbors.
|
|
73
|
+
|
|
74
|
+
- Hubs with degree ≤ bridge_degree_cap: bridge top-max_bridge neighbors
|
|
75
|
+
to preserve local connectivity, then remove the hub.
|
|
76
|
+
- Hubs with degree > bridge_degree_cap: remove without bridging. These
|
|
77
|
+
are file-scope container nodes (e.g. ``(top)``) whose neighbors are
|
|
78
|
+
already connected through the content graph — bridging them would
|
|
79
|
+
create O(N²) new edges for no retrieval benefit.
|
|
80
|
+
|
|
81
|
+
Returns the number of hub nodes removed.
|
|
82
|
+
"""
|
|
83
|
+
removed = 0
|
|
84
|
+
# Snapshot node list — graph mutates during iteration
|
|
85
|
+
candidates = [
|
|
86
|
+
nid for nid in list(G.nodes()) if not _has_content(G.nodes[nid]) and _neighbor_count(G, nid) >= min_degree
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
for nid in candidates:
|
|
90
|
+
if nid not in G:
|
|
91
|
+
continue # already removed by a prior collapse
|
|
92
|
+
|
|
93
|
+
ncnt = _neighbor_count(G, nid)
|
|
94
|
+
neighbors = list(set(G.neighbors(nid)))
|
|
95
|
+
if not neighbors:
|
|
96
|
+
G.remove_node(nid)
|
|
97
|
+
removed += 1
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
# High-degree super-hubs: remove without bridging
|
|
101
|
+
if ncnt > bridge_degree_cap:
|
|
102
|
+
G.remove_node(nid)
|
|
103
|
+
removed += 1
|
|
104
|
+
continue
|
|
105
|
+
|
|
106
|
+
# Moderate hubs: bridge top neighbors then remove
|
|
107
|
+
def _edge_weight(nbr: str) -> float:
|
|
108
|
+
if not G.has_edge(nid, nbr):
|
|
109
|
+
return 1.0
|
|
110
|
+
edge_bundle = G[nid][nbr]
|
|
111
|
+
if isinstance(edge_bundle, dict) and "weight" in edge_bundle:
|
|
112
|
+
return float(edge_bundle.get("weight", 1.0))
|
|
113
|
+
try:
|
|
114
|
+
return max(
|
|
115
|
+
(float(d.get("weight", 1.0)) for d in edge_bundle.values() if isinstance(d, dict)),
|
|
116
|
+
default=1.0,
|
|
117
|
+
)
|
|
118
|
+
except (TypeError, AttributeError):
|
|
119
|
+
return 1.0
|
|
120
|
+
|
|
121
|
+
if len(neighbors) > max_bridge:
|
|
122
|
+
neighbors = sorted(neighbors, key=_edge_weight, reverse=True)[:max_bridge]
|
|
123
|
+
|
|
124
|
+
hub_label = G.nodes[nid].get("label", "")
|
|
125
|
+
for i, u in enumerate(neighbors):
|
|
126
|
+
for v in neighbors[i + 1 :]:
|
|
127
|
+
if not G.has_edge(u, v):
|
|
128
|
+
G.add_edge(
|
|
129
|
+
u,
|
|
130
|
+
v,
|
|
131
|
+
relation="implicit",
|
|
132
|
+
context="simplified",
|
|
133
|
+
confidence="INFERRED",
|
|
134
|
+
confidence_score=0.5,
|
|
135
|
+
weight=0.5,
|
|
136
|
+
_simplified_from=hub_label,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
G.remove_node(nid)
|
|
140
|
+
removed += 1
|
|
141
|
+
|
|
142
|
+
return removed
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ── Strategy 3: merge single-line headings into children ─────────────────
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def merge_single_line_headings(G: nx.Graph) -> int:
|
|
149
|
+
"""Merge single-line heading nodes into their first content-bearing neighbor.
|
|
150
|
+
|
|
151
|
+
The heading's label is preserved as a ``heading_context`` attribute on
|
|
152
|
+
the target node so retrieval signals are not lost.
|
|
153
|
+
|
|
154
|
+
Returns the number of heading nodes merged.
|
|
155
|
+
"""
|
|
156
|
+
merged = 0
|
|
157
|
+
candidates = [nid for nid in list(G.nodes()) if _is_single_line_heading(G.nodes[nid])]
|
|
158
|
+
|
|
159
|
+
for nid in candidates:
|
|
160
|
+
if nid not in G:
|
|
161
|
+
continue
|
|
162
|
+
|
|
163
|
+
ndata = G.nodes[nid]
|
|
164
|
+
label = ndata.get("label", "")
|
|
165
|
+
neighbors = list(G.neighbors(nid))
|
|
166
|
+
|
|
167
|
+
# Find the best merge target: prefer a neighbor with content
|
|
168
|
+
target = None
|
|
169
|
+
for nbr in neighbors:
|
|
170
|
+
if _has_content(G.nodes[nbr]):
|
|
171
|
+
target = nbr
|
|
172
|
+
break
|
|
173
|
+
|
|
174
|
+
if target is None:
|
|
175
|
+
# No content-bearing neighbor — skip, don't orphan edges
|
|
176
|
+
continue
|
|
177
|
+
|
|
178
|
+
# Transfer heading label as context
|
|
179
|
+
existing_ctx = G.nodes[target].get("heading_context", "")
|
|
180
|
+
if existing_ctx:
|
|
181
|
+
G.nodes[target]["heading_context"] = f"{existing_ctx} > {label}"
|
|
182
|
+
else:
|
|
183
|
+
G.nodes[target]["heading_context"] = label
|
|
184
|
+
|
|
185
|
+
# Re-wire: connect all other neighbors of the heading to the target
|
|
186
|
+
is_multi = isinstance(G, (nx.MultiGraph, nx.MultiDiGraph))
|
|
187
|
+
for nbr in neighbors:
|
|
188
|
+
if nbr == target:
|
|
189
|
+
continue
|
|
190
|
+
if G.has_edge(nid, nbr):
|
|
191
|
+
edge_bundle = G[nid][nbr]
|
|
192
|
+
if is_multi:
|
|
193
|
+
# MultiGraph: copy ALL parallel edges to preserve relations
|
|
194
|
+
for _key, ed in edge_bundle.items():
|
|
195
|
+
if isinstance(ed, dict):
|
|
196
|
+
G.add_edge(target, nbr, **dict(ed))
|
|
197
|
+
else:
|
|
198
|
+
# Simple Graph: only add if no existing edge
|
|
199
|
+
if not G.has_edge(target, nbr):
|
|
200
|
+
G.add_edge(target, nbr, **dict(edge_bundle))
|
|
201
|
+
|
|
202
|
+
G.remove_node(nid)
|
|
203
|
+
merged += 1
|
|
204
|
+
|
|
205
|
+
return merged
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# ── Strategy 4: merge duplicate edges ────────────────────────────────────
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def merge_duplicate_edges(G: nx.Graph) -> int:
|
|
212
|
+
"""Merge parallel edges with the same (u, v, relation) in a MultiGraph.
|
|
213
|
+
|
|
214
|
+
Keeps the edge with the highest weight; discards the rest.
|
|
215
|
+
For non-MultiGraph instances, this is a no-op.
|
|
216
|
+
|
|
217
|
+
Returns the number of redundant edges removed.
|
|
218
|
+
"""
|
|
219
|
+
if not isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)):
|
|
220
|
+
return 0
|
|
221
|
+
|
|
222
|
+
removed = 0
|
|
223
|
+
# Snapshot: collect all node pairs that have edges
|
|
224
|
+
for u, v in list(set(G.edges())):
|
|
225
|
+
edge_bundle = G[u][v]
|
|
226
|
+
if len(edge_bundle) <= 1:
|
|
227
|
+
continue
|
|
228
|
+
|
|
229
|
+
# Group parallel edges by relation
|
|
230
|
+
by_relation: dict[str, list[tuple]] = {}
|
|
231
|
+
for key, edata in edge_bundle.items():
|
|
232
|
+
rel = edata.get("relation", "")
|
|
233
|
+
by_relation.setdefault(rel, []).append((key, edata))
|
|
234
|
+
|
|
235
|
+
for rel, group in by_relation.items():
|
|
236
|
+
if len(group) <= 1:
|
|
237
|
+
continue
|
|
238
|
+
# Keep the edge with the highest weight
|
|
239
|
+
group.sort(key=lambda x: float(x[1].get("weight", 1.0)), reverse=True)
|
|
240
|
+
for key, _ in group[1:]:
|
|
241
|
+
G.remove_edge(u, v, key=key)
|
|
242
|
+
removed += 1
|
|
243
|
+
|
|
244
|
+
return removed
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# ── Top-level entry point ────────────────────────────────────────────────
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def simplify_graph(G: nx.Graph, *, verbose: bool = True) -> dict[str, int]:
|
|
251
|
+
"""Run all simplification strategies on the graph in-place.
|
|
252
|
+
|
|
253
|
+
Returns a dict of statistics: {strategy_name: count_affected}.
|
|
254
|
+
"""
|
|
255
|
+
pre_nodes = G.number_of_nodes()
|
|
256
|
+
pre_edges = G.number_of_edges()
|
|
257
|
+
|
|
258
|
+
stats: dict[str, int] = {}
|
|
259
|
+
|
|
260
|
+
# Order matters: prune leaves first (cheap), then collapse hubs
|
|
261
|
+
# (may create new leaves but they have bridged edges so won't be
|
|
262
|
+
# re-pruned), then merge headings (operates on remaining structure).
|
|
263
|
+
stats["pruned_leaves"] = prune_empty_leaves(G)
|
|
264
|
+
stats["collapsed_hubs"] = collapse_empty_hubs(G)
|
|
265
|
+
stats["merged_headings"] = merge_single_line_headings(G)
|
|
266
|
+
stats["merged_edges"] = merge_duplicate_edges(G)
|
|
267
|
+
|
|
268
|
+
post_nodes = G.number_of_nodes()
|
|
269
|
+
post_edges = G.number_of_edges()
|
|
270
|
+
stats["nodes_before"] = pre_nodes
|
|
271
|
+
stats["nodes_after"] = post_nodes
|
|
272
|
+
stats["edges_before"] = pre_edges
|
|
273
|
+
stats["edges_after"] = post_edges
|
|
274
|
+
|
|
275
|
+
if verbose:
|
|
276
|
+
pct = (1 - post_nodes / pre_nodes) * 100 if pre_nodes else 0
|
|
277
|
+
print(
|
|
278
|
+
f"Simplify: {pre_nodes} → {post_nodes} nodes ({pct:.1f}% reduction), {pre_edges} → {post_edges} edges",
|
|
279
|
+
file=sys.stderr,
|
|
280
|
+
)
|
|
281
|
+
print(
|
|
282
|
+
f" pruned_leaves={stats['pruned_leaves']}, "
|
|
283
|
+
f"collapsed_hubs={stats['collapsed_hubs']}, "
|
|
284
|
+
f"merged_headings={stats['merged_headings']}, "
|
|
285
|
+
f"merged_edges={stats['merged_edges']}",
|
|
286
|
+
file=sys.stderr,
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
return stats
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: graphify_vault
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Fork of github.com/safishamsi/graphify with GRAPHIFY_WORKSPACE path resolution for distributed graph artifacts, same-label node merging, incremental build fixes, and local knowledge vault capabilities
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -31,6 +31,7 @@ graphify/query_log.py
|
|
|
31
31
|
graphify/report.py
|
|
32
32
|
graphify/security.py
|
|
33
33
|
graphify/serve.py
|
|
34
|
+
graphify/simplify.py
|
|
34
35
|
graphify/skill.md
|
|
35
36
|
graphify/transcribe.py
|
|
36
37
|
graphify/validate.py
|
|
@@ -89,6 +90,7 @@ tests/test_semantic_similarity.py
|
|
|
89
90
|
tests/test_serve.py
|
|
90
91
|
tests/test_serve_multigraph.py
|
|
91
92
|
tests/test_serve_multigraph_load.py
|
|
93
|
+
tests/test_simplify.py
|
|
92
94
|
tests/test_sort_by.py
|
|
93
95
|
tests/test_spatial_craft_build.py
|
|
94
96
|
tests/test_spatialsdk_query_regression.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "graphify_vault"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Fork of github.com/safishamsi/graphify with GRAPHIFY_WORKSPACE path resolution for distributed graph artifacts, same-label node merging, incremental build fixes, and local knowledge vault capabilities"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { file = "LICENSE" }
|