knowledge-master 0.3.0__tar.gz → 0.5.0__tar.gz

{knowledge_master-0.3.0 → knowledge_master-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: knowledge-master
-Version: 0.3.0
+Version: 0.5.0
 Summary: Local-first knowledge graph for developers. Your AI agent's permanent memory.
 Author: Milenko Mitrovic
 License: MIT

{knowledge_master-0.3.0 → knowledge_master-0.5.0}/knowledge_master/cli.py

@@ -460,5 +460,75 @@ def who_owns(file: str = typer.Argument(..., help="File path to check ownership"
         console.print("[dim]Run 'km index <repo>' first to extract ownership.[/]")
 
 
+@app.command()
+def upgrade():
+    """Upgrade graph schema to the latest version."""
+    from .migrations import check_and_migrate, get_schema_version, CURRENT_SCHEMA_VERSION
+
+    graph = store.get_graph()
+    current = get_schema_version(graph)
+    console.print(f"[bold]Current schema:[/] v{current}")
+    console.print(f"[bold]Target schema:[/]  v{CURRENT_SCHEMA_VERSION}")
+
+    if current == CURRENT_SCHEMA_VERSION:
+        console.print("[green]✓ Already up to date[/]")
+        return
+
+    result = check_and_migrate(graph, auto_migrate=True)
+    for step in result["steps"]:
+        console.print(f"  [green]✓[/] {step}")
+    console.print(f"\n[green]✓ Upgraded to v{CURRENT_SCHEMA_VERSION}[/]")
+
+
+@app.command()
+def prune(
+    older_than: int = typer.Option(30, help="Remove chunks not re-indexed in this many days"),
+    dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be removed"),
+):
+    """Remove stale/orphaned data from the knowledge graph."""
+    graph = store.get_graph()
+
+    # Find orphaned chunks (no PART_OF edge)
+    orphaned = graph.query("MATCH (c:Chunk) WHERE NOT (c)-[:PART_OF]->() RETURN count(c)")
+    orphan_count = orphaned.result_set[0][0] if orphaned.result_set else 0
+
+    # Find documents with no chunks
+    empty_docs = graph.query(
+        "MATCH (d:Document) WHERE NOT ()-[:PART_OF]->(d) AND NOT (d)-[:IN_REPO]->() RETURN count(d)"
+    )
+    empty_count = empty_docs.result_set[0][0] if empty_docs.result_set else 0
+
+    # Find stale chunks (indexed_at older than threshold)
+    # FalkorDB timestamp() returns ms since epoch
+    threshold_ms = older_than * 86400 * 1000
+    stale = graph.query(
+        """MATCH (c:Chunk)
+           WHERE c.indexed_at IS NOT NULL AND (timestamp() - c.indexed_at) > $threshold
+           RETURN count(c)""",
+        params={"threshold": threshold_ms},
+    )
+    stale_count = stale.result_set[0][0] if stale.result_set else 0
+
+    console.print("[bold]Prune analysis:[/]")
+    console.print(f"  Orphaned chunks (no document link): {orphan_count}")
+    console.print(f"  Empty documents (no chunks):        {empty_count}")
+    console.print(f"  Stale chunks (>{older_than} days):        {stale_count}")
+
+    if dry_run:
+        console.print("\n[yellow]Dry run — nothing removed.[/]")
+        return
+
+    total = 0
+    if orphan_count > 0:
+        graph.query("MATCH (c:Chunk) WHERE NOT (c)-[:PART_OF]->() DELETE c")
+        total += orphan_count
+
+    if empty_count > 0:
+        graph.query("MATCH (d:Document) WHERE NOT ()-[:PART_OF]->(d) AND NOT (d)-[:IN_REPO]->() DELETE d")
+        total += empty_count
+
+    console.print(f"\n[green]✓ Removed {total} stale nodes[/]")
+
+
 if __name__ == "__main__":
     app()

knowledge_master-0.5.0/knowledge_master/migrations.py

@@ -0,0 +1,89 @@
+"""Schema versioning and migrations for the knowledge graph."""
+
+CURRENT_SCHEMA_VERSION = 4  # v0.4.0
+
+# Migration definitions: version -> function that upgrades from previous version
+MIGRATIONS = {}
+
+
+def get_schema_version(graph) -> int:
+    """Get current schema version from graph metadata."""
+    try:
+        result = graph.query("MATCH (m:_Meta {key: 'schema_version'}) RETURN m.value")
+        if result.result_set:
+            return int(result.result_set[0][0])
+    except Exception:
+        pass
+    return 0  # no version = legacy graph
+
+
+def set_schema_version(graph, version: int):
+    """Store schema version in graph metadata."""
+    graph.query(
+        "MERGE (m:_Meta {key: 'schema_version'}) SET m.value = $version",
+        params={"version": version},
+    )
+
+
+def check_and_migrate(graph, auto_migrate: bool = True) -> dict:
+    """Check schema version and migrate if needed.
+
+    Returns: {"current": int, "target": int, "migrated": bool, "steps": list}
+    """
+    current = get_schema_version(graph)
+    target = CURRENT_SCHEMA_VERSION
+
+    if current == target:
+        return {"current": current, "target": target, "migrated": False, "steps": []}
+
+    if current > target:
+        raise RuntimeError(
+            f"Graph schema v{current} is newer than this version supports (v{target}). "
+            "Please upgrade knowledge-master."
+        )
+
+    if not auto_migrate:
+        raise RuntimeError(
+            f"Graph schema v{current} needs migration to v{target}. Run: km upgrade"
+        )
+
+    steps = []
+    for v in range(current + 1, target + 1):
+        migration_fn = MIGRATIONS.get(v)
+        if migration_fn:
+            migration_fn(graph)
+            steps.append(f"v{v-1} → v{v}: {migration_fn.__doc__ or 'applied'}")
+        else:
+            steps.append(f"v{v-1} → v{v}: no-op (compatible)")
+
+    set_schema_version(graph, target)
+    return {"current": current, "target": target, "migrated": True, "steps": steps}
+
+
+# --- Migrations ---
+
+def _migrate_to_v1(graph):
+    """Add indexed_at timestamp to existing chunks missing it."""
+    graph.query("MATCH (c:Chunk) WHERE c.indexed_at IS NULL SET c.indexed_at = timestamp()")
+
+
+def _migrate_to_v2(graph):
+    """Add OWNS relationships from ownership extraction."""
+    pass  # OWNS edges are created by extract_ownership, no schema change needed
+
+
+def _migrate_to_v3(graph):
+    """Add lang property to IMPORTS edges and Function nodes."""
+    graph.query("MATCH (f:Function) WHERE f.lang IS NULL SET f.lang = 'python'")
+    graph.query("MATCH ()-[e:IMPORTS]->() WHERE e.lang IS NULL SET e.lang = 'python'")
+
+
+def _migrate_to_v4(graph):
+    """Add content_hash to Chunk nodes for deduplication."""
+    pass  # New chunks will have hash, old ones are fine without
+
+
+MIGRATIONS[1] = _migrate_to_v1
+MIGRATIONS[2] = _migrate_to_v2
+MIGRATIONS[3] = _migrate_to_v3
+MIGRATIONS[4] = _migrate_to_v4

{knowledge_master-0.3.0 → knowledge_master-0.5.0}/knowledge_master/parsers/git_repo.py

@@ -46,6 +46,8 @@ def index_repo(repo_path: str, graph=None, branch: str = "HEAD", on_progress=Non
     tracked = repo.git.ls_files().splitlines()
     indexable = [f for f in tracked if _should_index(f)]
     total = len(indexable)
+    indexed_files = []
+    failed_files = []
 
     with Progress(disable=not sys.stdout.isatty()) as progress:
         task = progress.add_task(f"Indexing {repo_name}", total=total)
@@ -53,19 +55,26 @@ def index_repo(repo_path: str, graph=None, branch: str = "HEAD", on_progress=Non
             full_path = os.path.join(repo_path, filepath)
             try:
                 _index_file(graph, full_path, filepath, repo_name, repo)
+                indexed_files.append(filepath)
             except Exception as e:
+                failed_files.append((filepath, str(e)))
                 progress.console.print(f"  [yellow]skip {filepath}: {e}[/]")
             progress.advance(task)
             if on_progress:
                 on_progress(i + 1, total, filepath)
 
+    # If more than 50% failed, warn (possible systemic issue)
+    if total > 0 and len(failed_files) > total * 0.5:
+        import sys as _sys
+        print(f"WARNING: {len(failed_files)}/{total} files failed. Possible systemic issue.", file=_sys.stderr)
+
     # Run intelligence extraction
     intel = extract_all(repo_path, graph)
 
     # Run static analysis (import graph, symbols) — all languages
     static = build_import_graph_all(repo_path, graph)
 
-    return {"repo": repo_name, "files_indexed": total, "intelligence": intel, "static_analysis": static}
+    return {"repo": repo_name, "files_indexed": len(indexed_files), "files_failed": len(failed_files), "intelligence": intel, "static_analysis": static}
 
 
 def _should_index(filepath: str) -> bool:

{knowledge_master-0.3.0 → knowledge_master-0.5.0}/knowledge_master/store.py

@@ -1,5 +1,7 @@
 """FalkorDB graph store - nodes, edges, vector search, and graph traversal."""
 
+import hashlib
+
 from falkordb import FalkorDB
 
 GRAPH_NAME = "knowledge"
@@ -7,11 +9,35 @@ GRAPH_NAME = "knowledge"
 # Vector dimension for nomic-embed-text
 VECTOR_DIM = 768
 
+_graph_instance = None
+
 
 def get_graph(host: str = "localhost", port: int = 6379):
-    """Get FalkorDB graph instance."""
+    """Get FalkorDB graph instance with schema version check."""
+    global _graph_instance
+    if _graph_instance is not None:
+        return _graph_instance
+
     db = FalkorDB(host=host, port=port)
-    return db.select_graph(GRAPH_NAME)
+    graph = db.select_graph(GRAPH_NAME)
+
+    # Check and auto-migrate schema
+    from .migrations import check_and_migrate
+    check_and_migrate(graph, auto_migrate=True)
+
+    _graph_instance = graph
+    return graph
+
+
+def reset_graph_instance():
+    """Reset cached graph instance (for testing)."""
+    global _graph_instance
+    _graph_instance = None
+
+
+def content_hash(text: str) -> str:
+    """Compute content hash for deduplication."""
+    return hashlib.sha256(text.encode()).hexdigest()[:16]
 
 
 def init_schema(graph):
@@ -35,20 +61,32 @@ def init_schema(graph):
 
 
 def upsert_chunk(graph, chunk_id: str, text: str, embedding: list[float], metadata: dict):
-    """Insert or update a chunk node with embedding."""
+    """Insert or update a chunk node with embedding. Skips if content unchanged (dedup)."""
+    chash = content_hash(text)
+
+    # Check if chunk exists with same content hash — skip if unchanged
+    existing = graph.query(
+        "MATCH (c:Chunk {id: $id}) RETURN c.content_hash",
+        params={"id": chunk_id},
+    )
+    if existing.result_set and existing.result_set[0][0] == chash:
+        return False  # skip — content unchanged
+
     graph.query(
         """MERGE (c:Chunk {id: $id})
            SET c.text = $text, c.embedding = vecf32($embedding),
                c.source = $source, c.source_type = $source_type,
-               c.indexed_at = timestamp()""",
+               c.content_hash = $hash, c.indexed_at = timestamp()""",
         params={
             "id": chunk_id,
             "text": text,
             "embedding": embedding,
             "source": metadata.get("source", ""),
             "source_type": metadata.get("source_type", ""),
+            "hash": chash,
         },
     )
+    return True  # inserted/updated
 
 
 def upsert_document(graph, path: str, doc_type: str, metadata: dict):

{knowledge_master-0.3.0 → knowledge_master-0.5.0}/knowledge_master.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: knowledge-master
-Version: 0.3.0
+Version: 0.5.0
 Summary: Local-first knowledge graph for developers. Your AI agent's permanent memory.
 Author: Milenko Mitrovic
 License: MIT

{knowledge_master-0.3.0 → knowledge_master-0.5.0}/knowledge_master.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ knowledge_master/cli.py
 knowledge_master/connectors.py
 knowledge_master/embeddings.py
 knowledge_master/intelligence.py
+knowledge_master/migrations.py
 knowledge_master/rerank.py
 knowledge_master/server.py
 knowledge_master/static_analysis.py
@@ -28,5 +29,10 @@ knowledge_master/parsers/markdown.py
 tests/test_api.py
 tests/test_chunking.py
 tests/test_cli.py
+tests/test_connectors.py
+tests/test_dedup.py
 tests/test_intelligence.py
-tests/test_static_analysis.py
+tests/test_migrations.py
+tests/test_reranker.py
+tests/test_static_analysis.py
+tests/test_ts_parsers.py

{knowledge_master-0.3.0 → knowledge_master-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "knowledge-master"
-version = "0.3.0"
+version = "0.5.0"
 description = "Local-first knowledge graph for developers. Your AI agent's permanent memory."
 requires-python = ">=3.11"
 license = {text = "MIT"}

knowledge_master-0.5.0/tests/test_connectors.py

@@ -0,0 +1,58 @@
+"""Tests for connector framework."""
+from knowledge_master.connectors import MCPSource, SOURCES, add_custom_source, _parse_mcp_result
+
+
+def test_preconfigured_sources_exist():
+    assert "outlook" in SOURCES
+    assert "slack" in SOURCES
+
+
+def test_source_has_required_fields():
+    for name, source in SOURCES.items():
+        assert source.name, f"{name} missing name"
+        assert source.command, f"{name} missing command"
+        assert source.tool_name, f"{name} missing tool_name"
+        assert source.source_type, f"{name} missing source_type"
+
+
+def test_add_custom_source():
+    add_custom_source("test-source", ["echo", "hi"], "get_data", {"limit": 10}, "custom")
+    assert "test-source" in SOURCES
+    assert SOURCES["test-source"].tool_name == "get_data"
+    # Cleanup
+    del SOURCES["test-source"]
+
+
+def test_parse_mcp_result_json_list():
+    class FakeContent:
+        text = '[{"title": "hello", "body": "world"}]'
+
+    class FakeResult:
+        content = [FakeContent()]
+
+    items = _parse_mcp_result(FakeResult())
+    assert len(items) == 1
+    assert items[0]["title"] == "hello"
+
+
+def test_parse_mcp_result_json_dict_with_results():
+    class FakeContent:
+        text = '{"results": [{"text": "a"}, {"text": "b"}]}'
+
+    class FakeResult:
+        content = [FakeContent()]
+
+    items = _parse_mcp_result(FakeResult())
+    assert len(items) == 2
+
+
+def test_parse_mcp_result_plain_text():
+    class FakeContent:
+        text = "just plain text"
+
+    class FakeResult:
+        content = [FakeContent()]
+
+    items = _parse_mcp_result(FakeResult())
+    assert len(items) == 1
+    assert items[0]["text"] == "just plain text"

knowledge_master-0.5.0/tests/test_dedup.py

@@ -0,0 +1,24 @@
+"""Tests for store deduplication and content hashing."""
+from knowledge_master.store import content_hash
+
+
+def test_content_hash_deterministic():
+    h1 = content_hash("hello world")
+    h2 = content_hash("hello world")
+    assert h1 == h2
+
+
+def test_content_hash_different_for_different_content():
+    h1 = content_hash("hello world")
+    h2 = content_hash("hello world!")
+    assert h1 != h2
+
+
+def test_content_hash_length():
+    h = content_hash("test")
+    assert len(h) == 16  # sha256 truncated to 16 chars
+
+
+def test_content_hash_hex():
+    h = content_hash("test")
+    assert all(c in "0123456789abcdef" for c in h)

knowledge_master-0.5.0/tests/test_migrations.py

@@ -0,0 +1,23 @@
+"""Tests for schema migrations."""
+from knowledge_master.migrations import (
+    CURRENT_SCHEMA_VERSION,
+    MIGRATIONS,
+    _migrate_to_v1,
+    _migrate_to_v3,
+    _migrate_to_v4,
+)
+
+
+def test_current_schema_version():
+    assert CURRENT_SCHEMA_VERSION == 4
+
+
+def test_all_migrations_defined():
+    for v in range(1, CURRENT_SCHEMA_VERSION + 1):
+        assert v in MIGRATIONS, f"Migration to v{v} not defined"
+
+
+def test_migrations_are_callable():
+    for v, fn in MIGRATIONS.items():
+        assert callable(fn)
+        assert fn.__doc__ is not None, f"Migration v{v} has no docstring"

knowledge_master-0.5.0/tests/test_reranker.py

@@ -0,0 +1,43 @@
+"""Benchmark test for re-ranker — compare raw cosine vs re-ranked ordering.
+
+This is an integration test (requires Ollama) but demonstrates the value of re-ranking.
+Run with: pytest tests/integration/test_reranker_benchmark.py -v
+"""
+import pytest
+from knowledge_master.rerank import rerank, _cosine_sim
+
+
+def test_cosine_sim_identical():
+    a = [1.0, 0.0, 0.0]
+    assert _cosine_sim(a, a) == pytest.approx(1.0)
+
+
+def test_cosine_sim_orthogonal():
+    a = [1.0, 0.0, 0.0]
+    b = [0.0, 1.0, 0.0]
+    assert _cosine_sim(a, b) == pytest.approx(0.0)
+
+
+def test_cosine_sim_opposite():
+    a = [1.0, 0.0]
+    b = [-1.0, 0.0]
+    assert _cosine_sim(a, b) == pytest.approx(-1.0)
+
+
+def test_rerank_empty():
+    result = rerank("test query", [], top_k=5)
+    assert result == []
+
+
+def test_rerank_preserves_fields():
+    """Re-rank should keep all original result fields."""
+    results = [
+        {"text": "hello world", "source": "a.py", "score": 0.5, "extra": "keep"},
+    ]
+    # This would normally call Ollama — skip if not available
+    try:
+        ranked = rerank("hello", results, top_k=1)
+        assert "extra" in ranked[0]
+        assert "rerank_score" in ranked[0]
+    except Exception:
+        pytest.skip("Ollama not available")

knowledge_master-0.5.0/tests/test_ts_parsers.py

@@ -0,0 +1,96 @@
+"""Tests for tree-sitter based language parsers."""
+import tempfile
+from pathlib import Path
+
+from knowledge_master.ts_parsers import (
+    extract_typescript_graph,
+    extract_go_graph,
+    extract_rust_graph,
+    resolve_ts_import,
+)
+
+
+def test_typescript_import_extraction():
+    with tempfile.NamedTemporaryFile(suffix=".ts", mode="w", delete=False) as f:
+        f.write('import { useState } from "react";\n')
+        f.write('import { helper } from "./utils";\n')
+        f.write("export function App() { return null; }\n")
+        f.flush()
+        result = extract_typescript_graph(f.name)
+
+    assert len(result["imports"]) == 2
+    assert result["imports"][0]["module"] == "react"
+    assert result["imports"][1]["module"] == "./utils"
+    assert len(result["exports"]) >= 1
+    assert any(e["name"] == "App" for e in result["exports"])
+
+
+def test_typescript_require_extraction():
+    with tempfile.NamedTemporaryFile(suffix=".js", mode="w", delete=False) as f:
+        f.write('const fs = require("fs");\n')
+        f.flush()
+        result = extract_typescript_graph(f.name)
+
+    assert len(result["imports"]) >= 1
+    assert result["imports"][0]["module"] == "fs"
+
+
+def test_go_import_extraction():
+    with tempfile.NamedTemporaryFile(suffix=".go", mode="w", delete=False) as f:
+        f.write('package main\n\nimport (\n\t"fmt"\n\t"os"\n)\n\nfunc Main() {}\n')
+        f.flush()
+        result = extract_go_graph(f.name)
+
+    assert len(result["imports"]) >= 2
+    modules = [i["module"] for i in result["imports"]]
+    assert "fmt" in modules
+    assert "os" in modules
+
+
+def test_go_exported_functions():
+    with tempfile.NamedTemporaryFile(suffix=".go", mode="w", delete=False) as f:
+        f.write('package pkg\n\nfunc PublicFunc() {}\nfunc privateFunc() {}\n')
+        f.flush()
+        result = extract_go_graph(f.name)
+
+    names = [e["name"] for e in result["exports"]]
+    assert "PublicFunc" in names
+    assert "privateFunc" not in names
+
+
+def test_rust_use_extraction():
+    with tempfile.NamedTemporaryFile(suffix=".rs", mode="w", delete=False) as f:
+        f.write("use std::collections::HashMap;\nuse crate::utils;\n\npub fn hello() {}\nfn private() {}\n")
+        f.flush()
+        result = extract_rust_graph(f.name)
+
+    assert len(result["imports"]) >= 2
+    modules = [i["module"] for i in result["imports"]]
+    assert "std" in modules
+    assert any(e["name"] == "hello" for e in result["exports"])
+    assert not any(e["name"] == "private" for e in result["exports"])
+
+
+def test_rust_mod_detection():
+    with tempfile.NamedTemporaryFile(suffix=".rs", mode="w", delete=False) as f:
+        f.write("mod parser;\nmod utils;\n")
+        f.flush()
+        result = extract_rust_graph(f.name)
+
+    mod_imports = [i for i in result["imports"] if i.get("is_mod")]
+    assert len(mod_imports) == 2
+    assert mod_imports[0]["module"] == "parser"
+
+
+def test_ts_import_resolution():
+    with tempfile.TemporaryDirectory() as tmp:
+        (Path(tmp) / "src").mkdir()
+        (Path(tmp) / "src" / "utils.ts").write_text("export const x = 1;")
+        (Path(tmp) / "src" / "index.ts").write_text("")
+
+        result = resolve_ts_import("./utils", "src/index.ts", tmp)
+        assert result == "src/utils.ts"
+
+        # External package — should return None
+        result = resolve_ts_import("react", "src/index.ts", tmp)
+        assert result is None