knowledge-master 0.3.0__tar.gz → 0.4.0__tar.gz

{knowledge_master-0.3.0 → knowledge_master-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: knowledge-master
-Version: 0.3.0
+Version: 0.4.0
 Summary: Local-first knowledge graph for developers. Your AI agent's permanent memory.
 Author: Milenko Mitrovic
 License: MIT

{knowledge_master-0.3.0 → knowledge_master-0.4.0}/knowledge_master/cli.py

@@ -460,5 +460,75 @@ def who_owns(file: str = typer.Argument(..., help="File path to check ownership"
         console.print("[dim]Run 'km index <repo>' first to extract ownership.[/]")
 
 
+@app.command()
+def upgrade():
+    """Upgrade graph schema to the latest version."""
+    from .migrations import check_and_migrate, get_schema_version, CURRENT_SCHEMA_VERSION
+
+    graph = store.get_graph()
+    current = get_schema_version(graph)
+    console.print(f"[bold]Current schema:[/] v{current}")
+    console.print(f"[bold]Target schema:[/]  v{CURRENT_SCHEMA_VERSION}")
+
+    if current == CURRENT_SCHEMA_VERSION:
+        console.print("[green]✓ Already up to date[/]")
+        return
+
+    result = check_and_migrate(graph, auto_migrate=True)
+    for step in result["steps"]:
+        console.print(f"  [green]✓[/] {step}")
+    console.print(f"\n[green]✓ Upgraded to v{CURRENT_SCHEMA_VERSION}[/]")
+
+
+@app.command()
+def prune(
+    older_than: int = typer.Option(30, help="Remove chunks not re-indexed in this many days"),
+    dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be removed"),
+):
+    """Remove stale/orphaned data from the knowledge graph."""
+    graph = store.get_graph()
+
+    # Find orphaned chunks (no PART_OF edge)
+    orphaned = graph.query("MATCH (c:Chunk) WHERE NOT (c)-[:PART_OF]->() RETURN count(c)")
+    orphan_count = orphaned.result_set[0][0] if orphaned.result_set else 0
+
+    # Find documents with no chunks
+    empty_docs = graph.query(
+        "MATCH (d:Document) WHERE NOT ()-[:PART_OF]->(d) AND NOT (d)-[:IN_REPO]->() RETURN count(d)"
+    )
+    empty_count = empty_docs.result_set[0][0] if empty_docs.result_set else 0
+
+    # Find stale chunks (indexed_at older than threshold)
+    # FalkorDB timestamp() returns ms since epoch
+    threshold_ms = older_than * 86400 * 1000
+    stale = graph.query(
+        """MATCH (c:Chunk)
+           WHERE c.indexed_at IS NOT NULL AND (timestamp() - c.indexed_at) > $threshold
+           RETURN count(c)""",
+        params={"threshold": threshold_ms},
+    )
+    stale_count = stale.result_set[0][0] if stale.result_set else 0
+
+    console.print("[bold]Prune analysis:[/]")
+    console.print(f"  Orphaned chunks (no document link): {orphan_count}")
+    console.print(f"  Empty documents (no chunks):        {empty_count}")
+    console.print(f"  Stale chunks (>{older_than} days):        {stale_count}")
+
+    if dry_run:
+        console.print("\n[yellow]Dry run — nothing removed.[/]")
+        return
+
+    total = 0
+    if orphan_count > 0:
+        graph.query("MATCH (c:Chunk) WHERE NOT (c)-[:PART_OF]->() DELETE c")
+        total += orphan_count
+
+    if empty_count > 0:
+        graph.query("MATCH (d:Document) WHERE NOT ()-[:PART_OF]->(d) AND NOT (d)-[:IN_REPO]->() DELETE d")
+        total += empty_count
+
+    console.print(f"\n[green]✓ Removed {total} stale nodes[/]")
+
+
 if __name__ == "__main__":
     app()

knowledge_master-0.4.0/knowledge_master/migrations.py

@@ -0,0 +1,89 @@
+"""Schema versioning and migrations for the knowledge graph."""
+
+CURRENT_SCHEMA_VERSION = 4  # v0.4.0
+
+# Migration definitions: version -> function that upgrades from previous version
+MIGRATIONS = {}
+
+
+def get_schema_version(graph) -> int:
+    """Get current schema version from graph metadata."""
+    try:
+        result = graph.query("MATCH (m:_Meta {key: 'schema_version'}) RETURN m.value")
+        if result.result_set:
+            return int(result.result_set[0][0])
+    except Exception:
+        pass
+    return 0  # no version = legacy graph
+
+
+def set_schema_version(graph, version: int):
+    """Store schema version in graph metadata."""
+    graph.query(
+        "MERGE (m:_Meta {key: 'schema_version'}) SET m.value = $version",
+        params={"version": version},
+    )
+
+
+def check_and_migrate(graph, auto_migrate: bool = True) -> dict:
+    """Check schema version and migrate if needed.
+
+    Returns: {"current": int, "target": int, "migrated": bool, "steps": list}
+    """
+    current = get_schema_version(graph)
+    target = CURRENT_SCHEMA_VERSION
+
+    if current == target:
+        return {"current": current, "target": target, "migrated": False, "steps": []}
+
+    if current > target:
+        raise RuntimeError(
+            f"Graph schema v{current} is newer than this version supports (v{target}). "
+            "Please upgrade knowledge-master."
+        )
+
+    if not auto_migrate:
+        raise RuntimeError(
+            f"Graph schema v{current} needs migration to v{target}. Run: km upgrade"
+        )
+
+    steps = []
+    for v in range(current + 1, target + 1):
+        migration_fn = MIGRATIONS.get(v)
+        if migration_fn:
+            migration_fn(graph)
+            steps.append(f"v{v-1} → v{v}: {migration_fn.__doc__ or 'applied'}")
+        else:
+            steps.append(f"v{v-1} → v{v}: no-op (compatible)")
+
+    set_schema_version(graph, target)
+    return {"current": current, "target": target, "migrated": True, "steps": steps}
+
+
+# --- Migrations ---
+
+def _migrate_to_v1(graph):
+    """Add indexed_at timestamp to existing chunks missing it."""
+    graph.query("MATCH (c:Chunk) WHERE c.indexed_at IS NULL SET c.indexed_at = timestamp()")
+
+
+def _migrate_to_v2(graph):
+    """Add OWNS relationships from ownership extraction."""
+    pass  # OWNS edges are created by extract_ownership, no schema change needed
+
+
+def _migrate_to_v3(graph):
+    """Add lang property to IMPORTS edges and Function nodes."""
+    graph.query("MATCH (f:Function) WHERE f.lang IS NULL SET f.lang = 'python'")
+    graph.query("MATCH ()-[e:IMPORTS]->() WHERE e.lang IS NULL SET e.lang = 'python'")
+
+
+def _migrate_to_v4(graph):
+    """Add content_hash to Chunk nodes for deduplication."""
+    pass  # New chunks will have hash, old ones are fine without
+
+
+MIGRATIONS[1] = _migrate_to_v1
+MIGRATIONS[2] = _migrate_to_v2
+MIGRATIONS[3] = _migrate_to_v3
+MIGRATIONS[4] = _migrate_to_v4

{knowledge_master-0.3.0 → knowledge_master-0.4.0}/knowledge_master/parsers/git_repo.py

@@ -46,6 +46,8 @@ def index_repo(repo_path: str, graph=None, branch: str = "HEAD", on_progress=Non
     tracked = repo.git.ls_files().splitlines()
     indexable = [f for f in tracked if _should_index(f)]
     total = len(indexable)
+    indexed_files = []
+    failed_files = []
 
     with Progress(disable=not sys.stdout.isatty()) as progress:
         task = progress.add_task(f"Indexing {repo_name}", total=total)
@@ -53,19 +55,26 @@ def index_repo(repo_path: str, graph=None, branch: str = "HEAD", on_progress=Non
             full_path = os.path.join(repo_path, filepath)
             try:
                 _index_file(graph, full_path, filepath, repo_name, repo)
+                indexed_files.append(filepath)
             except Exception as e:
+                failed_files.append((filepath, str(e)))
                 progress.console.print(f"  [yellow]skip {filepath}: {e}[/]")
             progress.advance(task)
             if on_progress:
                 on_progress(i + 1, total, filepath)
 
+    # If more than 50% failed, warn (possible systemic issue)
+    if total > 0 and len(failed_files) > total * 0.5:
+        import sys as _sys
+        print(f"WARNING: {len(failed_files)}/{total} files failed. Possible systemic issue.", file=_sys.stderr)
+
     # Run intelligence extraction
     intel = extract_all(repo_path, graph)
 
     # Run static analysis (import graph, symbols) — all languages
     static = build_import_graph_all(repo_path, graph)
 
-    return {"repo": repo_name, "files_indexed": total, "intelligence": intel, "static_analysis": static}
+    return {"repo": repo_name, "files_indexed": len(indexed_files), "files_failed": len(failed_files), "intelligence": intel, "static_analysis": static}
 
 
 def _should_index(filepath: str) -> bool:

{knowledge_master-0.3.0 → knowledge_master-0.4.0}/knowledge_master/store.py

@@ -1,5 +1,7 @@
 """FalkorDB graph store - nodes, edges, vector search, and graph traversal."""
 
+import hashlib
+
 from falkordb import FalkorDB
 
 GRAPH_NAME = "knowledge"
@@ -7,11 +9,35 @@ GRAPH_NAME = "knowledge"
 # Vector dimension for nomic-embed-text
 VECTOR_DIM = 768
 
+_graph_instance = None
+
 
 def get_graph(host: str = "localhost", port: int = 6379):
-    """Get FalkorDB graph instance."""
+    """Get FalkorDB graph instance with schema version check."""
+    global _graph_instance
+    if _graph_instance is not None:
+        return _graph_instance
+
     db = FalkorDB(host=host, port=port)
-    return db.select_graph(GRAPH_NAME)
+    graph = db.select_graph(GRAPH_NAME)
+
+    # Check and auto-migrate schema
+    from .migrations import check_and_migrate
+    check_and_migrate(graph, auto_migrate=True)
+
+    _graph_instance = graph
+    return graph
+
+
+def reset_graph_instance():
+    """Reset cached graph instance (for testing)."""
+    global _graph_instance
+    _graph_instance = None
+
+
+def content_hash(text: str) -> str:
+    """Compute content hash for deduplication."""
+    return hashlib.sha256(text.encode()).hexdigest()[:16]
 
 
 def init_schema(graph):
@@ -35,20 +61,32 @@ def init_schema(graph):
 
 
 def upsert_chunk(graph, chunk_id: str, text: str, embedding: list[float], metadata: dict):
-    """Insert or update a chunk node with embedding."""
+    """Insert or update a chunk node with embedding. Skips if content unchanged (dedup)."""
+    chash = content_hash(text)
+
+    # Check if chunk exists with same content hash — skip if unchanged
+    existing = graph.query(
+        "MATCH (c:Chunk {id: $id}) RETURN c.content_hash",
+        params={"id": chunk_id},
+    )
+    if existing.result_set and existing.result_set[0][0] == chash:
+        return False  # skip — content unchanged
+
     graph.query(
         """MERGE (c:Chunk {id: $id})
            SET c.text = $text, c.embedding = vecf32($embedding),
                c.source = $source, c.source_type = $source_type,
-               c.indexed_at = timestamp()""",
+               c.content_hash = $hash, c.indexed_at = timestamp()""",
         params={
             "id": chunk_id,
             "text": text,
             "embedding": embedding,
             "source": metadata.get("source", ""),
             "source_type": metadata.get("source_type", ""),
+            "hash": chash,
         },
     )
+    return True  # inserted/updated
 
 
 def upsert_document(graph, path: str, doc_type: str, metadata: dict):

{knowledge_master-0.3.0 → knowledge_master-0.4.0}/knowledge_master.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: knowledge-master
-Version: 0.3.0
+Version: 0.4.0
 Summary: Local-first knowledge graph for developers. Your AI agent's permanent memory.
 Author: Milenko Mitrovic
 License: MIT

{knowledge_master-0.3.0 → knowledge_master-0.4.0}/knowledge_master.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ knowledge_master/cli.py
 knowledge_master/connectors.py
 knowledge_master/embeddings.py
 knowledge_master/intelligence.py
+knowledge_master/migrations.py
 knowledge_master/rerank.py
 knowledge_master/server.py
 knowledge_master/static_analysis.py

{knowledge_master-0.3.0 → knowledge_master-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "knowledge-master"
-version = "0.3.0"
+version = "0.4.0"
 description = "Local-first knowledge graph for developers. Your AI agent's permanent memory."
 requires-python = ">=3.11"
 license = {text = "MIT"}