knowledge-master 0.1.0__py3-none-any.whl

knowledge_master/store.py

@@ -0,0 +1,164 @@
+"""FalkorDB graph store - nodes, edges, vector search, and graph traversal."""
+
+from falkordb import FalkorDB
+
+GRAPH_NAME = "knowledge"
+
+# Vector dimension for nomic-embed-text
+VECTOR_DIM = 768
+
+
+def get_graph(host: str = "localhost", port: int = 6379):
+    """Get FalkorDB graph instance."""
+    db = FalkorDB(host=host, port=port)
+    return db.select_graph(GRAPH_NAME)
+
+
+def init_schema(graph):
+    """Create indexes and constraints."""
+    commands = [
+        # Vector indexes
+        "CREATE VECTOR INDEX FOR (c:Chunk) ON (c.embedding) OPTIONS {dimension: 768, similarityFunction: 'cosine'}",
+        # Fulltext indexes
+        "CREATE FULLTEXT INDEX FOR (c:Chunk) ON (c.text)",
+        # Regular indexes for lookups
+        "CREATE INDEX FOR (d:Document) ON (d.path)",
+        "CREATE INDEX FOR (r:Repo) ON (r.name)",
+        "CREATE INDEX FOR (p:Person) ON (p.email)",
+        "CREATE INDEX FOR (f:File) ON (f.path)",
+    ]
+    for cmd in commands:
+        try:
+            graph.query(cmd)
+        except Exception:
+            pass  # index may already exist
+
+
+def upsert_chunk(graph, chunk_id: str, text: str, embedding: list[float], metadata: dict):
+    """Insert or update a chunk node with embedding."""
+    graph.query(
+        """MERGE (c:Chunk {id: $id})
+           SET c.text = $text, c.embedding = vecf32($embedding),
+               c.source = $source, c.source_type = $source_type,
+               c.indexed_at = timestamp()""",
+        params={
+            "id": chunk_id,
+            "text": text,
+            "embedding": embedding,
+            "source": metadata.get("source", ""),
+            "source_type": metadata.get("source_type", ""),
+        },
+    )
+
+
+def upsert_document(graph, path: str, doc_type: str, metadata: dict):
+    """Insert or update a document node."""
+    graph.query(
+        """MERGE (d:Document {path: $path})
+           SET d.type = $type, d.title = $title, d.indexed_at = timestamp()""",
+        params={"path": path, "type": doc_type, "title": metadata.get("title", "")},
+    )
+
+
+def upsert_repo(graph, name: str, path: str):
+    """Insert or update a repo node."""
+    graph.query(
+        "MERGE (r:Repo {name: $name}) SET r.path = $path",
+        params={"name": name, "path": path},
+    )
+
+
+def upsert_person(graph, name: str, email: str):
+    """Insert or update a person node."""
+    graph.query(
+        "MERGE (p:Person {email: $email}) SET p.name = $name",
+        params={"name": name, "email": email},
+    )
+
+
+def link_chunk_to_document(graph, chunk_id: str, doc_path: str):
+    """Create PART_OF edge from chunk to document."""
+    graph.query(
+        """MATCH (c:Chunk {id: $chunk_id}), (d:Document {path: $doc_path})
+           MERGE (c)-[:PART_OF]->(d)""",
+        params={"chunk_id": chunk_id, "doc_path": doc_path},
+    )
+
+
+def link_document_to_repo(graph, doc_path: str, repo_name: str):
+    """Create IN_REPO edge."""
+    graph.query(
+        """MATCH (d:Document {path: $doc_path}), (r:Repo {name: $repo_name})
+           MERGE (d)-[:IN_REPO]->(r)""",
+        params={"doc_path": doc_path, "repo_name": repo_name},
+    )
+
+
+def link_person_authored(graph, email: str, doc_path: str):
+    """Create AUTHORED edge."""
+    graph.query(
+        """MATCH (p:Person {email: $email}), (d:Document {path: $doc_path})
+           MERGE (p)-[:AUTHORED]->(d)""",
+        params={"email": email, "doc_path": doc_path},
+    )
+
+
+def vector_search(graph, query_embedding: list[float], top_k: int = 10, filters: dict = None):
+    """Semantic vector search across chunks."""
+    filter_clause = ""
+    params = {"embedding": query_embedding, "top_k": top_k}
+
+    if filters and filters.get("source_type"):
+        filter_clause = "WHERE c.source_type = $source_type"
+        params["source_type"] = filters["source_type"]
+
+    result = graph.query(
+        f"""CALL db.idx.vector.queryNodes('Chunk', 'embedding', $top_k, vecf32($embedding))
+            YIELD node AS c, score
+            {filter_clause}
+            RETURN c.id AS id, c.text AS text, c.source AS source,
+                   c.source_type AS source_type, score
+            ORDER BY score DESC""",
+        params=params,
+    )
+    return [
+        {"id": r[0], "text": r[1], "source": r[2], "source_type": r[3], "score": r[4]}
+        for r in result.result_set
+    ]
+
+
+def graph_context_search(graph, query_embedding: list[float], top_k: int = 5):
+    """Hybrid search: vector find + graph traversal for related context."""
+    result = graph.query(
+        """CALL db.idx.vector.queryNodes('Chunk', 'embedding', $top_k, vecf32($embedding))
+           YIELD node AS c, score
+           OPTIONAL MATCH (c)-[:PART_OF]->(d:Document)-[:IN_REPO]->(r:Repo)
+           OPTIONAL MATCH (p:Person)-[:AUTHORED]->(d)
+           RETURN c.text AS text, c.source AS source, score,
+                  d.path AS doc_path, r.name AS repo, p.name AS author
+           ORDER BY score DESC""",
+        params={"embedding": query_embedding, "top_k": top_k},
+    )
+    return [
+        {
+            "text": r[0],
+            "source": r[1],
+            "score": r[2],
+            "doc_path": r[3],
+            "repo": r[4],
+            "author": r[5],
+        }
+        for r in result.result_set
+    ]
+
+
+def get_stats(graph):
+    """Get graph statistics."""
+    result = graph.query(
+        """MATCH (c:Chunk) WITH count(c) AS chunks
+           MATCH (d:Document) WITH chunks, count(d) AS docs
+           MATCH (r:Repo) WITH chunks, docs, count(r) AS repos
+           RETURN chunks, docs, repos"""
+    )
+    row = result.result_set[0] if result.result_set else [0, 0, 0]
+    return {"chunks": row[0], "documents": row[1], "repos": row[2]}

knowledge_master/watcher.py

@@ -0,0 +1,104 @@
+"""Incremental indexing — git hook and file watcher support."""
+
+import os
+import sys
+from pathlib import Path
+
+from . import store
+from .parsers.git_repo import _index_file, _should_index, INDEXABLE_EXTENSIONS
+
+from git import Repo
+
+
+def index_changed_files(repo_path: str, since: str = "HEAD~1"):
+    """Index only files that changed since a given ref."""
+    repo_path = str(Path(repo_path).expanduser().resolve())
+    repo = Repo(repo_path)
+    repo_name = Path(repo_path).name
+    graph = store.get_graph()
+    store.init_schema(graph)
+    store.upsert_repo(graph, repo_name, repo_path)
+
+    # Get changed files
+    try:
+        diff_output = repo.git.diff("--name-only", since)
+        changed = [f for f in diff_output.splitlines() if _should_index(f)]
+    except Exception:
+        changed = []
+
+    indexed = 0
+    for filepath in changed:
+        full_path = os.path.join(repo_path, filepath)
+        if os.path.exists(full_path):
+            try:
+                _index_file(graph, full_path, filepath, repo_name, repo)
+                indexed += 1
+            except Exception:
+                pass
+        else:
+            # File was deleted — remove its chunks
+            graph.query(
+                """MATCH (d:Document {path: $path})
+                   OPTIONAL MATCH (c:Chunk)-[:PART_OF]->(d)
+                   DELETE c, d""",
+                params={"path": filepath},
+            )
+
+    return {"repo": repo_name, "changed": len(changed), "indexed": indexed}
+
+
+def install_git_hook(repo_path: str):
+    """Install a post-commit git hook that triggers incremental indexing."""
+    repo_path = str(Path(repo_path).expanduser().resolve())
+    hook_dir = Path(repo_path) / ".git" / "hooks"
+    hook_file = hook_dir / "post-commit"
+
+    # Find km executable
+    km_bin = Path(sys.executable).parent / "km"
+    if not km_bin.exists():
+        km_bin = f"{sys.executable} -m knowledge_master.cli"
+
+    hook_content = f"""#!/bin/sh
+# Knowledge Master — auto-index on commit
+{km_bin} index {repo_path} --type repo 2>/dev/null &
+"""
+
+    hook_file.write_text(hook_content)
+    hook_file.chmod(0o755)
+    return str(hook_file)
+
+
+def watch_directory(path: str, callback=None):
+    """Watch a directory for changes and re-index (uses polling for cross-platform)."""
+    import time
+    path = str(Path(path).expanduser().resolve())
+    last_mtimes = {}
+
+    def scan():
+        current = {}
+        for ext in INDEXABLE_EXTENSIONS:
+            for f in Path(path).rglob(f"*{ext}"):
+                if ".git" in f.parts or ".venv" in f.parts:
+                    continue
+                current[str(f)] = f.stat().st_mtime
+        return current
+
+    last_mtimes = scan()
+    print(f"Watching {path} for changes... (Ctrl+C to stop)")
+
+    while True:
+        time.sleep(2)
+        current = scan()
+        changed = [f for f, mtime in current.items() if last_mtimes.get(f) != mtime]
+        deleted = [f for f in last_mtimes if f not in current]
+
+        if changed or deleted:
+            if callback:
+                callback(changed, deleted)
+            else:
+                print(f"  Changed: {len(changed)}, Deleted: {len(deleted)}")
+                if (Path(path) / ".git").exists():
+                    result = index_changed_files(path, "HEAD~1")
+                    print(f"  Indexed: {result}")
+
+        last_mtimes = current