knowledge-master 0.1.0__py3-none-any.whl

knowledge_master/intelligence.py

@@ -0,0 +1,254 @@
+"""Intelligence extraction — detect tech stack, patterns, conventions, service topology."""
+
+import json
+import os
+import re
+from pathlib import Path
+
+
+def extract_tech_stack(repo_path: str, graph):
+    """Detect technologies from dependency files and create Tech nodes + relationships."""
+    repo_name = Path(repo_path).name
+    techs = set()
+
+    # Python
+    req = _read(repo_path, "requirements.txt")
+    if req:
+        techs.add(("Python", "language"))
+        for line in req.splitlines():
+            pkg = re.split(r"[=<>!~]", line.strip())[0].strip()
+            if pkg and not pkg.startswith("#"):
+                techs.add((pkg, "python-package"))
+
+    pyproject = _read(repo_path, "pyproject.toml")
+    if pyproject:
+        techs.add(("Python", "language"))
+        for m in re.findall(r'"([a-zA-Z0-9_-]+)(?:[=<>!~]|$)', pyproject):
+            techs.add((m, "python-package"))
+
+    # Node.js
+    pkg_json = _read(repo_path, "package.json")
+    if pkg_json:
+        techs.add(("Node.js", "runtime"))
+        try:
+            pkg = json.loads(pkg_json)
+            for dep in list(pkg.get("dependencies", {})) + list(pkg.get("devDependencies", {})):
+                techs.add((dep, "npm-package"))
+        except json.JSONDecodeError:
+            pass
+
+    # Rust
+    cargo = _read(repo_path, "Cargo.toml")
+    if cargo:
+        techs.add(("Rust", "language"))
+        for m in re.findall(r'^\s*([a-zA-Z0-9_-]+)\s*=', cargo, re.MULTILINE):
+            if m not in ("name", "version", "edition", "authors", "description", "license", "repository"):
+                techs.add((m, "rust-crate"))
+
+    # Go
+    gomod = _read(repo_path, "go.mod")
+    if gomod:
+        techs.add(("Go", "language"))
+        for m in re.findall(r'^\s+([\w./-]+)', gomod, re.MULTILINE):
+            techs.add((m.split("/")[-1], "go-module"))
+
+    # Docker
+    if _exists(repo_path, "Dockerfile") or _exists(repo_path, "docker-compose.yml"):
+        techs.add(("Docker", "infrastructure"))
+
+    # Kubernetes
+    for f in Path(repo_path).rglob("*.yaml"):
+        content = f.read_text(errors="ignore")[:500]
+        if "apiVersion:" in content and "kind:" in content:
+            techs.add(("Kubernetes", "infrastructure"))
+            break
+
+    # Terraform
+    if any(Path(repo_path).rglob("*.tf")):
+        techs.add(("Terraform", "infrastructure"))
+
+    # Store in graph
+    for name, category in techs:
+        graph.query(
+            "MERGE (t:Tech {name: $name}) SET t.category = $cat",
+            params={"name": name, "cat": category},
+        )
+        graph.query(
+            """MATCH (r:Repo {name: $repo}), (t:Tech {name: $tech})
+               MERGE (r)-[:USES_TECH]->(t)""",
+            params={"repo": repo_name, "tech": name},
+        )
+
+    return list(techs)
+
+
+def extract_services(repo_path: str, graph):
+    """Parse docker-compose and K8s manifests to build service dependency graph."""
+    repo_name = Path(repo_path).name
+    services = []
+
+    # Docker Compose
+    for compose_file in ("docker-compose.yml", "docker-compose.yaml", "compose.yml"):
+        content = _read(repo_path, compose_file)
+        if not content:
+            continue
+        # Simple YAML parsing for services (avoid PyYAML dep)
+        in_services = False
+        current_svc = None
+        svc_indent = None
+        for line in content.splitlines():
+            if not line.strip() or line.strip().startswith("#"):
+                continue
+            indent = len(line) - len(line.lstrip())
+            stripped = line.strip()
+            if stripped == "services:":
+                in_services = True
+                svc_indent = indent + 2
+                continue
+            if in_services:
+                if indent <= indent - 2 and stripped and indent == 0 and not stripped.startswith("#"):
+                    break
+                if indent == svc_indent and stripped.endswith(":") and not stripped.startswith("-"):
+                    current_svc = stripped.rstrip(": ")
+                    if not current_svc.startswith('"') and not current_svc.startswith("'"):
+                        services.append(current_svc)
+                        graph.query(
+                            "MERGE (s:Service {name: $name}) SET s.source = 'docker-compose'",
+                            params={"name": current_svc},
+                        )
+                        graph.query(
+                            """MATCH (r:Repo {name: $repo}), (s:Service {name: $svc})
+                               MERGE (r)-[:DEFINES_SERVICE]->(s)""",
+                            params={"repo": repo_name, "svc": current_svc},
+                        )
+                # Detect depends_on
+                if current_svc and "depends_on" in stripped:
+                    pass  # next lines will have deps
+                if current_svc and stripped.startswith("- ") and indent > svc_indent + 2:
+                    dep = stripped.lstrip("- ").strip().rstrip(":")
+                    if dep in services or dep:
+                        graph.query(
+                            "MERGE (s:Service {name: $name})",
+                            params={"name": dep},
+                        )
+                        graph.query(
+                            """MATCH (a:Service {name: $svc}), (b:Service {name: $dep})
+                               MERGE (a)-[:DEPENDS_ON]->(b)""",
+                            params={"svc": current_svc, "dep": dep},
+                        )
+
+    # K8s Deployments/Services
+    for yaml_file in Path(repo_path).rglob("*.yaml"):
+        content = yaml_file.read_text(errors="ignore")[:2000]
+        if "kind: Deployment" in content or "kind: StatefulSet" in content:
+            name_match = re.search(r'name:\s*(\S+)', content)
+            if name_match:
+                svc_name = name_match.group(1)
+                services.append(svc_name)
+                graph.query(
+                    "MERGE (s:Service {name: $name}) SET s.source = 'kubernetes'",
+                    params={"name": svc_name},
+                )
+                graph.query(
+                    """MATCH (r:Repo {name: $repo}), (s:Service {name: $svc})
+                       MERGE (r)-[:DEFINES_SERVICE]->(s)""",
+                    params={"repo": repo_name, "svc": svc_name},
+                )
+
+    return services
+
+
+def extract_conventions(repo_path: str, graph):
+    """Analyze naming patterns, folder structure, and coding conventions."""
+    repo_name = Path(repo_path).name
+    conventions = []
+
+    # File naming convention
+    files = [f.name for f in Path(repo_path).rglob("*") if f.is_file() and not any(
+        p in f.parts for p in (".git", "node_modules", "__pycache__", ".venv", "target")
+    )]
+
+    code_files = [f for f in files if Path(f).suffix in (".py", ".ts", ".js", ".rs", ".go")]
+    if code_files:
+        snake = sum(1 for f in code_files if "_" in Path(f).stem)
+        kebab = sum(1 for f in code_files if "-" in Path(f).stem)
+        camel = sum(1 for f in code_files if re.match(r'^[a-z]+[A-Z]', Path(f).stem))
+        pascal = sum(1 for f in code_files if re.match(r'^[A-Z][a-z]+[A-Z]', Path(f).stem))
+        total = len(code_files)
+
+        if snake / total > 0.5:
+            conventions.append(("snake_case files", "file-naming"))
+        elif kebab / total > 0.3:
+            conventions.append(("kebab-case files", "file-naming"))
+        elif pascal / total > 0.3:
+            conventions.append(("PascalCase files", "file-naming"))
+        elif camel / total > 0.3:
+            conventions.append(("camelCase files", "file-naming"))
+
+    # Folder structure patterns
+    dirs = set()
+    for f in Path(repo_path).iterdir():
+        if f.is_dir() and not f.name.startswith("."):
+            dirs.add(f.name)
+
+    if "src" in dirs:
+        conventions.append(("src/ directory", "structure"))
+    if "lib" in dirs:
+        conventions.append(("lib/ directory", "structure"))
+    if "tests" in dirs or "test" in dirs:
+        conventions.append(("separate test directory", "testing"))
+    if "docs" in dirs:
+        conventions.append(("docs/ directory", "documentation"))
+    if "infra" in dirs or "deploy" in dirs or "k8s" in dirs:
+        conventions.append(("infra as code", "infrastructure"))
+
+    # Detect patterns from code
+    for py_file in list(Path(repo_path).rglob("*.py"))[:50]:
+        content = py_file.read_text(errors="ignore")[:3000]
+        if "class" in content and "Repository" in content:
+            conventions.append(("Repository pattern", "design-pattern"))
+        if "@app.route" in content or "@router" in content:
+            conventions.append(("Route decorators", "design-pattern"))
+        if "class" in content and ("Mixin" in content or "Base" in content):
+            conventions.append(("Mixin/Base classes", "design-pattern"))
+
+    # Deduplicate
+    conventions = list(set(conventions))
+
+    # Store in graph
+    for name, category in conventions:
+        graph.query(
+            "MERGE (c:Convention {name: $name}) SET c.category = $cat",
+            params={"name": name, "cat": category},
+        )
+        graph.query(
+            """MATCH (r:Repo {name: $repo}), (c:Convention {name: $conv})
+               MERGE (r)-[:FOLLOWS]->(c)""",
+            params={"repo": repo_name, "conv": name},
+        )
+
+    return conventions
+
+
+def extract_all(repo_path: str, graph):
+    """Run all extraction passes on a repo."""
+    techs = extract_tech_stack(repo_path, graph)
+    services = extract_services(repo_path, graph)
+    conventions = extract_conventions(repo_path, graph)
+    return {
+        "techs": len(techs),
+        "services": len(services),
+        "conventions": len(conventions),
+    }
+
+
+def _read(repo_path: str, filename: str) -> str | None:
+    p = os.path.join(repo_path, filename)
+    if os.path.exists(p):
+        with open(p, "r", errors="ignore") as f:
+            return f.read()
+    return None
+
+
+def _exists(repo_path: str, filename: str) -> bool:
+    return os.path.exists(os.path.join(repo_path, filename))

knowledge_master/parsers/git_repo.py

@@ -0,0 +1,115 @@
+"""Git repository parser - indexes files and extracts graph relationships."""
+
+import os
+import sys
+from pathlib import Path
+
+from git import Repo
+from rich.progress import Progress
+
+from .. import chunking, embeddings, store
+from ..intelligence import extract_all
+
+INDEXABLE_EXTENSIONS = {
+    ".py", ".ts", ".tsx", ".js", ".rs", ".go", ".java",
+    ".md", ".markdown", ".txt", ".yaml", ".yml", ".json",
+    ".toml", ".cfg", ".ini", ".sh", ".bash", ".dockerfile",
+}
+
+SKIP_DIRS = {".git", "node_modules", "target", "dist", "build", "__pycache__", ".venv", "venv"}
+
+MAX_FILE_SIZE = 512_000  # 500KB
+
+
+def index_repo(repo_path: str, graph=None, branch: str = "HEAD", on_progress=None):
+    """Index an entire git repository into the knowledge graph."""
+    repo_path = os.path.expanduser(repo_path)
+    repo = Repo(repo_path)
+    repo_name = Path(repo_path).name
+
+    if graph is None:
+        graph = store.get_graph()
+
+    store.init_schema(graph)
+    store.upsert_repo(graph, repo_name, repo_path)
+
+    # Extract authors from recent commits
+    authors = set()
+    for commit in repo.iter_commits(branch, max_count=100):
+        authors.add((commit.author.name, commit.author.email))
+
+    for name, email in authors:
+        store.upsert_person(graph, name, email)
+
+    # Get tracked files
+    tracked = repo.git.ls_files().splitlines()
+    indexable = [f for f in tracked if _should_index(f)]
+    total = len(indexable)
+
+    with Progress(disable=not sys.stdout.isatty()) as progress:
+        task = progress.add_task(f"Indexing {repo_name}", total=total)
+        for i, filepath in enumerate(indexable):
+            full_path = os.path.join(repo_path, filepath)
+            try:
+                _index_file(graph, full_path, filepath, repo_name, repo)
+            except Exception as e:
+                progress.console.print(f"  [yellow]skip {filepath}: {e}[/]")
+            progress.advance(task)
+            if on_progress:
+                on_progress(i + 1, total, filepath)
+
+    # Run intelligence extraction
+    intel = extract_all(repo_path, graph)
+
+    return {"repo": repo_name, "files_indexed": total, "intelligence": intel}
+
+
+def _should_index(filepath: str) -> bool:
+    """Check if file should be indexed."""
+    ext = Path(filepath).suffix.lower()
+    parts = Path(filepath).parts
+    if any(d in SKIP_DIRS for d in parts):
+        return False
+    if ext not in INDEXABLE_EXTENSIONS:
+        return False
+    return True
+
+
+def _index_file(graph, full_path: str, relative_path: str, repo_name: str, repo: Repo):
+    """Index a single file: chunk, embed, store with relationships."""
+    if os.path.getsize(full_path) > MAX_FILE_SIZE:
+        return
+
+    with open(full_path, "r", errors="ignore") as f:
+        content = f.read()
+
+    if not content.strip():
+        return
+
+    ext = Path(full_path).suffix.lower()
+    chunks = chunking.chunk_file(content, ext)
+    if not chunks:
+        return
+
+    # Embed all chunks
+    vectors = embeddings.embed_batch(chunks)
+
+    # Store document node
+    store.upsert_document(graph, relative_path, ext.lstrip("."), {"title": relative_path})
+    store.link_document_to_repo(graph, relative_path, repo_name)
+
+    # Get last author for this file
+    try:
+        last_commit = next(repo.iter_commits(paths=relative_path, max_count=1))
+        store.link_person_authored(graph, last_commit.author.email, relative_path)
+    except StopIteration:
+        pass
+
+    # Store chunks with embeddings
+    for i, (chunk_text, vector) in enumerate(zip(chunks, vectors)):
+        cid = chunking.chunk_id(relative_path, i)
+        store.upsert_chunk(
+            graph, cid, chunk_text, vector,
+            {"source": relative_path, "source_type": "code" if ext != ".md" else "docs"},
+        )
+        store.link_chunk_to_document(graph, cid, relative_path)

knowledge_master/parsers/markdown.py

@@ -0,0 +1,58 @@
+"""Markdown file/directory parser."""
+
+import os
+import sys
+from pathlib import Path
+
+from rich.progress import Progress
+
+from .. import chunking, embeddings, store
+
+
+def index_directory(directory: str, graph=None, patterns: list[str] = None):
+    """Index markdown files from a directory."""
+    directory = os.path.expanduser(directory)
+    if graph is None:
+        graph = store.get_graph()
+
+    store.init_schema(graph)
+    patterns = patterns or ["*.md", "*.markdown", "*.txt"]
+
+    files = []
+    for pattern in patterns:
+        files.extend(Path(directory).rglob(pattern))
+
+    with Progress(disable=not sys.stdout.isatty()) as progress:
+        task = progress.add_task(f"Indexing {directory}", total=len(files))
+        for filepath in files:
+            try:
+                _index_markdown(graph, str(filepath), directory)
+            except Exception as e:
+                progress.console.print(f"  [yellow]skip {filepath}: {e}[/]")
+            progress.advance(task)
+
+    return {"directory": directory, "files_indexed": len(files)}
+
+
+def _index_markdown(graph, filepath: str, base_dir: str):
+    """Index a single markdown file."""
+    with open(filepath, "r", errors="ignore") as f:
+        content = f.read()
+
+    if not content.strip():
+        return
+
+    relative = os.path.relpath(filepath, base_dir)
+    chunks = chunking.chunk_markdown(content)
+    if not chunks:
+        return
+
+    vectors = embeddings.embed_batch(chunks)
+    store.upsert_document(graph, relative, "markdown", {"title": relative})
+
+    for i, (text, vector) in enumerate(zip(chunks, vectors)):
+        cid = chunking.chunk_id(relative, i)
+        store.upsert_chunk(
+            graph, cid, text, vector, {"source": relative, "source_type": "docs"}
+        )
+        store.link_chunk_to_document(graph, cid, relative)

knowledge_master/server.py

@@ -0,0 +1,194 @@
+"""MCP Server exposing knowledge base tools for AI agents."""
+
+import json
+
+from mcp.server import Server
+from mcp.server.stdio import stdio_server
+from mcp.types import TextContent, Tool
+
+from . import embeddings, store
+from .parsers import git_repo, markdown
+
+server = Server("knowledge-master")
+
+
+@server.list_tools()
+async def list_tools() -> list[Tool]:
+    return [
+        Tool(
+            name="search",
+            description="Semantic search across the knowledge base. Returns relevant chunks with source info and graph context (author, repo, related docs).",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "query": {"type": "string", "description": "Natural language search query"},
+                    "top_k": {"type": "integer", "default": 10, "description": "Number of results"},
+                    "source_type": {"type": "string", "enum": ["code", "docs", "email", "infra"], "description": "Filter by source type"},
+                    "use_graph": {"type": "boolean", "default": True, "description": "Include graph context (author, repo relationships)"},
+                },
+                "required": ["query"],
+            },
+        ),
+        Tool(
+            name="index_repo",
+            description="Index a git repository into the knowledge graph. Parses code files, extracts authors, builds relationships.",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "path": {"type": "string", "description": "Path to git repository"},
+                    "branch": {"type": "string", "default": "HEAD", "description": "Branch to index"},
+                },
+                "required": ["path"],
+            },
+        ),
+        Tool(
+            name="index_directory",
+            description="Index markdown/text files from a directory.",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "path": {"type": "string", "description": "Directory path to index"},
+                    "patterns": {"type": "array", "items": {"type": "string"}, "description": "Glob patterns (default: *.md, *.txt)"},
+                },
+                "required": ["path"],
+            },
+        ),
+        Tool(
+            name="get_status",
+            description="Get knowledge base statistics: number of chunks, documents, repos indexed.",
+            inputSchema={"type": "object", "properties": {}},
+        ),
+        Tool(
+            name="blast_radius",
+            description="Show what depends on a target (service, tech, or file). Returns all entities that would be affected by changing the target.",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "target": {"type": "string", "description": "Service name, technology, or file to check dependencies for"},
+                },
+                "required": ["target"],
+            },
+        ),
+        Tool(
+            name="check_conventions",
+            description="Check if a repo or path follows the detected coding conventions (naming, structure, patterns). Returns pass/fail for each convention.",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "path": {"type": "string", "description": "Path to repo or directory to check"},
+                },
+                "required": ["path"],
+            },
+        ),
+    ]
+
+
+@server.call_tool()
+async def call_tool(name: str, arguments: dict) -> list[TextContent]:
+    graph = store.get_graph()
+
+    if name == "search":
+        query = arguments["query"]
+        top_k = arguments.get("top_k", 10)
+        use_graph = arguments.get("use_graph", True)
+        filters = {}
+        if arguments.get("source_type"):
+            filters["source_type"] = arguments["source_type"]
+
+        query_vector = embeddings.embed(query)
+
+        if use_graph:
+            results = store.graph_context_search(graph, query_vector, top_k)
+        else:
+            results = store.vector_search(graph, query_vector, top_k, filters)
+
+        return [TextContent(type="text", text=json.dumps(results, indent=2, default=str))]
+
+    elif name == "index_repo":
+        result = git_repo.index_repo(arguments["path"], graph, arguments.get("branch", "HEAD"))
+        return [TextContent(type="text", text=json.dumps(result))]
+
+    elif name == "index_directory":
+        result = markdown.index_directory(
+            arguments["path"], graph, arguments.get("patterns")
+        )
+        return [TextContent(type="text", text=json.dumps(result))]
+
+    elif name == "get_status":
+        stats = store.get_stats(graph)
+        return [TextContent(type="text", text=json.dumps(stats))]
+
+    elif name == "blast_radius":
+        target = arguments["target"]
+        # Try Service
+        result = graph.query(
+            """MATCH (t:Service {name: $name})
+               OPTIONAL MATCH (other)-[*1..3]->(t)
+               WHERE other <> t
+               RETURN labels(other)[0] AS type, other.name AS name, type(last(relationships(path))) AS rel""",
+            params={"name": target},
+        )
+        if not result.result_set or all(r[1] is None for r in result.result_set):
+            # Try Tech
+            result = graph.query(
+                """MATCH (t:Tech {name: $name})
+                   OPTIONAL MATCH (r:Repo)-[:USES_TECH]->(t)
+                   RETURN 'Repo' AS type, r.name AS name, 'USES_TECH' AS rel""",
+                params={"name": target},
+            )
+        affected = [{"type": r[0], "name": r[1], "relationship": r[2]}
+                    for r in (result.result_set or []) if r[1]]
+        output = {"target": target, "affected_count": len(affected), "affected": affected}
+        return [TextContent(type="text", text=json.dumps(output, indent=2))]
+
+    elif name == "check_conventions":
+        from pathlib import Path as P
+        path = str(P(arguments["path"]).expanduser().resolve())
+        repo_name = P(path).name
+        result = graph.query(
+            """MATCH (r:Repo)-[:FOLLOWS]->(c:Convention)
+               WHERE r.name = $name
+               RETURN c.name, c.category""",
+            params={"name": repo_name},
+        )
+        if not result.result_set:
+            result = graph.query("MATCH (c:Convention) RETURN c.name, c.category")
+
+        checks = []
+        for conv_name, category in (result.result_set or []):
+            passed = _check_convention_simple(path, conv_name)
+            checks.append({"convention": conv_name, "category": category, "passed": passed})
+
+        return [TextContent(type="text", text=json.dumps({"path": path, "checks": checks}, indent=2))]
+
+    return [TextContent(type="text", text=f"Unknown tool: {name}")]
+
+
+def _check_convention_simple(path: str, convention: str) -> bool:
+    """Check a single convention."""
+    from pathlib import Path as P
+    p = P(path)
+    if convention == "src/ directory":
+        return (p / "src").is_dir()
+    elif convention == "separate test directory":
+        return (p / "tests").is_dir() or (p / "test").is_dir()
+    elif convention == "docs/ directory":
+        return (p / "docs").is_dir()
+    elif convention == "snake_case files":
+        files = [f for f in p.rglob("*.py") if ".venv" not in str(f)]
+        return not any("-" in f.stem for f in files)
+    return True
+
+
+async def run():
+    async with stdio_server() as (read_stream, write_stream):
+        await server.run(read_stream, write_stream, server.create_initialization_options())
+
+
+def main():
+    import asyncio
+    asyncio.run(run())
+
+
+if __name__ == "__main__":
+    main()