contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/ingestion/universal/operators/scanners/llm_enrichment.py

@@ -0,0 +1,82 @@
+"""LLM enrichment — deep code summaries for CodeModule nodes."""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any, Dict, List
+
+from .helpers import _safe_read
+
+logger = logging.getLogger(__name__)
+
+_MAX_LLM_FILES = 15
+
+_LLM_PROMPT = """\
+Summarize this source code module in 2-3 sentences:
+1. What it does (purpose)
+2. Key dependencies it imports
+3. What could break (risks)
+
+File: {path}
+
+```
+{code}
+```
+
+Reply with ONLY the summary, no markdown headings."""
+
+
+def get_llm_client():
+    """Import and return the LLM client. Raises if unavailable."""
+    from contextsynapse.llm.client import get_llm_client as _get
+    return _get()
+
+
+def scan_with_llm(repo_path: Path, code_nodes: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Enrich CodeModule nodes with LLM-generated summaries.
+
+    Returns the same nodes with richer ``summary`` properties.
+    If LLM is unavailable, returns nodes unchanged.
+    """
+    try:
+        client = get_llm_client()
+    except Exception:
+        logger.debug("scan_with_llm: no LLM available — returning nodes unchanged")
+        return code_nodes
+
+    enriched = []
+    llm_count = 0
+
+    for node in code_nodes:
+        if llm_count >= _MAX_LLM_FILES:
+            enriched.append(node)
+            continue
+
+        fpath = repo_path / node["properties"].get("path", "")
+        if not fpath.is_file():
+            enriched.append(node)
+            continue
+
+        code = _safe_read(fpath, limit=3000)
+        if not code or len(code) < 10:
+            enriched.append(node)
+            continue
+
+        try:
+            prompt = _LLM_PROMPT.format(path=node["properties"].get("path", ""), code=code)
+            summary = client.generate(prompt, system="You are a senior code reviewer. Be concise.", max_tokens=200)
+            llm_count += 1
+
+            if summary and len(summary.strip()) > 10:
+                new_node = dict(node)
+                new_node["properties"] = dict(node["properties"])
+                new_node["properties"]["summary"] = summary.strip()[:400]
+                enriched.append(new_node)
+            else:
+                enriched.append(node)
+        except Exception as exc:
+            logger.debug("scan_with_llm: failed for %s — %s", fpath, exc)
+            enriched.append(node)
+
+    logger.info("scan_with_llm: enriched %d / %d modules", llm_count, len(code_nodes))
+    return enriched

contextsynapse/ingestion/universal/operators/scanners/repo.py

@@ -0,0 +1,149 @@
+"""Repo file scanners — README, source, tests, docs, API routes."""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any, Dict, List
+
+from .helpers import _safe_id, _safe_read, _find_src_dirs, _extract_summary, _SKIP_DIRS, _CODE_EXTENSIONS
+
+
+def scan_readme(repo_path: Path) -> List[Dict[str, Any]]:
+    nodes = []
+    for name in ("README.md", "README.rst", "readme.md"):
+        readme = repo_path / name
+        if readme.exists():
+            text = readme.read_text(encoding="utf-8", errors="replace")
+            break
+    else:
+        return nodes
+
+    sections = re.split(r'\n#{1,3}\s+', text)
+    for section in sections[1:]:
+        lines = section.strip().split("\n")
+        title = lines[0].strip().rstrip("#").strip()
+        body = "\n".join(lines[1:]).strip()[:500]
+        if len(body) < 20:
+            continue
+        nodes.append({
+            "id": f"req:{_safe_id(title)}",
+            "label": "Requirement",
+            "properties": {"content": f"{title}: {body[:300]}", "source": "README.md", "version": 1},
+        })
+    return nodes
+
+
+def scan_source_modules(repo_path: Path) -> List[Dict[str, Any]]:
+    nodes = []
+    for src_dir in _find_src_dirs(repo_path):
+        for fpath in sorted(src_dir.rglob("*")):
+            if not fpath.is_file() or fpath.suffix not in _CODE_EXTENSIONS:
+                continue
+            if any(p in fpath.parts for p in _SKIP_DIRS):
+                continue
+            if fpath.name.startswith("_") and fpath.name != "__init__.py":
+                continue
+            rel = fpath.relative_to(repo_path)
+            text = _safe_read(fpath, limit=2000)
+            if not text or len(text) < 50:
+                continue
+            summary = _extract_summary(text, fpath.suffix) or f"Module at {rel}"
+            nodes.append({
+                "id": f"code:{_safe_id(str(rel))}",
+                "label": "CodeModule",
+                "properties": {"summary": summary[:400], "path": str(rel), "version": 1},
+            })
+    return nodes
+
+
+def scan_tests(repo_path: Path) -> List[Dict[str, Any]]:
+    nodes = []
+    test_dirs = []
+    for d in ("tests", "test", "spec", "specs", "__tests__"):
+        td = repo_path / d
+        if td.is_dir():
+            test_dirs.append(td)
+
+    seen = set()
+    for td in test_dirs:
+        for fpath in sorted(td.rglob("*")):
+            if not fpath.is_file() or fpath.suffix not in _CODE_EXTENSIONS:
+                continue
+            if fpath in seen:
+                continue
+            seen.add(fpath)
+            rel = fpath.relative_to(repo_path)
+            text = _safe_read(fpath, limit=1500)
+            if not text:
+                continue
+            test_funcs = re.findall(r'(?:def test_|it\(|test\(|func Test)', text)
+            if not test_funcs:
+                continue
+            summary = _extract_summary(text, fpath.suffix)
+            nodes.append({
+                "id": f"tc:{_safe_id(str(rel))}",
+                "label": "TestCase",
+                "properties": {
+                    "what": summary[:200] if summary else f"Tests in {rel}",
+                    "how": f"{len(test_funcs)} test function(s) in {rel}",
+                    "version": 1,
+                },
+            })
+    return nodes
+
+
+def scan_docs(repo_path: Path) -> List[Dict[str, Any]]:
+    nodes = []
+    for d in ("docs", "doc", "documentation", "adr", "adrs", "decisions"):
+        dd = repo_path / d
+        if not dd.is_dir():
+            continue
+        for fpath in sorted(dd.rglob("*.md")):
+            if any(p in fpath.parts for p in _SKIP_DIRS):
+                continue
+            rel = fpath.relative_to(repo_path)
+            text = _safe_read(fpath, limit=2000)
+            if not text or len(text) < 50:
+                continue
+            paragraphs = text.split("\n\n")
+            title = paragraphs[0].strip().lstrip("#").strip()
+            body = paragraphs[1].strip() if len(paragraphs) > 1 else ""
+            nodes.append({
+                "id": f"arch:{_safe_id(str(rel))}",
+                "label": "ArchDecision",
+                "properties": {"decision": title[:200], "rationale": body[:300], "source": str(rel), "version": 1},
+            })
+    return nodes
+
+
+def scan_api_routes(repo_path: Path) -> List[Dict[str, Any]]:
+    nodes = []
+    patterns = [
+        r'@\w+\.(get|post|put|delete|patch)\(["\']([^"\']+)',
+        r'router\.(get|post|put|delete|patch)\(["\']([^"\']+)',
+        r'@(Get|Post|Put|Delete|Patch)Mapping\(["\']([^"\']+)',
+    ]
+    seen_ids = set()
+    for fpath in repo_path.rglob("*"):
+        if not fpath.is_file() or fpath.suffix not in (".py", ".ts", ".js", ".java"):
+            continue
+        if any(p in fpath.parts for p in _SKIP_DIRS):
+            continue
+        text = _safe_read(fpath, limit=5000)
+        if not text:
+            continue
+        rel = fpath.relative_to(repo_path)
+        for pattern in patterns:
+            for match in re.finditer(pattern, text):
+                method = match.group(1).upper()
+                path = match.group(2)
+                node_id = f"api:{_safe_id(f'{method}-{path}')}"
+                if node_id in seen_ids:
+                    continue
+                seen_ids.add(node_id)
+                nodes.append({
+                    "id": node_id,
+                    "label": "APIContract",
+                    "properties": {"method": method, "path": path, "source": str(rel), "version": 1},
+                })
+    return nodes

contextsynapse/ingestion/universal/operators/sdlc_scan.py

@@ -0,0 +1,213 @@
+"""SDLCScanOperator — orchestrates SDLC repo scanning via modular scanners.
+
+Two modes:
+  Quick Scan (no clone) — GitHub API only, ~5 seconds
+  Full Scan  (clone)    — git clone + file scanning + API, ~30-120 seconds
+
+Delegates to scanners/ package:
+  scanners.repo          — README, source, tests, docs, API routes (Full only)
+  scanners.github        — GitHub issues via .git/config (Full only)
+  scanners.github_api    — GitHub API: README, issues, PRs, contributors, metadata (Quick)
+  scanners.git           — git history, repo metadata (Full only)
+  scanners.edges         — edge inference (7 types)
+  scanners.llm_enrichment — LLM-powered code summaries (Full only, optional)
+  scanners.jira          — Jira ticket import (both modes)
+"""
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, TYPE_CHECKING
+
+from .base import StageOperator
+
+# Re-export everything from scanners for backward compatibility
+from .scanners import (
+    scan_readme, scan_source_modules, scan_tests, scan_docs, scan_api_routes,
+    scan_github_issues_full, scan_git_history, scan_repo_metadata,
+    scan_with_llm, scan_jira_issues, infer_edges, quick_scan,
+    _safe_id, _issue_to_node, _extract_github_owner_repo,
+)
+
+if TYPE_CHECKING:
+    from ..ingest_content import Chunk
+    from ..stage_executor import GraphContext
+
+logger = logging.getLogger(__name__)
+
+
+class SDLCScanOperator(StageOperator):
+    """Scan a repository into typed SDLC graph nodes.
+
+    Modes:
+        quick (default): GitHub API only — no clone, ~5 seconds
+        full:            git clone + file scan + API — ~30-120 seconds
+
+    Config (from pipeline_params):
+        scan_mode: "quick" or "full" (default: "quick")
+        skip_llm:  skip LLM enrichment in full mode (default: True)
+    """
+
+    name = "sdlc_scan"
+
+    def __init__(self, config: Dict[str, Any] = None):
+        self.config = config or {}
+
+    def process(self, chunks: List["Chunk"], graph_ctx: "GraphContext") -> List["Chunk"]:
+        cfg = self.config
+        for chunk in chunks:
+            if chunk.metadata.get("pipeline_params"):
+                cfg = {**cfg, **chunk.metadata["pipeline_params"]}
+                break
+
+        # Get source URL or repo path
+        source_url = ""
+        repo_path_str = ""
+        for chunk in chunks:
+            source_url = chunk.metadata.get("source_url", "")
+            repo_path_str = chunk.metadata.get("repo_path", "")
+            if source_url or repo_path_str:
+                break
+
+        # Determine scan mode: auto-detect from input if not explicitly set
+        scan_mode = cfg.get("scan_mode", "")
+        if not scan_mode:
+            if repo_path_str and os.path.isdir(repo_path_str):
+                scan_mode = "full"  # local directory → full scan
+            elif source_url and source_url.startswith("https://"):
+                scan_mode = "quick"  # URL → quick scan (no clone by default)
+            else:
+                scan_mode = "full"  # fallback
+
+        if scan_mode == "quick":
+            return self._quick_scan(source_url or repo_path_str, graph_ctx, cfg)
+        else:
+            return self._full_scan(repo_path_str, source_url, chunks, graph_ctx, cfg)
+
+    def _write_nodes_and_catch_orphans(
+        self, all_nodes: List[Dict], edges: List[Dict],
+        graph_ctx: "GraphContext", project_name: str,
+    ) -> None:
+        """Write nodes + edges, then connect orphans to a root Project node.
+
+        Most nodes connect to each other via inferred edges.
+        Only nodes with NO connections get linked to the root as a safety net.
+        """
+        # Write all nodes
+        for node_def in all_nodes:
+            graph_ctx.add_node(
+                label=node_def["label"],
+                properties=dict(node_def.get("properties", {})),
+                node_id=node_def["id"],
+            )
+
+        # Write all edges
+        for edge_def in edges:
+            graph_ctx.add_edge(
+                source_id=edge_def["source"],
+                target_id=edge_def["target"],
+                label=edge_def["label"],
+            )
+
+        # Find orphans — nodes with no edges at all
+        connected = set()
+        for e in edges:
+            connected.add(e["source"])
+            connected.add(e["target"])
+
+        orphans = [n for n in all_nodes if n["id"] not in connected]
+
+        if orphans:
+            # Create root node and link orphans to it
+            root_id = f"project:{_safe_id(project_name)}"
+            graph_ctx.add_node(
+                label="Project",
+                properties={"name": project_name, "version": 1},
+                node_id=root_id,
+            )
+            for orphan in orphans:
+                graph_ctx.add_edge(
+                    source_id=root_id,
+                    target_id=orphan["id"],
+                    label="HAS_NODE",
+                )
+            logger.info("sdlc_scan: %d orphans connected to root node %s", len(orphans), root_id)
+
+    def _quick_scan(
+        self, url: str, graph_ctx: "GraphContext", cfg: Dict,
+    ) -> List["Chunk"]:
+        """Quick Scan — GitHub API only, no clone."""
+        if not url:
+            logger.warning("sdlc_scan[quick]: no URL provided — skipping")
+            return []
+
+        logger.info("sdlc_scan[quick]: scanning %s via API (no clone)", url)
+
+        all_nodes = quick_scan(url)
+        all_nodes += scan_jira_issues()
+
+        # Infer cross-connections
+        edges = infer_edges(all_nodes)
+
+        # Write nodes + edges, orphans fall back to root
+        from .scanners.github_api import _parse_owner_repo
+        project_name = _parse_owner_repo(url) or url.split("/")[-1] or "project"
+        self._write_nodes_and_catch_orphans(all_nodes, edges, graph_ctx, project_name)
+
+        logger.info("sdlc_scan[quick]: %d nodes, %d edges", len(all_nodes), len(edges))
+        return []
+
+    def _full_scan(
+        self, repo_path_str: str, source_url: str,
+        chunks: List["Chunk"], graph_ctx: "GraphContext", cfg: Dict,
+    ) -> List["Chunk"]:
+        """Full Scan — clone + file scanning + API."""
+        if not repo_path_str:
+            logger.warning("sdlc_scan[full]: no repo_path — skipping")
+            return chunks
+
+        repo_path = Path(repo_path_str)
+        if not repo_path.is_dir():
+            logger.warning("sdlc_scan[full]: %s is not a directory — skipping", repo_path)
+            return chunks
+
+        logger.info("sdlc_scan[full]: scanning %s (clone mode)", repo_path)
+
+        # File-based scanners
+        all_nodes = (
+            scan_readme(repo_path)
+            + scan_source_modules(repo_path)
+            + scan_tests(repo_path)
+            + scan_docs(repo_path)
+            + scan_api_routes(repo_path)
+        )
+
+        # LLM enrichment (optional)
+        skip_llm = cfg.get("skip_llm", False) or os.environ.get("SDLC_SKIP_LLM")
+        if not skip_llm:
+            code_nodes = [n for n in all_nodes if n["label"] == "CodeModule"]
+            non_code = [n for n in all_nodes if n["label"] != "CodeModule"]
+            all_nodes = non_code + scan_with_llm(repo_path, code_nodes)
+        else:
+            logger.info("sdlc_scan[full]: LLM enrichment skipped")
+
+        # API-based scanners
+        all_nodes += scan_github_issues_full(repo_path, source_url=source_url or repo_path_str)
+        all_nodes += scan_jira_issues()
+        all_nodes += scan_repo_metadata(repo_path)
+
+        # Git history
+        history_nodes, history_edges = scan_git_history(repo_path)
+        all_nodes += history_nodes
+
+        # Infer cross-connections + history edges
+        edges = infer_edges(all_nodes)
+        edges += history_edges
+
+        # Write nodes + edges, orphans fall back to root
+        project_name = repo_path.name or "project"
+        self._write_nodes_and_catch_orphans(all_nodes, edges, graph_ctx, project_name)
+
+        logger.info("sdlc_scan[full]: %d nodes, %d edges from %s", len(all_nodes), len(edges), repo_path)
+        return chunks

contextsynapse/ingestion/universal/operators/spec_scanner.py

@@ -0,0 +1,149 @@
+"""SpecScanner — parse structured spec documents into SDLC nodes.
+
+Parses markdown/text specs (from ChatGPT, Claude, PRDs, etc.) into typed
+SDLC nodes by recognizing section headers:
+
+  ## Requirements / ## Functional Requirements  → Requirement
+  ## Architecture / ## Design Decisions          → ArchDecision
+  ## Constraints / ## Non-Functional              → Constraint
+  ## Acceptance Criteria / ## Test Cases          → TestCase
+  ## Goals / ## Objectives                        → Goal
+  ## User Stories                                 → UserStory
+
+Usage::
+
+    from contextsynapse.ingestion.universal.operators.spec_scanner import scan_spec_document
+    nodes = scan_spec_document(spec_text, source="chatgpt-spec.md")
+"""
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any, Dict, List, Tuple
+
+logger = logging.getLogger(__name__)
+
+# ── Section → Node Type mapping (loaded from schema) ─────────────────────────
+
+from contextsynapse.project.sdlc_schema import SPEC_SECTION_MAP as _SECTION_MAP
+
+_MAX_SPEC_NODES = 50
+
+
+def _safe_id(text: str) -> str:
+    """Convert text to a safe node ID slug."""
+    s = re.sub(r'[^a-zA-Z0-9_-]', '-', text.lower())
+    s = re.sub(r'-+', '-', s).strip('-')
+    return s[:60]
+
+
+def _classify_section(header: str) -> str:
+    """Map a section header to an SDLC node type. Returns '' if no match."""
+    header_lower = header.lower().strip()
+    for keywords, node_type in _SECTION_MAP:
+        for kw in keywords:
+            if kw in header_lower:
+                return node_type
+    return ""
+
+
+def _extract_items(body: str) -> List[str]:
+    """Extract individual items from a section body.
+
+    Handles:
+    - Bullet lists (- item, * item, • item)
+    - Numbered lists (1. item, 1) item)
+    - Paragraphs separated by blank lines
+    """
+    # Try bullet/numbered list first
+    items = re.findall(r'(?:^|\n)\s*(?:[-*•]|\d+[.)]\s)\s*(.+)', body)
+    if items:
+        return [i.strip() for i in items if len(i.strip()) > 10]
+
+    # Fall back to paragraphs
+    paragraphs = [p.strip() for p in body.split("\n\n") if p.strip()]
+    return [p for p in paragraphs if len(p) > 10]
+
+
+def _make_node(node_type: str, item_text: str, index: int, source: str) -> Dict[str, Any]:
+    """Create an SDLC node dict from a parsed spec item."""
+    node_id = f"spec:{_safe_id(item_text[:40])}-{index}"
+
+    if node_type == "Requirement":
+        props = {"content": item_text[:500], "source": source, "version": 1}
+    elif node_type == "ArchDecision":
+        props = {"decision": item_text[:500], "rationale": "", "source": source, "version": 1}
+    elif node_type == "Constraint":
+        props = {"description": item_text[:500], "source": source, "version": 1}
+    elif node_type == "TestCase":
+        props = {"what": item_text[:250], "how": "", "source": source, "version": 1}
+    elif node_type == "Goal":
+        props = {"content": item_text[:500], "source": source, "version": 1}
+    elif node_type == "UserStory":
+        props = {"content": item_text[:500], "source": source, "version": 1}
+    elif node_type == "DataModel":
+        props = {"summary": item_text[:500], "source": source, "version": 1}
+    elif node_type == "APIContract":
+        props = {"method": "", "path": "", "summary": item_text[:500], "source": source, "version": 1}
+    else:
+        props = {"content": item_text[:500], "source": source, "version": 1}
+
+    return {"id": node_id, "label": node_type, "properties": props}
+
+
+def scan_spec_document(
+    text: str,
+    source: str = "spec",
+) -> List[Dict[str, Any]]:
+    """Parse a structured spec document into SDLC nodes.
+
+    Recognizes markdown headers (## Section Name) and maps them to SDLC
+    node types based on keywords. Items within each section become individual
+    nodes.
+
+    Args:
+        text: the spec document text (markdown or plain text)
+        source: source identifier for traceability
+
+    Returns:
+        List of SDLC node dicts: [{id, label, properties}, ...]
+    """
+    if not text or not text.strip():
+        return []
+
+    # Split by markdown headers (## or ###)
+    sections = re.split(r'\n#{2,3}\s+', text)
+    if len(sections) <= 1:
+        # No markdown headers — try plain text with "Section:" pattern
+        sections = re.split(r'\n(?=[A-Z][A-Za-z\s]+:)', text)
+
+    nodes: List[Dict[str, Any]] = []
+    node_index = 0
+
+    for section in sections:
+        lines = section.strip().split("\n")
+        if not lines:
+            continue
+
+        header = lines[0].strip().rstrip(":")
+        body = "\n".join(lines[1:]).strip()
+
+        node_type = _classify_section(header)
+        if not node_type:
+            continue
+
+        # Extract individual items from the section
+        items = _extract_items(body)
+        if not items:
+            # Treat the whole body as one item if no list structure
+            if body and len(body) > 10:
+                items = [body[:500]]
+
+        for item_text in items:
+            if node_index >= _MAX_SPEC_NODES:
+                break
+            nodes.append(_make_node(node_type, item_text, node_index, source))
+            node_index += 1
+
+    logger.info("scan_spec_document: %d nodes from '%s'", len(nodes), source)
+    return nodes

contextsynapse/ingestion/universal/operators/store_documents.py

@@ -0,0 +1,72 @@
+"""StoreDocumentsOperator -- move heavy text to keep graph lean."""
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import List, TYPE_CHECKING
+
+from .base import StageOperator
+
+if TYPE_CHECKING:
+    from ..ingest_content import Chunk
+    from ..stage_executor import GraphContext
+
+logger = logging.getLogger(__name__)
+
+_MAX_INLINE_LENGTH = 200
+_CONTENT_FIELDS = ("content", "statement", "description")
+
+
+class StoreDocumentsOperator(StageOperator):
+    name = "store_documents"
+
+    def process(self, chunks, graph_ctx):
+        # Try to get document store
+        doc_store = None
+        try:
+            from contextsynapse.storage.namespace_store import NamespaceStore
+            ns = NamespaceStore(graph_ctx.namespace or "default")
+            doc_store = ns
+        except Exception:
+            pass
+
+        count = 0
+        for node_id in list(graph_ctx.node_ids):
+            try:
+                node = graph_ctx.get_node(node_id)
+                if not node:
+                    continue
+                props = getattr(node, "properties", {}) or {}
+                updates = {}
+                has_long = False
+
+                for field in _CONTENT_FIELDS:
+                    val = props.get(field, "")
+                    if isinstance(val, str) and len(val) > _MAX_INLINE_LENGTH:
+                        has_long = True
+                        if doc_store:
+                            doc_id = f"doc_{uuid.uuid4().hex[:12]}"
+                            try:
+                                doc_store.store_document(doc_id, {
+                                    "node_id": node_id, "field": field,
+                                    "text": val, "label": getattr(node, "label", ""),
+                                })
+                                updates["_doc_ref"] = doc_id
+                            except Exception:
+                                pass
+                        updates[field] = val[:_MAX_INLINE_LENGTH] + "..."
+
+                if has_long and updates:
+                    try:
+                        adapter = getattr(graph_ctx.db, "csr_adapter", graph_ctx.db)
+                        if hasattr(adapter, "update_node_properties"):
+                            adapter.update_node_properties(node_id, updates)
+                        count += 1
+                    except Exception:
+                        pass
+            except Exception:
+                continue
+
+        if count:
+            logger.info("store_documents: truncated %d nodes", count)
+        return chunks