codegraph-engine 3.0.0__1-py3-none-any.whl

codegraph/architecture.py

@@ -0,0 +1,371 @@
+"""Repository architecture intelligence with structured node/edge graph projection.
+
+Returns a structured view of the repository: entry points, endpoints, controllers,
+services, models, APIs, tests, config, and framework integration edges.
+
+All results are source-derived — no fictional architecture is generated.
+"""
+from __future__ import annotations
+
+import re
+import sqlite3
+from pathlib import Path
+
+_ENTRY_PATTERNS = re.compile(
+    r"(^|[_/])(main|app|server|index|wsgi|asgi|manage|run)\.(py|js|ts)$",
+    re.IGNORECASE,
+)
+_TEST_PATTERNS = re.compile(
+    r"(^|[_/])test[_s]?[_/]|test[_s]?\.(py|js|ts)$|spec\.(py|js|ts)$",
+    re.IGNORECASE,
+)
+_MODEL_PATTERNS = re.compile(
+    r"(^|[_/])(model|schema|entity|orm|db)\.(py|js|ts)$|models?(\.py|\.ts|\.js|/)$",
+    re.IGNORECASE,
+)
+_CONFIG_PATTERNS = re.compile(
+    r"(^|[_/])(config|settings|conf|env)\.(py|js|ts)$",
+    re.IGNORECASE,
+)
+_API_PATTERNS = re.compile(
+    r"(^|[_/])(api|routes?|views?|controllers?|handlers?)\.(py|js|ts)$|/api/",
+    re.IGNORECASE,
+)
+_REPO_PATTERNS = re.compile(
+    r"(^|[_/])(repo|repository|store|dao)\.(py|js|ts)$",
+    re.IGNORECASE,
+)
+_SERVICE_PATTERNS = re.compile(
+    r"(^|[_/])(service|svc|manager|provider)\.(py|js|ts)$",
+    re.IGNORECASE,
+)
+_INTEGRATION_PATTERNS = re.compile(
+    r"(database|redis|celery|kafka|rabbitmq|s3|stripe|twilio|sendgrid|oauth|jwt|openai)",
+    re.IGNORECASE,
+)
+
+_FRAMEWORK_IMPORT_PATTERNS: dict[str, re.Pattern[str]] = {
+    "flask": re.compile(r"^flask(?:\.|$)", re.IGNORECASE),
+    "fastapi": re.compile(r"^fastapi(?:\.|$)", re.IGNORECASE),
+    "django": re.compile(r"^django(?:\.|$)", re.IGNORECASE),
+    "express": re.compile(r"^express(?:\/|$)", re.IGNORECASE),
+    "nextjs": re.compile(r"^next(?:\/|$)", re.IGNORECASE),
+    "tornado": re.compile(r"^tornado(?:\.|$)", re.IGNORECASE),
+    "aiohttp": re.compile(r"^aiohttp(?:\.|$)", re.IGNORECASE),
+    "starlette": re.compile(r"^starlette(?:\.|$)", re.IGNORECASE),
+}
+
+_MANIFEST_NAMES = (
+    "pyproject.toml",
+    "requirements.txt",
+    "package.json",
+    "setup.py",
+    "setup.cfg",
+    "Pipfile",
+)
+
+
+def detect_frameworks(
+    con: sqlite3.Connection,
+    repository: Path,
+) -> list[str]:
+    """Detect application frameworks using routes, AST imports, and bounded manifest discovery."""
+    detected: set[str] = set()
+
+    # 1. Concrete routes registered in database
+    try:
+        fw_rows = con.execute("SELECT DISTINCT framework FROM framework_routes").fetchall()
+        for r in fw_rows:
+            if r[0]:
+                detected.add(str(r[0]).lower())
+    except sqlite3.OperationalError:
+        pass
+
+    # 2. AST imports from indexed source files
+    try:
+        imp_rows = con.execute("SELECT DISTINCT module FROM imports WHERE module != ''").fetchall()
+        for r in imp_rows:
+            mod = str(r[0]).strip()
+            for fw, pat in _FRAMEWORK_IMPORT_PATTERNS.items():
+                if pat.search(mod):
+                    detected.add(fw)
+    except sqlite3.OperationalError:
+        pass
+
+    # 3. Bounded recursive manifest discovery (depth <= 3)
+    try:
+        root = repository.resolve()
+        dirs_to_check: list[Path] = [root]
+
+        def _subdirs(d: Path) -> list[Path]:
+            res: list[Path] = []
+            try:
+                for entry in d.iterdir():
+                    if entry.is_dir() and not entry.name.startswith(".") and entry.name != "node_modules":
+                        res.append(entry)
+            except OSError:
+                pass
+            return res
+
+        level1 = _subdirs(root)
+        dirs_to_check.extend(level1)
+        for d1 in level1:
+            level2 = _subdirs(d1)
+            dirs_to_check.extend(level2)
+
+        for d in dirs_to_check:
+            try:
+                for item in d.iterdir():
+                    if not item.is_file():
+                        continue
+                    nm = item.name.lower()
+                    if nm in _MANIFEST_NAMES or (nm.startswith("requirements") and nm.endswith(".txt")):
+                        text = item.read_text(encoding="utf-8", errors="replace").lower()
+                        for fw in ("flask", "fastapi", "django", "express", "tornado", "aiohttp", "starlette"):
+                            if re.search(rf"(?:^|[^\w-]){fw}(?:[^\w-]|$)", text):
+                                detected.add(fw)
+                        if "next" in text and ("\"next\"" in text or "'next'" in text or "nextjs" in text):
+                            detected.add("nextjs")
+            except OSError:
+                pass
+    except Exception:
+        pass
+
+    return sorted(detected)
+
+
+def get_architecture(
+    con: sqlite3.Connection,
+    repository: Path,
+    max_per_category: int = 20,
+) -> dict[str, object]:
+    """Return a structured, source-derived architecture overview with nodes and edges."""
+    files: list[str] = [
+        r[0] for r in con.execute("SELECT path FROM files WHERE status='ok' ORDER BY path LIMIT 500")
+    ]
+
+    entry_points = [f for f in files if _ENTRY_PATTERNS.search(f)]
+    test_files = [f for f in files if _TEST_PATTERNS.search(f)]
+    model_files = [f for f in files if _MODEL_PATTERNS.search(f)]
+    config_files = [f for f in files if _CONFIG_PATTERNS.search(f)]
+    api_files = [f for f in files if _API_PATTERNS.search(f)]
+    repo_files = [f for f in files if _REPO_PATTERNS.search(f)]
+    service_files = [f for f in files if _SERVICE_PATTERNS.search(f)]
+
+    # Integration hints from imports
+    integrations: list[str] = []
+    try:
+        imp_rows = con.execute("SELECT DISTINCT module FROM imports LIMIT 500").fetchall()
+        for row in imp_rows:
+            m = _INTEGRATION_PATTERNS.search(row[0])
+            if m:
+                integrations.append(m.group(0).lower())
+    except sqlite3.OperationalError:
+        pass
+
+    # Top-level classes
+    class_rows = con.execute(
+        "SELECT canonical_id, qualified_name, path, kind FROM symbols WHERE kind='class' LIMIT 100"
+    ).fetchall()
+    classes = [
+        {"name": r["qualified_name"], "canonical_id": r["canonical_id"], "file": r["path"]}
+        for r in class_rows
+    ]
+
+    # Concrete framework routes
+    endpoints: list[dict[str, object]] = []
+    nodes: list[dict[str, object]] = []
+    edges: list[dict[str, object]] = []
+
+    try:
+        route_rows = con.execute(
+            "SELECT endpoint_id, framework, http_method, route_path, normalized_route, "
+            "handler_name, handler_canonical_id, file_path, line, evidence, confidence "
+            "FROM framework_routes LIMIT 100"
+        ).fetchall()
+        for rr in route_rows:
+            ep = {
+                "endpoint_id": rr["endpoint_id"],
+                "framework": rr["framework"],
+                "http_method": rr["http_method"],
+                "route_path": rr["route_path"],
+                "normalized_route": rr["normalized_route"],
+                "handler_name": rr["handler_name"],
+                "handler_canonical_id": rr["handler_canonical_id"],
+                "file": rr["file_path"],
+                "line": rr["line"],
+                "evidence": rr["evidence"],
+                "confidence": rr["confidence"],
+            }
+            endpoints.append(ep)
+            nodes.append(
+                {
+                    "id": rr["endpoint_id"],
+                    "name": f"{rr['http_method']} {rr['normalized_route']}",
+                    "layer": "endpoint",
+                    "file": rr["file_path"],
+                    "line": rr["line"],
+                    "confidence": rr["confidence"],
+                }
+            )
+            if rr["handler_canonical_id"]:
+                edges.append(
+                    {
+                        "source": rr["endpoint_id"],
+                        "target": rr["handler_canonical_id"],
+                        "relationship": "HANDLED_BY",
+                        "confidence": rr["confidence"],
+                        "evidence": rr["evidence"],
+                    }
+                )
+    except sqlite3.OperationalError:
+        pass
+
+    # Frameworks
+    frameworks = detect_frameworks(con, repository)
+
+    from codegraph.monorepo import detect_workspace
+    workspace_info = detect_workspace(repository, con=con)
+
+    # Top-level modules (excluding build artifacts, vendors, and bundles, enriched with package boundaries)
+    modules: list[str] = []
+    try:
+        mod_rows = con.execute(
+            """
+            SELECT DISTINCT s.module
+            FROM symbols s
+            LEFT JOIN files f ON s.path = f.path
+            WHERE s.module != ''
+              AND (f.category IS NULL OR f.category NOT IN ('BUILD_ARTIFACT', 'VENDOR', 'BUNDLE', 'MINIFIED'))
+            ORDER BY s.module LIMIT 20
+            """
+        ).fetchall()
+        modules = [r[0] for r in mod_rows]
+        if workspace_info.packages:
+            for p in workspace_info.packages.values():
+                if p.root_path and p.root_path not in modules:
+                    modules.append(p.root_path)
+            modules = sorted(modules)[:max_per_category]
+    except sqlite3.OperationalError:
+        pass
+
+    # Dependencies summary
+    dependencies: list[dict[str, object]] = []
+    try:
+        dep_rows = con.execute(
+            "SELECT module, count(*) AS c FROM imports GROUP BY module ORDER BY c DESC LIMIT 20"
+        ).fetchall()
+        dependencies = [{"module": r[0], "count": r[1]} for r in dep_rows]
+    except sqlite3.OperationalError:
+        pass
+
+    # Artifact directory separation
+    import posixpath
+    source_dirs: set[str] = set()
+    test_dirs: set[str] = set()
+    build_dirs: set[str] = set()
+    vendor_dirs: set[str] = set()
+    generated_dirs: set[str] = set()
+    generated_files: list[str] = []
+
+    try:
+        file_rows = con.execute("SELECT path, category FROM files WHERE status='ok'").fetchall()
+        for fr in file_rows:
+            p_dir = posixpath.dirname(fr["path"]) or "."
+            cat = str(fr["category"])
+            if cat in ("SOURCE", ""):
+                source_dirs.add(p_dir)
+            elif cat == "TEST":
+                test_dirs.add(p_dir)
+            elif cat in ("BUILD_ARTIFACT", "BUNDLE", "MINIFIED"):
+                build_dirs.add(p_dir)
+            elif cat == "VENDOR":
+                vendor_dirs.add(p_dir)
+            elif cat == "GENERATED":
+                generated_dirs.add(p_dir)
+                generated_files.append(fr["path"])
+    except sqlite3.OperationalError:
+        pass
+
+    # Test summary with test framework detection
+    from codegraph.indexing.test_framework import detect_test_framework
+    test_framework, tests_present, test_evidence = detect_test_framework(con, repository)
+    test_summary = {
+        "framework": test_framework.value,
+        "tests_present": tests_present,
+        "evidence": test_evidence,
+        "test_files_count": len(test_files),
+        "test_files": test_files[:max_per_category],
+    }
+
+    # Route summary
+    route_summary = {
+        "total_routes": len(endpoints),
+        "frameworks": frameworks,
+        "methods": {
+            m: len([e for e in endpoints if e.get("http_method") == m])
+            for m in sorted({str(e.get("http_method", "")) for e in endpoints if e.get("http_method")})
+        },
+    }
+
+    # Data / model summary
+    data_model_summary = {
+        "model_files": model_files[:max_per_category],
+        "model_classes": [c["name"] for c in classes[:max_per_category]],
+        "total_classes": len(classes),
+    }
+
+    # Module count by language
+    lang_counts: dict[str, int] = {}
+    for r in con.execute("SELECT language, count(*) AS n FROM files GROUP BY language"):
+        lang_counts[r["language"]] = r["n"]
+
+    total_symbols = con.execute("SELECT count(*) FROM symbols").fetchone()[0]
+    total_files = len(files)
+
+    return {
+        "source": "deterministic — parser-extracted from repository index",
+        "repository": str(repository),
+        "summary": {
+            "total_files": total_files,
+            "total_symbols": total_symbols,
+            "total_endpoints": len(endpoints),
+            "languages": lang_counts,
+        },
+        "languages": lang_counts,
+        "frameworks": frameworks,
+        "workspace": workspace_info.as_dict(),
+        "packages": [p.as_dict() for p in workspace_info.packages.values()],
+        "package_dependencies": [d.as_dict() for d in workspace_info.dependencies],
+        "entry_points": entry_points[:max_per_category],
+        "top_level_modules": modules,
+        "dependency_summary": dependencies,
+        "route_summary": route_summary,
+        "test_summary": test_summary,
+        "data_model_summary": data_model_summary,
+        "external_services": sorted(set(integrations))[:max_per_category],
+        "source_directories": sorted(source_dirs)[:max_per_category],
+        "test_directories": sorted(test_dirs)[:max_per_category],
+        "build_directories": sorted(build_dirs)[:max_per_category],
+        "vendor_directories": sorted(vendor_dirs)[:max_per_category],
+        "generated_directories": sorted(generated_dirs)[:max_per_category],
+        "generated_artifact_summary": {
+            "count": len(generated_files),
+            "files": generated_files,
+        },
+        "endpoints": endpoints[:max_per_category],
+        "services": service_files[:max_per_category],
+        "repositories": repo_files[:max_per_category],
+        "models": model_files[:max_per_category],
+        "apis": api_files[:max_per_category],
+        "tests": test_files[:max_per_category],
+        "config": config_files[:max_per_category],
+        "classes": classes[:max_per_category],
+        "external_integrations": sorted(set(integrations))[:max_per_category],
+        "architecture_nodes": nodes[:max_per_category * 2],
+        "architecture_edges": edges[:max_per_category * 2],
+        "note": (
+            "Architecture is inferred from source declarations, routes, and import patterns. "
+            "Framework endpoints are verified from AST route registrations."
+        ),
+    }

codegraph/async_queue/__init__.py

@@ -0,0 +1,50 @@
+"""Asynchronous message queue and cross-service distributed task intelligence.
+
+Provides static detection, queue linking, and OpenTelemetry distributed tracing
+reconciliation for Kafka, Celery, AWS SQS, RabbitMQ, Redis Pub/Sub, and Streams.
+"""
+from __future__ import annotations
+
+from codegraph.async_queue.models import (
+    AsyncConsumerRecord,
+    AsyncMessagingSystem,
+    AsyncProducerRecord,
+    AsyncQueueEntity,
+    AsyncQueueEvidenceClass,
+    AsyncQueueLink,
+)
+from codegraph.async_queue.otel_models import (
+    OTelEvidenceStatus,
+    OTelSpanRecord,
+    TraceReconciliationResult,
+)
+from codegraph.async_queue.otel_parser import parse_otel_spans
+from codegraph.async_queue.reconciler import OTelTraceReconciler, ensure_otel_tables
+from codegraph.async_queue.scanner import AsyncQueueScanner, ensure_async_tables
+from codegraph.async_queue.traversal import (
+    find_queue_consumers,
+    find_queue_producers,
+    list_async_queues,
+    trace_async_flow,
+)
+
+__all__ = [
+    "AsyncConsumerRecord",
+    "AsyncMessagingSystem",
+    "AsyncProducerRecord",
+    "AsyncQueueEntity",
+    "AsyncQueueEvidenceClass",
+    "AsyncQueueLink",
+    "AsyncQueueScanner",
+    "OTelEvidenceStatus",
+    "OTelSpanRecord",
+    "OTelTraceReconciler",
+    "TraceReconciliationResult",
+    "ensure_async_tables",
+    "ensure_otel_tables",
+    "find_queue_consumers",
+    "find_queue_producers",
+    "list_async_queues",
+    "parse_otel_spans",
+    "trace_async_flow",
+]

codegraph/async_queue/detectors/__init__.py

@@ -0,0 +1,48 @@
+"""Detectors for asynchronous message brokers and task dispatchers."""
+from __future__ import annotations
+
+import ast
+
+from codegraph.async_queue.detectors.celery import detect_celery
+from codegraph.async_queue.detectors.kafka import detect_kafka
+from codegraph.async_queue.detectors.rabbitmq import detect_rabbitmq
+from codegraph.async_queue.detectors.redis import detect_redis
+from codegraph.async_queue.detectors.sqs import detect_sqs
+from codegraph.async_queue.models import AsyncConsumerRecord, AsyncProducerRecord
+
+ALL_DETECTORS = [
+    detect_celery,
+    detect_kafka,
+    detect_sqs,
+    detect_rabbitmq,
+    detect_redis,
+]
+
+
+def run_all_detectors(
+    tree: ast.AST,
+    content: str,
+    file_path: str,
+    module_name: str,
+) -> tuple[list[AsyncProducerRecord], list[AsyncConsumerRecord]]:
+    """Run all framework detectors on parsed Python AST."""
+    all_producers: list[AsyncProducerRecord] = []
+    all_consumers: list[AsyncConsumerRecord] = []
+
+    for detector in ALL_DETECTORS:
+        prods, cons = detector(tree, content, file_path, module_name)
+        all_producers.extend(prods)
+        all_consumers.extend(cons)
+
+    return all_producers, all_consumers
+
+
+__all__ = [
+    "ALL_DETECTORS",
+    "detect_celery",
+    "detect_kafka",
+    "detect_rabbitmq",
+    "detect_redis",
+    "detect_sqs",
+    "run_all_detectors",
+]

codegraph/async_queue/detectors/celery.py

@@ -0,0 +1,160 @@
+"""Celery asynchronous task and canvas detector.
+
+Detects:
+  - Workers / Consumers: @app.task, @celery.task, @shared_task, @celery_app.task
+  - Publishers / Producers: task.delay(), task.apply_async(), app.send_task()
+"""
+from __future__ import annotations
+
+import ast
+from typing import Any
+
+from codegraph.async_queue.models import (
+    AsyncConsumerRecord,
+    AsyncMessagingSystem,
+    AsyncProducerRecord,
+)
+
+_CELERY_TASK_DECORATORS = {
+    "task",
+    "shared_task",
+}
+
+
+def detect_celery(
+    tree: ast.AST,
+    content: str,
+    file_path: str,
+    module_name: str,
+) -> tuple[list[AsyncProducerRecord], list[AsyncConsumerRecord]]:
+    producers: list[AsyncProducerRecord] = []
+    consumers: list[AsyncConsumerRecord] = []
+    lines = content.splitlines()
+
+    def get_line_snippet(lineno: int) -> str:
+        if 1 <= lineno <= len(lines):
+            return lines[lineno - 1].strip()
+        return ""
+
+    # Map of function names that are Celery tasks in this file
+    local_tasks: dict[str, str] = {}  # fn_name -> task_name
+
+    # 1. Detect consumers (task definitions)
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            for dec in node.decorator_list:
+                dec_func = dec.func if isinstance(dec, ast.Call) else dec
+                dec_id = getattr(dec_func, "id", "")
+                dec_attr = getattr(dec_func, "attr", "")
+
+                if dec_id in _CELERY_TASK_DECORATORS or dec_attr in _CELERY_TASK_DECORATORS:
+                    # Found a Celery task definition
+                    task_name = f"{module_name}.{node.name}"
+                    metadata: dict[str, Any] = {}
+
+                    if isinstance(dec, ast.Call):
+                        for kw in dec.keywords:
+                            if kw.arg == "name" and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
+                                task_name = kw.value.value
+                            elif kw.arg == "queue" and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
+                                metadata["queue"] = kw.value.value
+                            elif kw.arg == "max_retries" and isinstance(kw.value, ast.Constant):
+                                metadata["max_retries"] = kw.value.value
+                            elif kw.arg == "autoretry_for":
+                                metadata["autoretry"] = True
+                            elif kw.arg == "retry_backoff":
+                                metadata["retry_backoff"] = True
+
+                    local_tasks[node.name] = task_name
+                    canonical_id = f"{module_name}.{node.name}"
+                    queue_id = f"celery:{task_name}"
+
+                    consumers.append(
+                        AsyncConsumerRecord(
+                            system=AsyncMessagingSystem.CELERY,
+                            destination=task_name,
+                            queue_id=queue_id,
+                            handler_canonical_id=canonical_id,
+                            file_path=file_path,
+                            line=node.lineno,
+                            handler_snippet=get_line_snippet(node.lineno),
+                            confidence="HIGH",
+                            metadata=metadata,
+                        )
+                    )
+
+    # 2. Detect producers (task invocations: .delay, .apply_async, app.send_task)
+    class CallVisitor(ast.NodeVisitor):
+        def __init__(self) -> None:
+            self.current_function: str = f"{module_name}.<module>"
+
+        def visit_FunctionDef(self, fn: ast.FunctionDef) -> None:
+            prev = self.current_function
+            self.current_function = f"{module_name}.{fn.name}"
+            self.generic_visit(fn)
+            self.current_function = prev
+
+        def visit_AsyncFunctionDef(self, fn: ast.AsyncFunctionDef) -> None:
+            prev = self.current_function
+            self.current_function = f"{module_name}.{fn.name}"
+            self.generic_visit(fn)
+            self.current_function = prev
+
+        def visit_Call(self, node: ast.Call) -> None:
+            if isinstance(node.func, ast.Attribute):
+                method_name = node.func.attr
+                # Case A: task.delay(...) or task.apply_async(...)
+                if method_name in ("delay", "apply_async"):
+                    caller_val = node.func.value
+                    task_target_name = ""
+                    if isinstance(caller_val, ast.Name):
+                        task_target_name = caller_val.id
+                    elif isinstance(caller_val, ast.Attribute):
+                        task_target_name = caller_val.attr
+
+                    if task_target_name:
+                        resolved_dest = local_tasks.get(task_target_name, task_target_name)
+                        meta: dict[str, Any] = {"method": method_name}
+                        if method_name == "apply_async":
+                            for kw in node.keywords:
+                                if kw.arg in ("countdown", "eta", "queue", "retry"):
+                                    meta[kw.arg] = True
+
+                        producers.append(
+                            AsyncProducerRecord(
+                                system=AsyncMessagingSystem.CELERY,
+                                destination=resolved_dest,
+                                queue_id=f"celery:{resolved_dest}",
+                                caller_canonical_id=self.current_function,
+                                file_path=file_path,
+                                line=node.lineno,
+                                call_snippet=get_line_snippet(node.lineno),
+                                confidence="HIGH",
+                                metadata=meta,
+                            )
+                        )
+
+                # Case B: app.send_task("task_name", ...)
+                elif method_name == "send_task" and node.args:
+                    first_arg = node.args[0]
+                    if isinstance(first_arg, ast.Constant) and isinstance(first_arg.value, str):
+                        dest = first_arg.value
+                        producers.append(
+                            AsyncProducerRecord(
+                                system=AsyncMessagingSystem.CELERY,
+                                destination=dest,
+                                queue_id=f"celery:{dest}",
+                                caller_canonical_id=self.current_function,
+                                file_path=file_path,
+                                line=node.lineno,
+                                call_snippet=get_line_snippet(node.lineno),
+                                confidence="HIGH",
+                                metadata={"method": "send_task"},
+                            )
+                        )
+
+            self.generic_visit(node)
+
+    CallVisitor().visit(tree)
+
+    return producers, consumers