codegraph-engine 2.2.1__py3-none-any.whl → 3.0.0__py3-none-any.whl

codegraph/runtime/port_inspector.py

@@ -0,0 +1,304 @@
+"""Localhost Port and Process Inspector for Multi-Service Architectures.
+
+Deterministically detects, fingerprints, and attributes running localhost services:
+1. Port <-> PID mapping via lsof / netstat / /proc.
+2. Process CWD and command-line attribution to workspace packages.
+3. Static port inference from docker-compose.yml, package.json, and .env.
+4. Framework fingerprinting (Next.js, Vite, FastAPI, NestJS, Express, PostgreSQL, MongoDB).
+"""
+from __future__ import annotations
+
+import re
+import shutil
+import subprocess
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ServiceEndpoint:
+    port: int
+    host: str = "localhost"
+    pid: int | None = None
+    service_name: str = "unknown"
+    framework: str = "unknown"
+    process_cwd: str = ""
+    relative_repo_path: str = ""
+    command: str = ""
+    is_live: bool = False
+    attributes: dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "port": self.port,
+            "host": self.host,
+            "pid": self.pid,
+            "service_name": self.service_name,
+            "framework": self.framework,
+            "process_cwd": self.process_cwd,
+            "relative_repo_path": self.relative_repo_path,
+            "command": self.command,
+            "is_live": self.is_live,
+            "attributes": dict(self.attributes),
+        }
+
+
+def _infer_framework_from_cmd(cmd: str) -> str:
+    cmd_lower = cmd.lower()
+    if "next" in cmd_lower:
+        return "Next.js"
+    if "vite" in cmd_lower:
+        return "Vite"
+    if "uvicorn" in cmd_lower or "fastapi" in cmd_lower:
+        return "FastAPI"
+    if "nest" in cmd_lower:
+        return "NestJS"
+    if "express" in cmd_lower or "node" in cmd_lower:
+        return "Node/Express"
+    if "postgres" in cmd_lower:
+        return "PostgreSQL"
+    if "mongod" in cmd_lower or "mongo" in cmd_lower:
+        return "MongoDB"
+    if "redis" in cmd_lower:
+        return "Redis"
+    if "celery" in cmd_lower:
+        return "Celery"
+    return "GenericService"
+
+
+def _infer_service_name_from_path(repo_root: Path, cwd_path: Path, framework: str) -> str:
+    try:
+        rel = cwd_path.relative_to(repo_root)
+        parts = rel.parts
+        if parts:
+            if parts[0] in ("apps", "packages", "services") and len(parts) > 1:
+                return parts[1]
+            return parts[-1]
+    except Exception:
+        pass
+    return framework.lower().replace(".", "").replace("/", "-")
+
+
+def discover_static_configured_ports(repo_root: Path) -> dict[int, ServiceEndpoint]:
+    """Scan docker-compose, package.json, and .env files with in-memory TTL caching."""
+    global _STATIC_PORT_CACHE
+    root_key = str(repo_root)
+    now = time.monotonic()
+    if root_key in _STATIC_PORT_CACHE:
+        last_t, cached_map = _STATIC_PORT_CACHE[root_key]
+        if now - last_t < _PORT_CACHE_TTL_SEC:
+            return cached_map
+
+    results: dict[int, ServiceEndpoint] = {}
+
+    # 1. docker-compose.yml / compose.yaml
+    for compose_name in ("docker-compose.yml", "docker-compose.yaml", "compose.yaml", "compose.yml"):
+        compose_file = repo_root / compose_name
+        if compose_file.exists():
+            try:
+                content = compose_file.read_text(encoding="utf-8", errors="replace")
+                # match ports: - "3000:3000" or - 8000:8000
+                for m in re.finditer(r"""['"]?(\d{2,5}):(\d{2,5})['"]?""", content):
+                    host_port = int(m.group(1))
+                    results[host_port] = ServiceEndpoint(
+                        port=host_port,
+                        service_name=f"docker-service-{host_port}",
+                        framework="DockerContainer",
+                        relative_repo_path=str(compose_file.relative_to(repo_root)),
+                        is_live=False,
+                    )
+            except Exception:
+                pass
+
+    # 2. package.json scripts (e.g. -p 3000, PORT=4000)
+    for pkg_file in repo_root.glob("**/package.json"):
+        if "node_modules" in pkg_file.parts:
+            continue
+        try:
+            content = pkg_file.read_text(encoding="utf-8", errors="replace")
+            for m in re.finditer(r"""(?:PORT\s*=\s*|-p\s+|--port\s+)(\d{2,5})""", content):
+                port_num = int(m.group(1))
+                rel_dir = pkg_file.parent.relative_to(repo_root)
+                s_name = pkg_file.parent.name or "web"
+                results[port_num] = ServiceEndpoint(
+                    port=port_num,
+                    service_name=s_name,
+                    framework="Node/Web",
+                    relative_repo_path=str(rel_dir),
+                    is_live=False,
+                )
+        except Exception:
+            pass
+
+    # 3. .env files
+    for env_file in (repo_root / ".env", repo_root / ".env.development", repo_root / ".env.local"):
+        if env_file.exists():
+            try:
+                content = env_file.read_text(encoding="utf-8", errors="replace")
+                for m in re.finditer(r"""^\s*(?:PORT|SERVER_PORT|API_PORT)\s*=\s*(\d{2,5})""", content, re.MULTILINE):
+                    port_num = int(m.group(1))
+                    results[port_num] = ServiceEndpoint(
+                        port=port_num,
+                        service_name="env-service",
+                        framework="ConfiguredService",
+                        relative_repo_path=str(env_file.relative_to(repo_root)),
+                        is_live=False,
+                    )
+            except Exception:
+                pass
+
+    _STATIC_PORT_CACHE[root_key] = (now, results)
+    return results
+
+
+_PORT_CACHE: tuple[float, list[ServiceEndpoint]] = (0.0, [])
+_STATIC_PORT_CACHE: dict[str, tuple[float, dict[int, ServiceEndpoint]]] = {}
+_PORT_CACHE_TTL_SEC = 5.0
+
+
+def discover_live_listening_ports(repo_root: Path | None = None) -> list[ServiceEndpoint]:
+    """Inspect active OS listening ports on localhost with 5-second TTL cache (Zero-CPU idle)."""
+    global _PORT_CACHE
+    now = time.monotonic()
+    last_time, cached_endpoints = _PORT_CACHE
+    if now - last_time < _PORT_CACHE_TTL_SEC and cached_endpoints:
+        return cached_endpoints
+
+    endpoints: list[ServiceEndpoint] = []
+    seen_ports: set[int] = set()
+
+    lsof_bin = shutil.which("lsof")
+    if lsof_bin:
+        try:
+            # lsof -iTCP -sTCP:LISTEN -n -P
+            proc = subprocess.run(
+                [lsof_bin, "-iTCP", "-sTCP:LISTEN", "-n", "-P"],
+                capture_output=True,
+                text=True,
+                timeout=3,
+                check=False,
+            )
+            if proc.returncode == 0:
+                for line in proc.stdout.splitlines()[1:]:
+                    parts = line.split()
+                    if len(parts) >= 9:
+                        cmd = parts[0]
+                        pid_str = parts[1]
+                        addr_str = parts[8]
+                        # address format: *:3000, 127.0.0.1:8000, [::1]:5432
+                        m = re.search(r":(\d+)$", addr_str)
+                        if m:
+                            port = int(m.group(1))
+                            if port in seen_ports:
+                                continue
+                            seen_ports.add(port)
+                            pid = int(pid_str) if pid_str.isdigit() else None
+                            cwd_str = ""
+                            rel_repo = ""
+                            if pid:
+                                cwd_str = _get_process_cwd(pid)
+                                if repo_root and cwd_str:
+                                    try:
+                                        rel_repo = str(Path(cwd_str).relative_to(repo_root))
+                                    except Exception:
+                                        rel_repo = ""
+                            framework = _infer_framework_from_cmd(cmd)
+                            svc_name = (
+                                _infer_service_name_from_path(repo_root, Path(cwd_str), framework)
+                                if repo_root and cwd_str
+                                else cmd
+                            )
+                            endpoints.append(
+                                ServiceEndpoint(
+                                    port=port,
+                                    pid=pid,
+                                    service_name=svc_name,
+                                    framework=framework,
+                                    process_cwd=cwd_str,
+                                    relative_repo_path=rel_repo,
+                                    command=cmd,
+                                    is_live=True,
+                                )
+                            )
+        except Exception:
+            pass
+
+    _PORT_CACHE = (now, endpoints)
+    return endpoints
+
+
+def _get_process_cwd(pid: int) -> str:
+    """Get process working directory securely across macOS and Linux."""
+    # 1. Linux /proc/<pid>/cwd
+    proc_cwd = Path(f"/proc/{pid}/cwd")
+    if proc_cwd.exists():
+        try:
+            return str(proc_cwd.resolve())
+        except Exception:
+            pass
+
+    # 2. macOS lsof -p <pid> | grep cwd
+    lsof_bin = shutil.which("lsof")
+    if lsof_bin:
+        try:
+            proc = subprocess.run(
+                [lsof_bin, "-a", "-p", str(pid), "-d", "cwd", "-Fn"],
+                capture_output=True,
+                text=True,
+                timeout=1,
+                check=False,
+            )
+            for line in proc.stdout.splitlines():
+                if line.startswith("n"):
+                    return line[1:].strip()
+        except Exception:
+            pass
+    return ""
+
+
+def resolve_port_to_service(
+    port: int,
+    repo_root: Path,
+    live_endpoints: list[ServiceEndpoint] | None = None,
+) -> ServiceEndpoint:
+    """Identify the exact service, package, and framework for a given localhost port."""
+    if live_endpoints is None:
+        live_endpoints = discover_live_listening_ports(repo_root)
+
+    for ep in live_endpoints:
+        if ep.port == port:
+            return ep
+
+    static_map = discover_static_configured_ports(repo_root)
+    if port in static_map:
+        return static_map[port]
+
+    # Common standard ports fallback
+    standard_defaults: dict[int, tuple[str, str]] = {
+        3000: ("frontend", "Next.js/React"),
+        5173: ("frontend", "Vite"),
+        8000: ("backend-api", "FastAPI/Django"),
+        4000: ("backend-api", "NestJS/Express"),
+        5000: ("backend-api", "Flask/Express"),
+        5432: ("database", "PostgreSQL"),
+        27017: ("database", "MongoDB"),
+        6379: ("cache", "Redis"),
+        3306: ("database", "MySQL"),
+    }
+    if port in standard_defaults:
+        s_name, fw = standard_defaults[port]
+        return ServiceEndpoint(
+            port=port,
+            service_name=s_name,
+            framework=fw,
+            is_live=False,
+        )
+
+    return ServiceEndpoint(
+        port=port,
+        service_name=f"service-{port}",
+        framework="Unknown",
+        is_live=False,
+    )

codegraph/runtime/runner.py

@@ -35,13 +35,35 @@ const fs = require('fs');
 
 const logPath = {escaped_log_path};
 
+let _cg_buffer = [];
+let _cg_last_flush = Date.now();
+const _CG_BATCH_SIZE = 50;
+const _CG_FLUSH_INTERVAL_MS = 1500;
+
+function _cg_flush_buffer() {{
+  if (_cg_buffer.length === 0) return;
+  const content = _cg_buffer.join('');
+  _cg_buffer = [];
+  _cg_last_flush = Date.now();
+  try {{
+    fs.appendFileSync(logPath, content);
+  }} catch (_) {{}}
+}}
+
 function emitEvent(event) {{
   try {{
-    fs.appendFileSync(logPath, JSON.stringify(event) + '\\n');
+    _cg_buffer.push(JSON.stringify(event) + '\\n');
+    if (_cg_buffer.length >= _CG_BATCH_SIZE || (Date.now() - _cg_last_flush) >= _CG_FLUSH_INTERVAL_MS) {{
+      _cg_flush_buffer();
+    }}
   }} catch (_) {{}}
 }}
 
+setInterval(_cg_flush_buffer, _CG_FLUSH_INTERVAL_MS);
+process.on('exit', _cg_flush_buffer);
+
 function hookServer(serverProto) {{
+
   const origEmit = serverProto.emit;
   serverProto.emit = function(event, req, res) {{
     if (event === 'request' && req && res) {{
@@ -87,23 +109,64 @@ process.on('uncaughtExceptionMonitor', (err) => {{
 
 
 def _generate_python_interceptor(output_jsonl: Path) -> str:
-    """Generate lightweight Python WSGI/ASGI/FastAPI request interceptor."""
+    """Generate lightweight Python WSGI/ASGI/FastAPI request interceptor with in-memory buffering."""
     escaped_log_path = repr(str(output_jsonl))
     return f"""# CodeGraph Python Runtime Interceptor
-import sys
+import atexit
 import json
+import os
+import sys
+import threading
 import time
 
 LOG_PATH = {escaped_log_path}
 
+_cg_buffer = []
+_cg_lock = threading.Lock()
+_cg_last_flush = time.time()
+_cg_batch_size = 50
+_cg_flush_interval = 1.5
+
+def _cg_flush():
+    global _cg_last_flush
+    lines = None
+    with _cg_lock:
+        if _cg_buffer:
+            lines = "".join(_cg_buffer)
+            _cg_buffer.clear()
+            _cg_last_flush = time.time()
+    if lines:
+        try:
+            with open(LOG_PATH, "a", encoding="utf-8") as f:
+                f.write(lines)
+        except Exception:
+            pass
+
 def emit_event(event):
     try:
-        with open(LOG_PATH, "a", encoding="utf-8") as f:
-            f.write(json.dumps(event) + "\\n")
+        line = json.dumps(event) + "\\n"
+        should_flush = False
+        with _cg_lock:
+            _cg_buffer.append(line)
+            if len(_cg_buffer) >= _cg_batch_size or (time.time() - _cg_last_flush) >= _cg_flush_interval:
+                should_flush = True
+        if should_flush:
+            _cg_flush()
     except Exception:
         pass
 
-# Intercept uncaught exceptions
+def _cg_bg_flusher():
+    while True:
+        time.sleep(_cg_flush_interval)
+        if _cg_buffer:
+            _cg_flush()
+
+_cg_flusher_thread = threading.Thread(target=_cg_bg_flusher, daemon=True)
+_cg_flusher_thread.start()
+atexit.register(_cg_flush)
+
+
+# 1. Intercept uncaught exceptions
 _orig_excepthook = sys.excepthook
 def _cg_excepthook(exc_type, exc_value, exc_traceback):
     emit_event({{
@@ -118,6 +181,92 @@ def _cg_excepthook(exc_type, exc_value, exc_traceback):
     return _orig_excepthook(exc_type, exc_value, exc_traceback)
 
 sys.excepthook = _cg_excepthook
+
+# 2. Intercept SQLite query executions
+try:
+    import sqlite3
+    _orig_connect = sqlite3.connect
+
+    def _cg_connect(*args, **kwargs):
+        con = _orig_connect(*args, **kwargs)
+        def _sql_trace(statement):
+            if statement and isinstance(statement, str) and statement.strip():
+                emit_event({{
+                    "trace_id": "tr_sql_" + str(int(time.time())),
+                    "span_id": "sp_sql_" + str(id(con)),
+                    "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                    "sql": statement,
+                    "duration_ms": 1.0,
+                    "service_name": "python_dev_server",
+                    "source_format": "otel",
+                }})
+        try:
+            con.set_trace_callback(_sql_trace)
+        except Exception:
+            pass
+        return con
+
+    sqlite3.connect = _cg_connect
+except Exception:
+    pass
+
+
+# 3. Intercept SQLAlchemy engine query executions if imported
+try:
+    import sqlalchemy.event
+    import sqlalchemy.engine
+
+    @sqlalchemy.event.listens_for(sqlalchemy.engine.Engine, "after_cursor_execute")
+    def _cg_after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
+        if statement and str(statement).strip():
+            emit_event({{
+                "trace_id": "tr_sql_" + str(int(time.time())),
+                "span_id": "sp_sql_" + str(id(cursor)),
+                "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                "sql": str(statement),
+                "duration_ms": 1.0,
+                "service_name": "python_dev_server",
+                "source_format": "otel",
+            }})
+except Exception:
+    pass
+
+# 4. Intercept Werkzeug/Flask dev server requests
+try:
+    import werkzeug.serving
+    _orig_run_simple = werkzeug.serving.run_simple
+
+    def _cg_run_simple(hostname, port, application, *args, **kwargs):
+        def _wrapped_app(environ, start_response):
+            path = environ.get("PATH_INFO", "/")
+            method = environ.get("REQUEST_METHOD", "GET")
+            status_box = [200]
+            def _wrapped_sr(status, headers, exc_info=None):
+                try:
+                    status_box[0] = int(status.split()[0])
+                except Exception:
+                    pass
+                return start_response(status, headers, exc_info)
+            st = time.time()
+            try:
+                return application(environ, _wrapped_sr)
+            finally:
+                emit_event({{
+                    "trace_id": "tr_http_" + str(int(time.time())),
+                    "span_id": "sp_http_" + str(int(time.time())),
+                    "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                    "http_method": method,
+                    "route": path,
+                    "status_code": status_box[0],
+                    "duration_ms": (time.time() - st) * 1000.0,
+                    "service_name": "python_dev_server",
+                    "source_format": "otel",
+                }})
+        return _orig_run_simple(hostname, port, _wrapped_app, *args, **kwargs)
+
+    werkzeug.serving.run_simple = _cg_run_simple
+except Exception:
+    pass
 """
 
 

codegraph/runtime/trace_linker.py

@@ -0,0 +1,175 @@
+"""Cross-Service & Multi-Localhost Distributed Trace Linker.
+
+Stitches W3C distributed traces across multiple localhost ports:
+- Frontend (e.g. Next.js on localhost:3000)
+- Backend API (e.g. FastAPI / NestJS on localhost:8000)
+- Database mutation (e.g. PostgreSQL on localhost:5432)
+
+Attributes every step to:
+1. Participating localhost service & port.
+2. Static AST symbol / route handler.
+3. Actual latency and HTTP status code.
+"""
+from __future__ import annotations
+
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class TraceSpanNode:
+    span_id: str
+    parent_span_id: str
+    service_name: str
+    port: int | None
+    route_path: str
+    handler_symbol: str
+    db_operation: str
+    db_table: str
+    status_code: int | None
+    duration_ms: float
+    children: tuple[TraceSpanNode, ...] = ()
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "span_id": self.span_id,
+            "parent_span_id": self.parent_span_id,
+            "service_name": self.service_name,
+            "port": self.port,
+            "route_path": self.route_path,
+            "handler_symbol": self.handler_symbol,
+            "db_operation": self.db_operation,
+            "db_table": self.db_table,
+            "status_code": self.status_code,
+            "duration_ms": self.duration_ms,
+            "children": [c.as_dict() for c in self.children],
+        }
+
+
+@dataclass(frozen=True)
+class DistributedTraceReport:
+    trace_id: str
+    total_duration_ms: float
+    services_involved: tuple[dict[str, Any], ...]
+    root_spans: tuple[TraceSpanNode, ...]
+    db_queries_count: int
+    has_errors: bool
+    summary: str
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "trace_id": self.trace_id,
+            "total_duration_ms": self.total_duration_ms,
+            "services_involved": list(self.services_involved),
+            "root_spans": [s.as_dict() for s in self.root_spans],
+            "db_queries_count": self.db_queries_count,
+            "has_errors": self.has_errors,
+            "summary": self.summary,
+        }
+
+
+def link_distributed_trace(
+    con: sqlite3.Connection,
+    repo_root: Path,
+    trace_id: str,
+) -> DistributedTraceReport | None:
+    """Reconstruct multi-service distributed execution graph for a given trace_id."""
+    clean_tid = trace_id.strip()
+    if not clean_tid:
+        return None
+
+    rows = con.execute(
+        """SELECT event_id, trace_id, span_id, parent_span_id, http_method, route_path,
+                  handler_symbol, service_name, db_operation, db_table, status_code,
+                  duration_ms
+           FROM runtime_observations
+           WHERE trace_id = ?
+           ORDER BY duration_ms DESC""",
+        (clean_tid,),
+    ).fetchall()
+
+    if not rows:
+        return None
+
+    # Group spans
+    spans_by_id: dict[str, dict[str, Any]] = {}
+    children_by_parent: dict[str, list[str]] = {}
+    services_set: set[str] = set()
+    db_queries = 0
+    has_err = False
+    total_time = 0.0
+
+    for r in rows:
+        sid = str(r["span_id"] or r["event_id"])
+        pid = str(r["parent_span_id"] or "")
+        s_name = str(r["service_name"] or "unknown_service")
+        services_set.add(s_name)
+        dur = float(r["duration_ms"] or 0.0)
+        total_time = max(total_time, dur)
+        sc = r["status_code"]
+        if sc and int(sc) >= 400:
+            has_err = True
+        if r["db_operation"]:
+            db_queries += 1
+
+        spans_by_id[sid] = {
+            "span_id": sid,
+            "parent_span_id": pid,
+            "service_name": s_name,
+            "port": None,
+            "route_path": str(r["route_path"] or ""),
+            "handler_symbol": str(r["handler_symbol"] or ""),
+            "db_operation": str(r["db_operation"] or ""),
+            "db_table": str(r["db_table"] or ""),
+            "status_code": sc,
+            "duration_ms": dur,
+        }
+        if pid not in children_by_parent:
+            children_by_parent[pid] = []
+        children_by_parent[pid].append(sid)
+
+    # Build tree
+    def _build_node(sid: str) -> TraceSpanNode:
+        data = spans_by_id[sid]
+        child_sids = children_by_parent.get(sid, [])
+        children = tuple(_build_node(cid) for cid in child_sids if cid in spans_by_id)
+        return TraceSpanNode(
+            span_id=data["span_id"],
+            parent_span_id=data["parent_span_id"],
+            service_name=data["service_name"],
+            port=data["port"],
+            route_path=data["route_path"],
+            handler_symbol=data["handler_symbol"],
+            db_operation=data["db_operation"],
+            db_table=data["db_table"],
+            status_code=data["status_code"],
+            duration_ms=data["duration_ms"],
+            children=children,
+        )
+
+    # Root spans: parent_span_id is empty or not in spans_by_id
+    roots: list[TraceSpanNode] = []
+    for sid, data in spans_by_id.items():
+        pid = data["parent_span_id"]
+        if not pid or pid not in spans_by_id:
+            roots.append(_build_node(sid))
+
+    # Resolve services
+    services_list: list[dict[str, Any]] = [{"name": s} for s in sorted(services_set)]
+
+    summary = (
+        f"Trace {clean_tid[:8]}: {len(rows)} spans across {len(services_set)} service(s), "
+        f"{db_queries} DB queries, total duration {total_time:.1f}ms."
+    )
+
+    return DistributedTraceReport(
+        trace_id=clean_tid,
+        total_duration_ms=total_time,
+        services_involved=tuple(services_list),
+        root_spans=tuple(roots),
+        db_queries_count=db_queries,
+        has_errors=has_err,
+        summary=summary,
+    )