codegraph-engine 2.3.0__py3-none-any.whl → 3.0.0__py3-none-any.whl

codegraph/async_queue/detectors/sqs.py

@@ -0,0 +1,135 @@
+"""AWS SQS message producer and consumer detector.
+
+Detects:
+  - Publishers / Producers: sqs.send_message(QueueUrl=...), sqs.send_message_batch()
+  - Consumers / Workers: sqs.receive_message(QueueUrl=...), @sqs_handler()
+"""
+from __future__ import annotations
+
+import ast
+from typing import Any
+
+from codegraph.async_queue.models import (
+    AsyncConsumerRecord,
+    AsyncMessagingSystem,
+    AsyncProducerRecord,
+)
+
+
+def _extract_queue_name(url_or_name: str) -> str:
+    """Extract queue name from SQS URL or name string."""
+    clean = url_or_name.strip().rstrip("/")
+    if "/" in clean:
+        return clean.split("/")[-1]
+    return clean
+
+
+def detect_sqs(
+    tree: ast.AST,
+    content: str,
+    file_path: str,
+    module_name: str,
+) -> tuple[list[AsyncProducerRecord], list[AsyncConsumerRecord]]:
+    producers: list[AsyncProducerRecord] = []
+    consumers: list[AsyncConsumerRecord] = []
+    lines = content.splitlines()
+
+    def get_line_snippet(lineno: int) -> str:
+        if 1 <= lineno <= len(lines):
+            return lines[lineno - 1].strip()
+        return ""
+
+    class SQSVisitor(ast.NodeVisitor):
+        def __init__(self) -> None:
+            self.current_function: str = f"{module_name}.<module>"
+
+        def visit_FunctionDef(self, fn: ast.FunctionDef) -> None:
+            prev = self.current_function
+            self.current_function = f"{module_name}.{fn.name}"
+            # Check for decorator like @sqs_handler
+            for dec in fn.decorator_list:
+                dec_func = dec.func if isinstance(dec, ast.Call) else dec
+                dec_name = getattr(dec_func, "id", "") or getattr(dec_func, "attr", "")
+                if "sqs" in dec_name.lower():
+                    q_name = ""
+                    if isinstance(dec, ast.Call):
+                        for kw in dec.keywords:
+                            if kw.arg in ("queue", "queue_name", "queue_url") and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
+                                q_name = _extract_queue_name(kw.value.value)
+                    if q_name:
+                        consumers.append(
+                            AsyncConsumerRecord(
+                                system=AsyncMessagingSystem.AWS_SQS,
+                                destination=q_name,
+                                queue_id=f"aws_sqs:{q_name}",
+                                handler_canonical_id=f"{module_name}.{fn.name}",
+                                file_path=file_path,
+                                line=fn.lineno,
+                                handler_snippet=get_line_snippet(fn.lineno),
+                                confidence="HIGH",
+                                metadata={"decorator": dec_name},
+                            )
+                        )
+            self.generic_visit(fn)
+            self.current_function = prev
+
+        def visit_AsyncFunctionDef(self, fn: ast.AsyncFunctionDef) -> None:
+            prev = self.current_function
+            self.current_function = f"{module_name}.{fn.name}"
+            self.generic_visit(fn)
+            self.current_function = prev
+
+        def visit_Call(self, node: ast.Call) -> None:
+            if isinstance(node.func, ast.Attribute):
+                attr = node.func.attr
+                # Producer: send_message, send_message_batch
+                if attr in ("send_message", "send_message_batch"):
+                    q_name = ""
+                    meta: dict[str, Any] = {"method": attr}
+                    for kw in node.keywords:
+                        if kw.arg == "QueueUrl" and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
+                            q_name = _extract_queue_name(kw.value.value)
+                        elif kw.arg in ("DelaySeconds", "MessageDeduplicationId", "MessageGroupId"):
+                            meta[kw.arg] = True
+
+                    if q_name:
+                        producers.append(
+                            AsyncProducerRecord(
+                                system=AsyncMessagingSystem.AWS_SQS,
+                                destination=q_name,
+                                queue_id=f"aws_sqs:{q_name}",
+                                caller_canonical_id=self.current_function,
+                                file_path=file_path,
+                                line=node.lineno,
+                                call_snippet=get_line_snippet(node.lineno),
+                                confidence="HIGH",
+                                metadata=meta,
+                            )
+                        )
+
+                # Consumer: receive_message
+                elif attr == "receive_message":
+                    q_name = ""
+                    for kw in node.keywords:
+                        if kw.arg == "QueueUrl" and isinstance(kw.value, ast.Constant) and isinstance(kw.value.value, str):
+                            q_name = _extract_queue_name(kw.value.value)
+                    if q_name:
+                        consumers.append(
+                            AsyncConsumerRecord(
+                                system=AsyncMessagingSystem.AWS_SQS,
+                                destination=q_name,
+                                queue_id=f"aws_sqs:{q_name}",
+                                handler_canonical_id=self.current_function,
+                                file_path=file_path,
+                                line=node.lineno,
+                                handler_snippet=get_line_snippet(node.lineno),
+                                confidence="HIGH",
+                                metadata={"method": "receive_message"},
+                            )
+                        )
+
+            self.generic_visit(node)
+
+    SQSVisitor().visit(tree)
+
+    return producers, consumers

codegraph/async_queue/models.py

@@ -0,0 +1,161 @@
+"""Data models for asynchronous message queues, task dispatches, and cross-service traces.
+
+Epistemic Invariants:
+1. Never collapse distributed message passing into a generic synchronous CALLS edge.
+2. Distinct semantic relationships:
+   - PRODUCES_TO_QUEUE: Producer AST call site -> Queue/Topic entity
+   - CONSUMES_FROM_QUEUE: Queue/Topic entity -> Consumer Worker AST handler
+   - STATIC_QUEUE_LINKED: Same-repository static topic match (static inference, not runtime proof)
+   - OTEL_DISTRIBUTED_VERIFIED: OpenTelemetry runtime-confirmed trace/span/link proof
+3. Queue identity: `f"{system}:{destination}"` (system + topic/queue name).
+4. Never persist raw message bodies or unredacted keys.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import StrEnum
+from typing import Any
+
+
+class AsyncMessagingSystem(StrEnum):
+    """Supported message brokers and asynchronous execution systems."""
+
+    KAFKA = "kafka"
+    CELERY = "celery"
+    AWS_SQS = "aws_sqs"
+    RABBITMQ = "rabbitmq"
+    REDIS = "redis"
+    GENERIC = "generic"
+
+
+class AsyncQueueEvidenceClass(StrEnum):
+    """Epistemic evidence classification for asynchronous queue links."""
+
+    STATIC_QUEUE_LINKED = "STATIC_QUEUE_LINKED"
+    OTEL_DISTRIBUTED_VERIFIED = "OTEL_DISTRIBUTED_VERIFIED"
+
+
+class AsyncOperationType(StrEnum):
+    """Operation performed on the message broker."""
+
+    PUBLISH = "publish"
+    CONSUME = "consume"
+
+
+@dataclass(frozen=True)
+class AsyncQueueEntity:
+    """Logical queue, topic, or task channel."""
+
+    queue_id: str  # f"{system}:{destination}"
+    system: AsyncMessagingSystem
+    destination: str  # topic name, queue URL, or celery task name
+    service: str = ""
+    environment: str = ""
+    is_dead_letter_queue: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class AsyncProducerRecord:
+    """Static or runtime-observed message publisher/producer call site."""
+
+    system: AsyncMessagingSystem
+    destination: str
+    queue_id: str
+    caller_canonical_id: str
+    file_path: str
+    line: int
+    call_snippet: str
+    confidence: str = "HIGH"
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "system": str(self.system),
+            "destination": self.destination,
+            "queue_id": self.queue_id,
+            "caller_canonical_id": self.caller_canonical_id,
+            "file_path": self.file_path,
+            "line": self.line,
+            "call_snippet": self.call_snippet,
+            "confidence": self.confidence,
+            "metadata": dict(self.metadata),
+        }
+
+
+@dataclass(frozen=True)
+class AsyncConsumerRecord:
+    """Static or runtime-observed message subscriber/consumer worker handler."""
+
+    system: AsyncMessagingSystem
+    destination: str
+    queue_id: str
+    handler_canonical_id: str
+    file_path: str
+    line: int
+    handler_snippet: str
+    confidence: str = "HIGH"
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "system": str(self.system),
+            "destination": self.destination,
+            "queue_id": self.queue_id,
+            "handler_canonical_id": self.handler_canonical_id,
+            "file_path": self.file_path,
+            "line": self.line,
+            "handler_snippet": self.handler_snippet,
+            "confidence": self.confidence,
+            "metadata": dict(self.metadata),
+        }
+
+
+@dataclass(frozen=True)
+class AsyncQueueLink:
+    """Bridged relationship between a producer and a consumer across an async boundary."""
+
+    link_id: str
+    queue_id: str
+    system: AsyncMessagingSystem
+    destination: str
+    producer_id: str
+    producer_file: str
+    producer_line: int
+    consumer_id: str
+    consumer_file: str
+    consumer_line: int
+    evidence_class: AsyncQueueEvidenceClass
+    confidence: str = "HIGH"
+    trace_count: int = 1
+    avg_latency_ms: float = 0.0
+    p50_ms: float = 0.0
+    p95_ms: float = 0.0
+    p99_ms: float = 0.0
+    error_count: int = 0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "link_id": self.link_id,
+            "queue_id": self.queue_id,
+            "system": str(self.system),
+            "destination": self.destination,
+            "producer_id": self.producer_id,
+            "producer_file": self.producer_file,
+            "producer_line": self.producer_line,
+            "consumer_id": self.consumer_id,
+            "consumer_file": self.consumer_file,
+            "consumer_line": self.consumer_line,
+            "evidence_class": str(self.evidence_class),
+            "confidence": self.confidence,
+            "trace_count": self.trace_count,
+            "avg_latency_ms": self.avg_latency_ms,
+            "p50_ms": self.p50_ms,
+            "p95_ms": self.p95_ms,
+            "p99_ms": self.p99_ms,
+            "error_count": self.error_count,
+            "metadata": dict(self.metadata),
+        }

codegraph/async_queue/otel_models.py

@@ -0,0 +1,74 @@
+"""Canonical OpenTelemetry models and reconciliation classifications.
+
+Epistemic Invariant:
+1. Static and runtime evidence are distinct:
+   - STATIC_QUEUE_LINKED: Same-repo static topic match.
+   - OTEL_DISTRIBUTED_VERIFIED: Real empirical telemetry links producer and consumer.
+   - RUNTIME_ONLY_OBSERVED: Observed in telemetry, but consumer or producer lives in another repository/service.
+   - STATIC_RUNTIME_CONFLICT: Static code points to queue X, but runtime points to queue Y or reports persistent failure.
+   - UNKNOWN: Inconclusive / unobserved.
+2. Zero payload retention: Never persist message bodies, tokens, or raw sensitive headers.
+3. Correlation IDs are stored strictly as salted SHA-256 digests.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import StrEnum
+from typing import Any
+
+
+class OTelEvidenceStatus(StrEnum):
+    """Epistemic evidence classification for static and distributed links."""
+
+    STATIC_QUEUE_LINKED = "STATIC_QUEUE_LINKED"
+    OTEL_DISTRIBUTED_VERIFIED = "OTEL_DISTRIBUTED_VERIFIED"
+    RUNTIME_ONLY_OBSERVED = "RUNTIME_ONLY_OBSERVED"
+    STATIC_RUNTIME_CONFLICT = "STATIC_RUNTIME_CONFLICT"
+    UNKNOWN = "UNKNOWN"
+
+
+@dataclass(frozen=True)
+class OTelSpanRecord:
+    """Sanitized, canonical representation of an ingested OpenTelemetry span."""
+
+    trace_id: str
+    span_id: str
+    parent_span_id: str = ""
+    service_name: str = ""
+    name: str = ""
+    kind: str = "INTERNAL"  # PRODUCER, CONSUMER, SERVER, CLIENT, INTERNAL
+    start_time_unix_nano: int = 0
+    end_time_unix_nano: int = 0
+    duration_ms: float = 0.0
+    status_code: str = "OK"  # OK, ERROR, UNSET
+    messaging_system: str = ""
+    messaging_destination: str = ""
+    messaging_operation: str = ""  # publish, process, receive
+    correlation_id_hash: str = ""  # Salted SHA-256 digest
+    attributes: dict[str, Any] = field(default_factory=dict)
+    links: tuple[tuple[str, str], ...] = ()  # ((trace_id, span_id), ...)
+
+    @property
+    def composite_id(self) -> str:
+        """Stable idempotent key for deduplication."""
+        return f"{self.trace_id}:{self.span_id}"
+
+    def to_dict(self) -> dict[str, Any]:
+        d = asdict(self)
+        d["links"] = [{"trace_id": t, "span_id": s} for t, s in self.links]
+        return d
+
+
+@dataclass(frozen=True)
+class TraceReconciliationResult:
+    """Summary of an idempotent OTel trace ingestion and reconciliation pass."""
+
+    spans_ingested: int
+    spans_deduplicated: int
+    links_upgraded: int
+    links_created_runtime_only: int
+    conflicts_detected: int
+    details: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)