codebase-retrieval-context-engine 2.0.0__py3-none-any.whl

corbell/core/graph/providers/__init__.py

@@ -0,0 +1,6 @@
+"""Cloud provider pattern modules for infrastructure scanning.
+
+Each sub-module exposes:
+- ``TF_RESOURCE_MAP``: mapping of Terraform resource type → (node_class, kind).
+- ``CDK_PATTERNS``: list of (regex, node_class, kind) tuples for CDK/Python detection.
+"""

corbell/core/graph/providers/aws_patterns.py

@@ -0,0 +1,62 @@
+"""AWS infrastructure pattern definitions.
+
+Covers:
+- Terraform (``aws_*`` resource types)
+- AWS CDK for TypeScript / Python (``new rds.DatabaseInstance``, etc.)
+- CDKTF (``Rds.DatabaseInstance(``))
+"""
+
+from __future__ import annotations
+
+# ---------------------------------------------------------------------------
+# Terraform resource type → (node_class_label, kind)
+# node_class_label: "datastore" or "queue"
+# ---------------------------------------------------------------------------
+TF_RESOURCE_MAP: dict[str, tuple[str, str]] = {
+    # RDS / relational databases
+    "aws_db_instance":            ("datastore", "rds"),
+    "aws_rds_cluster":            ("datastore", "rds"),
+    "aws_rds_cluster_instance":   ("datastore", "rds"),
+    # DynamoDB
+    "aws_dynamodb_table":         ("datastore", "dynamodb"),
+    # S3
+    "aws_s3_bucket":              ("datastore", "s3"),
+    # ElastiCache (Redis / Memcached)
+    "aws_elasticache_cluster":    ("datastore", "redis"),
+    "aws_elasticache_replication_group": ("datastore", "redis"),
+    # SQS
+    "aws_sqs_queue":              ("queue", "sqs"),
+    # SNS
+    "aws_sns_topic":              ("queue", "sns"),
+    # MSK (managed Kafka)
+    "aws_msk_cluster":            ("queue", "kafka"),
+    "aws_msk_serverless_cluster": ("queue", "kafka"),
+    # OpenSearch / Elasticsearch
+    "aws_opensearch_domain":      ("datastore", "opensearch"),
+    "aws_elasticsearch_domain":   ("datastore", "opensearch"),
+}
+
+# ---------------------------------------------------------------------------
+# CDK / CDKTF patterns for TypeScript and Python source files.
+# Each entry: (pattern_substring, node_class_label, kind)
+# ---------------------------------------------------------------------------
+CDK_PATTERNS: list[tuple[str, str, str]] = [
+    # TypeScript CDK
+    ("new rds.DatabaseInstance(",    "datastore", "rds"),
+    ("new rds.CfnDBInstance(",       "datastore", "rds"),
+    ("new rds.DatabaseCluster(",     "datastore", "rds"),
+    ("new dynamodb.Table(",          "datastore", "dynamodb"),
+    ("new s3.Bucket(",               "datastore", "s3"),
+    ("new elasticache.CfnCacheCluster(", "datastore", "redis"),
+    ("new sqs.Queue(",               "queue", "sqs"),
+    ("new sns.Topic(",               "queue", "sns"),
+    ("new msk.Cluster(",             "queue", "kafka"),
+    # Python CDK / CDKTF
+    ("rds.DatabaseInstance(",        "datastore", "rds"),
+    ("rds.CfnDBInstance(",           "datastore", "rds"),
+    ("dynamodb.Table(",              "datastore", "dynamodb"),
+    ("s3.Bucket(",                   "datastore", "s3"),
+    ("sqs.Queue(",                   "queue", "sqs"),
+    ("sns.Topic(",                   "queue", "sns"),
+    ("msk.Cluster(",                 "queue", "kafka"),
+]

corbell/core/graph/providers/azure_patterns.py

@@ -0,0 +1,64 @@
+"""Azure infrastructure pattern definitions.
+
+Covers:
+- Terraform AzureRM provider (``azurerm_*`` resource types)
+- Azure CDK for TypeScript / Python (``@cdktf/provider-azurerm``, ``azure-native``)
+"""
+
+from __future__ import annotations
+
+# ---------------------------------------------------------------------------
+# Terraform resource type → (node_class_label, kind)
+# ---------------------------------------------------------------------------
+TF_RESOURCE_MAP: dict[str, tuple[str, str]] = {
+    # SQL / Relational
+    "azurerm_sql_database":                    ("datastore", "azure_sql"),
+    "azurerm_sql_server":                      ("datastore", "azure_sql"),
+    "azurerm_mssql_database":                  ("datastore", "azure_sql"),
+    "azurerm_postgresql_server":               ("datastore", "azure_sql"),
+    "azurerm_postgresql_flexible_server":       ("datastore", "azure_sql"),
+    "azurerm_mysql_server":                    ("datastore", "azure_sql"),
+    "azurerm_mysql_flexible_server":           ("datastore", "azure_sql"),
+    # CosmosDB (NoSQL / multi-model)
+    "azurerm_cosmosdb_account":                ("datastore", "cosmosdb"),
+    "azurerm_cosmosdb_sql_database":           ("datastore", "cosmosdb"),
+    # Blob / data lake storage
+    "azurerm_storage_account":                 ("datastore", "azure_blob"),
+    "azurerm_storage_container":               ("datastore", "azure_blob"),
+    # Redis Cache
+    "azurerm_redis_cache":                     ("datastore", "redis"),
+    # Service Bus (queues and topics)
+    "azurerm_servicebus_namespace":            ("queue", "servicebus"),
+    "azurerm_servicebus_queue":                ("queue", "servicebus"),
+    "azurerm_servicebus_topic":                ("queue", "servicebus"),
+    # Event Hub (streaming / Kafka-compatible)
+    "azurerm_eventhub_namespace":              ("queue", "eventhub"),
+    "azurerm_eventhub":                        ("queue", "eventhub"),
+    # Event Grid
+    "azurerm_eventgrid_topic":                 ("queue", "eventgrid"),
+    "azurerm_eventgrid_domain":                ("queue", "eventgrid"),
+    # Azure Data Explorer (Kusto)
+    "azurerm_kusto_cluster":                   ("datastore", "kusto"),
+}
+
+# ---------------------------------------------------------------------------
+# CDK / pulumi-azure / azure-native patterns for TypeScript and Python.
+# ---------------------------------------------------------------------------
+CDK_PATTERNS: list[tuple[str, str, str]] = [
+    # TypeScript (azure-native SDK / Pulumi Azure)
+    ("new sql.Database(",            "datastore", "azure_sql"),
+    ("new postgres.Server(",         "datastore", "azure_sql"),
+    ("new cosmos.DatabaseAccount(",  "datastore", "cosmosdb"),
+    ("new storage.StorageAccount(",  "datastore", "azure_blob"),
+    ("new redis.Redis(",             "datastore", "redis"),
+    ("new servicebus.Queue(",        "queue", "servicebus"),
+    ("new servicebus.Topic(",        "queue", "servicebus"),
+    ("new eventhub.EventHub(",       "queue", "eventhub"),
+    ("new eventgrid.Topic(",         "queue", "eventgrid"),
+    # Python (pulumi-azure or azure-native)
+    ("sql.Database(",                "datastore", "azure_sql"),
+    ("cosmos.DatabaseAccount(",      "datastore", "cosmosdb"),
+    ("storage.StorageAccount(",      "datastore", "azure_blob"),
+    ("servicebus.Queue(",            "queue", "servicebus"),
+    ("eventhub.EventHub(",           "queue", "eventhub"),
+]

corbell/core/graph/providers/gcp_patterns.py

@@ -0,0 +1,59 @@
+"""GCP infrastructure pattern definitions.
+
+Covers:
+- Terraform Google provider (``google_*`` resource types)
+- GCP CDK for TypeScript / Python (``@cdktf/provider-google``, Pulumi GCP)
+"""
+
+from __future__ import annotations
+
+# ---------------------------------------------------------------------------
+# Terraform resource type → (node_class_label, kind)
+# ---------------------------------------------------------------------------
+TF_RESOURCE_MAP: dict[str, tuple[str, str]] = {
+    # Cloud SQL
+    "google_sql_database_instance": ("datastore", "cloud_sql"),
+    "google_sql_database":          ("datastore", "cloud_sql"),
+    # Bigtable
+    "google_bigtable_instance":     ("datastore", "bigtable"),
+    "google_bigtable_table":        ("datastore", "bigtable"),
+    # Firestore / Datastore
+    "google_firestore_document":    ("datastore", "firestore"),
+    "google_datastore_index":       ("datastore", "firestore"),
+    # Spanner
+    "google_spanner_instance":      ("datastore", "spanner"),
+    "google_spanner_database":      ("datastore", "spanner"),
+    # Memorystore (Redis)
+    "google_redis_instance":        ("datastore", "redis"),
+    # GCS (Cloud Storage)
+    "google_storage_bucket":        ("datastore", "gcs"),
+    # Pub/Sub
+    "google_pubsub_topic":          ("queue", "pubsub"),
+    "google_pubsub_subscription":   ("queue", "pubsub"),
+    # BiqQuery (analytical)
+    "google_bigquery_dataset":      ("datastore", "bigquery"),
+    "google_bigquery_table":        ("datastore", "bigquery"),
+}
+
+# ---------------------------------------------------------------------------
+# CDK / Pulumi GCP patterns for TypeScript and Python.
+# ---------------------------------------------------------------------------
+CDK_PATTERNS: list[tuple[str, str, str]] = [
+    # TypeScript (Pulumi GCP / CDKTF google)
+    ("new sql.DatabaseInstance(",   "datastore", "cloud_sql"),
+    ("new bigtable.Instance(",      "datastore", "bigtable"),
+    ("new firestore.Document(",     "datastore", "firestore"),
+    ("new spanner.Instance(",       "datastore", "spanner"),
+    ("new redis.Instance(",         "datastore", "redis"),
+    ("new storage.Bucket(",         "datastore", "gcs"),
+    ("new pubsub.Topic(",           "queue", "pubsub"),
+    ("new pubsub.Subscription(",    "queue", "pubsub"),
+    ("new bigquery.Dataset(",       "datastore", "bigquery"),
+    # Python
+    ("sql.DatabaseInstance(",       "datastore", "cloud_sql"),
+    ("bigtable.Instance(",          "datastore", "bigtable"),
+    ("storage.Bucket(",             "datastore", "gcs"),
+    ("pubsub.Topic(",               "queue", "pubsub"),
+    ("pubsub.Subscription(",        "queue", "pubsub"),
+    ("bigquery.Dataset(",           "datastore", "bigquery"),
+]

corbell/core/graph/schema.py

@@ -0,0 +1,175 @@
+"""Graph schema: nodes, edges, and the GraphStore abstract interface."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class ServiceNode:
+    """Represents a discovered service (microservice, API, worker, etc.)."""
+
+    id: str
+    name: str
+    repo: str
+    language: str = "python"
+    tags: List[str] = field(default_factory=list)
+    service_type: str = "api"  # api | worker | cron | cli | service | infrastructure
+
+
+@dataclass
+class DataStoreNode:
+    """Represents a data store (DB, cache, object store)."""
+
+    id: str
+    kind: str  # postgres | redis | s3 | kafka | dynamodb | sqlite | mongodb | neo4j | chromadb
+    name: str
+
+
+@dataclass
+class QueueNode:
+    """Represents a message queue."""
+
+    id: str
+    kind: str  # sqs | rabbitmq | kafka | pubsub
+    name: str
+
+
+@dataclass
+class MethodNode:
+    """Represents a function or method extracted from source code.
+
+    Node ID format: ``{repo}::{file_path}::{class_name}::{method_name}``
+    """
+
+    id: str
+    repo: str
+    file_path: str
+    class_name: Optional[str]
+    method_name: str
+    signature: str
+    docstring: Optional[str]
+    line_start: int
+    line_end: int
+    service_id: str
+    typed_signature: Optional[str] = None  # e.g. "validate(token: str) -> bool"
+
+
+@dataclass
+class FlowNode:
+    """Represents a named execution flow detected from entry points.
+
+    A flow is a BFS-traversal from an entry point (HTTP handler, CLI command,
+    worker) through the call graph.  It records which methods participate and
+    in which order, enabling Linear tasks to say "step 3 of LoginFlow".
+
+    Node ID format: ``flow::{service_id}::{flow_name}``
+    """
+
+    id: str
+    name: str          # e.g. "LoginFlow"
+    service_id: str
+    entry_method_id: str  # method ID of the detected entry point
+    step_count: int = 0
+
+
+@dataclass
+class DependencyEdge:
+    """A directed relationship between two graph nodes."""
+
+    source_id: str
+    target_id: str
+    kind: str  # http_call | grpc_call | db_read | db_write | queue_publish | queue_consume | import | method_call
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+class GraphStore(ABC):
+    """Abstract interface for the architecture graph store."""
+
+    @abstractmethod
+    def upsert_node(self, node: ServiceNode | DataStoreNode | QueueNode | MethodNode | FlowNode) -> None:
+        """Insert or update a node in the graph."""
+        ...
+
+    @abstractmethod
+    def upsert_edge(self, edge: DependencyEdge) -> None:
+        """Insert or update an edge in the graph."""
+        ...
+
+    @abstractmethod
+    def get_service(self, service_id: str) -> Optional[ServiceNode]:
+        """Retrieve a service node by ID."""
+        ...
+
+    @abstractmethod
+    def get_all_services(self) -> List[ServiceNode]:
+        """Return all service nodes."""
+        ...
+
+    @abstractmethod
+    def get_dependencies(self, service_id: str) -> List[DependencyEdge]:
+        """Return all edges where source_id == service_id."""
+        ...
+
+    @abstractmethod
+    def get_dependents(self, service_id: str) -> List[DependencyEdge]:
+        """Return all edges where target_id == service_id."""
+        ...
+
+    @abstractmethod
+    def get_method(self, method_id: str) -> Optional[MethodNode]:
+        """Retrieve a method node by ID."""
+        ...
+
+    @abstractmethod
+    def get_call_path(
+        self, from_method_id: str, to_method_id: str, max_depth: int = 5
+    ) -> List[List[str]]:
+        """Return all call paths from one method to another (BFS)."""
+        ...
+
+    @abstractmethod
+    def get_methods_for_service(self, service_id: str) -> List[MethodNode]:
+        """Return all method nodes belonging to a service."""
+        ...
+
+    @abstractmethod
+    def get_callers_of_method(self, method_id: str) -> List[MethodNode]:
+        """Return all MethodNodes that call the given method (reverse call lookup).
+
+        Uses ``method_call`` edges in the store where ``target_id == method_id``.
+        Works for all languages as long as call edges were built by
+        :class:`~corbell.core.graph.method_graph.MethodGraphBuilder`.
+        """
+        ...
+
+    @abstractmethod
+    def get_flows_for_method(self, method_id: str) -> List[Dict[str, Any]]:
+        """Return flows that include the given method as a step.
+
+        Returns a list of dicts with keys ``flow_id``, ``flow_name``,
+        ``step`` (1-based position in the flow), and ``entry_method_id``.
+        """
+        ...
+
+    @abstractmethod
+    def get_all_nodes_summary(self) -> Dict[str, Any]:
+        """Return a summary of all nodes and edges for display."""
+        ...
+
+    @abstractmethod
+    def to_mermaid(self) -> str:
+        """Return a Mermaid representation of the graph."""
+        ...
+
+    @abstractmethod
+    def to_json(self) -> str:
+        """Return the JSON representation of the service graph."""
+        ...
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove all data from the store."""
+        ...