graphrag-codeproperty-graph 0.1.0__py3-none-any.whl

codeproperty_graph/__init__.py

@@ -0,0 +1,20 @@
+"""Code Property Graph — domain layer for Joern CPG delta ingestion.
+
+Built on document-graph for typed property graph primitives.
+Adds CPG-specific models, delta comparison, manifest management,
+and tenant lifecycle for incremental code analysis.
+"""
+
+__version__ = "0.1.0"
+
+from .models import CPGNode, CPGEdge, Manifest
+from .graph_diff import GraphDiff
+from .manifest_manager import ManifestManager
+from .delta_ingestor import DeltaIngestor
+from .tenant_ops import delete_tenant
+
+__all__ = [
+    "CPGNode", "CPGEdge", "Manifest",
+    "GraphDiff", "ManifestManager", "DeltaIngestor",
+    "delete_tenant",
+]

codeproperty_graph/delta_ingestor.py

@@ -0,0 +1,117 @@
+"""Delta Ingestor — orchestrates skip-or-replace CPG ingestion."""
+
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from .graph_diff import GraphDiff
+from .manifest_manager import ManifestManager
+from .models import Manifest
+from .tenant_ops import delete_tenant
+
+logger = logging.getLogger(__name__)
+
+
+class DeltaIngestor:
+    """Orchestrates CPG delta ingestion: compare → skip/ingest → purge → update manifest.
+
+    Usage:
+        ingestor = DeltaIngestor(bucket="graphrag-artifacts-705909755305")
+        result = await ingestor.ingest(
+            repo="amigo-core",
+            job_id="uuid",
+            nodes_data=[...],
+            edges_data=[...],
+            graph_store=neptune_store,
+            write_fn=my_write_function,
+        )
+    """
+
+    def __init__(self, bucket: str, prefix: str = "cpg-exports", region: str = "us-east-1"):
+        self._manifest_mgr = ManifestManager(bucket, prefix, region)
+
+    async def ingest(
+        self,
+        repo: str,
+        job_id: str,
+        tenant_id: str,
+        nodes_data: list[dict],
+        edges_data: list[dict],
+        nodes_path: str,
+        edges_path: str,
+        graph_store: Any,
+        write_fn=None,
+    ) -> dict:
+        """Execute delta-aware ingestion.
+
+        Args:
+            repo: Repository name
+            job_id: Current job UUID
+            tenant_id: Derived tenant for this job
+            nodes_data: Parsed node records from Joern export
+            edges_data: Parsed edge records from Joern export
+            nodes_path: S3 URI to nodes.json
+            edges_path: S3 URI to edges.json
+            graph_store: Neptune graph store instance
+            write_fn: async callable(nodes_data, edges_data, tenant_id, graph_store) → dict
+
+        Returns:
+            Dict with status, nodes_written, etc.
+        """
+        # Extract method signatures from nodes
+        method_sigs = {
+            n["full_name"]: n.get("hash", "")
+            for n in nodes_data
+            if n.get("node_type") == "METHOD" and n.get("full_name")
+        }
+
+        # Check manifest
+        changed, previous = self._manifest_mgr.has_changes(repo, method_sigs)
+
+        if not changed:
+            logger.info(f"Delta check: no changes for {repo}, skipping ingest")
+            return {
+                "status": "SKIPPED",
+                "reason": "no_changes",
+                "tenant_id": previous.tenant_id,
+                "previous_job_id": previous.job_id,
+            }
+
+        # Log diff if previous exists
+        if previous:
+            diff = GraphDiff.compare(previous.method_signatures, method_sigs)
+            logger.info(f"Delta: {diff.summary} for {repo}")
+
+        # Perform full ingest
+        if write_fn:
+            result = await write_fn(nodes_data, edges_data, tenant_id, graph_store)
+        else:
+            result = {"nodes_written": len(nodes_data), "edges_written": len(edges_data)}
+
+        # Purge old tenant
+        if previous and previous.tenant_id != tenant_id:
+            try:
+                await delete_tenant(previous.tenant_id, graph_store)
+            except Exception:
+                pass  # logged inside delete_tenant
+
+        # Update manifest
+        new_manifest = Manifest(
+            repo=repo,
+            signature=self._manifest_mgr.compute_signature(method_sigs),
+            job_id=job_id,
+            tenant_id=tenant_id,
+            exported_at=datetime.now(timezone.utc).isoformat(),
+            nodes_path=nodes_path,
+            edges_path=edges_path,
+            method_signatures=method_sigs,
+        )
+        self._manifest_mgr.put(new_manifest)
+
+        result["status"] = "INGESTED"
+        result["tenant_id"] = tenant_id
+        result["delta"] = GraphDiff.compare(
+            previous.method_signatures if previous else {}, method_sigs
+        ).summary
+
+        return result

codeproperty_graph/graph_diff.py

@@ -0,0 +1,61 @@
+"""Graph Diff — compare two CPG states and compute delta."""
+
+from dataclasses import dataclass, field
+from typing import Dict
+
+
+@dataclass
+class DiffResult:
+    """Result of comparing two method signature sets."""
+
+    added: Dict[str, str] = field(default_factory=dict)      # full_name → hash (new methods)
+    removed: Dict[str, str] = field(default_factory=dict)    # full_name → hash (deleted methods)
+    modified: Dict[str, str] = field(default_factory=dict)   # full_name → new_hash (body changed)
+    unchanged: int = 0
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.added or self.removed or self.modified)
+
+    @property
+    def summary(self) -> str:
+        return f"+{len(self.added)} -{len(self.removed)} ~{len(self.modified)} ={self.unchanged}"
+
+
+class GraphDiff:
+    """Compare method signatures between two CPG exports."""
+
+    @staticmethod
+    def compare(
+        previous: Dict[str, str],
+        current: Dict[str, str],
+    ) -> DiffResult:
+        """Compare previous vs current method_signatures dicts.
+
+        Args:
+            previous: {full_name: hash} from manifest
+            current: {full_name: hash} from new export
+
+        Returns:
+            DiffResult with added/removed/modified/unchanged counts
+        """
+        prev_keys = set(previous.keys())
+        curr_keys = set(current.keys())
+
+        added = {k: current[k] for k in curr_keys - prev_keys}
+        removed = {k: previous[k] for k in prev_keys - curr_keys}
+
+        modified = {}
+        unchanged = 0
+        for k in prev_keys & curr_keys:
+            if previous[k] != current[k]:
+                modified[k] = current[k]
+            else:
+                unchanged += 1
+
+        return DiffResult(
+            added=added,
+            removed=removed,
+            modified=modified,
+            unchanged=unchanged,
+        )

codeproperty_graph/manifest_manager.py

@@ -0,0 +1,64 @@
+"""Manifest Manager — S3-backed CPG state tracking."""
+
+import hashlib
+import json
+import logging
+from typing import Optional
+
+import boto3
+
+from .models import Manifest
+
+logger = logging.getLogger(__name__)
+
+
+class ManifestManager:
+    """Read/write/compare CPG manifests on S3."""
+
+    def __init__(self, bucket: str, prefix: str = "cpg-exports", region: str = "us-east-1"):
+        self._bucket = bucket
+        self._prefix = prefix
+        self._s3 = boto3.client("s3", region_name=region)
+
+    def compute_signature(self, method_signatures: dict[str, str]) -> str:
+        """Compute sha256 signature from method full_name:hash pairs."""
+        payload = json.dumps(method_signatures, sort_keys=True)
+        return "sha256:" + hashlib.sha256(payload.encode()).hexdigest()
+
+    def get(self, repo: str) -> Optional[Manifest]:
+        """Read manifest for a repo from S3. Returns None if not found."""
+        key = f"{self._prefix}/{repo}/manifest.json"
+        try:
+            resp = self._s3.get_object(Bucket=self._bucket, Key=key)
+            data = json.loads(resp["Body"].read())
+            return Manifest(**data)
+        except Exception:
+            return None
+
+    def put(self, manifest: Manifest) -> None:
+        """Write manifest to S3."""
+        key = f"{self._prefix}/{manifest.repo}/manifest.json"
+        body = json.dumps({
+            "repo": manifest.repo,
+            "signature": manifest.signature,
+            "job_id": manifest.job_id,
+            "tenant_id": manifest.tenant_id,
+            "exported_at": manifest.exported_at,
+            "nodes_path": manifest.nodes_path,
+            "edges_path": manifest.edges_path,
+            "method_signatures": manifest.method_signatures,
+        })
+        self._s3.put_object(Bucket=self._bucket, Key=key, Body=body.encode(), ContentType="application/json")
+        logger.info(f"Manifest written: s3://{self._bucket}/{key}")
+
+    def has_changes(self, repo: str, current_signatures: dict[str, str]) -> tuple[bool, Optional[Manifest]]:
+        """Check if current export differs from stored manifest.
+
+        Returns:
+            (has_changes: bool, previous_manifest: Optional[Manifest])
+        """
+        previous = self.get(repo)
+        if not previous:
+            return True, None
+        new_sig = self.compute_signature(current_signatures)
+        return new_sig != previous.signature, previous

codeproperty_graph/models.py

@@ -0,0 +1,62 @@
+"""CPG domain models — typed representations of Joern output."""
+
+from dataclasses import dataclass, field
+from typing import Dict, Any, Optional
+
+
+@dataclass
+class CPGNode:
+    """A code property graph node with Joern metadata.
+
+    Identity: full_name (stable across line shifts)
+    Change detection: hash (content fingerprint from Joern)
+    """
+
+    id: str
+    node_type: str  # METHOD, CALL, IDENTIFIER, LITERAL, etc.
+    full_name: str = ""
+    hash: str = ""
+    filename: str = ""
+    name: str = ""
+    code: str = ""
+    signature: str = ""
+    line_number: Optional[int] = None
+    is_external: bool = False
+    properties: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def stable_id(self) -> str:
+        """Content-addressed identity for delta comparison."""
+        return self.full_name or self.id
+
+
+@dataclass
+class CPGEdge:
+    """A code property graph edge with semantic metadata.
+
+    Joern edge types: AST, CFG, CDG, REACHING_DEF, CALL, ARGUMENT, DOMINATE, etc.
+    """
+
+    source_id: str
+    target_id: str
+    edge_type: str  # AST, CFG, CDG, REACHING_DEF, CALL, ARGUMENT
+    properties: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def key(self) -> str:
+        """Unique edge identity for diff."""
+        return f"{self.source_id}->{self.edge_type}->{self.target_id}"
+
+
+@dataclass
+class Manifest:
+    """CPG extraction manifest — tracks the current graph state for a repo."""
+
+    repo: str
+    signature: str  # sha256 of sorted method full_name:hash pairs
+    job_id: str
+    tenant_id: str
+    exported_at: str
+    nodes_path: str
+    edges_path: str
+    method_signatures: Dict[str, str] = field(default_factory=dict)  # full_name → hash

codeproperty_graph/tenant_ops.py

@@ -0,0 +1,26 @@
+"""Tenant Operations — lifecycle management for CPG tenants in Neptune."""
+
+import asyncio
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+async def delete_tenant(tenant_id: str, graph_store) -> int:
+    """Purge all nodes and edges for a tenant from Neptune.
+
+    Args:
+        tenant_id: The tenant scope to delete
+        graph_store: Neptune graph store with execute_query method
+
+    Returns:
+        Number of nodes deleted (approximate)
+    """
+    cypher = f"MATCH (n) WHERE n.tenant_id = '{tenant_id}' DETACH DELETE n"
+    try:
+        await asyncio.to_thread(graph_store.execute_query, cypher, {})
+        logger.info(f"Tenant purged: {tenant_id}")
+        return -1  # Neptune doesn't return count on DELETE
+    except Exception as e:
+        logger.warning(f"Tenant purge failed for {tenant_id}: {e}")
+        raise

graphrag_codeproperty_graph-0.1.0.dist-info/METADATA

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: graphrag-codeproperty-graph
+Version: 0.1.0
+Summary: Code Property Graphs — Joern/Semgrep extraction, AI vulnerability enrichment, risk scoring
+Project-URL: Repository, https://github.com/RW-Lab/codeproperty-graph
+Author-email: Evan Erwee <evan@erwee.com>
+License-Expression: MIT
+License-File: LICENSE
+License-File: NOTICE
+Keywords: code-analysis,cpg,graph,joern,neptune,semgrep,vulnerability
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.11
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: document-graph>=0.1.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: graphrag
+Requires-Dist: graphrag-toolkit-lexical-graph>=3.18.0; extra == 'graphrag'
+Description-Content-Type: text/markdown
+
+# Code Property Graph
+
+Code Property Graphs — Joern/Semgrep extraction, AI vulnerability enrichment, risk scoring.
+
+> **This package depends on AWS GraphRAG Toolkit (graphrag-toolkit-lexical-graph) for graph storage, vector indexing, and retrieval.**
+
+## Quick Start
+
+### Install
+
+```bash
+pip install codeproperty-graph
+```
+
+### Python API Example
+
+```python
+from codeproperty_graph import DeltaIngestor, CPGNode, CPGEdge, GraphDiff
+
+# Delta ingestion — only writes to Neptune when code actually changed
+ingestor = DeltaIngestor(bucket="graphrag-artifacts-705909755305")
+
+result = await ingestor.ingest(
+    repo="my-service",
+    job_id="build-456",
+    tenant_id="tenant_abc123",
+    nodes_data=joern_nodes,       # Joern CPG export
+    edges_data=joern_edges,       # Joern CPG export
+    nodes_path="s3://bucket/cpg-exports/my-service/build-456/nodes.json",
+    edges_path="s3://bucket/cpg-exports/my-service/build-456/edges.json",
+    graph_store=neptune_store,
+    write_fn=batch_write_function,
+)
+
+# result: {"status": "SKIPPED"} or {"status": "INGESTED", "delta": "+5 -2 ~3 =150"}
+```
+
+### Graph Diff — Compare CPG States
+
+```python
+from codeproperty_graph import GraphDiff, CPGNode
+
+# Compare current vs previous code analysis
+diff = GraphDiff.compare(
+    current_nodes=current_cpg_nodes,
+    previous_nodes=previous_cpg_nodes,
+)
+
+print(f"Added: {len(diff.added)}")
+print(f"Removed: {len(diff.removed)}")
+print(f"Modified: {len(diff.modified)}")
+print(f"Unchanged: {len(diff.unchanged)}")
+```
+
+### Manifest Management
+
+```python
+from codeproperty_graph import ManifestManager
+
+# Track CPG state per repository in S3
+manager = ManifestManager(bucket="graphrag-artifacts-705909755305")
+
+# Save manifest after successful ingestion
+await manager.save(repo="my-service", job_id="build-456", signatures=method_signatures)
+
+# Load previous manifest for diff comparison
+previous = await manager.load(repo="my-service")
+```
+
+## Package Structure
+
+```
+src/codeproperty_graph/
+├── __init__.py           # Public API: CPGNode, CPGEdge, DeltaIngestor, etc.
+├── models.py             # CPGNode, CPGEdge, Manifest — Joern-specific types
+├── graph_diff.py         # Compare two CPG states by method signature
+├── manifest_manager.py   # S3-backed state tracking per repository
+├── delta_ingestor.py     # Skip-or-replace orchestration with tenant purge
+└── tenant_ops.py         # Clean lifecycle management (delete_tenant)
+```
+
+## Integration
+
+### Architecture Stack
+
+```
+┌─────────────────────────────────────────────────────┐
+│         codeproperty-graph (this package)            │
+│  Joern/Semgrep CPG, delta ingestion, risk scoring   │
+├─────────────────────────────────────────────────────┤
+│              document-graph (infra)                  │
+│  Node, Edge, CypherBuilder, PipelineExecutor        │
+│  Multi-tenancy, batch operations                    │
+├─────────────────────────────────────────────────────┤
+│     graphrag-toolkit-lexical-graph (foundation)     │
+│  GraphStore, Neptune writer, AOSS writer            │
+│  Lexical indexing, entity resolution, retrieval     │
+└─────────────────────────────────────────────────────┘
+```
+
+### Delta Logic
+
+1. Joern exports CPG → `nodes.json` + `edges.json`
+2. Extract METHOD node signatures: `{full_name: hash}`
+3. Compare against previous manifest in S3
+4. If identical → **SKIP** (no Neptune writes, saves cost)
+5. If changed → **INGEST** full graph under new tenant, purge old tenant, update manifest
+
+### With Document Graph
+
+Code Property Graph uses document-graph for typed property graph primitives:
+
+```python
+# document-graph provides the graph write infrastructure
+from graphrag_toolkit.document_graph.graph_build.cypher_builder import CypherBuilder
+from graphrag_toolkit.document_graph import Node, Edge
+
+# codeproperty-graph adds CPG-specific semantics on top
+from codeproperty_graph import CPGNode, CPGEdge
+```
+
+### AI Vulnerability Enrichment
+
+After CPG ingestion, enrich with AI-powered vulnerability analysis:
+
+```python
+# Query Neptune for high-risk patterns
+# Score methods by complexity, dependency depth, and known CVE proximity
+# Annotate graph with risk scores for downstream consumption
+```
+
+## Requirements
+
+- Python >= 3.11
+- `document-graph >= 0.1.0`
+- `boto3 >= 1.26.0`
+- Optional: `graphrag-toolkit-lexical-graph >= 3.18.0`
+
+## License
+
+MIT — see [LICENSE](LICENSE) for details.
+
+See [NOTICE](NOTICE) for third-party acknowledgments.

graphrag_codeproperty_graph-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+codeproperty_graph/__init__.py,sha256=Kbf6nqUzsvTDHcukATDRVIt2zmDaw6JQr527_YhiwU0,612
+codeproperty_graph/delta_ingestor.py,sha256=J-TD7Z7qxKzdhdBOa3sCHTZhJQMmCruXoNRuBK9d6o8,3885
+codeproperty_graph/graph_diff.py,sha256=e6blXSv1JhtsHShuJG_U_c5oRHBGV7yMiC4mBhRKKeM,1870
+codeproperty_graph/manifest_manager.py,sha256=G4R9WqUwN9RzPg3soNEJS-54ryqi82xVI7utxM6s7oU,2394
+codeproperty_graph/models.py,sha256=vsGb5nL8efuyqreqytezmvF4tZK1adn8PbXlb6giQpQ,1713
+codeproperty_graph/tenant_ops.py,sha256=v4zzQuwjbzpePRDlCoMdPMliVF10L4s8pArmAec__C8,837
+graphrag_codeproperty_graph-0.1.0.dist-info/METADATA,sha256=n9NeKdRbxERtJA1l-tNCO1wHUZYCVLQRD99IFJ6NKQw,5941
+graphrag_codeproperty_graph-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+graphrag_codeproperty_graph-0.1.0.dist-info/licenses/LICENSE,sha256=uwE9MC3_X48nBQ0WlXxXYbTknQYFB2qBeiKk6CCVxc8,1067
+graphrag_codeproperty_graph-0.1.0.dist-info/licenses/NOTICE,sha256=r_j1bkjjV4lqE3M64cDNd_-JluPv216nqr4qicJdGfI,773
+graphrag_codeproperty_graph-0.1.0.dist-info/RECORD,,

graphrag_codeproperty_graph-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

graphrag_codeproperty_graph-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Evan Erwee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

graphrag_codeproperty_graph-0.1.0.dist-info/licenses/NOTICE

@@ -0,0 +1,27 @@
+NOTICE
+
+Code Property Graph
+Copyright (c) 2026 Evan Erwee
+
+This project is licensed under the MIT License.
+
+---
+
+Third-Party Components
+======================
+
+This project builds upon and integrates with the AWS GraphRAG Toolkit
+(lexical-graph), which is licensed under the Apache License, Version 2.0.
+
+  AWS GraphRAG Toolkit (graphrag-toolkit-lexical-graph)
+  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+  Licensed under the Apache License, Version 2.0
+  https://github.com/awslabs/graphrag-toolkit
+
+You may obtain a copy of the Apache License at:
+  http://www.apache.org/licenses/LICENSE-2.0
+
+---
+
+Our original additions and modifications in this repository are released
+under the MIT License. See the LICENSE file for the full MIT License text.