graphrag-codeproperty-graph 0.4.4__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/PKG-INFO +4 -3
  2. graphrag_codeproperty_graph-0.5.0/examples/README.md +45 -0
  3. graphrag_codeproperty_graph-0.5.0/examples/notebooks/01-CPG-Models-and-GraphDiff.ipynb +309 -0
  4. graphrag_codeproperty_graph-0.5.0/examples/notebooks/02-Delta-Ingestion-Pipeline.ipynb +253 -0
  5. graphrag_codeproperty_graph-0.5.0/examples/notebooks/data/sample_edges.json +25 -0
  6. graphrag_codeproperty_graph-0.5.0/examples/notebooks/data/sample_nodes.json +21 -0
  7. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/pyproject.toml +3 -2
  8. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/__init__.py +63 -0
  9. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/artifact_reader.py +179 -0
  10. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/artifact_validator.py +162 -0
  11. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/code_slice_store.py +209 -0
  12. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/cpg_service.py +273 -0
  13. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/delta_ingestor.py +388 -0
  14. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/src/codeproperty_graph/graph_diff.py +3 -0
  15. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/graph_loader.py +248 -0
  16. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/graphson_converter.py +243 -0
  17. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/manifest_manager.py +174 -0
  18. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/models.py +313 -0
  19. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/schema.py +197 -0
  20. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/summary_overlay.py +149 -0
  21. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/tenant_ops.py +86 -0
  22. graphrag_codeproperty_graph-0.5.0/src/codeproperty_graph/vector_loader.py +307 -0
  23. graphrag_codeproperty_graph-0.5.0/tests/test_cpg_service.py +280 -0
  24. graphrag_codeproperty_graph-0.4.4/src/codeproperty_graph/__init__.py +0 -20
  25. graphrag_codeproperty_graph-0.4.4/src/codeproperty_graph/delta_ingestor.py +0 -117
  26. graphrag_codeproperty_graph-0.4.4/src/codeproperty_graph/manifest_manager.py +0 -64
  27. graphrag_codeproperty_graph-0.4.4/src/codeproperty_graph/models.py +0 -62
  28. graphrag_codeproperty_graph-0.4.4/src/codeproperty_graph/tenant_ops.py +0 -26
  29. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/.github/workflows/publish.yml +0 -0
  30. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/.gitignore +0 -0
  31. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/CHANGELOG.md +0 -0
  32. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/CONTRIBUTING.md +0 -0
  33. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/LICENSE +0 -0
  34. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/NOTICE +0 -0
  35. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/README.md +0 -0
  36. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/ARCHITECTURE.md +0 -0
  37. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/Defining the Master Evidence Graph: A Formal Model for Stateful Software Verdicts.md +0 -0
  38. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/HYBRID-GRAPH.md +0 -0
  39. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/NOTICE.md +0 -0
  40. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/Toward a Master Evidence Graph for Software Verdicts.md +0 -0
  41. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/graphrag_com_redrawn_architecture.md +0 -0
  42. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/graphrag_phased_architecture.md +0 -0
  43. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/neptune_vs_neptune_analytics_cpg_rag_architecture copy.md +0 -0
  44. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/docs/neptune_vs_neptune_analytics_cpg_rag_architecture.md +0 -0
  45. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/sbom/AWS-SERVICES.md +0 -0
  46. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/sbom/SBOM.md +0 -0
  47. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/sbom/codeproperty-graph.cdx.json +0 -0
  48. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/tests/__init__.py +0 -0
  49. {graphrag_codeproperty_graph-0.4.4 → graphrag_codeproperty_graph-0.5.0}/tests/test_codeproperty_graph.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: graphrag-codeproperty-graph
3
- Version: 0.4.4
3
+ Version: 0.5.0
4
4
  Summary: Code Property Graphs — Joern/Semgrep extraction, AI vulnerability enrichment, risk scoring
5
5
  Project-URL: Repository, https://github.com/kanjani-ai-research/codeproperty-graph
6
6
  Author-email: Evan Erwee <evan@erwee.com>
@@ -13,7 +13,8 @@ Classifier: Intended Audience :: Developers
13
13
  Classifier: Topic :: Security
14
14
  Classifier: Topic :: Software Development :: Quality Assurance
15
15
  Requires-Python: >=3.11
16
- Requires-Dist: graphrag-document-graph>=3.0.9
16
+ Requires-Dist: boto3>=1.28.0
17
+ Requires-Dist: graphrag-document-graph>=3.1.1
17
18
  Provides-Extra: dev
18
19
  Requires-Dist: black; extra == 'dev'
19
20
  Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
@@ -0,0 +1,45 @@
1
+ # Code Property Graph Examples
2
+
3
+ Example notebooks for `codeproperty-graph` — delta-aware code analysis ingestion into Neptune.
4
+
5
+ ## Prerequisites
6
+
7
+ - Amazon Neptune (DB or Analytics) for production notebooks
8
+ - Python 3.10+
9
+ - Joern (for generating real CPG exports; sample data provided for demos)
10
+
11
+ ## Setup
12
+
13
+ ```bash
14
+ pip install codeproperty-graph
15
+ ```
16
+
17
+ For Neptune integration:
18
+ ```bash
19
+ pip install codeproperty-graph graphrag-toolkit-document-graph[graphrag,neptune]
20
+ ```
21
+
22
+ ## Notebooks
23
+
24
+ | Notebook | Description | Neptune Required? |
25
+ |----------|-------------|-------------------|
26
+ | `01-CPG-Models-and-GraphDiff.ipynb` | Core models, method signatures, and diff comparison | No (local only) |
27
+ | `02-Delta-Ingestion-Pipeline.ipynb` | Full pipeline: manifest → diff → skip/ingest decision | Optional (has mock mode) |
28
+
29
+ ## Sample Data
30
+
31
+ `data/sample_nodes.json` — 8 CPG nodes (3 METHOD, 2 CALL, 2 IDENTIFIER, 1 LITERAL)
32
+ `data/sample_edges.json` — 10 edges (AST, CFG, CALL, ARGUMENT)
33
+
34
+ Represents a minimal Java project with `Main.main()`, `Parser.parse()`, and `Validator.validate()`.
35
+
36
+ ## Generating Real CPG Data
37
+
38
+ ```bash
39
+ # Install Joern: https://joern.io
40
+ joern-export --repr cpg14 --format json --out cpg-export/ /path/to/source
41
+
42
+ # Output:
43
+ # cpg-export/nodes.json
44
+ # cpg-export/edges.json
45
+ ```
@@ -0,0 +1,309 @@
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# CPG Schema, Models, and Graph Diff\n",
8
+ "\n",
9
+ "This notebook demonstrates:\n",
10
+ "- The full Joern CPG schema (20 node types, 14+ edge types, 9 languages)\n",
11
+ "- Parsing Joern exports with `CPGNode.from_joern()` and `CPGEdge.from_joern()`\n",
12
+ "- Graph analysis: type distribution, call graph, structure\n",
13
+ "- Change detection with `GraphDiff`\n",
14
+ "\n",
15
+ "**Prerequisites:**\n",
16
+ "- `pip install graphrag-toolkit-codeproperty-graph`\n",
17
+ "- No Neptune or AWS credentials required\n"
18
+ ]
19
+ },
20
+ {
21
+ "cell_type": "markdown",
22
+ "metadata": {},
23
+ "source": [
24
+ "## 1. The CPG Schema\n",
25
+ "\n",
26
+ "Explore the complete Joern type system.\n"
27
+ ]
28
+ },
29
+ {
30
+ "cell_type": "code",
31
+ "metadata": {},
32
+ "source": [
33
+ "from graphrag_toolkit.codeproperty_graph import (\n",
34
+ " NodeType, EdgeType, DELTA_RELEVANT_TYPES, SUPPORTED_LANGUAGES,\n",
35
+ " joern_export_command\n",
36
+ ")\n",
37
+ "\n",
38
+ "print(f'Node types ({len(NodeType)}):')\n",
39
+ "for nt in NodeType:\n",
40
+ " marker = ' \u2605' if nt in DELTA_RELEVANT_TYPES else ''\n",
41
+ " print(f' {nt.value:20s}{marker}')\n",
42
+ "\n",
43
+ "print(f'\\nEdge types ({len(EdgeType)}):')\n",
44
+ "for et in EdgeType:\n",
45
+ " print(f' {et.value}')\n",
46
+ "\n",
47
+ "print(f'\\nSupported languages ({len(SUPPORTED_LANGUAGES)}):')\n",
48
+ "for lang, desc in SUPPORTED_LANGUAGES.items():\n",
49
+ " print(f' {lang:12s} {desc}')\n",
50
+ "\n",
51
+ "print(f'\\n\u2605 = participates in delta comparison')\n"
52
+ ],
53
+ "outputs": [],
54
+ "execution_count": null
55
+ },
56
+ {
57
+ "cell_type": "markdown",
58
+ "metadata": {},
59
+ "source": [
60
+ "## 2. Joern Export Command\n",
61
+ "\n",
62
+ "Generate the CLI command to produce a CPG from source code.\n"
63
+ ]
64
+ },
65
+ {
66
+ "cell_type": "code",
67
+ "metadata": {},
68
+ "source": [
69
+ "# Generate export commands for different languages\n",
70
+ "for lang in ['java', 'python', 'javascript']:\n",
71
+ " cmd = joern_export_command('/src/my-service', output_dir='cpg-out', language=lang)\n",
72
+ " print(f'{lang:12s} {cmd}')\n",
73
+ "\n",
74
+ "# Auto-detect language\n",
75
+ "print(f'{\"auto\":12s} {joern_export_command(\"/src/my-service\")}')\n"
76
+ ],
77
+ "outputs": [],
78
+ "execution_count": null
79
+ },
80
+ {
81
+ "cell_type": "markdown",
82
+ "metadata": {},
83
+ "source": [
84
+ "## 3. Parse Joern Export with `from_joern()`\n",
85
+ "\n",
86
+ "Load raw Joern JSON and parse into typed CPGNode/CPGEdge objects.\n",
87
+ "The factory methods handle both UPPER_CASE and camelCase property formats.\n"
88
+ ]
89
+ },
90
+ {
91
+ "cell_type": "code",
92
+ "metadata": {},
93
+ "source": [
94
+ "import json\n",
95
+ "from graphrag_toolkit.codeproperty_graph import CPGNode, CPGEdge\n",
96
+ "\n",
97
+ "with open('data/sample_nodes.json') as f:\n",
98
+ " raw_nodes = json.load(f)\n",
99
+ "with open('data/sample_edges.json') as f:\n",
100
+ " raw_edges = json.load(f)\n",
101
+ "\n",
102
+ "# Parse with from_joern()\n",
103
+ "nodes = [CPGNode.from_joern(raw) for raw in raw_nodes]\n",
104
+ "edges = [CPGEdge.from_joern(raw) for raw in raw_edges]\n",
105
+ "\n",
106
+ "print(f'Parsed {len(nodes)} nodes, {len(edges)} edges')\n"
107
+ ],
108
+ "outputs": [],
109
+ "execution_count": null
110
+ },
111
+ {
112
+ "cell_type": "markdown",
113
+ "metadata": {},
114
+ "source": [
115
+ "## 4. Graph Analysis: Type Distribution\n",
116
+ "\n",
117
+ "See what node and edge types are in the graph.\n"
118
+ ]
119
+ },
120
+ {
121
+ "cell_type": "code",
122
+ "metadata": {},
123
+ "source": [
124
+ "from collections import Counter\n",
125
+ "\n",
126
+ "# Node type distribution\n",
127
+ "node_types = Counter(n.node_type for n in nodes)\n",
128
+ "print('Node types:')\n",
129
+ "for ntype, count in node_types.most_common():\n",
130
+ " print(f' {ntype:20s} {count:3d}')\n",
131
+ "\n",
132
+ "# Edge type distribution\n",
133
+ "edge_types = Counter(e.edge_type for e in edges)\n",
134
+ "print(f'\\nEdge types:')\n",
135
+ "for etype, count in edge_types.most_common():\n",
136
+ " print(f' {etype:20s} {count:3d}')\n"
137
+ ],
138
+ "outputs": [],
139
+ "execution_count": null
140
+ },
141
+ {
142
+ "cell_type": "markdown",
143
+ "metadata": {},
144
+ "source": [
145
+ "## 5. Explore Methods (Delta-Relevant Nodes)\n",
146
+ "\n",
147
+ "Only METHOD nodes participate in change detection.\n",
148
+ "Their `full_name \u2192 hash` mapping is the fingerprint for delta comparison.\n"
149
+ ]
150
+ },
151
+ {
152
+ "cell_type": "code",
153
+ "metadata": {},
154
+ "source": [
155
+ "methods = [n for n in nodes if n.node_type == NodeType.METHOD]\n",
156
+ "\n",
157
+ "print(f'Methods ({len(methods)}):')\n",
158
+ "for m in methods:\n",
159
+ " print(f' {m.full_name}')\n",
160
+ " print(f' signature: {m.signature}')\n",
161
+ " print(f' hash: {m.hash}')\n",
162
+ " print(f' file: {m.filename}:{m.line_number}-{m.line_number_end}')\n",
163
+ " print(f' external: {m.is_external}')\n",
164
+ " print()\n"
165
+ ],
166
+ "outputs": [],
167
+ "execution_count": null
168
+ },
169
+ {
170
+ "cell_type": "markdown",
171
+ "metadata": {},
172
+ "source": [
173
+ "## 6. Explore Structure (Classes, Files, Namespaces)\n"
174
+ ]
175
+ },
176
+ {
177
+ "cell_type": "code",
178
+ "metadata": {},
179
+ "source": [
180
+ "from graphrag_toolkit.codeproperty_graph.schema import STRUCTURE_TYPES\n",
181
+ "\n",
182
+ "structure_nodes = [n for n in nodes if n.node_type in STRUCTURE_TYPES]\n",
183
+ "print(f'Structure nodes ({len(structure_nodes)}):')\n",
184
+ "for n in structure_nodes:\n",
185
+ " print(f' {n.node_type:16s} {n.full_name or n.name}')\n"
186
+ ],
187
+ "outputs": [],
188
+ "execution_count": null
189
+ },
190
+ {
191
+ "cell_type": "markdown",
192
+ "metadata": {},
193
+ "source": [
194
+ "## 7. Call Graph\n",
195
+ "\n",
196
+ "Extract the CALL edges to see which methods call which.\n"
197
+ ]
198
+ },
199
+ {
200
+ "cell_type": "code",
201
+ "metadata": {},
202
+ "source": [
203
+ "call_edges = [e for e in edges if e.edge_type == EdgeType.CALL]\n",
204
+ "node_map = {n.id: n for n in nodes}\n",
205
+ "\n",
206
+ "print(f'Call graph ({len(call_edges)} edges):')\n",
207
+ "for e in call_edges:\n",
208
+ " src = node_map.get(e.source_id)\n",
209
+ " tgt = node_map.get(e.target_id)\n",
210
+ " src_name = src.full_name if src else e.source_id\n",
211
+ " tgt_name = tgt.full_name if tgt else e.target_id\n",
212
+ " print(f' {src_name} \u2192 {tgt_name}')\n"
213
+ ],
214
+ "outputs": [],
215
+ "execution_count": null
216
+ },
217
+ {
218
+ "cell_type": "markdown",
219
+ "metadata": {},
220
+ "source": [
221
+ "## 8. Delta Comparison (GraphDiff)\n",
222
+ "\n",
223
+ "Compare method signatures between two CPG snapshots.\n",
224
+ "This is how `DeltaIngestor` decides whether to skip or ingest.\n"
225
+ ]
226
+ },
227
+ {
228
+ "cell_type": "code",
229
+ "metadata": {},
230
+ "source": [
231
+ "from graphrag_toolkit.codeproperty_graph import GraphDiff\n",
232
+ "\n",
233
+ "# Current state\n",
234
+ "current_sigs = {m.full_name: m.hash for m in methods}\n",
235
+ "print(f'Current signatures:')\n",
236
+ "for name, h in current_sigs.items():\n",
237
+ " print(f' {name}: {h}')\n",
238
+ "\n",
239
+ "# Simulate next commit: parse() refactored, validate() deleted, check() added\n",
240
+ "next_sigs = {\n",
241
+ " 'com.example.Main.main': 'a1b2c3d4e5', # unchanged\n",
242
+ " 'com.example.Parser.parse': 'NEWH4SH999', # MODIFIED\n",
243
+ " 'com.example.Checker.check': 'p6q7r8s9t0', # ADDED\n",
244
+ "}\n",
245
+ "\n",
246
+ "diff = GraphDiff.compare(current_sigs, next_sigs)\n",
247
+ "print(f'\\nDelta: {diff.summary}')\n",
248
+ "print(f' Added: {list(diff.added.keys())}')\n",
249
+ "print(f' Removed: {list(diff.removed.keys())}')\n",
250
+ "print(f' Modified: {list(diff.modified.keys())}')\n",
251
+ "print(f' Unchanged: {diff.unchanged}')\n",
252
+ "print(f'\\n\u2192 DeltaIngestor would: {\"INGEST\" if diff.has_changes else \"SKIP\"}')\n"
253
+ ],
254
+ "outputs": [],
255
+ "execution_count": null
256
+ },
257
+ {
258
+ "cell_type": "markdown",
259
+ "metadata": {},
260
+ "source": [
261
+ "## 9. No-Change Case (SKIP)\n",
262
+ "\n",
263
+ "When code hasn't changed, `GraphDiff` returns no delta \u2014 zero Neptune writes.\n"
264
+ ]
265
+ },
266
+ {
267
+ "cell_type": "code",
268
+ "metadata": {},
269
+ "source": [
270
+ "no_diff = GraphDiff.compare(current_sigs, current_sigs)\n",
271
+ "print(f'Delta: {no_diff.summary}')\n",
272
+ "print(f'Has changes: {no_diff.has_changes}')\n",
273
+ "print(f'\\n\u2192 DeltaIngestor would: SKIP')\n",
274
+ "print(f' Zero Neptune writes. Zero cost. <1 second.')\n"
275
+ ],
276
+ "outputs": [],
277
+ "execution_count": null
278
+ },
279
+ {
280
+ "cell_type": "markdown",
281
+ "metadata": {},
282
+ "source": [
283
+ "## Summary\n",
284
+ "\n",
285
+ "| Feature | What It Does |\n",
286
+ "|---------|-------------|\n",
287
+ "| `NodeType` / `EdgeType` enums | Complete Joern schema (20 + 14 types) |\n",
288
+ "| `CPGNode.from_joern(raw)` | Parse Joern JSON export \u2192 typed objects |\n",
289
+ "| `DELTA_RELEVANT_TYPES` | Which types matter for change detection |\n",
290
+ "| `STRUCTURE_TYPES` | Types for architectural analysis |\n",
291
+ "| `GraphDiff.compare()` | Method signature delta \u2192 skip or ingest |\n",
292
+ "| `joern_export_command()` | Generate the CLI command for any language |\n"
293
+ ]
294
+ }
295
+ ],
296
+ "metadata": {
297
+ "kernelspec": {
298
+ "display_name": "Python 3",
299
+ "language": "python",
300
+ "name": "python3"
301
+ },
302
+ "language_info": {
303
+ "name": "python",
304
+ "version": "3.12.0"
305
+ }
306
+ },
307
+ "nbformat": 4,
308
+ "nbformat_minor": 4
309
+ }
@@ -0,0 +1,253 @@
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {},
6
+ "source": [
7
+ "# 02 \u2014 Delta Ingestion Pipeline\n",
8
+ "\n",
9
+ "This notebook demonstrates the full DeltaIngestor pipeline: manifest comparison, skip-or-ingest decision, and Neptune graph write.\n",
10
+ "\n",
11
+ "**What you'll learn:**\n",
12
+ "- How ManifestManager tracks CPG state in S3\n",
13
+ "- How DeltaIngestor decides to skip or ingest\n",
14
+ "- How tenant-based graph replacement works\n",
15
+ "\n",
16
+ "**Prerequisites:**\n",
17
+ "- Neptune Analytics or Neptune Database endpoint\n",
18
+ "- S3 bucket for manifests\n",
19
+ "- IAM permissions for Neptune and S3"
20
+ ]
21
+ },
22
+ {
23
+ "cell_type": "code",
24
+ "metadata": {},
25
+ "source": [
26
+ "# %pip install codeproperty-graph graphrag-toolkit-document-graph[graphrag]"
27
+ ]
28
+ },
29
+ {
30
+ "cell_type": "code",
31
+ "metadata": {},
32
+ "source": [
33
+ "import json\n",
34
+ "import asyncio\n",
35
+ "from codeproperty_graph import DeltaIngestor, ManifestManager, GraphDiff, CPGNode"
36
+ ]
37
+ },
38
+ {
39
+ "cell_type": "markdown",
40
+ "metadata": {},
41
+ "source": [
42
+ "## Configuration"
43
+ ]
44
+ },
45
+ {
46
+ "cell_type": "code",
47
+ "metadata": {},
48
+ "source": [
49
+ "# Replace with your values:\n",
50
+ "BUCKET = \"my-graphrag-artifacts\" # S3 bucket for manifests\n",
51
+ "REPO = \"example-service\" # Repository name\n",
52
+ "JOB_ID = \"build-001\" # CI/CD build identifier\n",
53
+ "NEPTUNE_ENDPOINT = \"neptune-analytics://my-graph-id\" # or neptune-db://...\n",
54
+ "\n",
55
+ "# Tenant ID (auto-generated in production, fixed here for demo)\n",
56
+ "TENANT_ID = \"demo123\""
57
+ ]
58
+ },
59
+ {
60
+ "cell_type": "markdown",
61
+ "metadata": {},
62
+ "source": [
63
+ "## Load Joern Export"
64
+ ]
65
+ },
66
+ {
67
+ "cell_type": "code",
68
+ "metadata": {},
69
+ "source": [
70
+ "import json\n",
71
+ "from graphrag_toolkit.codeproperty_graph import CPGNode, CPGEdge, NodeType\n",
72
+ "\n",
73
+ "with open('data/sample_nodes.json') as f:\n",
74
+ " nodes = [CPGNode.from_joern(raw) for raw in json.load(f)]\n",
75
+ "\n",
76
+ "with open('data/sample_edges.json') as f:\n",
77
+ " edges = [CPGEdge.from_joern(raw) for raw in json.load(f)]\n",
78
+ "\n",
79
+ "print(f'Parsed {len(nodes)} nodes ({len(set(n.node_type for n in nodes))} types)')\n",
80
+ "print(f'Parsed {len(edges)} edges ({len(set(e.edge_type for e in edges))} types)')\n"
81
+ ]
82
+ },
83
+ {
84
+ "cell_type": "markdown",
85
+ "metadata": {},
86
+ "source": [
87
+ "## Step 1: Extract Method Signatures\n",
88
+ "\n",
89
+ "Only METHOD nodes participate in change detection."
90
+ ]
91
+ },
92
+ {
93
+ "cell_type": "code",
94
+ "metadata": {},
95
+ "source": [
96
+ "# Only METHOD nodes participate in delta comparison\n",
97
+ "methods = [n for n in nodes if n.node_type == NodeType.METHOD]\n",
98
+ "current_sigs = {m.full_name: m.hash for m in methods}\n",
99
+ "\n",
100
+ "print(f'Extracted {len(current_sigs)} method signatures (from {len(nodes)} total nodes):')\n",
101
+ "for name, h in current_sigs.items():\n",
102
+ " print(f' {name}: {h}')\n",
103
+ "\n",
104
+ "print(f'\\nOther node types (not in delta): {set(n.node_type for n in nodes if n.node_type != NodeType.METHOD)}')\n"
105
+ ]
106
+ },
107
+ {
108
+ "cell_type": "markdown",
109
+ "metadata": {},
110
+ "source": [
111
+ "## Step 2: Check Previous Manifest\n",
112
+ "\n",
113
+ "ManifestManager retrieves the last known state from S3."
114
+ ]
115
+ },
116
+ {
117
+ "cell_type": "code",
118
+ "metadata": {},
119
+ "source": [
120
+ "manifest_mgr = ManifestManager(bucket=BUCKET)\n",
121
+ "\n",
122
+ "# In production this reads from S3. For demo, simulate:\n",
123
+ "# previous_manifest = manifest_mgr.get(REPO)\n",
124
+ "\n",
125
+ "# Simulate: first run (no previous manifest)\n",
126
+ "previous_manifest = None\n",
127
+ "\n",
128
+ "if previous_manifest is None:\n",
129
+ " print(\"No previous manifest \u2014 this is a fresh ingestion.\")\n",
130
+ " print(\"All methods will be treated as ADDED.\")\n",
131
+ "else:\n",
132
+ " print(f\"Previous manifest: tenant={previous_manifest.tenant_id}, job={previous_manifest.job_id}\")"
133
+ ]
134
+ },
135
+ {
136
+ "cell_type": "markdown",
137
+ "metadata": {},
138
+ "source": [
139
+ "## Step 3: Compute Diff"
140
+ ]
141
+ },
142
+ {
143
+ "cell_type": "code",
144
+ "metadata": {},
145
+ "source": [
146
+ "previous_sigs = previous_manifest.method_signatures if previous_manifest else {}\n",
147
+ "\n",
148
+ "diff = GraphDiff.compare(previous_sigs, current_sigs)\n",
149
+ "print(f\"Diff: {diff.summary}\")\n",
150
+ "print(f\"Has changes: {diff.has_changes}\")\n",
151
+ "\n",
152
+ "if diff.has_changes:\n",
153
+ " print(\"\\n\u2192 Will INGEST to Neptune\")\n",
154
+ "else:\n",
155
+ " print(\"\\n\u2192 Will SKIP (no Neptune writes)\")"
156
+ ]
157
+ },
158
+ {
159
+ "cell_type": "markdown",
160
+ "metadata": {},
161
+ "source": [
162
+ "## Step 4: Write to Neptune (if changed)\n",
163
+ "\n",
164
+ "In production, this uses the GraphStore. For this demo, we simulate the write."
165
+ ]
166
+ },
167
+ {
168
+ "cell_type": "code",
169
+ "metadata": {},
170
+ "source": [
171
+ "# Simulated write function (replace with actual Neptune write in production)\n",
172
+ "write_log = []\n",
173
+ "\n",
174
+ "async def mock_write_fn(query: str, params: dict):\n",
175
+ " \"\"\"Simulates a Neptune write \u2014 logs instead of executing.\"\"\"\n",
176
+ " write_log.append({'query': query[:80], 'param_count': len(params)})\n",
177
+ "\n",
178
+ "if diff.has_changes:\n",
179
+ " print(f\"Writing {len(nodes_data)} nodes and {len(edges_data)} edges...\")\n",
180
+ " print(f\"Tenant: {TENANT_ID}\")\n",
181
+ " print(f\"(Using mock write \u2014 replace with graph_store.execute_query in production)\")\n",
182
+ " \n",
183
+ " # In production:\n",
184
+ " # result = await ingestor.ingest(repo=REPO, job_id=JOB_ID, ...)\n",
185
+ " \n",
186
+ " # Simulated result:\n",
187
+ " result = {\n",
188
+ " \"status\": \"INGESTED\",\n",
189
+ " \"delta\": diff.summary,\n",
190
+ " \"tenant_id\": TENANT_ID,\n",
191
+ " \"nodes_written\": len(nodes_data),\n",
192
+ " \"edges_written\": len(edges_data),\n",
193
+ " }\n",
194
+ " print(f\"\\nResult: {json.dumps(result, indent=2)}\")\n",
195
+ "else:\n",
196
+ " result = {\"status\": \"SKIPPED\", \"reason\": \"no changes detected\"}\n",
197
+ " print(f\"Result: {json.dumps(result, indent=2)}\")"
198
+ ]
199
+ },
200
+ {
201
+ "cell_type": "markdown",
202
+ "metadata": {},
203
+ "source": [
204
+ "## Step 5: Simulate Second Run (No Changes)\n",
205
+ "\n",
206
+ "On the next CI/CD run with no code changes, the ingestor skips entirely."
207
+ ]
208
+ },
209
+ {
210
+ "cell_type": "code",
211
+ "metadata": {},
212
+ "source": [
213
+ "# Same signatures as before \u2014 simulates \"no code change\" commit\n",
214
+ "second_run_sigs = current_sigs.copy()\n",
215
+ "\n",
216
+ "diff2 = GraphDiff.compare(current_sigs, second_run_sigs)\n",
217
+ "print(f\"Second run diff: {diff2.summary}\")\n",
218
+ "print(f\"Has changes: {diff2.has_changes}\")\n",
219
+ "print()\n",
220
+ "print(\"\u2192 SKIPPED \u2014 zero Neptune writes, zero cost, <1 second\")\n",
221
+ "print(\" This is the common case in CI/CD (most commits don't change method bodies)\")"
222
+ ]
223
+ },
224
+ {
225
+ "cell_type": "markdown",
226
+ "metadata": {},
227
+ "source": [
228
+ "## Summary\n",
229
+ "\n",
230
+ "| Scenario | Action | Neptune Cost | Time |\n",
231
+ "|----------|--------|-------------|------|\n",
232
+ "| First run (no manifest) | Full ingest | N nodes + M edges | Seconds |\n",
233
+ "| Code changed | Full replace (new tenant, purge old) | N nodes + M edges + purge | Seconds |\n",
234
+ "| No code change | SKIP | Zero | <1s |\n",
235
+ "\n",
236
+ "The delta logic ensures you only pay for Neptune writes when code actually changes."
237
+ ]
238
+ }
239
+ ],
240
+ "metadata": {
241
+ "kernelspec": {
242
+ "display_name": "Python 3",
243
+ "language": "python",
244
+ "name": "python3"
245
+ },
246
+ "language_info": {
247
+ "name": "python",
248
+ "version": "3.12.0"
249
+ }
250
+ },
251
+ "nbformat": 4,
252
+ "nbformat_minor": 4
253
+ }
@@ -0,0 +1,25 @@
1
+ [
2
+ {"src": "5001", "dst": "4001", "label": "CONTAINS", "properties": {}},
3
+ {"src": "5002", "dst": "4002", "label": "CONTAINS", "properties": {}},
4
+ {"src": "5003", "dst": "4003", "label": "CONTAINS", "properties": {}},
5
+ {"src": "4001", "dst": "1001", "label": "CONTAINS", "properties": {}},
6
+ {"src": "4002", "dst": "1002", "label": "CONTAINS", "properties": {}},
7
+ {"src": "4003", "dst": "1003", "label": "CONTAINS", "properties": {}},
8
+ {"src": "1001", "dst": "2001", "label": "AST", "properties": {"ORDER": 1}},
9
+ {"src": "1001", "dst": "2002", "label": "AST", "properties": {"ORDER": 2}},
10
+ {"src": "1001", "dst": "3001", "label": "AST", "properties": {"ORDER": 3}},
11
+ {"src": "2001", "dst": "3001", "label": "ARGUMENT", "properties": {"ORDER": 1}},
12
+ {"src": "2001", "dst": "3002", "label": "ARGUMENT", "properties": {"ORDER": 2}},
13
+ {"src": "2002", "dst": "3003", "label": "ARGUMENT", "properties": {"ORDER": 1}},
14
+ {"src": "1001", "dst": "1002", "label": "CALL", "properties": {}},
15
+ {"src": "1001", "dst": "1003", "label": "CALL", "properties": {}},
16
+ {"src": "1001", "dst": "1002", "label": "CFG", "properties": {}},
17
+ {"src": "1002", "dst": "1003", "label": "CFG", "properties": {}},
18
+ {"src": "1001", "dst": "6001", "label": "AST", "properties": {"ORDER": 0}},
19
+ {"src": "1002", "dst": "6002", "label": "AST", "properties": {"ORDER": 0}},
20
+ {"src": "1003", "dst": "6003", "label": "AST", "properties": {"ORDER": 0}},
21
+ {"src": "3001", "dst": "6001", "label": "REF", "properties": {}},
22
+ {"src": "7001", "dst": "4001", "label": "CONTAINS", "properties": {}},
23
+ {"src": "7001", "dst": "4002", "label": "CONTAINS", "properties": {}},
24
+ {"src": "7001", "dst": "4003", "label": "CONTAINS", "properties": {}}
25
+ ]
@@ -0,0 +1,21 @@
1
+ [
2
+ {"id": "1001", "label": "METHOD", "properties": {"FULL_NAME": "com.example.Main.main", "SIGNATURE": "void(String[])", "HASH": "a1b2c3d4e5", "FILENAME": "src/Main.java", "LINE_NUMBER": 5, "LINE_NUMBER_END": 12, "CODE": "public static void main(String[] args) { Parser.parse(args[0]); }", "IS_EXTERNAL": false}},
3
+ {"id": "1002", "label": "METHOD", "properties": {"FULL_NAME": "com.example.Parser.parse", "SIGNATURE": "Document(String)", "HASH": "f6g7h8i9j0", "FILENAME": "src/Parser.java", "LINE_NUMBER": 12, "LINE_NUMBER_END": 18, "CODE": "public Document parse(String input) { return new Document(input); }", "IS_EXTERNAL": false}},
4
+ {"id": "1003", "label": "METHOD", "properties": {"FULL_NAME": "com.example.Validator.validate", "SIGNATURE": "boolean(Document)", "HASH": "k1l2m3n4o5", "FILENAME": "src/Validator.java", "LINE_NUMBER": 8, "LINE_NUMBER_END": 11, "CODE": "public boolean validate(Document doc) { return doc != null; }", "IS_EXTERNAL": false}},
5
+ {"id": "2001", "label": "CALL", "properties": {"NAME": "parse", "FULL_NAME": "com.example.Parser.parse", "CODE": "Parser.parse(args[0])", "LINE_NUMBER": 6, "DISPATCH_TYPE": "STATIC_DISPATCH"}},
6
+ {"id": "2002", "label": "CALL", "properties": {"NAME": "validate", "FULL_NAME": "com.example.Validator.validate", "CODE": "validator.validate(doc)", "LINE_NUMBER": 7, "DISPATCH_TYPE": "DYNAMIC_DISPATCH"}},
7
+ {"id": "3001", "label": "IDENTIFIER", "properties": {"NAME": "args", "CODE": "args", "LINE_NUMBER": 6, "ORDER": 1}},
8
+ {"id": "3002", "label": "LITERAL", "properties": {"CODE": "0", "LINE_NUMBER": 6, "ORDER": 2}},
9
+ {"id": "3003", "label": "IDENTIFIER", "properties": {"NAME": "doc", "CODE": "doc", "LINE_NUMBER": 7, "ORDER": 1}},
10
+ {"id": "4001", "label": "TYPE_DECL", "properties": {"FULL_NAME": "com.example.Main", "NAME": "Main", "IS_EXTERNAL": false, "ORDER": 1}},
11
+ {"id": "4002", "label": "TYPE_DECL", "properties": {"FULL_NAME": "com.example.Parser", "NAME": "Parser", "IS_EXTERNAL": false, "ORDER": 2}},
12
+ {"id": "4003", "label": "TYPE_DECL", "properties": {"FULL_NAME": "com.example.Validator", "NAME": "Validator", "IS_EXTERNAL": false, "ORDER": 3}},
13
+ {"id": "5001", "label": "FILE", "properties": {"NAME": "src/Main.java", "ORDER": 1}},
14
+ {"id": "5002", "label": "FILE", "properties": {"NAME": "src/Parser.java", "ORDER": 2}},
15
+ {"id": "5003", "label": "FILE", "properties": {"NAME": "src/Validator.java", "ORDER": 3}},
16
+ {"id": "6001", "label": "PARAMETER", "properties": {"NAME": "args", "CODE": "String[] args", "LINE_NUMBER": 5, "ORDER": 1, "TYPE_FULL_NAME": "java.lang.String[]"}},
17
+ {"id": "6002", "label": "PARAMETER", "properties": {"NAME": "input", "CODE": "String input", "LINE_NUMBER": 12, "ORDER": 1, "TYPE_FULL_NAME": "java.lang.String"}},
18
+ {"id": "6003", "label": "PARAMETER", "properties": {"NAME": "doc", "CODE": "Document doc", "LINE_NUMBER": 8, "ORDER": 1, "TYPE_FULL_NAME": "com.example.Document"}},
19
+ {"id": "7001", "label": "NAMESPACE", "properties": {"NAME": "com.example", "ORDER": 1}},
20
+ {"id": "8001", "label": "META_DATA", "properties": {"LANGUAGE": "java", "VERSION": "1.0"}}
21
+ ]