contextsynapse 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (776) hide show
  1. contextsynapse/__init__.py +107 -0
  2. contextsynapse/__main__.py +9 -0
  3. contextsynapse/a2a/__init__.py +28 -0
  4. contextsynapse/a2a/client.py +120 -0
  5. contextsynapse/a2a/discovery.py +128 -0
  6. contextsynapse/a2a/handlers.py +164 -0
  7. contextsynapse/a2a/models.py +246 -0
  8. contextsynapse/a2a/streaming.py +92 -0
  9. contextsynapse/a2a/task_manager.py +275 -0
  10. contextsynapse/adapters/__init__.py +98 -0
  11. contextsynapse/adapters/_base.py +298 -0
  12. contextsynapse/adapters/autogen/__init__.py +4 -0
  13. contextsynapse/adapters/autogen/tools.py +34 -0
  14. contextsynapse/adapters/crewai/__init__.py +4 -0
  15. contextsynapse/adapters/crewai/tools.py +63 -0
  16. contextsynapse/adapters/langchain/__init__.py +19 -0
  17. contextsynapse/adapters/langchain/message_history.py +17 -0
  18. contextsynapse/adapters/langchain/retriever.py +158 -0
  19. contextsynapse/adapters/langchain/tools.py +43 -0
  20. contextsynapse/adapters/langgraph/__init__.py +21 -0
  21. contextsynapse/adapters/langgraph/checkpoint.py +223 -0
  22. contextsynapse/adapters/langgraph/context_tools.py +341 -0
  23. contextsynapse/adapters/langgraph/message_history.py +138 -0
  24. contextsynapse/adapters/langgraph/tools.py +43 -0
  25. contextsynapse/adapters/llamaindex/__init__.py +4 -0
  26. contextsynapse/adapters/llamaindex/tools.py +64 -0
  27. contextsynapse/adapters/openai/__init__.py +24 -0
  28. contextsynapse/adapters/openai/tools.py +206 -0
  29. contextsynapse/adapters/pydantic_ai/__init__.py +4 -0
  30. contextsynapse/adapters/pydantic_ai/tools.py +36 -0
  31. contextsynapse/adapters/swarm/__init__.py +4 -0
  32. contextsynapse/adapters/swarm/tools.py +44 -0
  33. contextsynapse/agents/__init__.py +6 -0
  34. contextsynapse/agents/auto_capture.py +80 -0
  35. contextsynapse/agents/dispatcher.py +299 -0
  36. contextsynapse/agents/notifications.py +112 -0
  37. contextsynapse/aiql/__init__.py +71 -0
  38. contextsynapse/aiql/cache.py +341 -0
  39. contextsynapse/aiql/compiler/__init__.py +62 -0
  40. contextsynapse/aiql/compiler/optimizer.py +126 -0
  41. contextsynapse/aiql/compiler/planner.py +366 -0
  42. contextsynapse/aiql/compiler/statistics.py +235 -0
  43. contextsynapse/aiql/compiler/validator.py +425 -0
  44. contextsynapse/aiql/engine/__init__.py +11 -0
  45. contextsynapse/aiql/engine/chunk_storage.py +2584 -0
  46. contextsynapse/aiql/engine/executor.py +7737 -0
  47. contextsynapse/aiql/engine/extraction_integration.py +442 -0
  48. contextsynapse/aiql/engine/graph_builder.py +405 -0
  49. contextsynapse/aiql/engine/node_creation_from_normalized.py +691 -0
  50. contextsynapse/aiql/engine/performance_monitor.py +289 -0
  51. contextsynapse/aiql/engine/postprocessing.py +365 -0
  52. contextsynapse/aiql/engine/ray_stage_executors.py +918 -0
  53. contextsynapse/aiql/engine/redis_cache.py +106 -0
  54. contextsynapse/aiql/engine/streaming_json.py +366 -0
  55. contextsynapse/aiql/engine/streaming_parquet.py +568 -0
  56. contextsynapse/aiql/engine/streaming_ray_chunking.py +419 -0
  57. contextsynapse/aiql/grammar/__init__.py +14 -0
  58. contextsynapse/aiql/grammar/aiql_grammar.py +1822 -0
  59. contextsynapse/aiql/grammar/base.py +71 -0
  60. contextsynapse/aiql/parser/__init__.py +46 -0
  61. contextsynapse/aiql/parser/aiql_parser.py +11330 -0
  62. contextsynapse/aiql/parser/base_parser.py +47 -0
  63. contextsynapse/api/__init__.py +5 -0
  64. contextsynapse/api/a2a_router.py +233 -0
  65. contextsynapse/api/admin_router.py +140 -0
  66. contextsynapse/api/agent_worker_router.py +978 -0
  67. contextsynapse/api/algorithms_router.py +317 -0
  68. contextsynapse/api/api.py +4144 -0
  69. contextsynapse/api/audit.py +204 -0
  70. contextsynapse/api/auth.py +441 -0
  71. contextsynapse/api/auth_router.py +537 -0
  72. contextsynapse/api/billing.py +261 -0
  73. contextsynapse/api/billing_router.py +275 -0
  74. contextsynapse/api/boundary_router.py +452 -0
  75. contextsynapse/api/cognition_router.py +116 -0
  76. contextsynapse/api/connector_router.py +237 -0
  77. contextsynapse/api/context_router.py +1619 -0
  78. contextsynapse/api/dashboard_router.py +11367 -0
  79. contextsynapse/api/errors.py +183 -0
  80. contextsynapse/api/events.py +88 -0
  81. contextsynapse/api/experiment_bootstrap.py +321 -0
  82. contextsynapse/api/experiment_orchestrator.py +1078 -0
  83. contextsynapse/api/experiment_router.py +2089 -0
  84. contextsynapse/api/feature_gate.py +89 -0
  85. contextsynapse/api/federation_router.py +108 -0
  86. contextsynapse/api/integrations.py +567 -0
  87. contextsynapse/api/integrations_router.py +461 -0
  88. contextsynapse/api/intelligence_router.py +1028 -0
  89. contextsynapse/api/metering.py +155 -0
  90. contextsynapse/api/metrics.py +112 -0
  91. contextsynapse/api/models.py +38 -0
  92. contextsynapse/api/monitoring_router.py +174 -0
  93. contextsynapse/api/pipeline_router.py +1437 -0
  94. contextsynapse/api/playground_conn.py +116 -0
  95. contextsynapse/api/projects_router.py +131 -0
  96. contextsynapse/api/queries_router.py +282 -0
  97. contextsynapse/api/quota.py +95 -0
  98. contextsynapse/api/rate_limit.py +64 -0
  99. contextsynapse/api/search_router.py +261 -0
  100. contextsynapse/api/shield_router.py +447 -0
  101. contextsynapse/api/tenant_guard.py +103 -0
  102. contextsynapse/api/tenant_quotas.py +231 -0
  103. contextsynapse/api/tenants.py +305 -0
  104. contextsynapse/api/users.py +646 -0
  105. contextsynapse/api/v1.py +167 -0
  106. contextsynapse/api/vertical_builder_router.py +387 -0
  107. contextsynapse/benchmark/__init__.py +15 -0
  108. contextsynapse/benchmark/__main__.py +64 -0
  109. contextsynapse/benchmark/agents.py +338 -0
  110. contextsynapse/benchmark/live_ab.py +338 -0
  111. contextsynapse/benchmark/reporter.py +149 -0
  112. contextsynapse/benchmark/runner.py +264 -0
  113. contextsynapse/benchmark/savings.py +266 -0
  114. contextsynapse/benchmark/token_counter.py +212 -0
  115. contextsynapse/benchmark/workloads.py +289 -0
  116. contextsynapse/buffer/__init__.py +154 -0
  117. contextsynapse/buffer/base.py +257 -0
  118. contextsynapse/buffer/duckdb_buffer.py +246 -0
  119. contextsynapse/buffer/file_buffer.py +229 -0
  120. contextsynapse/buffer/lmdb_buffer.py +286 -0
  121. contextsynapse/buffer/local_buffer.py +154 -0
  122. contextsynapse/buffer/redis_buffer.py +211 -0
  123. contextsynapse/buffer/sqlite_buffer.py +266 -0
  124. contextsynapse/cache.py +2 -0
  125. contextsynapse/cli/__init__.py +40 -0
  126. contextsynapse/cli/cli.py +475 -0
  127. contextsynapse/cli/config_cli.py +177 -0
  128. contextsynapse/cli/scaffold.py +465 -0
  129. contextsynapse/cognition/__init__.py +31 -0
  130. contextsynapse/cognition/derivation.py +55 -0
  131. contextsynapse/cognition/events.py +135 -0
  132. contextsynapse/cognition/feedback.py +52 -0
  133. contextsynapse/cognition/hallucination.py +90 -0
  134. contextsynapse/cognition/invalidation.py +70 -0
  135. contextsynapse/cognition/read_tracker.py +50 -0
  136. contextsynapse/collaboration.py +2 -0
  137. contextsynapse/comparison.py +2 -0
  138. contextsynapse/config/__init__.py +10 -0
  139. contextsynapse/config/connectors/amfi_nav.yaml +18 -0
  140. contextsynapse/config/connectors/auto_sector.yaml +20 -0
  141. contextsynapse/config/connectors/banking_sector.yaml +20 -0
  142. contextsynapse/config/connectors/energy_sector.yaml +20 -0
  143. contextsynapse/config/connectors/fmcg_sector.yaml +20 -0
  144. contextsynapse/config/connectors/nsdl_fpi.yaml +9 -0
  145. contextsynapse/config/connectors/nse_price.yaml +61 -0
  146. contextsynapse/config/connectors/pharma_sector.yaml +20 -0
  147. contextsynapse/config/connectors/rbi_dbie.yaml +12 -0
  148. contextsynapse/config/context_presets/ai_research.json +25 -0
  149. contextsynapse/config/context_presets/chatgpt_conversations.json +19 -0
  150. contextsynapse/config/context_presets/claude_conversations.json +19 -0
  151. contextsynapse/config/context_presets/legal_compliance.json +21 -0
  152. contextsynapse/config/context_presets/meeting_notes.json +21 -0
  153. contextsynapse/config/context_presets/slack_engineering.json +21 -0
  154. contextsynapse/config/context_presets/tech_climate.json +24 -0
  155. contextsynapse/config/context_presets/toi_politics.json +25 -0
  156. contextsynapse/config/database_config.py +453 -0
  157. contextsynapse/config/deployment.py +221 -0
  158. contextsynapse/config/gpt_action_schema.json +164 -0
  159. contextsynapse/config/indicators/india_macro.yaml +213 -0
  160. contextsynapse/config/logging_config.py +88 -0
  161. contextsynapse/config/mode_config.py +223 -0
  162. contextsynapse/config/schemas/api_spec.yaml +48 -0
  163. contextsynapse/config/schemas/architecture_doc.yaml +51 -0
  164. contextsynapse/config/schemas/chat_history.yaml +90 -0
  165. contextsynapse/config/schemas/chatgpt_conversation.yaml +101 -0
  166. contextsynapse/config/schemas/claude_conversation.yaml +148 -0
  167. contextsynapse/config/schemas/contract.yaml +48 -0
  168. contextsynapse/config/schemas/conversation.yaml +167 -0
  169. contextsynapse/config/schemas/database.yaml +74 -0
  170. contextsynapse/config/schemas/decision.yaml +72 -0
  171. contextsynapse/config/schemas/ecommerce.yaml +56 -0
  172. contextsynapse/config/schemas/education.yaml +42 -0
  173. contextsynapse/config/schemas/healthcare.yaml +133 -0
  174. contextsynapse/config/schemas/hr_document.yaml +48 -0
  175. contextsynapse/config/schemas/invoice.yaml +40 -0
  176. contextsynapse/config/schemas/knowledge_base.yaml +75 -0
  177. contextsynapse/config/schemas/legal.yaml +42 -0
  178. contextsynapse/config/schemas/marketing_content.yaml +48 -0
  179. contextsynapse/config/schemas/meeting_notes.yaml +51 -0
  180. contextsynapse/config/schemas/news_article.yaml +61 -0
  181. contextsynapse/config/schemas/real_estate.yaml +49 -0
  182. contextsynapse/config/schemas/requirements_doc.yaml +48 -0
  183. contextsynapse/config/schemas/research_paper.yaml +56 -0
  184. contextsynapse/config/schemas/resume.yaml +49 -0
  185. contextsynapse/config/schemas/retail.yaml +42 -0
  186. contextsynapse/config/schemas/rule_meta.yaml +173 -0
  187. contextsynapse/config/schemas/rules.yaml +72 -0
  188. contextsynapse/config/schemas/screener_meta.yaml +236 -0
  189. contextsynapse/config/schemas/sdlc.yaml +103 -0
  190. contextsynapse/config/schemas/session_graph.yaml +172 -0
  191. contextsynapse/config/schemas/support_ticket.yaml +48 -0
  192. contextsynapse/config/schemas/web.yaml +64 -0
  193. contextsynapse/config/sensors/earnings_calendar.yaml +52 -0
  194. contextsynapse/config/sensors/global_risk.yaml +53 -0
  195. contextsynapse/config/sensors/government_policy.yaml +54 -0
  196. contextsynapse/config/sensors/supply_chain.yaml +50 -0
  197. contextsynapse/config/sensors/templates/behavioral.yaml +56 -0
  198. contextsynapse/config/sensors/templates/equity_sentiment.yaml +48 -0
  199. contextsynapse/config/sensors/templates/global_macro.yaml +47 -0
  200. contextsynapse/config/sensors/templates/market_data.yaml +118 -0
  201. contextsynapse/config/sensors/templates/sector_technical.yaml +47 -0
  202. contextsynapse/config/sensors/weather.yaml +52 -0
  203. contextsynapse/config/storage_strategy.py +202 -0
  204. contextsynapse/config/templates/competitive_intel.yaml +39 -0
  205. contextsynapse/config/templates/geopolitical_risk.yaml +46 -0
  206. contextsynapse/config/templates/market_analysis.yaml +58 -0
  207. contextsynapse/connectors/__init__.py +15 -0
  208. contextsynapse/connectors/api_router.py +121 -0
  209. contextsynapse/connectors/base.py +357 -0
  210. contextsynapse/connectors/financial/audio.py +170 -0
  211. contextsynapse/connectors/financial/youtube.py +272 -0
  212. contextsynapse/connectors/github.py +164 -0
  213. contextsynapse/connectors/jira.py +120 -0
  214. contextsynapse/connectors/kafka_connector.py +124 -0
  215. contextsynapse/connectors/pipeline_connector.py +747 -0
  216. contextsynapse/connectors/registry.py +116 -0
  217. contextsynapse/connectors/scheduler.py +87 -0
  218. contextsynapse/connectors/universal.py +360 -0
  219. contextsynapse/context/__init__.py +96 -0
  220. contextsynapse/context/__main__.py +4 -0
  221. contextsynapse/context/acl.py +125 -0
  222. contextsynapse/context/agent_card.py +201 -0
  223. contextsynapse/context/agent_memory.py +814 -0
  224. contextsynapse/context/agents.py +472 -0
  225. contextsynapse/context/agents_redis.py +391 -0
  226. contextsynapse/context/assembled.py +204 -0
  227. contextsynapse/context/attribution.py +113 -0
  228. contextsynapse/context/blob.py +184 -0
  229. contextsynapse/context/boundaries.py +187 -0
  230. contextsynapse/context/boundary.py +507 -0
  231. contextsynapse/context/bundle.py +155 -0
  232. contextsynapse/context/cli.py +519 -0
  233. contextsynapse/context/collaboration.py +165 -0
  234. contextsynapse/context/compiler.py +604 -0
  235. contextsynapse/context/composite.py +536 -0
  236. contextsynapse/context/compression.py +147 -0
  237. contextsynapse/context/context_manager.py +888 -0
  238. contextsynapse/context/context_schema.py +90 -0
  239. contextsynapse/context/context_schema.yaml +105 -0
  240. contextsynapse/context/context_units.py +891 -0
  241. contextsynapse/context/conversation.py +488 -0
  242. contextsynapse/context/dedup.py +207 -0
  243. contextsynapse/context/document_processor.py +698 -0
  244. contextsynapse/context/embedding_hooks.py +338 -0
  245. contextsynapse/context/execution_schema.py +274 -0
  246. contextsynapse/context/fan_out.py +138 -0
  247. contextsynapse/context/frozen.py +80 -0
  248. contextsynapse/context/gravity.py +227 -0
  249. contextsynapse/context/hub.py +1401 -0
  250. contextsynapse/context/ingest.py +391 -0
  251. contextsynapse/context/injection.py +226 -0
  252. contextsynapse/context/interaction_graphifier.py +562 -0
  253. contextsynapse/context/layers.py +574 -0
  254. contextsynapse/context/projection.py +1195 -0
  255. contextsynapse/context/promotion.py +634 -0
  256. contextsynapse/context/proof.py +131 -0
  257. contextsynapse/context/propagation.py +577 -0
  258. contextsynapse/context/pubsub.py +135 -0
  259. contextsynapse/context/quality.py +1054 -0
  260. contextsynapse/context/quality_tracker.py +270 -0
  261. contextsynapse/context/scoping.py +370 -0
  262. contextsynapse/context/seed.py +992 -0
  263. contextsynapse/context/session.py +1142 -0
  264. contextsynapse/context/session_graph.py +593 -0
  265. contextsynapse/context/session_memory.py +309 -0
  266. contextsynapse/context/session_resolver.py +111 -0
  267. contextsynapse/context/skill_learning.py +237 -0
  268. contextsynapse/context/smart_budget.py +129 -0
  269. contextsynapse/context/source_policy.py +242 -0
  270. contextsynapse/context/store_factory.py +53 -0
  271. contextsynapse/context/sync.py +648 -0
  272. contextsynapse/context/token.py +236 -0
  273. contextsynapse/context/vector_integration.py +297 -0
  274. contextsynapse/context/webhooks.py +194 -0
  275. contextsynapse/context/working_memory.py +224 -0
  276. contextsynapse/core/__init__.py +18 -0
  277. contextsynapse/core/archiver.py +237 -0
  278. contextsynapse/core/change_stream.py +282 -0
  279. contextsynapse/core/checkpoint.py +310 -0
  280. contextsynapse/core/cloud_storage.py +387 -0
  281. contextsynapse/core/context_state.py +651 -0
  282. contextsynapse/core/cron.py +324 -0
  283. contextsynapse/core/datasource.py +643 -0
  284. contextsynapse/core/db.py +289 -0
  285. contextsynapse/core/federation.py +283 -0
  286. contextsynapse/core/graph_coordinator.py +201 -0
  287. contextsynapse/core/graph_intelligence.py +388 -0
  288. contextsynapse/core/graph_structures.py +106 -0
  289. contextsynapse/core/graph_sync.py +157 -0
  290. contextsynapse/core/hybrid_graph_storage.py +2998 -0
  291. contextsynapse/core/lifecycle.py +691 -0
  292. contextsynapse/core/multiworker.py +378 -0
  293. contextsynapse/core/pruning.py +312 -0
  294. contextsynapse/core/redis_registry.py +524 -0
  295. contextsynapse/core/registry.py +1043 -0
  296. contextsynapse/core/registry_factory.py +51 -0
  297. contextsynapse/core/registry_metadata.py +294 -0
  298. contextsynapse/core/replay.py +354 -0
  299. contextsynapse/core/replication.py +339 -0
  300. contextsynapse/core/time_travel.py +260 -0
  301. contextsynapse/core/write_behind.py +152 -0
  302. contextsynapse/core/write_context.py +39 -0
  303. contextsynapse/db/__init__.py +1 -0
  304. contextsynapse/db/postgres.py +267 -0
  305. contextsynapse/db/profile.py +236 -0
  306. contextsynapse/db/rules.py +328 -0
  307. contextsynapse/demo/__init__.py +0 -0
  308. contextsynapse/demo/report.py +147 -0
  309. contextsynapse/demo/runner.py +520 -0
  310. contextsynapse/demo/scenario.py +165 -0
  311. contextsynapse/demo/session_demo.py +280 -0
  312. contextsynapse/demo/session_report.py +143 -0
  313. contextsynapse/demo/universal_runner.py +257 -0
  314. contextsynapse/engine/__init__.py +15 -0
  315. contextsynapse/engine/core.py +555 -0
  316. contextsynapse/extraction/__init__.py +69 -0
  317. contextsynapse/extraction/cli.py +274 -0
  318. contextsynapse/extraction/config.py +107 -0
  319. contextsynapse/extraction/examples/example_usage.py +118 -0
  320. contextsynapse/extraction/examples/sample_manifest.json +53 -0
  321. contextsynapse/extraction/examples/sample_normalized.json +185 -0
  322. contextsynapse/extraction/extract_engine.py +440 -0
  323. contextsynapse/extraction/extractors/__init__.py +37 -0
  324. contextsynapse/extraction/extractors/chat_export_extractor.py +189 -0
  325. contextsynapse/extraction/extractors/csv_extractor.py +83 -0
  326. contextsynapse/extraction/extractors/docx_extractor.py +129 -0
  327. contextsynapse/extraction/extractors/excel_extractor.py +152 -0
  328. contextsynapse/extraction/extractors/html_extractor.py +146 -0
  329. contextsynapse/extraction/extractors/pdf_extractor.py +635 -0
  330. contextsynapse/extraction/extractors/pdf_extractor_layoutparser.py +408 -0
  331. contextsynapse/extraction/extractors/register_extractors.py +63 -0
  332. contextsynapse/extraction/extractors/text_extractor.py +65 -0
  333. contextsynapse/extraction/extractors/txt_extractor.py +120 -0
  334. contextsynapse/extraction/extractors/website_extractor.py +246 -0
  335. contextsynapse/extraction/fact_extractor.py +236 -0
  336. contextsynapse/extraction/file_manager.py +353 -0
  337. contextsynapse/extraction/hierarchy.py +239 -0
  338. contextsynapse/extraction/id_generator.py +450 -0
  339. contextsynapse/extraction/layout_detector.py +409 -0
  340. contextsynapse/extraction/llm_entity_extractor.py +301 -0
  341. contextsynapse/extraction/normalized_store.py +480 -0
  342. contextsynapse/extraction/normalizer.py +594 -0
  343. contextsynapse/extraction/parquet_writer.py +598 -0
  344. contextsynapse/extraction/post_processors/__init__.py +40 -0
  345. contextsynapse/extraction/post_processors/column_reorganizer.py +507 -0
  346. contextsynapse/extraction/ray_normalization.py +233 -0
  347. contextsynapse/extraction/ray_runner.py +207 -0
  348. contextsynapse/extraction/redis_schema_registry.py +222 -0
  349. contextsynapse/extraction/registry.py +204 -0
  350. contextsynapse/extraction/schema_loader.py +663 -0
  351. contextsynapse/extraction/schema_registry.py +230 -0
  352. contextsynapse/extraction/semantic_normalizer.py +245 -0
  353. contextsynapse/extraction/version_manager.py +314 -0
  354. contextsynapse/gateway/__init__.py +24 -0
  355. contextsynapse/gateway/action_emitter.py +212 -0
  356. contextsynapse/gateway/cost_tracker.py +508 -0
  357. contextsynapse/gateway/gateway.py +377 -0
  358. contextsynapse/gateway/policy.py +303 -0
  359. contextsynapse/governance/__init__.py +13 -0
  360. contextsynapse/governance/layer.py +513 -0
  361. contextsynapse/ingestion/__init__.py +15 -0
  362. contextsynapse/ingestion/amplifier.py +288 -0
  363. contextsynapse/ingestion/async_ingest.py +463 -0
  364. contextsynapse/ingestion/chunker.py +303 -0
  365. contextsynapse/ingestion/chunking.py +58 -0
  366. contextsynapse/ingestion/cleaner.py +190 -0
  367. contextsynapse/ingestion/cleanse.py +252 -0
  368. contextsynapse/ingestion/connectors/__init__.py +38 -0
  369. contextsynapse/ingestion/connectors/base.py +300 -0
  370. contextsynapse/ingestion/connectors/chatgpt_memory.py +247 -0
  371. contextsynapse/ingestion/connectors/claude_memory.py +274 -0
  372. contextsynapse/ingestion/connectors/sap_ingestor.py +208 -0
  373. contextsynapse/ingestion/content_detector.py +80 -0
  374. contextsynapse/ingestion/content_monitor.py +502 -0
  375. contextsynapse/ingestion/dedup.py +106 -0
  376. contextsynapse/ingestion/feed.py +640 -0
  377. contextsynapse/ingestion/filters.py +461 -0
  378. contextsynapse/ingestion/graph_builder.py +705 -0
  379. contextsynapse/ingestion/input_classifier.py +307 -0
  380. contextsynapse/ingestion/job_manager.py +571 -0
  381. contextsynapse/ingestion/llm_extractor.py +610 -0
  382. contextsynapse/ingestion/orchestrator.py +116 -0
  383. contextsynapse/ingestion/parsers/__init__.py +19 -0
  384. contextsynapse/ingestion/parsers/graph_parser_registry.py +103 -0
  385. contextsynapse/ingestion/parsers/graphml_parser.py +141 -0
  386. contextsynapse/ingestion/parsers/jsonld_parser.py +171 -0
  387. contextsynapse/ingestion/parsers/rdf_parser.py +150 -0
  388. contextsynapse/ingestion/pipeline_context.py +228 -0
  389. contextsynapse/ingestion/pipeline_store.py +1195 -0
  390. contextsynapse/ingestion/queue.py +133 -0
  391. contextsynapse/ingestion/reprocess.py +179 -0
  392. contextsynapse/ingestion/scenario_router.py +224 -0
  393. contextsynapse/ingestion/schema_extractor.py +759 -0
  394. contextsynapse/ingestion/schema_validator.py +274 -0
  395. contextsynapse/ingestion/sdlc_ingest.py +87 -0
  396. contextsynapse/ingestion/smart_ingest.py +2451 -0
  397. contextsynapse/ingestion/source_connector.py +171 -0
  398. contextsynapse/ingestion/stage_executor.py +2095 -0
  399. contextsynapse/ingestion/stages/__init__.py +9 -0
  400. contextsynapse/ingestion/stages/tabular_to_graph.py +289 -0
  401. contextsynapse/ingestion/strategies.py +386 -0
  402. contextsynapse/ingestion/universal/__init__.py +4 -0
  403. contextsynapse/ingestion/universal/_operator_registry.py +59 -0
  404. contextsynapse/ingestion/universal/ingest_content.py +32 -0
  405. contextsynapse/ingestion/universal/operators/__init__.py +1 -0
  406. contextsynapse/ingestion/universal/operators/base.py +19 -0
  407. contextsynapse/ingestion/universal/operators/build_edges.py +75 -0
  408. contextsynapse/ingestion/universal/operators/cluster_topics.py +397 -0
  409. contextsynapse/ingestion/universal/operators/deduplicate.py +45 -0
  410. contextsynapse/ingestion/universal/operators/detect_signals.py +90 -0
  411. contextsynapse/ingestion/universal/operators/embed.py +80 -0
  412. contextsynapse/ingestion/universal/operators/extract_entities.py +253 -0
  413. contextsynapse/ingestion/universal/operators/extract_preferences.py +117 -0
  414. contextsynapse/ingestion/universal/operators/filter_content.py +69 -0
  415. contextsynapse/ingestion/universal/operators/index_bm25.py +52 -0
  416. contextsynapse/ingestion/universal/operators/infer_domains.py +69 -0
  417. contextsynapse/ingestion/universal/operators/link_cross_reference.py +210 -0
  418. contextsynapse/ingestion/universal/operators/parse_records.py +66 -0
  419. contextsynapse/ingestion/universal/operators/resolve_entities.py +261 -0
  420. contextsynapse/ingestion/universal/operators/scanners/__init__.py +16 -0
  421. contextsynapse/ingestion/universal/operators/scanners/edges.py +173 -0
  422. contextsynapse/ingestion/universal/operators/scanners/git.py +187 -0
  423. contextsynapse/ingestion/universal/operators/scanners/github.py +135 -0
  424. contextsynapse/ingestion/universal/operators/scanners/github_api.py +509 -0
  425. contextsynapse/ingestion/universal/operators/scanners/helpers.py +67 -0
  426. contextsynapse/ingestion/universal/operators/scanners/jira.py +65 -0
  427. contextsynapse/ingestion/universal/operators/scanners/llm_enrichment.py +82 -0
  428. contextsynapse/ingestion/universal/operators/scanners/repo.py +149 -0
  429. contextsynapse/ingestion/universal/operators/sdlc_scan.py +213 -0
  430. contextsynapse/ingestion/universal/operators/spec_scanner.py +149 -0
  431. contextsynapse/ingestion/universal/operators/store_documents.py +72 -0
  432. contextsynapse/ingestion/universal/operators/synthesize_cu.py +180 -0
  433. contextsynapse/ingestion/universal/operators/validate_gate.py +65 -0
  434. contextsynapse/ingestion/universal/parsers/__init__.py +5 -0
  435. contextsynapse/ingestion/universal/parsers/record_parser.py +203 -0
  436. contextsynapse/ingestion/universal/parsers/turn_parser.py +329 -0
  437. contextsynapse/ingestion/universal/shared_signals.py +161 -0
  438. contextsynapse/ingestion/universal/stage_executor.py +219 -0
  439. contextsynapse/ingestion/web_crawler.py +265 -0
  440. contextsynapse/intelligence/__init__.py +53 -0
  441. contextsynapse/intelligence/alerts.py +198 -0
  442. contextsynapse/intelligence/collector.py +655 -0
  443. contextsynapse/intelligence/comparison.py +198 -0
  444. contextsynapse/intelligence/config.py +56 -0
  445. contextsynapse/intelligence/conflict_detector.py +182 -0
  446. contextsynapse/intelligence/context_gaps.py +192 -0
  447. contextsynapse/intelligence/context_radar.py +135 -0
  448. contextsynapse/intelligence/correlation_engine.py +416 -0
  449. contextsynapse/intelligence/correlation_template.py +120 -0
  450. contextsynapse/intelligence/dedup.py +296 -0
  451. contextsynapse/intelligence/event_bus.py +91 -0
  452. contextsynapse/intelligence/feedback_loop.py +101 -0
  453. contextsynapse/intelligence/freshness.py +307 -0
  454. contextsynapse/intelligence/fundamental_feed.py +551 -0
  455. contextsynapse/intelligence/geo_data.py +170 -0
  456. contextsynapse/intelligence/geo_reference.py +112 -0
  457. contextsynapse/intelligence/graph_fusion.py +465 -0
  458. contextsynapse/intelligence/heuristics.py +176 -0
  459. contextsynapse/intelligence/impact_tracker.py +336 -0
  460. contextsynapse/intelligence/macro_indicators.py +319 -0
  461. contextsynapse/intelligence/ocr.py +104 -0
  462. contextsynapse/intelligence/persistence.py +267 -0
  463. contextsynapse/intelligence/pipeline_health.py +130 -0
  464. contextsynapse/intelligence/price_feed.py +221 -0
  465. contextsynapse/intelligence/rate_limiter.py +130 -0
  466. contextsynapse/intelligence/reactive.py +411 -0
  467. contextsynapse/intelligence/runtime_context.py +390 -0
  468. contextsynapse/intelligence/sentiment_decay.py +426 -0
  469. contextsynapse/intelligence/session.py +277 -0
  470. contextsynapse/intelligence/signal_hierarchy.py +438 -0
  471. contextsynapse/intelligence/signals.py +185 -0
  472. contextsynapse/intelligence/source_watcher.py +132 -0
  473. contextsynapse/intelligence/table_enricher.py +256 -0
  474. contextsynapse/intelligence/tagger.py +319 -0
  475. contextsynapse/intelligence/timeseries.py +209 -0
  476. contextsynapse/intelligence/tracker.py +318 -0
  477. contextsynapse/intelligence/watchdog.py +392 -0
  478. contextsynapse/intelligence/watchdog_manager.py +263 -0
  479. contextsynapse/intelligence/webhook_receiver.py +165 -0
  480. contextsynapse/intelligence/weight_learner.py +234 -0
  481. contextsynapse/llm/__init__.py +8 -0
  482. contextsynapse/llm/client.py +348 -0
  483. contextsynapse/llm/rate_limiter.py +166 -0
  484. contextsynapse/logging_config.py +2 -0
  485. contextsynapse/marketplace/__init__.py +7 -0
  486. contextsynapse/marketplace/registry.py +173 -0
  487. contextsynapse/mcp/__init__.py +11 -0
  488. contextsynapse/mcp/__main__.py +4 -0
  489. contextsynapse/mcp/auth_middleware.py +212 -0
  490. contextsynapse/mcp/connection_pool.py +128 -0
  491. contextsynapse/mcp/server.py +986 -0
  492. contextsynapse/mcp/session_context.py +67 -0
  493. contextsynapse/memory/__init__.py +34 -0
  494. contextsynapse/memory/engine.py +475 -0
  495. contextsynapse/memory/temporal.py +327 -0
  496. contextsynapse/metadata/__init__.py +12 -0
  497. contextsynapse/metadata/metadata_db.py +272 -0
  498. contextsynapse/metadata/metadata_query_tool.py +300 -0
  499. contextsynapse/metadata/metadata_tracker.py +425 -0
  500. contextsynapse/models/__init__.py +27 -0
  501. contextsynapse/models/embedding_service.py +360 -0
  502. contextsynapse/models/model_registry.py +228 -0
  503. contextsynapse/multiworker.py +2 -0
  504. contextsynapse/pipelines/__init__.py +1 -0
  505. contextsynapse/pipelines/executor.py +851 -0
  506. contextsynapse/pipelines/filters.py +189 -0
  507. contextsynapse/pipelines/models.py +176 -0
  508. contextsynapse/pipelines/scheduler.py +268 -0
  509. contextsynapse/playground/__init__.py +1 -0
  510. contextsynapse/playground/templates.py +85 -0
  511. contextsynapse/plugins/__init__.py +43 -0
  512. contextsynapse/plugins/api.py +66 -0
  513. contextsynapse/plugins/base.py +350 -0
  514. contextsynapse/plugins/domain_interface.py +508 -0
  515. contextsynapse/plugins/domain_loader.py +241 -0
  516. contextsynapse/plugins/loader.py +148 -0
  517. contextsynapse/plugins/registry.py +265 -0
  518. contextsynapse/project/__init__.py +11 -0
  519. contextsynapse/project/code_context.py +807 -0
  520. contextsynapse/project/connectors/__init__.py +4 -0
  521. contextsynapse/project/connectors/github_connector.py +113 -0
  522. contextsynapse/project/connectors/jira_connector.py +160 -0
  523. contextsynapse/project/graph.py +1023 -0
  524. contextsynapse/project/project_context.py +545 -0
  525. contextsynapse/project/scanner_registry.py +61 -0
  526. contextsynapse/project/schema_manager.py +322 -0
  527. contextsynapse/project/schemas/knowledge.yaml +59 -0
  528. contextsynapse/project/schemas/sdlc.yaml +369 -0
  529. contextsynapse/project/sdlc_schema.py +172 -0
  530. contextsynapse/project/spec_parser.py +85 -0
  531. contextsynapse/project/task_lock.py +181 -0
  532. contextsynapse/project/task_stream.py +203 -0
  533. contextsynapse/project/team_tools.py +537 -0
  534. contextsynapse/py.typed +0 -0
  535. contextsynapse/realtime/__init__.py +6 -0
  536. contextsynapse/realtime/collaboration.py +147 -0
  537. contextsynapse/rules/__init__.py +10 -0
  538. contextsynapse/rules/engine.py +116 -0
  539. contextsynapse/rules/evaluators/__init__.py +6 -0
  540. contextsynapse/rules/evaluators/aggregate.py +205 -0
  541. contextsynapse/rules/evaluators/custom.py +170 -0
  542. contextsynapse/rules/evaluators/pre_action.py +125 -0
  543. contextsynapse/rules/evaluators/temporal.py +129 -0
  544. contextsynapse/rules/loader.py +52 -0
  545. contextsynapse/rules/models.py +124 -0
  546. contextsynapse/rules/universal.py +393 -0
  547. contextsynapse/scheduler.py +229 -0
  548. contextsynapse/schema/__init__.py +42 -0
  549. contextsynapse/schema/compiler.py +252 -0
  550. contextsynapse/schema/composer.py +158 -0
  551. contextsynapse/schema/dedup.py +73 -0
  552. contextsynapse/schema/derivation.py +172 -0
  553. contextsynapse/schema/sdl.py +326 -0
  554. contextsynapse/schema/validation_gate.py +124 -0
  555. contextsynapse/sdk/__init__.py +63 -0
  556. contextsynapse/sdk/agent.py +214 -0
  557. contextsynapse/sdk/client.py +269 -0
  558. contextsynapse/sdk/exceptions.py +23 -0
  559. contextsynapse/sdk/models.py +78 -0
  560. contextsynapse/sdk/session.py +408 -0
  561. contextsynapse/sdk/tracked.py +218 -0
  562. contextsynapse/sdk/worker.py +266 -0
  563. contextsynapse/sdk/workspace_client.py +98 -0
  564. contextsynapse/sdk/ws.py +41 -0
  565. contextsynapse/search/__init__.py +20 -0
  566. contextsynapse/search/embedding_cache.py +140 -0
  567. contextsynapse/search/enhanced_search.py +152 -0
  568. contextsynapse/search/fulltext.py +130 -0
  569. contextsynapse/search/graph_search.py +634 -0
  570. contextsynapse/search/lmdb_index.py +1218 -0
  571. contextsynapse/search/nl_to_aiql.py +199 -0
  572. contextsynapse/search/rag.py +1346 -0
  573. contextsynapse/search/rag_cache.py +167 -0
  574. contextsynapse/search/reasoning_chain.py +231 -0
  575. contextsynapse/search/redis_search.py +207 -0
  576. contextsynapse/search/retrieval_quality.py +314 -0
  577. contextsynapse/search/semantic_query.py +455 -0
  578. contextsynapse/search/text_resolver.py +159 -0
  579. contextsynapse/search/whoosh_search.py +373 -0
  580. contextsynapse/security/__init__.py +17 -0
  581. contextsynapse/security/audit.py +297 -0
  582. contextsynapse/security/audit_logger.py +97 -0
  583. contextsynapse/security/audit_trail.py +270 -0
  584. contextsynapse/security/auth_unified.py +284 -0
  585. contextsynapse/security/auto_tagger.py +230 -0
  586. contextsynapse/security/data_security.py +218 -0
  587. contextsynapse/security/encryption.py +207 -0
  588. contextsynapse/security/identity.py +447 -0
  589. contextsynapse/security/jwt_identity.py +59 -0
  590. contextsynapse/security/middleware.py +336 -0
  591. contextsynapse/security/pii.py +233 -0
  592. contextsynapse/security/rbac.py +96 -0
  593. contextsynapse/security/rls.py +205 -0
  594. contextsynapse/security/sanitize.py +233 -0
  595. contextsynapse/security/scoped_encryption.py +131 -0
  596. contextsynapse/security/tenant.py +401 -0
  597. contextsynapse/shield/__init__.py +21 -0
  598. contextsynapse/shield/anomaly.py +56 -0
  599. contextsynapse/shield/permissions.py +99 -0
  600. contextsynapse/shield/profile.py +103 -0
  601. contextsynapse/shield/shield.py +108 -0
  602. contextsynapse/shield/trust_engine.py +69 -0
  603. contextsynapse/skills/__init__.py +14 -0
  604. contextsynapse/skills/engine.py +233 -0
  605. contextsynapse/skills/loader.py +59 -0
  606. contextsynapse/skills/models.py +139 -0
  607. contextsynapse/storage/__init__.py +7 -0
  608. contextsynapse/storage/cache.py +208 -0
  609. contextsynapse/storage/columnar_store_v2.py +178 -0
  610. contextsynapse/storage/csr_graph_storage.py +587 -0
  611. contextsynapse/storage/csr_store.py +388 -0
  612. contextsynapse/storage/document_store.py +491 -0
  613. contextsynapse/storage/hnsw_index.py +294 -0
  614. contextsynapse/storage/hybrid_store_v2.py +278 -0
  615. contextsynapse/storage/lmdb_graph_storage.py +366 -0
  616. contextsynapse/storage/migration.py +279 -0
  617. contextsynapse/storage/namespace_store.py +346 -0
  618. contextsynapse/storage/pure_graph_storage.py +185 -0
  619. contextsynapse/storage/redis_graph_adapter.py +186 -0
  620. contextsynapse/storage/redis_graph_storage.py +375 -0
  621. contextsynapse/storage/router/__init__.py +29 -0
  622. contextsynapse/storage/router/duckdb_store.py +398 -0
  623. contextsynapse/storage/router/factory.py +83 -0
  624. contextsynapse/storage/router/interface.py +125 -0
  625. contextsynapse/storage/router/postgres_store.py +339 -0
  626. contextsynapse/storage/router/resolver.py +192 -0
  627. contextsynapse/storage/router/router.py +300 -0
  628. contextsynapse/storage/single_file_storage.py +391 -0
  629. contextsynapse/storage/storage_manager.py +214 -0
  630. contextsynapse/storage/storage_query_optimizer.py +314 -0
  631. contextsynapse/storage/wal.py +359 -0
  632. contextsynapse/temporal/__init__.py +0 -0
  633. contextsynapse/temporal/storage.py +106 -0
  634. contextsynapse/tests/__init__.py +11 -0
  635. contextsynapse/tests/regression/__init__.py +1 -0
  636. contextsynapse/tests/regression/add_metadata.py +395 -0
  637. contextsynapse/tests/regression/add_skip_flags.py +85 -0
  638. contextsynapse/tests/regression/cases/test_basic_create_node.yaml +20 -0
  639. contextsynapse/tests/regression/cases/test_basic_select.yaml +21 -0
  640. contextsynapse/tests/regression/cases/test_blockchain_audit_trail.yaml +37 -0
  641. contextsynapse/tests/regression/cases/test_blockchain_audit_trail_filtered.yaml +39 -0
  642. contextsynapse/tests/regression/cases/test_blockchain_blocks_range.yaml +39 -0
  643. contextsynapse/tests/regression/cases/test_blockchain_get_block.yaml +35 -0
  644. contextsynapse/tests/regression/cases/test_blockchain_latest_block.yaml +35 -0
  645. contextsynapse/tests/regression/cases/test_blockchain_length.yaml +37 -0
  646. contextsynapse/tests/regression/cases/test_blockchain_merkle_root.yaml +37 -0
  647. contextsynapse/tests/regression/cases/test_blockchain_verify.yaml +35 -0
  648. contextsynapse/tests/regression/cases/test_blockchain_verify_block.yaml +35 -0
  649. contextsynapse/tests/regression/cases/test_create_edge.yaml +28 -0
  650. contextsynapse/tests/regression/cases/test_delete_edge_basic.yaml +29 -0
  651. contextsynapse/tests/regression/cases/test_delete_node_basic.yaml +29 -0
  652. contextsynapse/tests/regression/cases/test_evaluation_mrr.yaml +40 -0
  653. contextsynapse/tests/regression/cases/test_evaluation_ndcg.yaml +40 -0
  654. contextsynapse/tests/regression/cases/test_evaluation_precision.yaml +40 -0
  655. contextsynapse/tests/regression/cases/test_evaluation_recall.yaml +40 -0
  656. contextsynapse/tests/regression/cases/test_group_by_basic.yaml +56 -0
  657. contextsynapse/tests/regression/cases/test_match_node_basic.yaml +29 -0
  658. contextsynapse/tests/regression/cases/test_match_node_with_order_limit.yaml +31 -0
  659. contextsynapse/tests/regression/cases/test_namespace_switch.yaml +18 -0
  660. contextsynapse/tests/regression/cases/test_pipeline_cascaded.yaml +61 -0
  661. contextsynapse/tests/regression/cases/test_pipeline_fixed_size_chunking.yaml +48 -0
  662. contextsynapse/tests/regression/cases/test_pipeline_gpt4_chunking.yaml +54 -0
  663. contextsynapse/tests/regression/cases/test_pipeline_hybrid_indexing.yaml +57 -0
  664. contextsynapse/tests/regression/cases/test_pipeline_large_embedding.yaml +52 -0
  665. contextsynapse/tests/regression/cases/test_pipeline_paragraph_chunking.yaml +49 -0
  666. contextsynapse/tests/regression/cases/test_pipeline_recursive_chunking.yaml +47 -0
  667. contextsynapse/tests/regression/cases/test_pipeline_run_pipeline.yaml +56 -0
  668. contextsynapse/tests/regression/cases/test_pipeline_section_based_chunking.yaml +53 -0
  669. contextsynapse/tests/regression/cases/test_pipeline_semantic_chunking.yaml +51 -0
  670. contextsynapse/tests/regression/cases/test_pipeline_with_entity_extraction.yaml +57 -0
  671. contextsynapse/tests/regression/cases/test_pipeline_with_relationships.yaml +62 -0
  672. contextsynapse/tests/regression/cases/test_rag_evaluation.yaml +38 -0
  673. contextsynapse/tests/regression/cases/test_rag_graph_bfs_with_eval.yaml +48 -0
  674. contextsynapse/tests/regression/cases/test_rag_graph_centrality.yaml +40 -0
  675. contextsynapse/tests/regression/cases/test_rag_graph_centrality_eval.yaml +35 -0
  676. contextsynapse/tests/regression/cases/test_rag_graph_community.yaml +40 -0
  677. contextsynapse/tests/regression/cases/test_rag_graph_community_eval.yaml +35 -0
  678. contextsynapse/tests/regression/cases/test_rag_graph_dfs_with_eval.yaml +35 -0
  679. contextsynapse/tests/regression/cases/test_rag_graph_multi_hop_eval.yaml +35 -0
  680. contextsynapse/tests/regression/cases/test_rag_graph_path.yaml +40 -0
  681. contextsynapse/tests/regression/cases/test_rag_graph_search_basic.yaml +38 -0
  682. contextsynapse/tests/regression/cases/test_rag_graph_search_bfs.yaml +52 -0
  683. contextsynapse/tests/regression/cases/test_rag_graph_search_dfs.yaml +41 -0
  684. contextsynapse/tests/regression/cases/test_rag_graph_search_multi_hop.yaml +43 -0
  685. contextsynapse/tests/regression/cases/test_rag_graph_search_shortest_path.yaml +41 -0
  686. contextsynapse/tests/regression/cases/test_rag_graph_search_weighted.yaml +43 -0
  687. contextsynapse/tests/regression/cases/test_rag_graph_shortest_path_eval.yaml +35 -0
  688. contextsynapse/tests/regression/cases/test_rag_graph_traversal.yaml +30 -0
  689. contextsynapse/tests/regression/cases/test_rag_graph_weighted_eval.yaml +35 -0
  690. contextsynapse/tests/regression/cases/test_rag_hybrid_search.yaml +33 -0
  691. contextsynapse/tests/regression/cases/test_rag_hybrid_with_prompt_embedding.yaml +44 -0
  692. contextsynapse/tests/regression/cases/test_rag_keyword_search.yaml +32 -0
  693. contextsynapse/tests/regression/cases/test_rag_multi_hop_graph.yaml +32 -0
  694. contextsynapse/tests/regression/cases/test_rag_path_based.yaml +34 -0
  695. contextsynapse/tests/regression/cases/test_rag_profile_based.yaml +32 -0
  696. contextsynapse/tests/regression/cases/test_rag_vector_search.yaml +32 -0
  697. contextsynapse/tests/regression/cases/test_rag_weighted_graph.yaml +34 -0
  698. contextsynapse/tests/regression/cases/test_rag_weighted_hybrid.yaml +31 -0
  699. contextsynapse/tests/regression/cases/test_semantic_boundary_chunking.yaml +57 -0
  700. contextsynapse/tests/regression/cases/test_semantic_boundary_detection.yaml +33 -0
  701. contextsynapse/tests/regression/cases/test_semantic_hash_function.yaml +30 -0
  702. contextsynapse/tests/regression/cases/test_semantic_hash_with_model.yaml +31 -0
  703. contextsynapse/tests/regression/cases/test_tcs_ingestion_fixed_size.yaml +44 -0
  704. contextsynapse/tests/regression/cases/test_tcs_ingestion_recursive.yaml +46 -0
  705. contextsynapse/tests/regression/cases/test_tcs_ingestion_section.yaml +46 -0
  706. contextsynapse/tests/regression/cases/test_tcs_ingestion_semantic.yaml +44 -0
  707. contextsynapse/tests/regression/cases/test_time_travel_as_of.yaml +35 -0
  708. contextsynapse/tests/regression/cases/test_time_travel_at_commit.yaml +33 -0
  709. contextsynapse/tests/regression/cases/test_time_travel_at_timestamp.yaml +33 -0
  710. contextsynapse/tests/regression/cases/test_time_travel_for_system_time_all.yaml +37 -0
  711. contextsynapse/tests/regression/cases/test_time_travel_versions_between.yaml +39 -0
  712. contextsynapse/tests/regression/cases/test_traverse_basic.yaml +31 -0
  713. contextsynapse/tests/regression/cases/test_traverse_max_depth.yaml +34 -0
  714. contextsynapse/tests/regression/cases/test_update_edge_basic.yaml +29 -0
  715. contextsynapse/tests/regression/evaluation_metrics.py +227 -0
  716. contextsynapse/tests/regression/regression_runner.py +1081 -0
  717. contextsynapse/tests/regression/setup_regression_data.py +212 -0
  718. contextsynapse/tests/regression/test_basic_only.py +84 -0
  719. contextsynapse/tests/regression/test_namespace_creation.py +29 -0
  720. contextsynapse/tests/regression/test_single_fix.py +61 -0
  721. contextsynapse/tests/regression/test_single_query.py +20 -0
  722. contextsynapse/tests/regression/validate_test_syntax.py +179 -0
  723. contextsynapse/tests/test_advanced_features.py +347 -0
  724. contextsynapse/tests/test_aiql_summary.py +283 -0
  725. contextsynapse/tests/test_all_aiql_queries.py +948 -0
  726. contextsynapse/tests/test_all_create_queries.py +213 -0
  727. contextsynapse/tests/test_api_vs_native.py +280 -0
  728. contextsynapse/tests/test_chunking_dedup.py +199 -0
  729. contextsynapse/tests/test_connectors.py +171 -0
  730. contextsynapse/tests/test_edges_and_match.py +202 -0
  731. contextsynapse/tests/test_extraction_pipeline_integration.py +581 -0
  732. contextsynapse/tests/test_name_property.py +224 -0
  733. contextsynapse/tests/test_node_edge_metadata.py +250 -0
  734. contextsynapse/tests/test_playground_conn.py +114 -0
  735. contextsynapse/tests/test_query_cache.py +213 -0
  736. contextsynapse/tests/test_source_policy_auth.py +348 -0
  737. contextsynapse/tests/test_temporal_and_domain.py +224 -0
  738. contextsynapse/tests/test_time_travel_e2e.py +571 -0
  739. contextsynapse/tests/test_traversal_and_aggregation.py +275 -0
  740. contextsynapse/tools/__init__.py +36 -0
  741. contextsynapse/tools/a2a_tools.py +106 -0
  742. contextsynapse/tools/cognition_tools.py +118 -0
  743. contextsynapse/tools/context.py +905 -0
  744. contextsynapse/tools/formatting.py +92 -0
  745. contextsynapse/tools/gateway_tools.py +186 -0
  746. contextsynapse/tools/graph.py +1700 -0
  747. contextsynapse/tools/intelligence.py +235 -0
  748. contextsynapse/tools/memory.py +118 -0
  749. contextsynapse/tools/registry.py +373 -0
  750. contextsynapse/tools/shield_tools.py +159 -0
  751. contextsynapse/tools/tasks.py +341 -0
  752. contextsynapse/tools/workspace_tools.py +124 -0
  753. contextsynapse/utils/__init__.py +15 -0
  754. contextsynapse/utils/config_loader.py +198 -0
  755. contextsynapse/utils/env_loader.py +156 -0
  756. contextsynapse/utils/file_utils.py +288 -0
  757. contextsynapse/utils/logging_optimizer.py +119 -0
  758. contextsynapse/vector/__init__.py +30 -0
  759. contextsynapse/vector/chroma_vector_store.py +152 -0
  760. contextsynapse/vector/custom_vector_store.py +537 -0
  761. contextsynapse/vector/faiss_vector_store.py +296 -0
  762. contextsynapse/vector/qdrant_vector_store.py +163 -0
  763. contextsynapse/vector/vector_db.py +953 -0
  764. contextsynapse/vector/vector_db_manager.py +197 -0
  765. contextsynapse/workspace/__init__.py +27 -0
  766. contextsynapse/workspace/base.py +94 -0
  767. contextsynapse/workspace/git.py +228 -0
  768. contextsynapse/workspace/github.py +78 -0
  769. contextsynapse/workspace/local.py +73 -0
  770. contextsynapse/workspace/tools.py +222 -0
  771. contextsynapse-1.0.0.dist-info/METADATA +847 -0
  772. contextsynapse-1.0.0.dist-info/RECORD +776 -0
  773. contextsynapse-1.0.0.dist-info/WHEEL +5 -0
  774. contextsynapse-1.0.0.dist-info/entry_points.txt +4 -0
  775. contextsynapse-1.0.0.dist-info/licenses/LICENSE +200 -0
  776. contextsynapse-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2998 @@
1
+ """
2
+ CSR-Only Graph Storage Implementation
3
+ High-performance CSR storage backend - NetworkX fallback disabled
4
+ """
5
+
6
+ from typing import Dict, List, Any, Optional, Union, Set, Tuple
7
+ from dataclasses import dataclass
8
+ import uuid
9
+ import time
10
+ import logging
11
+ import json
12
+ from collections import defaultdict
13
+
14
+ from ..storage.csr_graph_storage import CSRGraphStorage, CSRGraphStorageAdapter, CSRNode, CSREdge
15
+ from .graph_structures import GraphNode, GraphEdge
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ # Optional imports (removed modules)
20
+ TEMPORAL_AVAILABLE = True
21
+ HASHING_AVAILABLE = False
22
+
23
+ # Try to import sortedcontainers for B-tree indexes
24
+ try:
25
+ from sortedcontainers import SortedDict
26
+ SORTED_CONTAINERS_AVAILABLE = True
27
+ except ImportError:
28
+ SORTED_CONTAINERS_AVAILABLE = False
29
+ # Fallback to dict
30
+ SortedDict = dict
31
+ logger.warning("sortedcontainers not available, using dict for indexes (slower)")
32
+
33
+ # RAG-critical node types that need immediate write-through (bypass buffer)
34
+ RAG_CRITICAL_NODE_TYPES = {
35
+ 'Document', 'Chunk', 'TextChunk', 'Passage', 'Fact', 'Entity',
36
+ 'Table', 'Image', 'Embedding', 'Index', 'KnowledgeGraph',
37
+ 'ContextBoundary', 'CategoryBoundary',
38
+ # Session graph conversation layer — need immediate visibility for
39
+ # cross-layer REFERENCES edge linking and context building
40
+ 'Turn', 'Decision', 'Question', 'Action', 'Topic', 'AgentPresence',
41
+ }
42
+
43
+ # Metadata node types (use fast buffer with 0.5s flush)
44
+ METADATA_NODE_TYPES = {
45
+ 'Metadata', 'Config', 'Schema', 'IndexMetadata', 'Pipeline'
46
+ }
47
+
48
+ class VersionConflictError(Exception):
49
+ """Raised when an optimistic-lock version check fails on node update."""
50
+
51
+ def __init__(self, current_version: int, provided_version: int):
52
+ self.current_version = current_version
53
+ self.provided_version = provided_version
54
+ super().__init__(
55
+ f"Version conflict: stored version is {current_version}, "
56
+ f"but update provided version {provided_version}. "
57
+ f"Re-fetch the node and retry."
58
+ )
59
+
60
+
61
+ class ContextSynapse:
62
+ """ContextSynapse - CSR-only graph storage for high-performance operations (NetworkX disabled)."""
63
+
64
+ def __init__(self, name: str = "default", config: Optional[Dict] = None, storage_backend: str = 'csr'):
65
+ """
66
+ Initialize Cognitive Database.
67
+
68
+ Args:
69
+ name: Name of the graph database
70
+ config: Optional configuration dictionary
71
+ storage_backend: 'csr' only (NetworkX fallback disabled)
72
+
73
+ Raises:
74
+ ValueError: If storage_backend is not 'csr'
75
+ """
76
+ self.name = name
77
+ self.config = config or {}
78
+
79
+ # CRITICAL: Force CSR-only mode - NetworkX fallback disabled
80
+ if storage_backend != 'csr':
81
+ logger.warning(f"NetworkX fallback disabled. Forcing CSR-only mode. Received: {storage_backend}")
82
+ storage_backend = 'csr'
83
+
84
+ self.storage_backend = 'csr' # Always CSR
85
+ self.use_csr = True # Always True - no fallback
86
+
87
+ # Enable single-file HDF5 storage by default (Oracle-type format)
88
+ if 'graph' not in self.config:
89
+ self.config['graph'] = {}
90
+ # NEO4J-LEVEL PERFORMANCE: Default to HDF5 single-file storage (faster than JSON)
91
+ if 'storage_strategy' not in self.config['graph']:
92
+ self.config['graph']['storage_strategy'] = 'single_file' # HDF5 single-file (faster)
93
+ if 'storage_format' not in self.config['graph']:
94
+ self.config['graph']['storage_format'] = 'single_file' # Backward compatibility
95
+
96
+ # CSR-only: No NetworkX storage
97
+ # Thread-safe write lock for concurrent ingestion
98
+ import threading
99
+ self._write_lock = threading.RLock()
100
+
101
+ # Maintain minimal compatibility structures for properties
102
+ self.node_properties = {} # Store node properties separately
103
+ self.edge_properties = {} # Store edge properties separately
104
+ self.edge_store = {} # Full edge metadata: edge_uuid -> {source, target, label, properties, weight}
105
+ self.unique_constraints = defaultdict(set)
106
+ # NOTE: alias_to_node removed - aliases are now query-scoped (stored in executor only)
107
+ self.global_strategy = 'DFS'
108
+
109
+ # Create node_index for compatibility
110
+ self.node_index = {}
111
+
112
+ # Create edge_index for compatibility
113
+ self.edge_index = {}
114
+
115
+ # NEW: Property indexes for fast filtering
116
+ if SORTED_CONTAINERS_AVAILABLE:
117
+ self._property_indexes: Dict[str, SortedDict] = {}
118
+ else:
119
+ self._property_indexes: Dict[str, Dict[Any, Set[str]]] = {}
120
+
121
+ # Performance metrics
122
+ self.operation_counts = {
123
+ 'add_node': 0,
124
+ 'add_edge': 0,
125
+ 'get_neighbors': 0,
126
+ 'traverse': 0
127
+ }
128
+
129
+ # Time travel / Temporal storage (always enabled)
130
+ self.temporal_enabled = True
131
+ self.temporal_storage = None
132
+ try:
133
+ from ..temporal.storage import TemporalStorage
134
+ base_path = self.config.get('base_path', 'contextcore_data')
135
+ self.temporal_storage = TemporalStorage(namespace=name, base_path=base_path)
136
+ self.temporal_storage.load_versions()
137
+ logger.debug(f"[TIME_TRAVEL] Temporal storage ready for namespace: {name}")
138
+ except Exception as e:
139
+ logger.warning(f"[TIME_TRAVEL] Failed to initialize temporal storage: {e}")
140
+ self.temporal_enabled = False
141
+ self.temporal_storage = None
142
+
143
+ # Semantic hashing (optional, enabled via config)
144
+ self.semantic_hashing_enabled = self.config.get('hashing', {}).get('enabled', False)
145
+ self.hasher = None
146
+ if self.semantic_hashing_enabled:
147
+ try:
148
+ from ..hashing.comprehensive_hashing import ComprehensiveHasher
149
+ self.hasher = ComprehensiveHasher()
150
+ logger.info(f"[SEMANTIC_HASH] Semantic hashing enabled for namespace: {name}")
151
+ except Exception as e:
152
+ logger.warning(f"[SEMANTIC_HASH] Failed to initialize semantic hasher: {e}")
153
+ self.semantic_hashing_enabled = False
154
+
155
+ # Initialize traceability layer (optional, for transparency and integrity)
156
+ # Check config file first, then local config
157
+ from ..config.database_config import DatabaseConfigManager
158
+ try:
159
+ config_manager = DatabaseConfigManager()
160
+ db_config = config_manager.config
161
+ traceability_enabled = db_config.traceability.enabled
162
+ except:
163
+ traceability_enabled = self.config.get('traceability', {}).get('enabled', False)
164
+
165
+ self.traceability_enabled = traceability_enabled
166
+ self.traceability_layer = None
167
+ if self.traceability_enabled:
168
+ try:
169
+ from ..traceability.traceability_layer import TraceabilityLayer
170
+ self.traceability_layer = TraceabilityLayer(name, base_path="contextcore_data")
171
+ logger.info(f"Traceability layer enabled for graph '{name}'")
172
+ except Exception as e:
173
+ logger.warning(f"Failed to initialize traceability layer: {e}")
174
+
175
+ # WAL (Write-Ahead Log) — lazy init on first write for speed
176
+ self.wal = None
177
+ self._wal_enabled = True
178
+ self._wal_name = name
179
+ try:
180
+ self._wal_enabled = self.config.get('wal', {}).get('enabled', True)
181
+ except Exception:
182
+ pass
183
+
184
+ # CRITICAL: Enable buffer manager by default with delayed block updates
185
+ # Buffer writes to memory first, then flushes to CSR storage after delay
186
+ buffer_config = self.config.get('buffer', {})
187
+ if 'enabled' not in buffer_config:
188
+ buffer_config['enabled'] = True # Enable by default
189
+ if 'config' not in buffer_config:
190
+ buffer_config['config'] = {}
191
+ buffer_config['config'].setdefault('max_size', 1000) # Max records before flush
192
+ buffer_config['config'].setdefault('flush_interval', 2.0) # Flush every 2 seconds (delayed block update)
193
+ buffer_config['config'].setdefault('batch_size', 100) # Records per batch
194
+ buffer_config['config'].setdefault('auto_flush', True) # Enable auto-flush
195
+ buffer_config['config'].setdefault('flush_threshold', 0.7) # Flush when buffer reaches 70% capacity (0.0-1.0)
196
+ buffer_config['config'].setdefault('threshold_flush_enabled', True) # Enable threshold-based flushing
197
+
198
+ self.buffer_enabled = buffer_config.get('enabled', True)
199
+ self.buffer_manager = None
200
+ # OPTIMIZATION: Pre-initialize buffer manager synchronously during __init__
201
+ # This avoids async initialization overhead during first node creation
202
+ if self.buffer_enabled:
203
+ try:
204
+ from ..buffer import create_buffer_manager
205
+ buffer_type = buffer_config.get('type', 'local')
206
+ buffer_params = buffer_config.get('config', {})
207
+ buffer_params['namespace'] = name # Add namespace for multi-tenant buffers
208
+ self.buffer_manager = create_buffer_manager(buffer_type, buffer_params)
209
+ # Store writer function reference for later use
210
+ self._buffer_writer_fn_ref = self._buffer_writer_fn
211
+
212
+ # Register cleanup on instance destruction
213
+ import atexit
214
+ atexit.register(self._cleanup_buffer_manager)
215
+
216
+ # OPTIMIZATION: Initialize buffer manager synchronously if possible
217
+ # This avoids async overhead during first node creation
218
+ try:
219
+ import asyncio
220
+ # Try to get existing event loop
221
+ try:
222
+ loop = asyncio.get_event_loop()
223
+ if loop.is_running():
224
+ # Loop is running, initialize async (will be done lazily)
225
+ self._buffer_initialized = False
226
+ else:
227
+ # Loop exists but not running, initialize synchronously
228
+ loop.run_until_complete(self.buffer_manager.initialize())
229
+ if buffer_params.get('auto_flush', True):
230
+ loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
231
+ self._buffer_initialized = True
232
+ logger.debug(f"Buffer manager pre-initialized for graph '{name}'")
233
+ except RuntimeError:
234
+ # No event loop, create one and initialize
235
+ loop = asyncio.new_event_loop()
236
+ asyncio.set_event_loop(loop)
237
+ loop.run_until_complete(self.buffer_manager.initialize())
238
+ if buffer_params.get('auto_flush', True):
239
+ loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
240
+ self._buffer_initialized = True
241
+ logger.debug(f"Buffer manager pre-initialized for graph '{name}' (new loop)")
242
+ except Exception as init_error:
243
+ # If sync initialization fails, fall back to lazy initialization
244
+ logger.debug(f"Buffer manager will be initialized lazily: {init_error}")
245
+ self._buffer_initialized = False
246
+
247
+ logger.debug(f"Buffer manager enabled for graph '{name}' (type: {buffer_type}, flush_interval: {buffer_params.get('flush_interval', 2.0)}s)")
248
+ except Exception as e:
249
+ logger.warning(f"Failed to initialize buffer manager: {e}")
250
+ self.buffer_enabled = False
251
+ self._buffer_initialized = False
252
+ else:
253
+ self._buffer_initialized = False
254
+
255
+ # Initialize graph storage backend: Redis (shared) or CSR (in-memory)
256
+ import os
257
+ graph_backend = os.environ.get("CONTEXTSYNAPSE_GRAPH_BACKEND") or os.environ.get("AICONTEXTDB_GRAPH_BACKEND", "csr")
258
+ self._using_redis_backend = False
259
+
260
+ if graph_backend == "redis" and os.environ.get("CONTEXTSYNAPSE_REDIS_URL") or os.environ.get("AICONTEXTDB_REDIS_URL"):
261
+ try:
262
+ from ..storage.redis_graph_adapter import RedisGraphAdapter
263
+ self.csr_adapter = RedisGraphAdapter(name)
264
+ self.csr_storage = self.csr_adapter # compatibility
265
+ self._using_redis_backend = True
266
+ # Disable buffer for Redis — Redis handles persistence directly
267
+ self.buffer_enabled = False
268
+ self.buffer_manager = None
269
+ self._buffer_initialized = True
270
+ logger.info("[REDIS] Graph '%s' using Redis backend (no in-memory loading)", name)
271
+ except Exception as e:
272
+ logger.warning("[REDIS] Redis graph backend failed, falling back to CSR: %s", e)
273
+ graph_backend = "csr"
274
+
275
+ # Check for LMDB backend
276
+ self._using_lmdb_backend = False
277
+ if not self._using_redis_backend:
278
+ storage_backend_env = os.environ.get("CONTEXTSYNAPSE_STORAGE_BACKEND") or os.environ.get("AICONTEXTDB_STORAGE_BACKEND", "csr")
279
+ if storage_backend_env == "lmdb":
280
+ try:
281
+ from ..storage.lmdb_graph_storage import LMDBGraphStorage, LMDB_AVAILABLE
282
+ if LMDB_AVAILABLE:
283
+ lmdb_path = os.environ.get("CONTEXTSYNAPSE_LMDB_PATH") or os.environ.get("AICONTEXTDB_LMDB_PATH", "contextcore_data/lmdb")
284
+ lmdb_dir = os.path.join(lmdb_path, self.name)
285
+ self.csr_storage = LMDBGraphStorage(lmdb_dir, namespace=self.name)
286
+ self.csr_adapter = self.csr_storage
287
+ self._using_lmdb_backend = True
288
+ logger.info("[STORAGE] Using LMDB backend: %s", lmdb_dir)
289
+ else:
290
+ logger.warning("[STORAGE] LMDB requested but lmdb/msgpack not installed, falling back to CSR")
291
+ except Exception as e:
292
+ logger.warning("[STORAGE] LMDB init failed (%s), falling back to CSR", e)
293
+
294
+ if not self._using_redis_backend and not self._using_lmdb_backend:
295
+ try:
296
+ self.csr_storage = CSRGraphStorage()
297
+ self.csr_adapter = CSRGraphStorageAdapter(self.csr_storage)
298
+ logger.debug("[CSR] Graph '%s' using in-memory CSR storage", name)
299
+ except Exception as e:
300
+ logger.error("[CSR] Failed to initialize CSR storage: %s", e)
301
+ raise RuntimeError(f"Graph storage initialization failed: {e}")
302
+
303
+ # Initialize Time Travel (always enabled)
304
+ self.temporal_enabled = True
305
+ self.temporal_storage = None
306
+ try:
307
+ from ..temporal.storage import TemporalStorage as _TemporalStorage
308
+ base_path = self.config.get('temporal', {}).get('base_path', self.config.get('base_path', 'contextcore_data'))
309
+ self.temporal_storage = _TemporalStorage(namespace=name, base_path=base_path)
310
+ self.temporal_storage.load_versions()
311
+ logger.debug(f"[TIME_TRAVEL] Temporal storage ready for namespace: {name}")
312
+ except Exception as e:
313
+ logger.warning(f"[TIME_TRAVEL] Failed to initialize temporal storage: {e}")
314
+ self.temporal_enabled = False
315
+ self.temporal_storage = None
316
+
317
+ # Initialize Semantic Hashing (if available and enabled)
318
+ self.hashing_enabled = self.config.get('hashing', {}).get('enabled', False) and HASHING_AVAILABLE
319
+ if self.hashing_enabled:
320
+ try:
321
+ self.hasher = get_global_hasher()
322
+ logger.info(f"[EMOJI] Semantic hashing enabled for namespace: {name}")
323
+ except Exception as e:
324
+ logger.warning(f"[EMOJI] Failed to initialize semantic hashing: {e}")
325
+ self.hashing_enabled = False
326
+ self.hasher = None
327
+ else:
328
+ self.hasher = None
329
+
330
+ def _clean_properties_for_indexing(self, properties: Dict[str, Any]) -> Dict[str, Any]:
331
+ """Recursively clean properties to remove unhashable types that could cause indexing errors.
332
+
333
+ Converts nested dicts and lists to JSON strings to make them hashable.
334
+ """
335
+ import copy
336
+ import json
337
+ cleaned = {}
338
+ for key, value in properties.items():
339
+ if isinstance(value, dict):
340
+ # Convert nested dict to JSON string
341
+ try:
342
+ cleaned[key] = json.dumps(value, sort_keys=True)
343
+ except (TypeError, ValueError):
344
+ # If can't serialize, skip it
345
+ continue
346
+ elif isinstance(value, list):
347
+ # Convert list to JSON string
348
+ try:
349
+ cleaned[key] = json.dumps(value, sort_keys=True)
350
+ except (TypeError, ValueError):
351
+ # If can't serialize, skip it
352
+ continue
353
+ else:
354
+ # Keep primitive types as-is
355
+ cleaned[key] = value
356
+ return cleaned
357
+
358
+ def _cleanup_buffer_manager(self):
359
+ """Cleanup buffer manager on instance destruction."""
360
+ if self.buffer_manager:
361
+ try:
362
+ # Stop auto-flush if running
363
+ if hasattr(self.buffer_manager, 'stop_auto_flush'):
364
+ try:
365
+ loop = asyncio.get_event_loop()
366
+ if loop.is_running():
367
+ # Schedule cleanup
368
+ asyncio.create_task(self.buffer_manager.stop_auto_flush())
369
+ else:
370
+ # Run cleanup synchronously
371
+ loop.run_until_complete(self.buffer_manager.stop_auto_flush())
372
+ except (RuntimeError, AttributeError):
373
+ # No event loop - task will be cleaned up by Python
374
+ pass
375
+
376
+ # Close buffer manager
377
+ if hasattr(self.buffer_manager, 'close'):
378
+ try:
379
+ loop = asyncio.get_event_loop()
380
+ if loop.is_running():
381
+ asyncio.create_task(self.buffer_manager.close())
382
+ else:
383
+ loop.run_until_complete(self.buffer_manager.close())
384
+ except (RuntimeError, AttributeError):
385
+ pass
386
+ except Exception as e:
387
+ logger.debug(f"Error cleaning up buffer manager for {self.name}: {e}")
388
+
389
+ def load(self, file_path: str) -> bool:
390
+ """
391
+ Load graph data from a file.
392
+ Tries to load CSR format first (for performance), falls back to JSON.
393
+
394
+ Args:
395
+ file_path: Path to the graph file
396
+
397
+ Returns:
398
+ True if successful, False otherwise
399
+ """
400
+ try:
401
+ from pathlib import Path
402
+ from scipy.sparse import load_npz
403
+
404
+ file_path_obj = Path(file_path)
405
+ base_dir = file_path_obj.parent
406
+ base_name = file_path_obj.stem
407
+
408
+ # Check storage strategy and load accordingly
409
+ graph_config = self.config.get('graph', {})
410
+ storage_strategy = graph_config.get('storage_strategy') or graph_config.get('storage_format', 'single_file')
411
+
412
+ # Try single-file HDF5 first if strategy is single_file
413
+ if storage_strategy == 'single_file':
414
+ h5_path = file_path_obj.with_suffix('.h5')
415
+ if h5_path.exists():
416
+ return self._load_single_file(str(h5_path))
417
+ # Try multi-file optimized if strategy is multi_file_optimized
418
+ elif storage_strategy == 'multi_file_optimized':
419
+ if self._load_multi_file_optimized(str(file_path_obj)):
420
+ return True
421
+ # Auto-detect: try single-file first, then multi-file
422
+ else:
423
+ h5_path = file_path_obj.with_suffix('.h5')
424
+ if h5_path.exists():
425
+ return self._load_single_file(str(h5_path))
426
+
427
+ # Try to load CSR format first (faster for large graphs)
428
+ csr_file = base_dir / "csr" / f"{base_name}_csr.npz"
429
+ csr_metadata_file = base_dir / "csr" / f"{base_name}_csr_metadata.json"
430
+
431
+ if csr_file.exists() and csr_metadata_file.exists():
432
+ try:
433
+ logger.info(f"Loading graph from CSR format: {csr_file}")
434
+
435
+ # Load CSR matrix
436
+ csr_matrix_data = load_npz(str(csr_file))
437
+
438
+ # Load CSR metadata
439
+ with open(csr_metadata_file, 'r') as f:
440
+ csr_metadata = json.load(f)
441
+
442
+ node_list = csr_metadata.get('node_list', [])
443
+ node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
444
+ idx_to_node = {idx: node_id for node_id, idx in node_to_idx.items()}
445
+ edge_labels = csr_metadata.get('edge_labels', [])
446
+ edge_properties_list = csr_metadata.get('edge_properties', [])
447
+
448
+ # Load nodes from metadata (we'll need to get properties from JSON)
449
+ # First, load JSON to get node properties
450
+ with open(file_path, 'r') as f:
451
+ json_data = json.load(f)
452
+
453
+ # CRITICAL: Ensure CSR adapter is initialized
454
+ if self.use_csr and not self.csr_adapter:
455
+ from ..storage.csr_graph_storage import CSRGraphStorageAdapter
456
+ if self.csr_storage:
457
+ self.csr_adapter = CSRGraphStorageAdapter(self.csr_storage)
458
+ logger.info("Initialized CSR adapter during CSR format load")
459
+
460
+ # Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
461
+
462
+ # Create node properties map
463
+ node_props_map = {}
464
+ if 'nodes' in json_data:
465
+ for node_data in json_data['nodes']:
466
+ node_id = node_data.get('id')
467
+ if node_id:
468
+ node_props_map[node_id] = node_data.get('properties', {})
469
+
470
+ # Add nodes
471
+ for node_id in node_list:
472
+ label = 'Node'
473
+ properties = node_props_map.get(node_id, {})
474
+ # Get label from JSON data if available
475
+ if 'nodes' in json_data:
476
+ for node_data in json_data['nodes']:
477
+ if node_data.get('id') == node_id:
478
+ label = node_data.get('label', 'Node')
479
+ break
480
+
481
+ node = GraphNode(id=node_id, label=label, properties=properties)
482
+ properties_cleaned = self._clean_properties_for_indexing(properties)
483
+ node.properties, original_props = properties_cleaned, node.properties
484
+ self.add_node(node)
485
+ self.node_properties[node_id] = original_props
486
+ if node_id in self.node_index:
487
+ self.node_index[node_id].properties = original_props
488
+
489
+ # Add edges from CSR matrix
490
+ csr_matrix_data = csr_matrix_data.tocoo() # Convert to COO for iteration
491
+ for i, (row_idx, col_idx) in enumerate(zip(csr_matrix_data.row, csr_matrix_data.col)):
492
+ if i < len(edge_labels):
493
+ src = idx_to_node.get(row_idx)
494
+ dst = idx_to_node.get(col_idx)
495
+ edge_label = edge_labels[i] if i < len(edge_labels) else 'Edge'
496
+ edge_props = edge_properties_list[i] if i < len(edge_properties_list) else {}
497
+
498
+ if src and dst:
499
+ self.add_edge(GraphEdge(
500
+ id=f"{src}_{dst}_{i}",
501
+ source=src,
502
+ target=dst,
503
+ label=edge_label,
504
+ properties=edge_props
505
+ ))
506
+
507
+ logger.info(f"[EMOJI] Loaded graph from CSR format: {len(node_list)} nodes, {len(edge_labels)} edges")
508
+ return True
509
+
510
+ except Exception as e:
511
+ logger.warning(f"Failed to load CSR format: {e}, falling back to JSON")
512
+
513
+ # Fallback to JSON format
514
+ logger.info(f"Loading graph from JSON format: {file_path}")
515
+ with open(file_path, 'r') as f:
516
+ data = json.load(f)
517
+
518
+ # Try to load node properties from Parquet (faster than JSON for large graphs)
519
+ parquet_file = base_dir / "node_properties" / f"{base_name}_nodes.parquet"
520
+ if parquet_file.exists():
521
+ try:
522
+ import pandas as pd
523
+ nodes_df = pd.read_parquet(parquet_file, engine='pyarrow')
524
+ logger.info(f"Loading node properties from Parquet: {parquet_file}")
525
+
526
+ # Update node properties from Parquet (if available)
527
+ parquet_nodes = {}
528
+ for _, row in nodes_df.iterrows():
529
+ node_id = row['node_id']
530
+ props = {}
531
+ for col in nodes_df.columns:
532
+ if col.startswith('prop_'):
533
+ prop_key = col.replace('prop_', '')
534
+ value = row[col]
535
+ # Try to parse JSON strings back to objects
536
+ if pd.notna(value) and isinstance(value, str) and (value.startswith('{') or value.startswith('[')):
537
+ try:
538
+ props[prop_key] = json.loads(value)
539
+ except:
540
+ props[prop_key] = value
541
+ elif pd.notna(value):
542
+ props[prop_key] = value
543
+ if props:
544
+ parquet_nodes[node_id] = props
545
+
546
+ # Merge Parquet properties into JSON data if they exist
547
+ if parquet_nodes:
548
+ logger.info(f"Loaded {len(parquet_nodes)} node properties from Parquet")
549
+ for node in data.get('nodes', []):
550
+ node_id = node.get('id')
551
+ if node_id in parquet_nodes:
552
+ # Merge Parquet properties (may be more up-to-date)
553
+ if 'properties' not in node:
554
+ node['properties'] = {}
555
+ node['properties'].update(parquet_nodes[node_id])
556
+ except ImportError:
557
+ logger.debug("PyArrow/pandas not available, using JSON node properties only")
558
+ except Exception as e:
559
+ logger.debug(f"Failed to load node properties from Parquet: {e}, using JSON")
560
+
561
+ # CRITICAL: Ensure CSR adapter is initialized
562
+ # CSR adapter is created in __init__, but ensure it exists
563
+ if self.use_csr and not self.csr_adapter:
564
+ from ..storage.csr_graph_storage import CSRGraphStorageAdapter
565
+ if self.csr_storage:
566
+ self.csr_adapter = CSRGraphStorageAdapter(self.csr_storage)
567
+ logger.info("Initialized CSR adapter during graph load")
568
+
569
+ # Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
570
+
571
+ # Load nodes
572
+ if 'nodes' in data:
573
+ for node_data in data['nodes']:
574
+ try:
575
+ node_id = node_data['id']
576
+ label = node_data.get('label', 'Node')
577
+ properties = node_data.get('properties', {})
578
+ # Ensure properties is a dict (not nested dicts that could cause unhashable errors)
579
+ if isinstance(properties, dict):
580
+ # Deep copy to avoid reference issues
581
+ import copy
582
+ properties = copy.deepcopy(properties)
583
+ # Clean properties for indexing (convert nested dicts/lists to JSON strings)
584
+ properties_cleaned = self._clean_properties_for_indexing(properties)
585
+ # Keep original properties for node, but use cleaned for indexing
586
+ node = GraphNode(id=node_id, label=label, properties=properties)
587
+ # Temporarily replace properties with cleaned version for add_node (to avoid indexing errors)
588
+ # Then restore original properties after indexing
589
+ node.properties, original_props = properties_cleaned, node.properties
590
+ self.add_node(node)
591
+ # Restore original properties after indexing
592
+ self.node_properties[node_id] = original_props
593
+ if node_id in self.node_index:
594
+ self.node_index[node_id].properties = original_props
595
+ else:
596
+ properties = {}
597
+ self.add_node(GraphNode(id=node_id, label=label, properties=properties))
598
+ except Exception as e:
599
+ logger.warning(f"[EMOJI][EMOJI] Skipping node {node_data.get('id', 'unknown')} due to error: {e}")
600
+ continue
601
+
602
+ # Load edges
603
+ if 'edges' in data:
604
+ for edge_data in data['edges']:
605
+ # Handle both 'src'/'dst' and 'source'/'target' formats
606
+ src = edge_data.get('src', edge_data.get('source', ''))
607
+ dest = edge_data.get('dst', edge_data.get('dest', edge_data.get('target', '')))
608
+ label = edge_data.get('label', edge_data.get('type', 'Edge'))
609
+ properties = edge_data.get('properties', {})
610
+ weight = edge_data.get('weight', 1.0)
611
+
612
+ if src and dest: # Only add edge if both source and target exist
613
+ self.add_edge(GraphEdge(
614
+ id=f"{src}_{dest}",
615
+ source=src,
616
+ target=dest,
617
+ label=label,
618
+ properties=properties
619
+ ))
620
+
621
+ return True
622
+ except Exception as e:
623
+ logger.error(f"Error loading graph from {file_path}: {e}")
624
+ return False
625
+
626
+ def _load_single_file(self, h5_path: str) -> bool:
627
+ """
628
+ Load graph from single-file HDF5 format.
629
+
630
+ Args:
631
+ h5_path: Path to HDF5 file
632
+
633
+ Returns:
634
+ True if successful
635
+ """
636
+ try:
637
+ from ..storage.single_file_storage import SingleFileStorage
638
+
639
+ with SingleFileStorage(h5_path, mode='r') as storage:
640
+ data = storage.load_graph()
641
+ if not data:
642
+ return False
643
+
644
+ # Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
645
+
646
+ # Load nodes (always write_through — bypass async buffer during load)
647
+ for node_data in data.get('nodes', []):
648
+ node_id = node_data.get('id')
649
+ label = node_data.get('label', 'Node')
650
+ properties = node_data.get('properties', {})
651
+
652
+ node = GraphNode(id=node_id, label=label, properties=properties)
653
+ properties_cleaned = self._clean_properties_for_indexing(properties)
654
+ node.properties, original_props = properties_cleaned, node.properties
655
+ self.add_node(node, write_through=True)
656
+ self.node_properties[node_id] = original_props
657
+ if node_id in self.node_index:
658
+ self.node_index[node_id].properties = original_props
659
+
660
+ # Load edges
661
+ for edge_data in data.get('edges', []):
662
+ src = edge_data.get('src', edge_data.get('source', ''))
663
+ dst = edge_data.get('dst', edge_data.get('target', ''))
664
+ label = edge_data.get('label', 'Edge')
665
+ properties = edge_data.get('properties', {})
666
+
667
+ if src and dst:
668
+ self.add_edge(GraphEdge(
669
+ id=f"{src}_{dst}",
670
+ source=src,
671
+ target=dst,
672
+ label=label,
673
+ properties=properties
674
+ ))
675
+
676
+ # Try to load CSR matrix
677
+ csr_matrix_data, csr_metadata = storage.load_csr_matrix()
678
+ if csr_matrix_data is not None:
679
+ logger.info("Loaded CSR matrix from HDF5")
680
+ # CSR matrix can be used for fast operations
681
+ # Note: Full CSR integration would require CSR adapter setup
682
+
683
+ logger.info(f"Loaded graph from single-file HDF5: {len(data.get('nodes', []))} nodes, {len(data.get('edges', []))} edges")
684
+ return True
685
+
686
+ except ImportError:
687
+ logger.error("h5py not available. Install with: pip install h5py")
688
+ return False
689
+ except Exception as e:
690
+ logger.error(f"Failed to load from single-file format: {e}")
691
+ return False
692
+
693
+ def flush(self):
694
+ """Flush any buffered writes to CSR storage for immediate visibility."""
695
+ if self.buffer_enabled and self.buffer_manager:
696
+ try:
697
+ import asyncio
698
+ loop = asyncio.new_event_loop()
699
+ loop.run_until_complete(self.buffer_manager.flush())
700
+ loop.close()
701
+ except Exception:
702
+ pass
703
+
704
+ def save(self, file_path: str) -> bool:
705
+ """
706
+ Save graph data to a file.
707
+ Saves both JSON format (for compatibility) and CSR format (for performance).
708
+ Optionally saves to single-file HDF5 format if configured.
709
+
710
+ This method is used by BOTH direct graph operations and API operations.
711
+ It ensures the same code path is used regardless of how the graph was created/modified.
712
+
713
+ Args:
714
+ file_path: Path to save the graph file
715
+
716
+ Returns:
717
+ True if successful, False otherwise
718
+ """
719
+ try:
720
+ from pathlib import Path
721
+ import numpy as np
722
+ from scipy.sparse import save_npz
723
+
724
+ file_path_obj = Path(file_path)
725
+ base_dir = file_path_obj.parent
726
+ base_name = file_path_obj.stem
727
+
728
+ # Check storage strategy (prefer storage_strategy over storage_format for backward compatibility)
729
+ graph_config = self.config.get('graph', {})
730
+ storage_strategy = graph_config.get('storage_strategy') or graph_config.get('storage_format', 'single_file')
731
+
732
+ if storage_strategy == 'single_file':
733
+ return self._save_single_file(file_path)
734
+ elif storage_strategy == 'multi_file_optimized':
735
+ return self._save_multi_file_optimized(file_path)
736
+ # Fallback to legacy multi-file format
737
+
738
+ # 1. Save JSON format (for compatibility and human readability)
739
+ # Note: Aliases are NOT saved (they are query-scoped only)
740
+
741
+ # CSR-only: Get node/edge counts from CSR storage
742
+ node_count = self.csr_storage.get_node_count() if self.csr_storage else 0
743
+ edge_count = self.csr_storage.get_edge_count() if self.csr_storage else 0
744
+
745
+ data = {
746
+ 'nodes': [],
747
+ 'edges': [],
748
+ # 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
749
+ 'metadata': {
750
+ 'name': self.name,
751
+ 'node_count': node_count,
752
+ 'edge_count': edge_count,
753
+ 'storage_backend': self.storage_backend,
754
+ 'saved_at': time.time(),
755
+ 'has_csr': self.csr_storage is not None and node_count > 0
756
+ }
757
+ }
758
+
759
+ # Save nodes from CSR storage
760
+ nodes_saved = 0
761
+ if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
762
+ nodes_dict = self.csr_storage.nodes if hasattr(self.csr_storage, 'nodes') else {}
763
+ logger.debug(f"[SAVE] CSR storage has {len(nodes_dict)} nodes in nodes dict")
764
+
765
+ for node_id, csr_node in nodes_dict.items():
766
+ # Handle both CSRNode objects and GraphNode objects
767
+ if hasattr(csr_node, 'node_type'):
768
+ node_type = csr_node.node_type
769
+ elif hasattr(csr_node, 'label'):
770
+ node_type = csr_node.label
771
+ else:
772
+ node_type = 'Node'
773
+
774
+ # Get properties from node_properties dict (stored separately)
775
+ node_props = self.node_properties.get(node_id, {})
776
+ if not node_props and hasattr(csr_node, 'properties'):
777
+ node_props = csr_node.properties
778
+
779
+ node_data = {
780
+ 'id': node_id,
781
+ 'label': node_type,
782
+ 'properties': node_props
783
+ }
784
+ data['nodes'].append(node_data)
785
+ nodes_saved += 1
786
+
787
+ logger.info(f"[SAVE] Saved {nodes_saved} nodes from CSR storage")
788
+
789
+ # FALLBACK: If CSR storage has no nodes but node_index does, use node_index
790
+ if not data['nodes'] and hasattr(self, 'node_index') and self.node_index:
791
+ logger.warning(f"[SAVE] CSR storage has no nodes, using node_index fallback ({len(self.node_index)} nodes)")
792
+ for node_id, node in self.node_index.items():
793
+ node_data = {
794
+ 'id': node_id,
795
+ 'label': node.label if hasattr(node, 'label') else 'Node',
796
+ 'properties': self.node_properties.get(node_id, node.properties if hasattr(node, 'properties') else {})
797
+ }
798
+ data['nodes'].append(node_data)
799
+ nodes_saved += 1
800
+ logger.info(f"[SAVE] Saved {nodes_saved} nodes from node_index fallback")
801
+
802
+ # Update node count in metadata
803
+ data['metadata']['node_count'] = len(data['nodes'])
804
+
805
+ # Save edges from CSR storage
806
+ saved_edge_pairs = set() # Track (source, target, label) to avoid duplicates
807
+ if self.csr_storage and hasattr(self.csr_storage, 'edge_data'):
808
+ for edge in self.csr_storage.edge_data:
809
+ edge_id = f"{edge.source_id}_{edge.target_id}"
810
+ edge_info = {
811
+ 'src': edge.source_id,
812
+ 'dst': edge.target_id,
813
+ 'label': edge.edge_type,
814
+ 'properties': self.edge_properties.get(edge_id, edge.properties),
815
+ 'weight': edge.weight
816
+ }
817
+ data['edges'].append(edge_info)
818
+ saved_edge_pairs.add((edge.source_id, edge.target_id, edge.edge_type))
819
+
820
+ # Fallback: save edges from edge_store that weren't in CSR
821
+ # This catches edges where CSR add_edge silently failed (e.g., node not in CSR)
822
+ for edge_id, edge_meta in self.edge_store.items():
823
+ pair = (edge_meta['source'], edge_meta['target'], edge_meta.get('label', 'EDGE'))
824
+ if pair in saved_edge_pairs:
825
+ continue
826
+ edge_info = {
827
+ 'src': edge_meta['source'],
828
+ 'dst': edge_meta['target'],
829
+ 'label': edge_meta.get('label', 'EDGE'),
830
+ 'properties': edge_meta.get('properties', {}),
831
+ 'weight': edge_meta.get('weight', 1.0)
832
+ }
833
+ data['edges'].append(edge_info)
834
+ saved_edge_pairs.add(pair)
835
+
836
+ # Write JSON file (for compatibility)
837
+ with open(file_path, 'w') as f:
838
+ json.dump(data, f, indent=2)
839
+
840
+ logger.info(f"Graph '{self.name}' saved to {file_path} (JSON format)")
841
+
842
+ # 3. Save node properties to Parquet (for fast property queries)
843
+ try:
844
+ import pandas as pd
845
+ parquet_dir = base_dir / "node_properties"
846
+ parquet_dir.mkdir(parents=True, exist_ok=True)
847
+
848
+ # Convert node properties to DataFrame
849
+ if data['nodes']:
850
+ nodes_df_data = []
851
+ for node in data['nodes']:
852
+ row = {
853
+ 'node_id': node['id'],
854
+ 'label': node['label']
855
+ }
856
+ # Flatten properties
857
+ props = node.get('properties', {})
858
+ for key, value in props.items():
859
+ # Handle nested properties by converting to string
860
+ if isinstance(value, (dict, list)):
861
+ row[f'prop_{key}'] = json.dumps(value)
862
+ else:
863
+ row[f'prop_{key}'] = value
864
+ nodes_df_data.append(row)
865
+
866
+ if nodes_df_data:
867
+ nodes_df = pd.DataFrame(nodes_df_data)
868
+ parquet_file = parquet_dir / f"{base_name}_nodes.parquet"
869
+ nodes_df.to_parquet(parquet_file, index=False, engine='pyarrow', compression='snappy')
870
+ logger.info(f"Node properties saved to Parquet: {parquet_file} ({len(nodes_df)} nodes)")
871
+ except ImportError:
872
+ logger.warning("PyArrow/pandas not available, skipping Parquet node properties storage")
873
+ except Exception as e:
874
+ logger.warning(f"Failed to save node properties to Parquet: {e}")
875
+
876
+ # 2. Save CSR format (CRITICAL: Always save CSR when enabled, not just when _should_use_csr())
877
+ # CSR is the primary storage format for performance
878
+ if self.csr_storage and self.use_csr:
879
+ try:
880
+ csr_dir = base_dir / "csr"
881
+ csr_dir.mkdir(parents=True, exist_ok=True)
882
+
883
+ # Save CSR matrix and metadata
884
+ csr_file = csr_dir / f"{base_name}_csr.npz"
885
+ csr_metadata_file = csr_dir / f"{base_name}_csr_metadata.json"
886
+
887
+ # Build CSR matrix from CSR storage edges
888
+ if not self.csr_storage or not hasattr(self.csr_storage, 'nodes'):
889
+ raise RuntimeError("CSR storage not available")
890
+
891
+ node_list = list(self.csr_storage.nodes.keys())
892
+ node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
893
+ num_nodes = len(node_list)
894
+
895
+ # Build edge lists for CSR
896
+ row_indices = []
897
+ col_indices = []
898
+ edge_labels = []
899
+ edge_properties_list = []
900
+
901
+ if hasattr(self.csr_storage, 'edge_data'):
902
+ for edge in self.csr_storage.edge_data:
903
+ if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
904
+ row_indices.append(node_to_idx[edge.source_id])
905
+ col_indices.append(node_to_idx[edge.target_id])
906
+ edge_labels.append(edge.edge_type)
907
+ edge_id = f"{edge.source_id}_{edge.target_id}"
908
+ edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
909
+
910
+ # CRITICAL: Always save CSR format, even if no edges (for node-only graphs)
911
+ # Create CSR matrix (use zeros if no edges, but still create the matrix)
912
+ from scipy.sparse import csr_matrix
913
+ if row_indices:
914
+ csr_matrix_data = csr_matrix(
915
+ (np.ones(len(row_indices)), (row_indices, col_indices)),
916
+ shape=(num_nodes, num_nodes)
917
+ )
918
+ else:
919
+ # Empty graph - create empty CSR matrix
920
+ csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
921
+
922
+ # Save CSR matrix
923
+ save_npz(str(csr_file), csr_matrix_data)
924
+
925
+ # Save CSR metadata (without aliases - they are query-scoped)
926
+ csr_metadata = {
927
+ 'node_list': node_list,
928
+ 'node_to_idx': node_to_idx,
929
+ 'edge_labels': edge_labels,
930
+ 'edge_properties': edge_properties_list,
931
+ 'row_indices': row_indices,
932
+ 'col_indices': col_indices,
933
+ 'num_nodes': num_nodes,
934
+ 'num_edges': len(row_indices),
935
+ # 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
936
+ 'saved_at': time.time()
937
+ }
938
+
939
+ with open(csr_metadata_file, 'w') as f:
940
+ json.dump(csr_metadata, f, indent=2, default=str)
941
+
942
+ logger.info(f"CSR format saved to {csr_file} ({num_nodes} nodes, {len(row_indices)} edges)")
943
+ except Exception as e:
944
+ logger.error(f"[EMOJI] Failed to save CSR format: {e}")
945
+ import traceback
946
+ logger.debug(f"CSR save traceback: {traceback.format_exc()}")
947
+ # Don't fail completely - JSON is still saved as fallback
948
+
949
+ return True
950
+ except Exception as e:
951
+ logger.error(f"Failed to save graph '{self.name}': {e}")
952
+ return False
953
+
954
+ def _save_single_file(self, file_path: str) -> bool:
955
+ """
956
+ Save graph to single-file HDF5 format (like Oracle's .dbf).
957
+
958
+ Args:
959
+ file_path: Path to save (will use .h5 extension)
960
+
961
+ Returns:
962
+ True if successful
963
+ """
964
+ try:
965
+ from pathlib import Path
966
+ import numpy as np
967
+ from scipy.sparse import csr_matrix
968
+ from ..storage.single_file_storage import SingleFileStorage
969
+
970
+ # Use .h5 extension for single-file format
971
+ h5_path = Path(file_path).with_suffix('.h5')
972
+
973
+ # Prepare graph data (without aliases - they are query-scoped)
974
+ # CSR-only: Get node/edge counts from CSR storage
975
+ node_count = self.csr_storage.get_node_count() if self.csr_storage else 0
976
+ edge_count = self.csr_storage.get_edge_count() if self.csr_storage else 0
977
+
978
+ data = {
979
+ 'nodes': [],
980
+ 'edges': [],
981
+ # 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
982
+ 'metadata': {
983
+ 'name': self.name,
984
+ 'node_count': node_count,
985
+ 'edge_count': edge_count,
986
+ 'storage_backend': self.storage_backend,
987
+ 'saved_at': time.time(),
988
+ 'has_csr': self.csr_storage is not None and node_count > 0
989
+ }
990
+ }
991
+
992
+ # Collect nodes from CSR storage
993
+ if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
994
+ for node_id, csr_node in self.csr_storage.nodes.items():
995
+ node_data = {
996
+ 'id': node_id,
997
+ 'label': csr_node.node_type,
998
+ 'properties': self.node_properties.get(node_id, {})
999
+ }
1000
+ data['nodes'].append(node_data)
1001
+
1002
+ # Collect edges from CSR storage
1003
+ saved_edge_pairs = set()
1004
+ if self.csr_storage and hasattr(self.csr_storage, 'edge_data'):
1005
+ for edge in self.csr_storage.edge_data:
1006
+ edge_id = f"{edge.source_id}_{edge.target_id}"
1007
+ edge_info = {
1008
+ 'src': edge.source_id,
1009
+ 'dst': edge.target_id,
1010
+ 'label': edge.edge_type,
1011
+ 'properties': self.edge_properties.get(edge_id, edge.properties),
1012
+ 'weight': edge.weight
1013
+ }
1014
+ data['edges'].append(edge_info)
1015
+ saved_edge_pairs.add((edge.source_id, edge.target_id, edge.edge_type))
1016
+
1017
+ # Fallback: save edges from edge_store not in CSR
1018
+ for edge_id, edge_meta in self.edge_store.items():
1019
+ pair = (edge_meta['source'], edge_meta['target'], edge_meta.get('label', 'EDGE'))
1020
+ if pair in saved_edge_pairs:
1021
+ continue
1022
+ edge_info = {
1023
+ 'src': edge_meta['source'],
1024
+ 'dst': edge_meta['target'],
1025
+ 'label': edge_meta.get('label', 'EDGE'),
1026
+ 'properties': edge_meta.get('properties', {}),
1027
+ 'weight': edge_meta.get('weight', 1.0)
1028
+ }
1029
+ data['edges'].append(edge_info)
1030
+ saved_edge_pairs.add(pair)
1031
+
1032
+ # Save to HDF5
1033
+ # NEO4J-LEVEL PERFORMANCE: Use 'a' mode and delete/recreate groups to avoid file locking issues
1034
+ # This allows concurrent saves without "file already open" errors
1035
+ storage = None
1036
+ try:
1037
+ with SingleFileStorage(str(h5_path), mode='a') as storage:
1038
+ storage.save_graph(data)
1039
+
1040
+ # Save CSR matrix if available (CRITICAL: Inside try block, not except!)
1041
+ if self.csr_storage and self.use_csr:
1042
+ try:
1043
+ # Build CSR matrix from CSR storage
1044
+ if not hasattr(self.csr_storage, 'nodes'):
1045
+ raise RuntimeError("CSR storage nodes not available")
1046
+
1047
+ node_list = list(self.csr_storage.nodes.keys())
1048
+ node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
1049
+ num_nodes = len(node_list)
1050
+
1051
+ row_indices = []
1052
+ col_indices = []
1053
+ edge_labels = []
1054
+ edge_properties_list = []
1055
+
1056
+ if hasattr(self.csr_storage, 'edge_data'):
1057
+ for edge in self.csr_storage.edge_data:
1058
+ if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
1059
+ row_indices.append(node_to_idx[edge.source_id])
1060
+ col_indices.append(node_to_idx[edge.target_id])
1061
+ edge_labels.append(edge.edge_type)
1062
+ edge_id = f"{edge.source_id}_{edge.target_id}"
1063
+ edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
1064
+
1065
+ if row_indices:
1066
+ csr_matrix_data = csr_matrix(
1067
+ (np.ones(len(row_indices)), (row_indices, col_indices)),
1068
+ shape=(num_nodes, num_nodes)
1069
+ )
1070
+ else:
1071
+ csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
1072
+
1073
+ csr_metadata = {
1074
+ 'node_list': node_list,
1075
+ 'node_to_idx': node_to_idx,
1076
+ 'edge_labels': edge_labels,
1077
+ 'edge_properties': edge_properties_list,
1078
+ 'num_nodes': num_nodes,
1079
+ 'num_edges': len(row_indices)
1080
+ # 'alias_to_node': {} # REMOVED: Aliases are query-scoped, not persisted
1081
+ }
1082
+
1083
+ storage.save_csr_matrix(csr_matrix_data, csr_metadata)
1084
+ logger.info(f"CSR matrix saved to HDF5: {num_nodes} nodes, {len(row_indices)} edges")
1085
+ except Exception as csr_error:
1086
+ logger.warning(f"Failed to save CSR matrix to HDF5: {csr_error}")
1087
+ except Exception as e:
1088
+ # If append mode fails (file locked), try write mode with retry
1089
+ logger.warning(f"Append mode failed, trying write mode: {e}")
1090
+ max_retries = 3
1091
+ for retry in range(max_retries):
1092
+ try:
1093
+ with SingleFileStorage(str(h5_path), mode='w') as storage:
1094
+ storage.save_graph(data)
1095
+
1096
+ # Save CSR matrix if available (also in retry path)
1097
+ if self.csr_storage and self.use_csr:
1098
+ try:
1099
+ # Build CSR matrix from CSR storage
1100
+ if not hasattr(self.csr_storage, 'nodes'):
1101
+ raise RuntimeError("CSR storage nodes not available")
1102
+
1103
+ node_list = list(self.csr_storage.nodes.keys())
1104
+ node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
1105
+ num_nodes = len(node_list)
1106
+
1107
+ row_indices = []
1108
+ col_indices = []
1109
+ edge_labels = []
1110
+ edge_properties_list = []
1111
+
1112
+ if hasattr(self.csr_storage, 'edge_data'):
1113
+ for edge in self.csr_storage.edge_data:
1114
+ if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
1115
+ row_indices.append(node_to_idx[edge.source_id])
1116
+ col_indices.append(node_to_idx[edge.target_id])
1117
+ edge_labels.append(edge.edge_type)
1118
+ edge_id = f"{edge.source_id}_{edge.target_id}"
1119
+ edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
1120
+
1121
+ if row_indices:
1122
+ csr_matrix_data = csr_matrix(
1123
+ (np.ones(len(row_indices)), (row_indices, col_indices)),
1124
+ shape=(num_nodes, num_nodes)
1125
+ )
1126
+ else:
1127
+ csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
1128
+
1129
+ csr_metadata = {
1130
+ 'node_list': node_list,
1131
+ 'node_to_idx': node_to_idx,
1132
+ 'edge_labels': edge_labels,
1133
+ 'edge_properties': edge_properties_list,
1134
+ 'num_nodes': num_nodes,
1135
+ 'num_edges': len(row_indices)
1136
+ }
1137
+
1138
+ storage.save_csr_matrix(csr_matrix_data, csr_metadata)
1139
+ logger.info(f"CSR matrix saved to HDF5 (retry): {num_nodes} nodes, {len(row_indices)} edges")
1140
+ except Exception as csr_error:
1141
+ logger.warning(f"Failed to save CSR matrix to HDF5 (retry): {csr_error}")
1142
+ break
1143
+ except Exception as retry_error:
1144
+ if retry < max_retries - 1:
1145
+ time.sleep(0.1) # Wait 100ms before retry
1146
+ else:
1147
+ raise retry_error
1148
+
1149
+ logger.info(f"Graph '{self.name}' saved to single-file HDF5: {h5_path}")
1150
+ return True
1151
+
1152
+ except ImportError:
1153
+ logger.error("h5py not available. Install with: pip install h5py")
1154
+ return False
1155
+ except Exception as e:
1156
+ logger.error(f"Failed to save to single-file format: {e}")
1157
+ return False
1158
+
1159
+ def _save_multi_file_optimized(self, file_path: str) -> bool:
1160
+ """
1161
+ Save graph using multi-file optimized format (different formats for different data types).
1162
+
1163
+ Uses:
1164
+ - JSON for graph structure (human-readable)
1165
+ - Parquet for node/edge properties (columnar, fast queries)
1166
+ - NPZ for CSR matrices (optimized for sparse graphs)
1167
+ - NPY for vectors (fast NumPy arrays)
1168
+ - HDF5 for images (efficient binary)
1169
+ - Parquet for tables (columnar analytics)
1170
+
1171
+ Args:
1172
+ file_path: Base path for saving (will create multiple files)
1173
+
1174
+ Returns:
1175
+ True if successful
1176
+ """
1177
+ try:
1178
+ from pathlib import Path
1179
+ import numpy as np
1180
+ from scipy.sparse import save_npz
1181
+
1182
+ file_path_obj = Path(file_path)
1183
+ base_dir = file_path_obj.parent
1184
+ base_name = file_path_obj.stem
1185
+
1186
+ # Get multi-file optimized config
1187
+ graph_config = self.config.get('graph', {})
1188
+ multi_config = graph_config.get('multi_file_optimized', {})
1189
+ graph_format = multi_config.get('graph_format', 'csr') # Default: CSR for graph structure
1190
+ properties_format = multi_config.get('properties_format', 'parquet')
1191
+ csr_format = multi_config.get('csr_format', 'npz')
1192
+
1193
+ # 1. Save graph structure (CSR by default for multi-file optimized)
1194
+ # Note: Aliases are NOT saved (they are query-scoped only)
1195
+
1196
+ node_count = self.csr_storage.get_node_count() if self.csr_storage else 0
1197
+ edge_count = self.csr_storage.get_edge_count() if self.csr_storage else 0
1198
+
1199
+ graph_data = {
1200
+ 'nodes': [],
1201
+ 'edges': [],
1202
+ # 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
1203
+ 'metadata': {
1204
+ 'name': self.name,
1205
+ 'node_count': node_count,
1206
+ 'edge_count': edge_count,
1207
+ 'storage_backend': self.storage_backend,
1208
+ 'saved_at': time.time()
1209
+ }
1210
+ }
1211
+
1212
+ # Collect nodes from CSR storage
1213
+ if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
1214
+ for node_id, csr_node in self.csr_storage.nodes.items():
1215
+ node_data = {
1216
+ 'id': node_id,
1217
+ 'label': csr_node.node_type,
1218
+ 'properties': self.node_properties.get(node_id, {})
1219
+ }
1220
+ graph_data['nodes'].append(node_data)
1221
+
1222
+ # Collect edges from CSR storage
1223
+ if self.csr_storage and hasattr(self.csr_storage, 'edge_data'):
1224
+ for edge in self.csr_storage.edge_data:
1225
+ edge_id = f"{edge.source_id}_{edge.target_id}"
1226
+ edge_info = {
1227
+ 'src': edge.source_id,
1228
+ 'dst': edge.target_id,
1229
+ 'label': edge.edge_type,
1230
+ 'properties': self.edge_properties.get(edge_id, edge.properties),
1231
+ 'weight': edge.weight
1232
+ }
1233
+ graph_data['edges'].append(edge_info)
1234
+
1235
+ # Save graph structure (CSR format for graph, JSON as fallback/metadata)
1236
+ if graph_format == 'csr':
1237
+ # Graph structure is saved as CSR (see step 3 below)
1238
+ # Also save minimal JSON for metadata
1239
+ graph_file = file_path_obj.with_suffix('.json')
1240
+ minimal_graph_data = {
1241
+ # 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
1242
+ 'metadata': graph_data['metadata']
1243
+ }
1244
+ with open(graph_file, 'w') as f:
1245
+ json.dump(minimal_graph_data, f, indent=2)
1246
+ logger.debug(f"Graph metadata saved to JSON: {graph_file}")
1247
+ elif graph_format == 'json':
1248
+ # Full JSON format (fallback)
1249
+ graph_file = file_path_obj.with_suffix('.json')
1250
+ with open(graph_file, 'w') as f:
1251
+ json.dump(graph_data, f, indent=2)
1252
+ logger.debug(f"Graph structure saved to JSON: {graph_file}")
1253
+
1254
+ # 2. Save node properties to Parquet (optimized for property queries)
1255
+ if properties_format == 'parquet':
1256
+ try:
1257
+ import pandas as pd
1258
+ parquet_dir = base_dir / "node_properties"
1259
+ parquet_dir.mkdir(parents=True, exist_ok=True)
1260
+
1261
+ if graph_data['nodes']:
1262
+ nodes_df_data = []
1263
+ for node in graph_data['nodes']:
1264
+ row = {'node_id': node['id'], 'label': node['label']}
1265
+ props = node.get('properties', {})
1266
+ for key, value in props.items():
1267
+ if isinstance(value, (dict, list)):
1268
+ row[f'prop_{key}'] = json.dumps(value)
1269
+ else:
1270
+ row[f'prop_{key}'] = value
1271
+ nodes_df_data.append(row)
1272
+
1273
+ if nodes_df_data:
1274
+ nodes_df = pd.DataFrame(nodes_df_data)
1275
+ parquet_file = parquet_dir / f"{base_name}_nodes.parquet"
1276
+ compression = multi_config.get('properties_compression', 'snappy')
1277
+ nodes_df.to_parquet(parquet_file, index=False, engine='pyarrow', compression=compression)
1278
+ logger.debug(f"Node properties saved to Parquet: {parquet_file}")
1279
+ except ImportError:
1280
+ logger.warning("PyArrow/pandas not available, skipping Parquet storage")
1281
+ except Exception as e:
1282
+ logger.warning(f"Failed to save node properties to Parquet: {e}")
1283
+
1284
+ # 3. Save CSR matrix (NPZ format - optimized for sparse graphs)
1285
+ # This is the PRIMARY graph structure storage for multi-file optimized strategy
1286
+ if (graph_format == 'csr' or csr_format == 'npz') and self.csr_storage and self.use_csr:
1287
+ try:
1288
+ csr_dir = base_dir / "csr"
1289
+ csr_dir.mkdir(parents=True, exist_ok=True)
1290
+
1291
+ node_list = list(self.csr_storage.nodes.keys())
1292
+ node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
1293
+ num_nodes = len(node_list)
1294
+
1295
+ row_indices = []
1296
+ col_indices = []
1297
+ edge_labels = []
1298
+ edge_properties_list = []
1299
+
1300
+ if hasattr(self.csr_storage, 'edge_data'):
1301
+ for edge in self.csr_storage.edge_data:
1302
+ if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
1303
+ row_indices.append(node_to_idx[edge.source_id])
1304
+ col_indices.append(node_to_idx[edge.target_id])
1305
+ edge_labels.append(edge.edge_type)
1306
+ edge_id = f"{edge.source_id}_{edge.target_id}"
1307
+ edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
1308
+
1309
+ from scipy.sparse import csr_matrix
1310
+ if row_indices:
1311
+ csr_matrix_data = csr_matrix(
1312
+ (np.ones(len(row_indices)), (row_indices, col_indices)),
1313
+ shape=(num_nodes, num_nodes)
1314
+ )
1315
+ else:
1316
+ csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
1317
+
1318
+ csr_file = csr_dir / f"{base_name}_csr.npz"
1319
+ save_npz(str(csr_file), csr_matrix_data)
1320
+
1321
+ csr_metadata = {
1322
+ 'node_list': node_list,
1323
+ 'node_to_idx': node_to_idx,
1324
+ 'edge_labels': edge_labels,
1325
+ 'edge_properties': edge_properties_list,
1326
+ 'num_nodes': num_nodes,
1327
+ 'num_edges': len(row_indices)
1328
+ # 'alias_to_node': {} # REMOVED: Aliases are query-scoped, not persisted
1329
+ }
1330
+
1331
+ csr_metadata_file = csr_dir / f"{base_name}_csr_metadata.json"
1332
+ with open(csr_metadata_file, 'w') as f:
1333
+ json.dump(csr_metadata, f, indent=2, default=str)
1334
+
1335
+ logger.debug(f"CSR matrix saved to NPZ: {csr_file}")
1336
+ except Exception as e:
1337
+ logger.warning(f"Failed to save CSR format: {e}")
1338
+
1339
+ logger.info(f"Graph '{self.name}' saved using multi-file optimized format")
1340
+ return True
1341
+
1342
+ except Exception as e:
1343
+ logger.error(f"Failed to save multi-file optimized format: {e}")
1344
+ return False
1345
+
1346
+ def _load_multi_file_optimized(self, file_path: str) -> bool:
1347
+ """
1348
+ Load graph from multi-file optimized format.
1349
+
1350
+ Args:
1351
+ file_path: Base path for loading
1352
+
1353
+ Returns:
1354
+ True if successful
1355
+ """
1356
+ try:
1357
+ from pathlib import Path
1358
+ from scipy.sparse import load_npz
1359
+
1360
+ file_path_obj = Path(file_path)
1361
+ base_dir = file_path_obj.parent
1362
+ base_name = file_path_obj.stem
1363
+
1364
+ # Get config
1365
+ graph_config = self.config.get('graph', {})
1366
+ multi_config = graph_config.get('multi_file_optimized', {})
1367
+ graph_format = multi_config.get('graph_format', 'csr') # Default: CSR for graph structure
1368
+ csr_format = multi_config.get('csr_format', 'npz')
1369
+
1370
+ # 1. Try loading CSR first (primary graph structure format for multi-file optimized)
1371
+ csr_file = base_dir / "csr" / f"{base_name}_csr.npz"
1372
+ csr_metadata_file = base_dir / "csr" / f"{base_name}_csr_metadata.json"
1373
+
1374
+ if csr_file.exists() and csr_metadata_file.exists() and (graph_format == 'csr' or csr_format == 'npz'):
1375
+ try:
1376
+ csr_matrix_data = load_npz(str(csr_file))
1377
+ with open(csr_metadata_file, 'r') as f:
1378
+ csr_metadata = json.load(f)
1379
+
1380
+ node_list = csr_metadata.get('node_list', [])
1381
+ edge_labels = csr_metadata.get('edge_labels', [])
1382
+ edge_properties_list = csr_metadata.get('edge_properties', [])
1383
+ # Note: aliases are NOT loaded (they are query-scoped only)
1384
+
1385
+ # Restore nodes to CSR storage
1386
+ for node_id in node_list:
1387
+ self.csr_storage.add_node(node_id, 'Node', {})
1388
+
1389
+ # Restore edges
1390
+ row_indices, col_indices = csr_matrix_data.nonzero()
1391
+ for i, (row_idx, col_idx) in enumerate(zip(row_indices, col_indices)):
1392
+ source_id = node_list[row_idx]
1393
+ target_id = node_list[col_idx]
1394
+ edge_label = edge_labels[i] if i < len(edge_labels) else 'RELATED_TO'
1395
+ edge_props = edge_properties_list[i] if i < len(edge_properties_list) else {}
1396
+ self.csr_storage.add_edge(source_id, target_id, edge_label, edge_props)
1397
+
1398
+ # Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
1399
+
1400
+ logger.debug(f"Loaded graph from multi-file optimized format (CSR)")
1401
+ return True
1402
+ except Exception as e:
1403
+ logger.warning(f"Failed to load CSR format: {e}")
1404
+
1405
+ # 2. Fallback to JSON graph structure
1406
+ graph_file = file_path_obj.with_suffix('.json')
1407
+ if graph_file.exists() and graph_format == 'json':
1408
+ with open(graph_file, 'r') as f:
1409
+ graph_data = json.load(f)
1410
+
1411
+ # Restore nodes
1412
+ for node in graph_data.get('nodes', []):
1413
+ node_id = node['id']
1414
+ node_type = node.get('label', 'Node')
1415
+ properties = node.get('properties', {})
1416
+ self.csr_storage.add_node(node_id, node_type, properties)
1417
+ self.node_properties[node_id] = properties
1418
+
1419
+ # Restore edges
1420
+ for edge in graph_data.get('edges', []):
1421
+ source_id = edge['src']
1422
+ target_id = edge['dst']
1423
+ edge_type = edge.get('label', 'RELATED_TO')
1424
+ edge_props = edge.get('properties', {})
1425
+ self.csr_storage.add_edge(source_id, target_id, edge_type, edge_props)
1426
+ edge_id = f"{source_id}_{target_id}"
1427
+ self.edge_properties[edge_id] = edge_props
1428
+
1429
+ # Note: aliases are NOT loaded (they are query-scoped only)
1430
+ # No need to initialize alias_to_node - aliases are query-scoped only
1431
+
1432
+ logger.debug(f"Loaded graph from multi-file optimized format (JSON)")
1433
+ return True
1434
+
1435
+ return False
1436
+
1437
+ except Exception as e:
1438
+ logger.error(f"Failed to load multi-file optimized format: {e}")
1439
+ return False
1440
+
1441
+ def _should_use_csr(self) -> bool:
1442
+ """Determine if CSR storage should be used. Always True - NetworkX disabled."""
1443
+ return True # CSR-only mode - always use CSR
1444
+
1445
+ def add_node(self, node: GraphNode, alias: Optional[str] = None, user_id: Optional[str] = None, write_through: Optional[bool] = None, _skip_temporal: bool = False):
1446
+ """
1447
+ Add a node to the graph with smart routing based on node type.
1448
+
1449
+ RAG-critical nodes (Document, Chunk, Entity) are written immediately (write-through)
1450
+ to ensure they're visible for RAG queries without waiting for buffer flush.
1451
+
1452
+ This method is used by BOTH direct graph operations and API operations.
1453
+ It ensures the same code path is used regardless of how the node was created.
1454
+
1455
+ Args:
1456
+ node: GraphNode to add
1457
+ alias: Optional alias for the node
1458
+ user_id: Optional user ID for traceability
1459
+ write_through: Force write-through (bypass buffer). If None, auto-detect based on node type.
1460
+ """
1461
+
1462
+ # STRATEGY: Default to write-through for immediate visibility.
1463
+ # Buffer is an optimization for bulk ingestion pipelines that explicitly
1464
+ # set write_through=False. Interactive AIQL queries and API operations
1465
+ # need nodes visible immediately for queries, saves, and multi-worker sync.
1466
+ if write_through is None:
1467
+ write_through = True
1468
+
1469
+
1470
+ # If write-through is enabled, write directly (bypass buffer)
1471
+ if write_through:
1472
+ logger.debug(f"Write-through enabled for node type '{node.label}' (RAG-critical)")
1473
+ self._add_node_direct(node, alias, user_id, _skip_temporal=_skip_temporal)
1474
+ return True
1475
+
1476
+ # If buffer is enabled, add to buffer instead of directly to graph
1477
+ if self.buffer_enabled and self.buffer_manager:
1478
+ # OPTIMIZATION: Check if buffer was pre-initialized during __init__
1479
+ # If not, initialize lazily (should be rare now)
1480
+ if not getattr(self, '_buffer_initialized', False) and not self.buffer_manager._initialized:
1481
+ import asyncio
1482
+ try:
1483
+ loop = asyncio.get_event_loop()
1484
+ if loop.is_running():
1485
+ # If loop is running, schedule initialization
1486
+ asyncio.create_task(self._init_buffer_async())
1487
+ else:
1488
+ loop.run_until_complete(self.buffer_manager.initialize())
1489
+ if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
1490
+ loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
1491
+ self._buffer_initialized = True
1492
+ except RuntimeError:
1493
+ loop = asyncio.new_event_loop()
1494
+ asyncio.set_event_loop(loop)
1495
+ loop.run_until_complete(self.buffer_manager.initialize())
1496
+ if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
1497
+ loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
1498
+ self._buffer_initialized = True
1499
+
1500
+ # Add to buffer (will be flushed later)
1501
+ record = {
1502
+ 'operation': 'CREATE_NODE',
1503
+ 'node_id': node.id,
1504
+ 'label': node.label,
1505
+ 'properties': node.properties,
1506
+ 'alias': alias,
1507
+ 'user_id': user_id
1508
+ }
1509
+ import asyncio
1510
+ try:
1511
+ loop = asyncio.get_event_loop()
1512
+ if loop.is_running():
1513
+ # If loop is running, schedule add_record
1514
+ asyncio.create_task(self.buffer_manager.add_record(record))
1515
+ else:
1516
+ loop.run_until_complete(self.buffer_manager.add_record(record))
1517
+ except RuntimeError:
1518
+ loop = asyncio.new_event_loop()
1519
+ asyncio.set_event_loop(loop)
1520
+ loop.run_until_complete(self.buffer_manager.add_record(record))
1521
+
1522
+ # STRATEGY: Check if buffer reached threshold and trigger immediate flush
1523
+ if self.config.get('buffer', {}).get('config', {}).get('threshold_flush_enabled', True):
1524
+ try:
1525
+ pending_count = loop.run_until_complete(self.buffer_manager.get_pending_count()) if not loop.is_running() else 0
1526
+ max_size = self.config.get('buffer', {}).get('config', {}).get('max_size', 1000)
1527
+ flush_threshold = self.config.get('buffer', {}).get('config', {}).get('flush_threshold', 0.7)
1528
+ threshold = int(max_size * flush_threshold)
1529
+
1530
+ if pending_count >= threshold:
1531
+ logger.debug(f"Buffer threshold reached ({pending_count}/{max_size} = {pending_count/max_size*100:.1f}%) - triggering flush")
1532
+ # Trigger immediate flush (non-blocking)
1533
+ if loop.is_running():
1534
+ asyncio.create_task(self.buffer_manager.flush_to_db(writer_fn=self._buffer_writer_fn_ref))
1535
+ else:
1536
+ loop.run_until_complete(self.buffer_manager.flush_to_db(writer_fn=self._buffer_writer_fn_ref))
1537
+ except Exception as e:
1538
+ logger.debug(f"Error checking buffer threshold: {e}")
1539
+
1540
+ logger.debug(f"Added node {node.id} to buffer")
1541
+ return True # Return True to indicate success (node is buffered)
1542
+
1543
+ # Direct addition (buffer disabled or called from buffer writer)
1544
+ self._add_node_direct(node, alias, user_id, _skip_temporal=_skip_temporal)
1545
+
1546
+ async def _init_buffer_async(self):
1547
+ """Initialize buffer asynchronously."""
1548
+ await self.buffer_manager.initialize()
1549
+ if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
1550
+ await self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref)
1551
+
1552
+ async def _buffer_writer_fn(self, batch: List[Dict[str, Any]]) -> None:
1553
+ """
1554
+ Writer function for buffer manager.
1555
+ Writes buffered records to the graph (CSR storage in memory).
1556
+
1557
+ NOTE: This writes to CSR storage in memory. To persist to disk, call save() separately.
1558
+
1559
+ Args:
1560
+ batch: List of buffered records
1561
+ """
1562
+ if not batch:
1563
+ logger.debug("Buffer writer called with empty batch")
1564
+ return
1565
+
1566
+ logger.info(f"Buffer writer: Processing {len(batch)} records for graph '{self.name}'")
1567
+
1568
+ nodes_written = 0
1569
+ edges_written = 0
1570
+ errors = []
1571
+
1572
+ for record in batch:
1573
+ try:
1574
+ operation = record.get('operation')
1575
+ if operation == 'CREATE_NODE':
1576
+ # Create node from buffer record
1577
+ node = GraphNode(
1578
+ id=record['node_id'],
1579
+ label=record['label'],
1580
+ properties=record.get('properties', {})
1581
+ )
1582
+ alias = record.get('alias')
1583
+ user_id = record.get('user_id')
1584
+ # Call direct addition (bypassing buffer to avoid recursion)
1585
+ self._add_node_direct(node, alias, user_id)
1586
+ nodes_written += 1
1587
+ logger.debug(f"Buffer writer: Wrote node {node.id[:8]}... (label: {node.label})")
1588
+ elif operation == 'CREATE_EDGE':
1589
+ # Create edge from buffer record
1590
+ edge = GraphEdge(
1591
+ id=record.get('edge_id', f"{record['source']}_{record['target']}"),
1592
+ source=record['source'],
1593
+ target=record['target'],
1594
+ label=record['label'],
1595
+ properties=record.get('properties', {})
1596
+ )
1597
+ user_id = record.get('user_id')
1598
+ # Call direct addition (bypassing buffer to avoid recursion)
1599
+ self._add_edge_direct(edge, user_id)
1600
+ edges_written += 1
1601
+ logger.debug(f"Buffer writer: Wrote edge {edge.id[:8]}... ({edge.source} -> {edge.target})")
1602
+ else:
1603
+ logger.warning(f"Buffer writer: Unknown operation '{operation}' in record")
1604
+ errors.append(f"Unknown operation: {operation}")
1605
+ except Exception as e:
1606
+ error_msg = f"Error writing record {record.get('node_id') or record.get('edge_id', 'unknown')}: {e}"
1607
+ logger.error(error_msg)
1608
+ errors.append(error_msg)
1609
+ # Continue processing other records
1610
+
1611
+ logger.info(f"Buffer writer: Completed - {nodes_written} nodes, {edges_written} edges written to CSR storage (graph: '{self.name}')")
1612
+ if errors:
1613
+ logger.warning(f"Buffer writer: {len(errors)} errors occurred during flush")
1614
+
1615
+ # CRITICAL: Nodes are now in CSR storage (memory), but NOT persisted to disk yet
1616
+ # Trigger background save to persist flushed records to disk (non-blocking)
1617
+ # Only save if we wrote something and enough time has passed since last save
1618
+ if (nodes_written > 0 or edges_written > 0):
1619
+ try:
1620
+ # Import registry to trigger save
1621
+ from ..core.registry import graph_registry
1622
+
1623
+ # Trigger background save (non-blocking)
1624
+ import asyncio
1625
+ try:
1626
+ loop = asyncio.get_event_loop()
1627
+ if loop.is_running():
1628
+ # Schedule background save
1629
+ asyncio.create_task(self._background_save_async())
1630
+ else:
1631
+ # Run in executor if no event loop
1632
+ loop.run_in_executor(None, self._save_to_disk)
1633
+ except RuntimeError:
1634
+ # No event loop, run synchronously in thread
1635
+ import threading
1636
+ threading.Thread(target=self._save_to_disk, daemon=True).start()
1637
+
1638
+ logger.debug(f"Buffer writer: Triggered background save for graph '{self.name}' ({nodes_written} nodes, {edges_written} edges)")
1639
+ except Exception as e:
1640
+ logger.warning(f"Buffer writer: Failed to trigger background save: {e}")
1641
+ # Don't fail the flush if save fails - nodes are still in memory
1642
+
1643
+ def _save_to_disk(self):
1644
+ """Synchronous save to disk (runs in background thread)."""
1645
+ try:
1646
+ from ..core.registry import graph_registry
1647
+ # Re-register in registry if evicted (LRU cleanup can remove graphs
1648
+ # while their pipeline is still running in a background thread)
1649
+ if self.name not in graph_registry.graphs:
1650
+ graph_registry.graphs[self.name] = self
1651
+ success = graph_registry.save_graph(self.name)
1652
+ if success:
1653
+ logger.debug(f"Buffer writer: Successfully saved graph '{self.name}' to disk")
1654
+ else:
1655
+ logger.warning(f"Buffer writer: Failed to save graph '{self.name}' to disk")
1656
+ except Exception as e:
1657
+ logger.error(f"Buffer writer: Error saving graph '{self.name}' to disk: {e}")
1658
+
1659
+ async def _background_save_async(self):
1660
+ """Async background save (runs in event loop)."""
1661
+ try:
1662
+ from ..core.registry import graph_registry
1663
+ import asyncio
1664
+ # Re-register in registry if evicted by LRU cleanup
1665
+ if self.name not in graph_registry.graphs:
1666
+ graph_registry.graphs[self.name] = self
1667
+ loop = asyncio.get_event_loop()
1668
+ # Run save in executor to avoid blocking
1669
+ success = await loop.run_in_executor(None, graph_registry.save_graph, self.name)
1670
+ if success:
1671
+ logger.debug(f"Buffer writer: Successfully saved graph '{self.name}' to disk (async)")
1672
+ else:
1673
+ logger.warning(f"Buffer writer: Failed to save graph '{self.name}' to disk (async)")
1674
+ except Exception as e:
1675
+ logger.error(f"Buffer writer: Error saving graph '{self.name}' to disk (async): {e}")
1676
+
1677
+ def _ensure_wal(self):
1678
+ """Lazy-init WAL on first write."""
1679
+ if self.wal is None and self._wal_enabled:
1680
+ try:
1681
+ from ..storage.wal import AIContextDBWALSystem, WALConfig
1682
+ import re as _re
1683
+ safe = _re.sub(r'[<>:"/\\|?*]', '_', self._wal_name)
1684
+ self.wal = AIContextDBWALSystem(WALConfig(wal_dir=f"contextcore_data/wal/{safe}"))
1685
+ except Exception:
1686
+ self._wal_enabled = False
1687
+
1688
+ def _add_node_direct(self, node: GraphNode, alias: Optional[str] = None, user_id: Optional[str] = None, _skip_temporal: bool = False):
1689
+ """Direct node addition (bypasses buffer). WAL-logged for durability. Thread-safe."""
1690
+ with self._write_lock:
1691
+ return self._add_node_direct_inner(node, alias, user_id, _skip_temporal=_skip_temporal)
1692
+
1693
+ def _add_node_direct_inner(self, node: GraphNode, alias: Optional[str] = None, user_id: Optional[str] = None, _skip_temporal: bool = False):
1694
+ # WAL: log before writing to memory (write-ahead)
1695
+ self._ensure_wal()
1696
+ if self.wal:
1697
+ try:
1698
+ from ..storage.wal import OperationType
1699
+ self.wal.log_operation(OperationType.CREATE_NODE, {
1700
+ 'node_id': node.id, 'label': node.label,
1701
+ 'properties': node.properties,
1702
+ })
1703
+ except Exception:
1704
+ pass # WAL failure should not block writes
1705
+
1706
+ # Record operation in traceability layer
1707
+ if self.traceability_layer:
1708
+ self.traceability_layer.record_operation(
1709
+ operation_type='CREATE',
1710
+ entity_type='node',
1711
+ entity_id=node.id,
1712
+ data={'label': node.label, 'properties': node.properties, 'alias': alias},
1713
+ user_id=user_id,
1714
+ namespace=self.name
1715
+ )
1716
+
1717
+ # Add domain metadata if not present (default to namespace as domain)
1718
+ if 'domain' not in node.properties:
1719
+ node.properties['domain'] = self.name
1720
+
1721
+ # Provenance: inject write metadata
1722
+ from .write_context import get_write_context
1723
+ from datetime import datetime as _dt, timezone as _tz
1724
+ _wctx = get_write_context()
1725
+ if '_created_at' not in node.properties:
1726
+ node.properties['_created_at'] = _dt.now(_tz.utc).isoformat()
1727
+ if '_agent_id' not in node.properties:
1728
+ node.properties['_agent_id'] = user_id or (_wctx.agent_id if _wctx else "") or ""
1729
+ if '_origin' not in node.properties:
1730
+ node.properties['_origin'] = (_wctx.origin if _wctx else "unknown")
1731
+ if '_verified' not in node.properties:
1732
+ node.properties['_verified'] = (_wctx.verified if _wctx else False)
1733
+ if '_region' not in node.properties:
1734
+ node.properties['_region'] = (_wctx.region if _wctx else "local")
1735
+
1736
+ # Provenance defaults (A4)
1737
+ if 'source_type' not in node.properties:
1738
+ node.properties['source_type'] = 'unknown'
1739
+ if 'confidence' not in node.properties:
1740
+ node.properties['confidence'] = 1.0
1741
+ if 'created_by' not in node.properties:
1742
+ node.properties['created_by'] = node.properties.get('_agent_id', '')
1743
+
1744
+ # Temporal defaults (A2)
1745
+ if 'valid_from' not in node.properties:
1746
+ node.properties['valid_from'] = node.properties.get('_created_at', _dt.now(_tz.utc).isoformat())
1747
+ if 'valid_to' not in node.properties:
1748
+ node.properties['valid_to'] = None
1749
+ if 'version' not in node.properties:
1750
+ node.properties['version'] = 1
1751
+ if 'status' not in node.properties:
1752
+ node.properties['status'] = 'active'
1753
+
1754
+ # Semantic hashing: Calculate content hash for deduplication
1755
+ if self.hashing_enabled and self.hasher:
1756
+ try:
1757
+ hash_result = self.hasher.hash_node(node.id, node.label, node.properties)
1758
+ # Store hash in properties for later deduplication
1759
+ if 'content_hash' not in node.properties:
1760
+ node.properties['content_hash'] = hash_result.hash_value
1761
+ if 'hash_algorithm' not in node.properties:
1762
+ node.properties['hash_algorithm'] = hash_result.algorithm
1763
+ except Exception as e:
1764
+ logger.warning(f"[SEMANTIC_HASH] Failed to hash node {node.id}: {e}")
1765
+
1766
+ # Time travel: Create version for new node (skip when called internally)
1767
+ if self.temporal_enabled and self.temporal_storage and not _skip_temporal:
1768
+ try:
1769
+ from datetime import datetime
1770
+ self.temporal_storage.create_version(
1771
+ entity_id=node.id,
1772
+ entity_type='node',
1773
+ properties=node.properties.copy(),
1774
+ operation='CREATE',
1775
+ timestamp=datetime.now()
1776
+ )
1777
+ except Exception as e:
1778
+ logger.warning(f"[TIME_TRAVEL] Failed to create version for node {node.id}: {e}")
1779
+
1780
+ # Debug - log what we're adding
1781
+ logger.debug(f"add_node called: node_id={node.id[:8]}..., alias={alias} (query-scoped, not persisted), label={node.label}, graph='{self.name}'")
1782
+
1783
+ # CSR-only: Store properties separately (no NetworkX)
1784
+ self.node_properties[node.id] = node.properties.copy()
1785
+
1786
+ # Store in node_index for compatibility
1787
+ self.node_index[node.id] = node
1788
+
1789
+ # NOTE: Aliases are query-scoped only (stored in executor.node_aliases, not here)
1790
+ # We accept the alias parameter for compatibility but don't store it persistently
1791
+
1792
+ # NEW: Update property indexes (use cleaned properties to avoid unhashable errors)
1793
+ # Clean properties before indexing to handle nested dicts/lists
1794
+ properties_cleaned = self._clean_properties_for_indexing(node.properties)
1795
+ self._update_property_indexes(node.id, properties_cleaned)
1796
+ # Store original properties (with nested structures) separately
1797
+ self.node_properties[node.id] = node.properties
1798
+
1799
+ # CRITICAL: Always add to CSR (mandatory - NetworkX disabled)
1800
+ if not self.csr_adapter:
1801
+ raise RuntimeError("CSR adapter not initialized. NetworkX fallback disabled.")
1802
+ # Add to CSR (pass None for alias since we don't persist aliases)
1803
+ success = self.csr_adapter.add_node(
1804
+ node.id,
1805
+ node.label,
1806
+ node.properties,
1807
+ None # Don't persist alias - query-scoped only
1808
+ )
1809
+ if not success:
1810
+ logger.warning(f"Failed to add node {node.id} to CSR storage")
1811
+
1812
+ # Update ContextMeta counters (A1)
1813
+ if node.label != "ContextMeta" and success:
1814
+ try:
1815
+ meta_node = self.csr_adapter.get_node("_context_meta")
1816
+ if meta_node:
1817
+ meta_node.properties["node_count"] = meta_node.properties.get("node_count", 0) + 1
1818
+ meta_node.properties["updated_at"] = _dt.now(_tz.utc).isoformat()
1819
+ existing = set(meta_node.properties.get("schema_summary", "").split(", ")) - {""}
1820
+ existing.add(node.label)
1821
+ meta_node.properties["schema_summary"] = ", ".join(sorted(existing))
1822
+ self.csr_adapter.update_node_properties("_context_meta", meta_node.properties)
1823
+ except Exception:
1824
+ pass # ContextMeta update failure should never block writes
1825
+ else:
1826
+ logger.debug(f"Successfully added node {node.id} to CSR storage")
1827
+
1828
+ # Graph intelligence: auto-link, quality score, topic tag
1829
+ try:
1830
+ from .graph_intelligence import get_graph_intelligence
1831
+ gi = get_graph_intelligence(self)
1832
+ gi.on_node_added(node, namespace=self.name)
1833
+ except Exception as e:
1834
+ logger.debug(f"Graph intelligence hook failed: {e}")
1835
+
1836
+ self.operation_counts['add_node'] += 1
1837
+ return True # Return True to indicate success
1838
+
1839
+ def remove_node(self, node_id: str, user_id: Optional[str] = None) -> bool:
1840
+ """Remove a node from the graph. Thread-safe."""
1841
+ with self._write_lock:
1842
+ return self._remove_node_inner(node_id, user_id)
1843
+
1844
+ def _remove_node_inner(self, node_id: str, user_id: Optional[str] = None) -> bool:
1845
+ try:
1846
+ # Record operation in traceability layer
1847
+ if self.traceability_layer:
1848
+ node_data = {}
1849
+ # Get node from CSR storage
1850
+ csr_node = self.csr_adapter.get_node(node_id) if self.csr_adapter else None
1851
+ if csr_node:
1852
+ node_data = {
1853
+ 'label': csr_node.node_type,
1854
+ 'properties': self.node_properties.get(node_id, {})
1855
+ }
1856
+ self.traceability_layer.record_operation(
1857
+ operation_type='DELETE',
1858
+ entity_type='node',
1859
+ entity_id=node_id,
1860
+ data=node_data,
1861
+ user_id=user_id,
1862
+ namespace=self.name
1863
+ )
1864
+ # Check if node exists in CSR storage
1865
+ if not self.csr_adapter or not self.csr_adapter.get_node(node_id):
1866
+ return False
1867
+
1868
+ # Remove from CSR storage (manual removal since CSR doesn't have remove_node method)
1869
+ if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
1870
+ if node_id in self.csr_storage.nodes:
1871
+ # Remove node from CSR storage
1872
+ csr_node = self.csr_storage.nodes[node_id]
1873
+ node_index = csr_node.index
1874
+ del self.csr_storage.nodes[node_id]
1875
+ del self.csr_storage.node_id_to_index[node_id]
1876
+ if node_index in self.csr_storage.index_to_node_id:
1877
+ del self.csr_storage.index_to_node_id[node_index]
1878
+
1879
+ # Remove from node types
1880
+ node_type = csr_node.node_type
1881
+ if node_type in self.csr_storage.node_types:
1882
+ self.csr_storage.node_types[node_type].discard(node_id)
1883
+ if not self.csr_storage.node_types[node_type]:
1884
+ del self.csr_storage.node_types[node_type]
1885
+
1886
+ # Remove from property index
1887
+ for prop_name, prop_index in self.csr_storage.property_index.items():
1888
+ for prop_value, node_set in list(prop_index.items()):
1889
+ node_set.discard(node_id)
1890
+ if not node_set:
1891
+ del prop_index[prop_value]
1892
+
1893
+ # Note: Edges pointing to/from this node should also be removed
1894
+ # This is a simplified removal - full implementation would require edge cleanup
1895
+ if hasattr(self.csr_storage, 'edge_data'):
1896
+ # Remove edges where this node is source or target
1897
+ edges_to_remove = []
1898
+ for i, edge in enumerate(self.csr_storage.edge_data):
1899
+ if edge.source_id == node_id or edge.target_id == node_id:
1900
+ edges_to_remove.append(i)
1901
+ # Remove in reverse order to maintain indices
1902
+ for i in reversed(edges_to_remove):
1903
+ edge = self.csr_storage.edge_data.pop(i)
1904
+ # Update edge types
1905
+ if edge.edge_type in self.csr_storage.edge_types:
1906
+ self.csr_storage.edge_types[edge.edge_type].discard(i)
1907
+ if not self.csr_storage.edge_types[edge.edge_type]:
1908
+ del self.csr_storage.edge_types[edge.edge_type]
1909
+ # Note: CSR row_ptr and col_indices would need to be rebuilt after edge removal
1910
+ # This is a simplified implementation
1911
+
1912
+ # Remove from properties
1913
+ if node_id in self.node_properties:
1914
+ del self.node_properties[node_id]
1915
+
1916
+ # Remove from alias mapping
1917
+ # NOTE: Aliases are query-scoped only, so no need to remove from persistent storage
1918
+ # (alias_to_node is kept for backward compatibility but is not used)
1919
+
1920
+ # Remove from CSR if enabled
1921
+ if self.csr_storage and hasattr(self.csr_storage, 'remove_node'):
1922
+ self.csr_storage.remove_node(node_id)
1923
+
1924
+ logger.info(f"Node '{node_id}' removed successfully")
1925
+ return True
1926
+
1927
+ except Exception as e:
1928
+ logger.error(f"Failed to remove node '{node_id}': {e}")
1929
+ return False
1930
+
1931
+ def get_node_by_alias(self, alias: str) -> Optional[str]:
1932
+ """
1933
+ Get node ID by alias.
1934
+
1935
+ NOTE: Aliases are now query-scoped only (not persisted).
1936
+ This method returns None for query-scoped aliases.
1937
+ Use executor.node_aliases during query execution instead.
1938
+ """
1939
+ # Aliases are query-scoped only - not persisted in graph
1940
+ # This method is kept for backward compatibility but will return None
1941
+ return None
1942
+
1943
+ def add_edge(self, edge: GraphEdge, user_id: Optional[str] = None):
1944
+ """Add an edge to the graph."""
1945
+ from datetime import datetime as _dt, timezone as _tz
1946
+ # Add domain metadata if not present (default to namespace as domain)
1947
+ if 'domain' not in edge.properties:
1948
+ edge.properties['domain'] = self.name
1949
+ # Edge metadata defaults (A5)
1950
+ if 'weight' not in edge.properties:
1951
+ edge.properties['weight'] = 1.0
1952
+ if 'confidence' not in edge.properties:
1953
+ edge.properties['confidence'] = 1.0
1954
+ if 'created_at' not in edge.properties:
1955
+ edge.properties['created_at'] = _dt.now(_tz.utc).isoformat()
1956
+
1957
+ # If buffer is enabled, add to buffer instead of directly to graph
1958
+ if self.buffer_enabled and self.buffer_manager:
1959
+ # Initialize buffer if not already initialized
1960
+ if not self.buffer_manager._initialized:
1961
+ import asyncio
1962
+ try:
1963
+ loop = asyncio.get_event_loop()
1964
+ if loop.is_running():
1965
+ asyncio.create_task(self._init_buffer_async())
1966
+ else:
1967
+ loop.run_until_complete(self.buffer_manager.initialize())
1968
+ if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
1969
+ loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
1970
+ except RuntimeError:
1971
+ loop = asyncio.new_event_loop()
1972
+ asyncio.set_event_loop(loop)
1973
+ loop.run_until_complete(self.buffer_manager.initialize())
1974
+ if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
1975
+ loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
1976
+
1977
+ # Add to buffer (will be flushed later)
1978
+ record = {
1979
+ 'operation': 'CREATE_EDGE',
1980
+ 'edge_id': edge.id,
1981
+ 'source': edge.source,
1982
+ 'target': edge.target,
1983
+ 'label': edge.label,
1984
+ 'properties': edge.properties,
1985
+ 'user_id': user_id
1986
+ }
1987
+ import asyncio
1988
+ try:
1989
+ loop = asyncio.get_event_loop()
1990
+ if loop.is_running():
1991
+ asyncio.create_task(self.buffer_manager.add_record(record))
1992
+ else:
1993
+ loop.run_until_complete(self.buffer_manager.add_record(record))
1994
+ except RuntimeError:
1995
+ loop = asyncio.new_event_loop()
1996
+ asyncio.set_event_loop(loop)
1997
+ loop.run_until_complete(self.buffer_manager.add_record(record))
1998
+ logger.debug(f"Added edge {edge.id} to buffer")
1999
+ # WRITE-THROUGH: Also add directly to CSR so edges are immediately queryable
2000
+ # (buffer reads fail in async context since loop.is_running() returns True)
2001
+ self._add_edge_direct(edge, user_id)
2002
+ return True # Return True to indicate success
2003
+
2004
+ # Direct addition (buffer disabled or called from buffer writer)
2005
+ return self._add_edge_direct(edge, user_id)
2006
+
2007
+ def _add_edge_direct(self, edge: GraphEdge, user_id: Optional[str] = None):
2008
+ """Direct edge addition (bypasses buffer). WAL-logged for durability. Thread-safe."""
2009
+ with self._write_lock:
2010
+ return self._add_edge_direct_inner(edge, user_id)
2011
+
2012
+ def _add_edge_direct_inner(self, edge: GraphEdge, user_id: Optional[str] = None):
2013
+ try:
2014
+ # WAL: log before writing to memory (write-ahead)
2015
+ self._ensure_wal()
2016
+ if self.wal:
2017
+ try:
2018
+ from ..storage.wal import OperationType
2019
+ self.wal.log_operation(OperationType.CREATE_EDGE, {
2020
+ 'edge_id': edge.id, 'source': edge.source,
2021
+ 'target': edge.target, 'label': edge.label,
2022
+ 'properties': edge.properties,
2023
+ })
2024
+ except Exception:
2025
+ pass # WAL failure should not block writes
2026
+
2027
+ # Add domain metadata if not present (default to namespace as domain)
2028
+ if 'domain' not in edge.properties:
2029
+ edge.properties['domain'] = self.name
2030
+
2031
+ # Provenance: inject write metadata
2032
+ from .write_context import get_write_context
2033
+ from datetime import datetime as _dt, timezone as _tz
2034
+ _wctx = get_write_context()
2035
+ if '_created_at' not in edge.properties:
2036
+ edge.properties['_created_at'] = _dt.now(_tz.utc).isoformat()
2037
+ if '_agent_id' not in edge.properties:
2038
+ edge.properties['_agent_id'] = user_id or (_wctx.agent_id if _wctx else "") or ""
2039
+ if '_origin' not in edge.properties:
2040
+ edge.properties['_origin'] = (_wctx.origin if _wctx else "unknown")
2041
+ if '_verified' not in edge.properties:
2042
+ edge.properties['_verified'] = (_wctx.verified if _wctx else False)
2043
+ if '_region' not in edge.properties:
2044
+ edge.properties['_region'] = (_wctx.region if _wctx else "local")
2045
+
2046
+ # Time travel: Create version for new edge
2047
+ if self.temporal_enabled and self.temporal_storage:
2048
+ try:
2049
+ from datetime import datetime
2050
+ self.temporal_storage.create_version(
2051
+ entity_id=edge.id,
2052
+ entity_type='edge',
2053
+ properties=edge.properties.copy(),
2054
+ operation='CREATE',
2055
+ timestamp=datetime.now()
2056
+ )
2057
+ except Exception as e:
2058
+ logger.warning(f"[TIME_TRAVEL] Failed to create version for edge {edge.id}: {e}")
2059
+
2060
+ # Semantic hashing: Calculate content hash for deduplication
2061
+ if self.hashing_enabled and self.hasher:
2062
+ try:
2063
+ hash_result = self.hasher.hash_edge(edge.source, edge.target, edge.label, edge.properties)
2064
+ if 'content_hash' not in edge.properties:
2065
+ edge.properties['content_hash'] = hash_result.hash_value
2066
+ if 'hash_algorithm' not in edge.properties:
2067
+ edge.properties['hash_algorithm'] = hash_result.algorithm
2068
+ except Exception as e:
2069
+ logger.warning(f"[SEMANTIC_HASH] Failed to hash edge {edge.id}: {e}")
2070
+
2071
+ # Record operation in traceability layer
2072
+ if self.traceability_layer:
2073
+ self.traceability_layer.record_operation(
2074
+ operation_type='CREATE',
2075
+ entity_type='edge',
2076
+ entity_id=edge.id,
2077
+ data={
2078
+ 'source': edge.source,
2079
+ 'target': edge.target,
2080
+ 'label': edge.label,
2081
+ 'properties': edge.properties
2082
+ },
2083
+ user_id=user_id,
2084
+ namespace=self.name
2085
+ )
2086
+ # CSR-only: Store edge properties separately (no NetworkX)
2087
+ # Include _source/_target so get_all_edges can match CSR entries back to this edge
2088
+ props_copy = edge.properties.copy()
2089
+ props_copy['_source'] = edge.source
2090
+ props_copy['_target'] = edge.target
2091
+ self.edge_properties[edge.id] = props_copy
2092
+ # Store full edge metadata for reliable save/load
2093
+ self.edge_store[edge.id] = {
2094
+ 'source': edge.source,
2095
+ 'target': edge.target,
2096
+ 'label': edge.label,
2097
+ 'properties': edge.properties.copy(),
2098
+ 'weight': getattr(edge, 'weight', 1.0) if hasattr(edge, 'weight') else 1.0
2099
+ }
2100
+
2101
+ # CRITICAL: Always add to CSR (mandatory - NetworkX disabled)
2102
+ if not self.csr_adapter:
2103
+ raise RuntimeError("CSR adapter not initialized. NetworkX fallback disabled.")
2104
+ self.csr_adapter.add_edge(
2105
+ edge.source,
2106
+ edge.target,
2107
+ edge.label,
2108
+ edge.properties
2109
+ )
2110
+
2111
+ self.operation_counts['add_edge'] += 1
2112
+
2113
+ # Update ContextMeta edge_count
2114
+ try:
2115
+ meta_node = self.csr_adapter.get_node("_context_meta")
2116
+ if meta_node:
2117
+ meta_node.properties["edge_count"] = meta_node.properties.get("edge_count", 0) + 1
2118
+ meta_node.properties["updated_at"] = _dt.now(_tz.utc).isoformat()
2119
+ self.csr_adapter.update_node_properties("_context_meta", meta_node.properties)
2120
+ except Exception:
2121
+ pass
2122
+ # Only log edge additions in debug mode to reduce verbosity
2123
+ if logger.isEnabledFor(logging.DEBUG):
2124
+ logger.debug(f"Edge '{edge.source}' -> '{edge.target}' added successfully")
2125
+ return True
2126
+
2127
+ except Exception as e:
2128
+ logger.error(f"Failed to add edge '{edge.source}' -> '{edge.target}': {e}")
2129
+ return False
2130
+
2131
+ def get_node(self, node_id: str) -> Optional[GraphNode]:
2132
+ """Get a node by ID. Checks both persistent storage and buffer (like Neo4j's page cache)."""
2133
+ # CSR-only: Get node from CSR storage first
2134
+ if not self.csr_adapter:
2135
+ # If no CSR adapter, check buffer only
2136
+ if self.buffer_enabled and self.buffer_manager:
2137
+ return self._get_buffered_node_by_id(node_id)
2138
+ return None
2139
+
2140
+ csr_node = self.csr_adapter.get_node(node_id)
2141
+ if csr_node:
2142
+ properties = self.node_properties.get(node_id, {})
2143
+ # node_id is now the UUID - use it directly
2144
+ # Extract name and alias from properties if present
2145
+ node_uuid = node_id # node_id is the UUID
2146
+ node_name = properties.get('name')
2147
+ node_alias = properties.get('alias')
2148
+ return GraphNode(
2149
+ id=node_id,
2150
+ label=csr_node.node_type,
2151
+ properties=properties,
2152
+ uuid=node_uuid,
2153
+ name=node_name,
2154
+ alias=node_alias
2155
+ )
2156
+
2157
+ # OPTIMIZATION: Also check buffer memory for pending nodes (immediate consistency)
2158
+ # This ensures nodes created in buffer are immediately visible (like Neo4j's page cache)
2159
+ if self.buffer_enabled and self.buffer_manager:
2160
+ buffered_node = self._get_buffered_node_by_id(node_id)
2161
+ if buffered_node:
2162
+ return buffered_node
2163
+
2164
+ return None
2165
+
2166
+ def update_node(
2167
+ self,
2168
+ node_id: str,
2169
+ properties: dict,
2170
+ expected_version: Optional[int] = None,
2171
+ author: Optional[str] = None,
2172
+ reason: Optional[str] = None,
2173
+ ) -> "GraphNode":
2174
+ """Update a node's properties with optional optimistic locking.
2175
+
2176
+ Snapshots old content in TemporalStorage before overwriting.
2177
+ Propagates stale flag to directly downstream nodes.
2178
+
2179
+ Args:
2180
+ node_id: ID of the node to update.
2181
+ properties: Property dict to merge into the existing node.
2182
+ expected_version: If set, raises VersionConflictError on mismatch.
2183
+ author: Agent/user making the change (stored as _updated_by).
2184
+ reason: Why this change is being made (stored as _change_reason).
2185
+
2186
+ Raises:
2187
+ KeyError: If the node does not exist.
2188
+ VersionConflictError: If expected_version is provided and does not
2189
+ match the stored version.
2190
+ """
2191
+ from datetime import datetime as _dt, timezone as _tz
2192
+
2193
+ existing = self.get_node(node_id)
2194
+ if existing is None:
2195
+ raise KeyError(f"Node {node_id!r} not found")
2196
+
2197
+ stored_version = existing.properties.get("version", 1)
2198
+ if expected_version is not None and expected_version != stored_version:
2199
+ raise VersionConflictError(
2200
+ current_version=stored_version,
2201
+ provided_version=expected_version,
2202
+ )
2203
+
2204
+ # Snapshot old content before overwriting
2205
+ if self.temporal_enabled and self.temporal_storage:
2206
+ try:
2207
+ self.temporal_storage.create_version(
2208
+ entity_id=node_id,
2209
+ entity_type="node",
2210
+ properties=existing.properties.copy(),
2211
+ operation="UPDATE",
2212
+ timestamp=_dt.now(),
2213
+ author=author or "",
2214
+ reason=reason or "",
2215
+ )
2216
+ except Exception as e:
2217
+ logger.warning(f"[TIME_TRAVEL] Failed to snapshot node {node_id}: {e}")
2218
+
2219
+ merged = dict(existing.properties)
2220
+ merged.update(properties)
2221
+ merged["version"] = stored_version + 1
2222
+ merged["_updated_at"] = _dt.now(_tz.utc).isoformat()
2223
+ if author:
2224
+ merged["_updated_by"] = author
2225
+ if reason:
2226
+ merged["_change_reason"] = reason
2227
+
2228
+ updated = GraphNode(id=node_id, label=existing.label, properties=merged)
2229
+ self.add_node(updated, _skip_temporal=True)
2230
+
2231
+ # Snapshot NEW content too (v2 in temporal storage)
2232
+ if self.temporal_enabled and self.temporal_storage:
2233
+ try:
2234
+ self.temporal_storage.create_version(
2235
+ entity_id=node_id,
2236
+ entity_type="node",
2237
+ properties=updated.properties.copy(),
2238
+ operation="UPDATE",
2239
+ timestamp=_dt.now(),
2240
+ author=author or "",
2241
+ reason=reason or "",
2242
+ )
2243
+ except Exception as e:
2244
+ logger.warning(f"[TIME_TRAVEL] Failed to snapshot updated node {node_id}: {e}")
2245
+
2246
+ # Propagate stale to directly downstream nodes
2247
+ self._propagate_stale(node_id, merged["version"])
2248
+
2249
+ return updated
2250
+
2251
+ def _propagate_stale(self, source_node_id: str, new_version: int) -> None:
2252
+ """Flag all directly-downstream nodes as stale (one hop only).
2253
+
2254
+ Called automatically by update_node(). Never recursive — only
2255
+ direct outgoing neighbors are flagged.
2256
+ """
2257
+ from datetime import datetime as _dt, timezone as _tz
2258
+
2259
+ source_node = self.get_node(source_node_id)
2260
+ if not source_node:
2261
+ return
2262
+
2263
+ source_name = source_node.properties.get("name", source_node_id[:8])
2264
+ stale_reason = f"{source_name} updated to v{new_version}"
2265
+ stale_at = _dt.now(_tz.utc).isoformat()
2266
+
2267
+ try:
2268
+ neighbors = self.get_neighbors(source_node_id, direction="OUTGOING")
2269
+ except Exception as e:
2270
+ logger.warning(f"[STALE] Failed to get neighbors of {source_node_id}: {e}")
2271
+ return
2272
+
2273
+ for neighbor_id, _ in neighbors:
2274
+ neighbor = self.get_node(neighbor_id)
2275
+ if not neighbor:
2276
+ continue
2277
+ props = dict(neighbor.properties)
2278
+ props["_stale"] = True
2279
+ props["_stale_reason"] = stale_reason
2280
+ props["_stale_since"] = stale_at
2281
+ # Write directly via add_node to avoid triggering another update_node
2282
+ # cascade — stale writes are not versioned changes
2283
+ self.add_node(GraphNode(id=neighbor_id, label=neighbor.label, properties=props), _skip_temporal=True)
2284
+
2285
+ def confirm_current(self, node_id: str, author: str = "") -> "GraphNode":
2286
+ """Clear the stale flag on a node — agent confirms it is still valid.
2287
+
2288
+ Args:
2289
+ node_id: UUID of the node to confirm.
2290
+ author: Agent or user making the confirmation.
2291
+
2292
+ Raises:
2293
+ KeyError: If node_id does not exist.
2294
+ """
2295
+ from datetime import datetime as _dt, timezone as _tz
2296
+
2297
+ node = self.get_node(node_id)
2298
+ if node is None:
2299
+ raise KeyError(f"Node {node_id!r} not found")
2300
+
2301
+ props = dict(node.properties)
2302
+ props["_stale"] = False
2303
+ props.pop("_stale_reason", None)
2304
+ props.pop("_stale_since", None)
2305
+ if author:
2306
+ props["_stale_confirmed_by"] = author
2307
+ props["_stale_confirmed_at"] = _dt.now(_tz.utc).isoformat()
2308
+
2309
+ updated = GraphNode(id=node_id, label=node.label, properties=props)
2310
+ self.add_node(updated, _skip_temporal=True)
2311
+ return updated
2312
+
2313
+ def get_neighbors(self, node_id: str, edge_label: Optional[str] = None, direction: str = "OUTGOING") -> List[Tuple[str, str]]:
2314
+ """Get neighbors of a node."""
2315
+ self.operation_counts['get_neighbors'] += 1
2316
+
2317
+ # CSR-only: Use CSR adapter (NetworkX disabled)
2318
+ if not self.csr_adapter:
2319
+ return []
2320
+
2321
+ if direction == "OUTGOING":
2322
+ csr_neighbors = self.csr_adapter.get_neighbors(node_id, edge_label)
2323
+ # Convert CSR format to expected format
2324
+ return [(neighbor_id, edge.source_id + "_" + edge.target_id) for neighbor_id, edge in csr_neighbors]
2325
+ else: # INCOMING
2326
+ # For incoming edges, traverse all edges to find those pointing to this node
2327
+ neighbors = []
2328
+ # Get all edges from CSR storage
2329
+ if hasattr(self.csr_storage, 'edge_data'):
2330
+ for edge in self.csr_storage.edge_data:
2331
+ if edge.target_id == node_id:
2332
+ if edge_label is None or edge.edge_type == edge_label:
2333
+ edge_id = f"{edge.source_id}_{edge.target_id}"
2334
+ neighbors.append((edge.source_id, edge_id))
2335
+ return neighbors
2336
+
2337
+ def check_unique_constraint(self, constraint_name: str, value: Any) -> bool:
2338
+ """Check if a unique constraint is satisfied."""
2339
+ return value not in self.unique_constraints[constraint_name]
2340
+
2341
+ def add_unique_constraint(self, constraint_name: str, value: Any):
2342
+ """Add a value to a unique constraint."""
2343
+ self.unique_constraints[constraint_name].add(value)
2344
+
2345
+ def get_all_nodes(self, label: Optional[str] = None, *, limit: Optional[int] = None, offset: int = 0) -> List[GraphNode]:
2346
+ """Get all nodes, optionally filtered by label. Includes buffered nodes.
2347
+
2348
+ Args:
2349
+ label: Optional label filter.
2350
+ limit: Max number of nodes to return (None = unlimited).
2351
+ offset: Number of matching nodes to skip before collecting results.
2352
+ """
2353
+ seen_ids = set()
2354
+ nodes = []
2355
+ skipped = 0
2356
+
2357
+ def _should_stop():
2358
+ return limit is not None and len(nodes) >= limit
2359
+
2360
+ def _collect(node):
2361
+ nonlocal skipped
2362
+ if _should_stop():
2363
+ return
2364
+ if offset > 0 and skipped < offset:
2365
+ skipped += 1
2366
+ return
2367
+ nodes.append(node)
2368
+
2369
+ # Primary source: storage adapter
2370
+ if self.csr_adapter:
2371
+ # Redis backend: use label index when available (O(k) vs O(n))
2372
+ if self._using_redis_backend:
2373
+ adapter_nodes = self.csr_adapter.get_all_nodes(node_type=label) if label else self.csr_adapter.get_all_nodes()
2374
+ for gn in adapter_nodes:
2375
+ if gn.id not in seen_ids:
2376
+ seen_ids.add(gn.id)
2377
+ _collect(gn)
2378
+ return nodes
2379
+
2380
+ # CSR backend: iterate node IDs
2381
+ if label:
2382
+ node_ids = self.csr_adapter.get_nodes_by_type(label)
2383
+ else:
2384
+ node_ids = list(self.csr_storage.nodes.keys()) if hasattr(self.csr_storage, 'nodes') else []
2385
+
2386
+ for node_id in node_ids:
2387
+ if _should_stop():
2388
+ break
2389
+ node = self.get_node(node_id)
2390
+ if node and node.id not in seen_ids:
2391
+ seen_ids.add(node.id)
2392
+ _collect(node)
2393
+
2394
+ # Secondary source: node_index (catches nodes not yet in CSR)
2395
+ if not _should_stop() and hasattr(self, 'node_index'):
2396
+ for node_id, node in self.node_index.items():
2397
+ if _should_stop():
2398
+ break
2399
+ if node_id in seen_ids:
2400
+ continue
2401
+ if label is None or node.label == label:
2402
+ seen_ids.add(node_id)
2403
+ _collect(node)
2404
+
2405
+ # Tertiary source: write buffer (unflushed nodes)
2406
+ if not _should_stop():
2407
+ buffered_nodes = self._get_buffered_nodes(label)
2408
+ for buffered_node in buffered_nodes:
2409
+ if _should_stop():
2410
+ break
2411
+ if buffered_node.id not in seen_ids:
2412
+ seen_ids.add(buffered_node.id)
2413
+ _collect(buffered_node)
2414
+
2415
+ return nodes
2416
+
2417
+ def _get_buffered_nodes(self, label: Optional[str] = None) -> List[GraphNode]:
2418
+ """Get nodes from buffer memory that haven't been flushed yet."""
2419
+ buffered_nodes = []
2420
+
2421
+ if not self.buffer_enabled or not self.buffer_manager:
2422
+ return buffered_nodes
2423
+
2424
+ try:
2425
+ # Get pending records from buffer
2426
+ import asyncio
2427
+
2428
+ # Try to get records synchronously
2429
+ try:
2430
+ loop = asyncio.get_event_loop()
2431
+ if loop.is_running():
2432
+ # Can't run sync in async context, return empty
2433
+ return buffered_nodes
2434
+ else:
2435
+ # Run async method synchronously
2436
+ records = loop.run_until_complete(self._get_buffer_records_async())
2437
+ except RuntimeError:
2438
+ # No event loop, create one
2439
+ loop = asyncio.new_event_loop()
2440
+ asyncio.set_event_loop(loop)
2441
+ records = loop.run_until_complete(self._get_buffer_records_async())
2442
+ loop.close()
2443
+
2444
+ # Convert buffer records to GraphNode objects
2445
+ for record in records:
2446
+ operation = record.get('operation', '')
2447
+
2448
+ if operation == 'CREATE_NODE':
2449
+ node_label = record.get('label', '')
2450
+ node_id = record.get('node_id', '')
2451
+ properties = record.get('properties', {})
2452
+
2453
+ # Filter by label if specified
2454
+ if label is None or node_label == label:
2455
+ node = GraphNode(
2456
+ id=node_id,
2457
+ label=node_label,
2458
+ properties=properties
2459
+ )
2460
+ buffered_nodes.append(node)
2461
+
2462
+ except Exception as e:
2463
+ logger.debug(f"Error getting buffered nodes: {e}")
2464
+
2465
+ return buffered_nodes
2466
+
2467
+ def _get_buffered_node_by_id(self, node_id: str) -> Optional[GraphNode]:
2468
+ """Get a specific node from buffer by ID (for immediate consistency)."""
2469
+ if not self.buffer_enabled or not self.buffer_manager:
2470
+ return None
2471
+
2472
+ try:
2473
+ # Get pending records from buffer synchronously
2474
+ records = self._get_buffer_records_sync()
2475
+
2476
+ for record in records:
2477
+ if record.get('operation') == 'CREATE_NODE':
2478
+ if record.get('node_id') == node_id:
2479
+ return GraphNode(
2480
+ id=record.get('node_id'),
2481
+ label=record.get('label', ''),
2482
+ properties=record.get('properties', {})
2483
+ )
2484
+ except Exception as e:
2485
+ logger.debug(f"Error getting buffered node by ID: {e}")
2486
+
2487
+ return None
2488
+
2489
+ def _get_buffer_records_sync(self) -> List[Dict[str, Any]]:
2490
+ """Get buffer records synchronously (helper for sync contexts)."""
2491
+ import asyncio
2492
+
2493
+ if not self.buffer_manager:
2494
+ return []
2495
+
2496
+ try:
2497
+ loop = asyncio.get_event_loop()
2498
+ if loop.is_running():
2499
+ # Can't run sync in async context, return empty (will be available after flush)
2500
+ return []
2501
+ else:
2502
+ return loop.run_until_complete(self._get_buffer_records_async())
2503
+ except RuntimeError:
2504
+ # No event loop, create one
2505
+ loop = asyncio.new_event_loop()
2506
+ asyncio.set_event_loop(loop)
2507
+ try:
2508
+ return loop.run_until_complete(self._get_buffer_records_async())
2509
+ finally:
2510
+ loop.close()
2511
+
2512
+ async def _get_buffer_records_async(self) -> List[Dict[str, Any]]:
2513
+ """Get all pending records from buffer asynchronously."""
2514
+ if not self.buffer_manager:
2515
+ return []
2516
+
2517
+ try:
2518
+ # Check buffer type and get records accordingly
2519
+ if hasattr(self.buffer_manager, '_records'):
2520
+ # Local buffer - direct access
2521
+ async with self.buffer_manager._lock:
2522
+ return list(self.buffer_manager._records)
2523
+ elif hasattr(self.buffer_manager, 'get_pending_records'):
2524
+ # Buffer manager with get_pending_records method
2525
+ return await self.buffer_manager.get_pending_records()
2526
+ else:
2527
+ # For other buffer types, we'd need to implement get_pending_records
2528
+ # For now, return empty list
2529
+ return []
2530
+ except Exception as e:
2531
+ logger.debug(f"Error getting buffer records: {e}")
2532
+ return []
2533
+
2534
+ def get_all_edges(self, label: Optional[str] = None, *, limit: Optional[int] = None, offset: int = 0) -> List[GraphEdge]:
2535
+ """Get all edges, optionally filtered by label. Includes buffered edges.
2536
+
2537
+ Args:
2538
+ label: Optional label filter.
2539
+ limit: Max number of edges to return (None = unlimited).
2540
+ offset: Number of matching edges to skip before collecting results.
2541
+ """
2542
+ edges = []
2543
+ _skipped = 0
2544
+
2545
+ def _at_limit():
2546
+ return limit is not None and len(edges) >= limit
2547
+ # Redis backend: use adapter directly
2548
+ if self._using_redis_backend and self.csr_adapter:
2549
+ for edge in self.csr_adapter.get_all_edges():
2550
+ if _at_limit():
2551
+ break
2552
+ edge_label = getattr(edge, 'edge_type', getattr(edge, 'label', ''))
2553
+ if label and edge_label != label:
2554
+ continue
2555
+ if offset > 0 and _skipped < offset:
2556
+ _skipped += 1
2557
+ continue
2558
+ edges.append(GraphEdge(
2559
+ id=getattr(edge, 'id', ''),
2560
+ source=getattr(edge, 'source_id', getattr(edge, 'source', '')),
2561
+ target=getattr(edge, 'target_id', getattr(edge, 'target', '')),
2562
+ label=edge_label,
2563
+ properties=getattr(edge, 'properties', {}),
2564
+ ))
2565
+ return edges
2566
+
2567
+ # CSR backend: Get edges from CSR storage
2568
+ if not self.csr_storage or not hasattr(self.csr_storage, 'edge_data'):
2569
+ pass # Continue to check buffer
2570
+ else:
2571
+ for edge in self.csr_storage.edge_data:
2572
+ if _at_limit():
2573
+ break
2574
+ edge_label = edge.edge_type
2575
+ if label is None or edge_label == label:
2576
+ import uuid as uuid_module
2577
+
2578
+ # Try to find edge in edge_properties by matching source/target
2579
+ # Since edge_properties now uses UUID as key, we need to search
2580
+ edge_id = None
2581
+ properties = edge.properties.copy() if edge.properties else {}
2582
+
2583
+ # Search edge_properties for matching source/target
2584
+ # This handles both new (UUID key) and old (source_target key) formats
2585
+ old_format_id = f"{edge.source_id}_{edge.target_id}"
2586
+ if old_format_id in self.edge_properties:
2587
+ # Old format - migrate to UUID
2588
+ edge_id = str(uuid_module.uuid4())
2589
+ properties = self.edge_properties[old_format_id].copy()
2590
+ # Migrate to new format
2591
+ self.edge_properties[edge_id] = properties
2592
+ self.edge_properties.pop(old_format_id, None)
2593
+ else:
2594
+ # Search for edge with matching source/target in edge_properties
2595
+ for stored_id, stored_props in self.edge_properties.items():
2596
+ # Check if properties indicate this is our edge
2597
+ if (stored_props.get('_source') == edge.source_id or
2598
+ stored_props.get('source') == edge.source_id):
2599
+ if (stored_props.get('_target') == edge.target_id or
2600
+ stored_props.get('target') == edge.target_id):
2601
+ edge_id = stored_id
2602
+ properties = stored_props.copy()
2603
+ break
2604
+
2605
+ # If not found, generate new UUID (new edge or migration)
2606
+ if edge_id is None:
2607
+ edge_id = str(uuid_module.uuid4())
2608
+ # Store properties with new UUID key
2609
+ self.edge_properties[edge_id] = properties
2610
+
2611
+ # Remove uuid from properties if present (to avoid duplication)
2612
+ properties.pop('uuid', None)
2613
+
2614
+ # edge_id is now the UUID
2615
+ edge_uuid = edge_id
2616
+ edge_name = properties.get('name')
2617
+ edge_alias = properties.get('alias')
2618
+
2619
+ graph_edge = GraphEdge(
2620
+ id=edge_id, # This is the UUID
2621
+ source=edge.source_id, # Source node UUID
2622
+ target=edge.target_id, # Target node UUID
2623
+ label=edge_label,
2624
+ properties=properties,
2625
+ uuid=edge_uuid,
2626
+ name=edge_name,
2627
+ alias=edge_alias
2628
+ )
2629
+ if offset > 0 and _skipped < offset:
2630
+ _skipped += 1
2631
+ continue
2632
+ edges.append(graph_edge)
2633
+
2634
+ # OPTIMIZATION: Also check buffer memory for pending edges
2635
+ if not _at_limit():
2636
+ buffered_edges = self._get_buffered_edges(label)
2637
+ if buffered_edges:
2638
+ existing_ids = {edge.id for edge in edges}
2639
+ # Also deduplicate by (source, target, label) to handle write-through duplicates
2640
+ existing_triples = {(edge.source, edge.target, edge.label) for edge in edges}
2641
+ for buffered_edge in buffered_edges:
2642
+ if _at_limit():
2643
+ break
2644
+ triple = (buffered_edge.source, buffered_edge.target, buffered_edge.label)
2645
+ if buffered_edge.id not in existing_ids and triple not in existing_triples:
2646
+ if offset > 0 and _skipped < offset:
2647
+ _skipped += 1
2648
+ continue
2649
+ edges.append(buffered_edge)
2650
+
2651
+ return edges
2652
+
2653
+ def get_statistics(self) -> Dict[str, Any]:
2654
+ """Return graph statistics for query optimization.
2655
+
2656
+ Returns node/edge counts by type using the label index (O(1) per label
2657
+ on Redis, O(k) on CSR) — avoids full scan.
2658
+ """
2659
+ node_counts: Dict[str, int] = {}
2660
+ edge_counts: Dict[str, int] = {}
2661
+ total_nodes = 0
2662
+ total_edges = 0
2663
+
2664
+ if self._using_redis_backend and self.csr_adapter:
2665
+ # Use Redis SETs for O(1) count per label
2666
+ r = getattr(self.csr_adapter, '_r', None)
2667
+ prefix = getattr(self.csr_adapter, '_prefix', '')
2668
+ if r:
2669
+ total_nodes = r.scard(f"{prefix}nodes")
2670
+ total_edges = r.scard(f"{prefix}edges")
2671
+ # Scan ntype:* keys for label counts
2672
+ for key in r.scan_iter(f"{prefix}ntype:*", count=100):
2673
+ label_name = key.split("ntype:")[-1] if "ntype:" in key else ""
2674
+ if label_name:
2675
+ node_counts[label_name] = r.scard(key)
2676
+ for key in r.scan_iter(f"{prefix}etype:*", count=100):
2677
+ etype = key.split("etype:")[-1] if "etype:" in key else ""
2678
+ if etype:
2679
+ edge_counts[etype] = r.scard(key)
2680
+ elif self.csr_adapter:
2681
+ # CSR: use in-memory node_types dict
2682
+ nt = getattr(self.csr_adapter, 'node_types', {})
2683
+ for lbl, ids in nt.items():
2684
+ node_counts[lbl] = len(ids)
2685
+ total_nodes = sum(node_counts.values())
2686
+ total_edges = len(self.get_all_edges())
2687
+
2688
+ return {
2689
+ "total_nodes": total_nodes,
2690
+ "total_edges": total_edges,
2691
+ "node_count_by_label": node_counts,
2692
+ "edge_count_by_type": edge_counts,
2693
+ "label_index_available": True,
2694
+ }
2695
+
2696
+ def _get_buffered_edges(self, label: Optional[str] = None) -> List[GraphEdge]:
2697
+ """Get edges from buffer memory that haven't been flushed yet."""
2698
+ buffered_edges = []
2699
+
2700
+ if not self.buffer_enabled or not self.buffer_manager:
2701
+ return buffered_edges
2702
+
2703
+ try:
2704
+ import asyncio
2705
+
2706
+ # Try to get records synchronously
2707
+ try:
2708
+ loop = asyncio.get_event_loop()
2709
+ if loop.is_running():
2710
+ return buffered_edges # Can't run sync in async context
2711
+ else:
2712
+ records = loop.run_until_complete(self._get_buffer_records_async())
2713
+ except RuntimeError:
2714
+ loop = asyncio.new_event_loop()
2715
+ asyncio.set_event_loop(loop)
2716
+ records = loop.run_until_complete(self._get_buffer_records_async())
2717
+ loop.close()
2718
+
2719
+ # Convert buffer records to GraphEdge objects
2720
+ for record in records:
2721
+ operation = record.get('operation', '')
2722
+
2723
+ if operation == 'CREATE_EDGE':
2724
+ edge_label = record.get('label', '')
2725
+ edge_id = record.get('edge_id', f"{record.get('source')}_{record.get('target')}")
2726
+ source = record.get('source', '')
2727
+ target = record.get('target', '')
2728
+ properties = record.get('properties', {})
2729
+
2730
+ # Filter by label if specified
2731
+ if label is None or edge_label == label:
2732
+ edge = GraphEdge(
2733
+ id=edge_id,
2734
+ source=source,
2735
+ target=target,
2736
+ label=edge_label,
2737
+ properties=properties
2738
+ )
2739
+ buffered_edges.append(edge)
2740
+
2741
+ except Exception as e:
2742
+ logger.debug(f"Error getting buffered edges: {e}")
2743
+
2744
+ return buffered_edges
2745
+
2746
+ def get_node_types(self) -> Set[str]:
2747
+ """Get all unique node types (labels) in the graph."""
2748
+ # CSR-only: Get node types from CSR storage
2749
+ if not self.csr_storage:
2750
+ return set()
2751
+ return self.csr_storage.get_node_types()
2752
+
2753
+ def get_edge_types(self) -> Set[str]:
2754
+ """Get all unique edge types (labels) in the graph."""
2755
+ # CSR-only: Get edge types from CSR storage
2756
+ if not self.csr_storage:
2757
+ return set()
2758
+ return self.csr_storage.get_edge_types()
2759
+
2760
+ def traverse_graph(self, start_node_id: str, edge_label: Optional[str] = None,
2761
+ max_depth: int = 1, direction: str = "OUTGOING") -> List[str]:
2762
+ """Traverse the graph from a starting node."""
2763
+ self.operation_counts['traverse'] += 1
2764
+
2765
+ # CSR-only: Use CSR adapter (NetworkX disabled)
2766
+ if not self.csr_adapter:
2767
+ return []
2768
+ return self.csr_adapter.traverse(start_node_id, edge_label, max_depth, direction)
2769
+
2770
+ def get_nodes_by_property(self, property_name: str, property_value: Any) -> List[str]:
2771
+ """Get nodes by property value."""
2772
+ # CSR-only: Use CSR adapter (NetworkX disabled)
2773
+ if not self.csr_adapter:
2774
+ return []
2775
+ return self.csr_adapter.get_nodes_by_property(property_name, property_value)
2776
+
2777
+ def _update_property_indexes(self, node_id: str, properties: Dict[str, Any]):
2778
+ """Update property indexes when a node is added."""
2779
+ for prop_name, prop_value in properties.items():
2780
+ # Skip complex types (dict, list) that can't be indexed
2781
+ if isinstance(prop_value, (dict, list)):
2782
+ continue
2783
+
2784
+ # Skip None values
2785
+ if prop_value is None:
2786
+ continue
2787
+
2788
+ # Test hashability to prevent "unhashable type" errors
2789
+ try:
2790
+ # Try to use value as dict key (tests hashability)
2791
+ _ = {prop_value: True}
2792
+ except TypeError:
2793
+ # Value is not hashable, skip indexing
2794
+ continue
2795
+
2796
+ if prop_name not in self._property_indexes:
2797
+ if SORTED_CONTAINERS_AVAILABLE:
2798
+ self._property_indexes[prop_name] = SortedDict()
2799
+ else:
2800
+ self._property_indexes[prop_name] = {}
2801
+
2802
+ if SORTED_CONTAINERS_AVAILABLE:
2803
+ # Use SortedDict — coerce key to str to avoid mixed-type comparison errors
2804
+ _idx_key = str(prop_value) if not isinstance(prop_value, str) else prop_value
2805
+ if _idx_key not in self._property_indexes[prop_name]:
2806
+ self._property_indexes[prop_name][_idx_key] = set()
2807
+ self._property_indexes[prop_name][_idx_key].add(node_id)
2808
+ else:
2809
+ # Use regular dict
2810
+ if prop_value not in self._property_indexes[prop_name]:
2811
+ self._property_indexes[prop_name][prop_value] = set()
2812
+ self._property_indexes[prop_name][prop_value].add(node_id)
2813
+
2814
+ def filter_by_property(self, prop_name: str, prop_value: Any) -> Set[str]:
2815
+ """Fast property filtering using index."""
2816
+ if prop_name not in self._property_indexes:
2817
+ return set()
2818
+
2819
+ index = self._property_indexes[prop_name]
2820
+ return index.get(prop_value, set())
2821
+
2822
+ def filter_by_property_range(self, prop_name: str, op: str, value: Any) -> Set[str]:
2823
+ """Filter by property range using index."""
2824
+ if prop_name not in self._property_indexes:
2825
+ return set()
2826
+
2827
+ index = self._property_indexes[prop_name]
2828
+ filtered = set()
2829
+
2830
+ if op == '=':
2831
+ return index.get(value, set())
2832
+ elif op == '>':
2833
+ # For now, use dict fallback for range queries
2834
+ for k, node_ids in index.items():
2835
+ if k > value:
2836
+ filtered.update(node_ids)
2837
+ elif op == '<':
2838
+ for k, node_ids in index.items():
2839
+ if k < value:
2840
+ filtered.update(node_ids)
2841
+ elif op == '>=':
2842
+ for k, node_ids in index.items():
2843
+ if k >= value:
2844
+ filtered.update(node_ids)
2845
+ elif op == '<=':
2846
+ for k, node_ids in index.items():
2847
+ if k <= value:
2848
+ filtered.update(node_ids)
2849
+
2850
+ return filtered
2851
+
2852
+ def get_statistics(self) -> Dict[str, Any]:
2853
+ """Get comprehensive graph statistics."""
2854
+ # CSR-only: Get statistics from CSR storage
2855
+ if not self.csr_storage:
2856
+ return {
2857
+ 'node_count': 0,
2858
+ 'edge_count': 0,
2859
+ 'node_types': 0,
2860
+ 'edge_types': 0,
2861
+ 'storage_backend': 'CSR',
2862
+ 'operation_counts': self.operation_counts
2863
+ }
2864
+
2865
+ csr_stats = self.csr_storage.get_statistics()
2866
+ csr_stats['storage_backend'] = 'CSR'
2867
+ csr_stats['operation_counts'] = self.operation_counts
2868
+ return csr_stats
2869
+
2870
+ def optimize(self):
2871
+ """Optimize the graph storage."""
2872
+ # CSR-only: Optimize CSR storage
2873
+ if self.csr_storage:
2874
+ self.csr_storage.optimize()
2875
+ logger.info("CSR storage optimized")
2876
+
2877
+ def migrate_to_csr(self):
2878
+ """Migrate existing data to CSR storage. No-op in CSR-only mode."""
2879
+ logger.info("CSR-only mode: No migration needed (NetworkX disabled)")
2880
+ return True
2881
+
2882
+ def clear(self):
2883
+ """Clear all data."""
2884
+ # CSR-only: Clear CSR storage and properties
2885
+ self.node_properties.clear()
2886
+ self.edge_properties.clear()
2887
+ self.unique_constraints.clear()
2888
+ # NOTE: alias_to_node removed - aliases are query-scoped only
2889
+
2890
+ if self.csr_storage:
2891
+ self.csr_storage.clear()
2892
+
2893
+ # NOTE: alias_mapping in csr_adapter is not used (aliases are query-scoped only)
2894
+
2895
+ self.operation_counts = {k: 0 for k in self.operation_counts}
2896
+ logger.info("Graph storage cleared (CSR-only mode)")
2897
+
2898
+ # Performance comparison utility
2899
+ def compare_storage_performance():
2900
+ """Compare performance between NetworkX and CSR storage."""
2901
+ import time
2902
+ import random
2903
+
2904
+ print("Storage Performance Comparison")
2905
+ print("=" * 50)
2906
+
2907
+ # Test parameters
2908
+ num_nodes = 1000
2909
+ num_edges = 2000
2910
+
2911
+ # Generate test data
2912
+ nodes = []
2913
+ edges = []
2914
+
2915
+ for i in range(num_nodes):
2916
+ nodes.append(GraphNode(
2917
+ id=f"node_{i}",
2918
+ label="Person",
2919
+ properties={"id": i, "name": f"Person_{i}", "age": random.randint(18, 80)}
2920
+ ))
2921
+
2922
+ for i in range(num_edges):
2923
+ source = f"node_{random.randint(0, num_nodes-1)}"
2924
+ target = f"node_{random.randint(0, num_nodes-1)}"
2925
+ edges.append(GraphEdge(
2926
+ id=f"edge_{i}",
2927
+ source=source,
2928
+ target=target,
2929
+ label="KNOWS",
2930
+ properties={"weight": random.random()}
2931
+ ))
2932
+
2933
+ # Test NetworkX-only storage
2934
+ print("\nTesting NetworkX-only storage:")
2935
+ nx_storage = HybridGraphStorage('networkx')
2936
+
2937
+ start_time = time.time()
2938
+ for node in nodes:
2939
+ nx_storage.add_node(node)
2940
+ nx_node_time = time.time() - start_time
2941
+
2942
+ start_time = time.time()
2943
+ for edge in edges:
2944
+ nx_storage.add_edge(edge)
2945
+ nx_edge_time = time.time() - start_time
2946
+
2947
+ start_time = time.time()
2948
+ for i in range(100):
2949
+ start_node = f"node_{random.randint(0, num_nodes-1)}"
2950
+ nx_storage.traverse_graph(start_node, max_depth=2)
2951
+ nx_traverse_time = time.time() - start_time
2952
+
2953
+ # Test CSR storage
2954
+ print("\nTesting CSR storage:")
2955
+ csr_storage = HybridGraphStorage('csr')
2956
+
2957
+ start_time = time.time()
2958
+ for node in nodes:
2959
+ csr_storage.add_node(node)
2960
+ csr_node_time = time.time() - start_time
2961
+
2962
+ start_time = time.time()
2963
+ for edge in edges:
2964
+ csr_storage.add_edge(edge)
2965
+ csr_edge_time = time.time() - start_time
2966
+
2967
+ start_time = time.time()
2968
+ for i in range(100):
2969
+ start_node = f"node_{random.randint(0, num_nodes-1)}"
2970
+ csr_storage.traverse_graph(start_node, max_depth=2)
2971
+ csr_traverse_time = time.time() - start_time
2972
+
2973
+ # Results
2974
+ print(f"\nResults ({num_nodes} nodes, {num_edges} edges):")
2975
+ print(f"Add Nodes:")
2976
+ print(f" NetworkX: {nx_node_time:.4f}s")
2977
+ print(f" CSR: {csr_node_time:.4f}s")
2978
+ print(f" Speedup: {nx_node_time/csr_node_time:.2f}x")
2979
+
2980
+ print(f"\nAdd Edges:")
2981
+ print(f" NetworkX: {nx_edge_time:.4f}s")
2982
+ print(f" CSR: {csr_edge_time:.4f}s")
2983
+ print(f" Speedup: {nx_edge_time/csr_edge_time:.2f}x")
2984
+
2985
+ print(f"\nTraverse (100 iterations):")
2986
+ print(f" NetworkX: {nx_traverse_time:.4f}s")
2987
+ print(f" CSR: {csr_traverse_time:.4f}s")
2988
+ print(f" Speedup: {nx_traverse_time/csr_traverse_time:.2f}x")
2989
+
2990
+ print(f"\nStorage Statistics:")
2991
+ print("NetworkX:", nx_storage.get_statistics())
2992
+ print("CSR:", csr_storage.get_statistics())
2993
+
2994
+ # Backward compatibility alias
2995
+ AIContextDB = ContextSynapse
2996
+
2997
+ if __name__ == "__main__":
2998
+ compare_storage_performance()