contextsynapse 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (776) hide show
  1. contextsynapse/__init__.py +107 -0
  2. contextsynapse/__main__.py +9 -0
  3. contextsynapse/a2a/__init__.py +28 -0
  4. contextsynapse/a2a/client.py +120 -0
  5. contextsynapse/a2a/discovery.py +128 -0
  6. contextsynapse/a2a/handlers.py +164 -0
  7. contextsynapse/a2a/models.py +246 -0
  8. contextsynapse/a2a/streaming.py +92 -0
  9. contextsynapse/a2a/task_manager.py +275 -0
  10. contextsynapse/adapters/__init__.py +98 -0
  11. contextsynapse/adapters/_base.py +298 -0
  12. contextsynapse/adapters/autogen/__init__.py +4 -0
  13. contextsynapse/adapters/autogen/tools.py +34 -0
  14. contextsynapse/adapters/crewai/__init__.py +4 -0
  15. contextsynapse/adapters/crewai/tools.py +63 -0
  16. contextsynapse/adapters/langchain/__init__.py +19 -0
  17. contextsynapse/adapters/langchain/message_history.py +17 -0
  18. contextsynapse/adapters/langchain/retriever.py +158 -0
  19. contextsynapse/adapters/langchain/tools.py +43 -0
  20. contextsynapse/adapters/langgraph/__init__.py +21 -0
  21. contextsynapse/adapters/langgraph/checkpoint.py +223 -0
  22. contextsynapse/adapters/langgraph/context_tools.py +341 -0
  23. contextsynapse/adapters/langgraph/message_history.py +138 -0
  24. contextsynapse/adapters/langgraph/tools.py +43 -0
  25. contextsynapse/adapters/llamaindex/__init__.py +4 -0
  26. contextsynapse/adapters/llamaindex/tools.py +64 -0
  27. contextsynapse/adapters/openai/__init__.py +24 -0
  28. contextsynapse/adapters/openai/tools.py +206 -0
  29. contextsynapse/adapters/pydantic_ai/__init__.py +4 -0
  30. contextsynapse/adapters/pydantic_ai/tools.py +36 -0
  31. contextsynapse/adapters/swarm/__init__.py +4 -0
  32. contextsynapse/adapters/swarm/tools.py +44 -0
  33. contextsynapse/agents/__init__.py +6 -0
  34. contextsynapse/agents/auto_capture.py +80 -0
  35. contextsynapse/agents/dispatcher.py +299 -0
  36. contextsynapse/agents/notifications.py +112 -0
  37. contextsynapse/aiql/__init__.py +71 -0
  38. contextsynapse/aiql/cache.py +341 -0
  39. contextsynapse/aiql/compiler/__init__.py +62 -0
  40. contextsynapse/aiql/compiler/optimizer.py +126 -0
  41. contextsynapse/aiql/compiler/planner.py +366 -0
  42. contextsynapse/aiql/compiler/statistics.py +235 -0
  43. contextsynapse/aiql/compiler/validator.py +425 -0
  44. contextsynapse/aiql/engine/__init__.py +11 -0
  45. contextsynapse/aiql/engine/chunk_storage.py +2584 -0
  46. contextsynapse/aiql/engine/executor.py +7737 -0
  47. contextsynapse/aiql/engine/extraction_integration.py +442 -0
  48. contextsynapse/aiql/engine/graph_builder.py +405 -0
  49. contextsynapse/aiql/engine/node_creation_from_normalized.py +691 -0
  50. contextsynapse/aiql/engine/performance_monitor.py +289 -0
  51. contextsynapse/aiql/engine/postprocessing.py +365 -0
  52. contextsynapse/aiql/engine/ray_stage_executors.py +918 -0
  53. contextsynapse/aiql/engine/redis_cache.py +106 -0
  54. contextsynapse/aiql/engine/streaming_json.py +366 -0
  55. contextsynapse/aiql/engine/streaming_parquet.py +568 -0
  56. contextsynapse/aiql/engine/streaming_ray_chunking.py +419 -0
  57. contextsynapse/aiql/grammar/__init__.py +14 -0
  58. contextsynapse/aiql/grammar/aiql_grammar.py +1822 -0
  59. contextsynapse/aiql/grammar/base.py +71 -0
  60. contextsynapse/aiql/parser/__init__.py +46 -0
  61. contextsynapse/aiql/parser/aiql_parser.py +11330 -0
  62. contextsynapse/aiql/parser/base_parser.py +47 -0
  63. contextsynapse/api/__init__.py +5 -0
  64. contextsynapse/api/a2a_router.py +233 -0
  65. contextsynapse/api/admin_router.py +140 -0
  66. contextsynapse/api/agent_worker_router.py +978 -0
  67. contextsynapse/api/algorithms_router.py +317 -0
  68. contextsynapse/api/api.py +4144 -0
  69. contextsynapse/api/audit.py +204 -0
  70. contextsynapse/api/auth.py +441 -0
  71. contextsynapse/api/auth_router.py +537 -0
  72. contextsynapse/api/billing.py +261 -0
  73. contextsynapse/api/billing_router.py +275 -0
  74. contextsynapse/api/boundary_router.py +452 -0
  75. contextsynapse/api/cognition_router.py +116 -0
  76. contextsynapse/api/connector_router.py +237 -0
  77. contextsynapse/api/context_router.py +1619 -0
  78. contextsynapse/api/dashboard_router.py +11367 -0
  79. contextsynapse/api/errors.py +183 -0
  80. contextsynapse/api/events.py +88 -0
  81. contextsynapse/api/experiment_bootstrap.py +321 -0
  82. contextsynapse/api/experiment_orchestrator.py +1078 -0
  83. contextsynapse/api/experiment_router.py +2089 -0
  84. contextsynapse/api/feature_gate.py +89 -0
  85. contextsynapse/api/federation_router.py +108 -0
  86. contextsynapse/api/integrations.py +567 -0
  87. contextsynapse/api/integrations_router.py +461 -0
  88. contextsynapse/api/intelligence_router.py +1028 -0
  89. contextsynapse/api/metering.py +155 -0
  90. contextsynapse/api/metrics.py +112 -0
  91. contextsynapse/api/models.py +38 -0
  92. contextsynapse/api/monitoring_router.py +174 -0
  93. contextsynapse/api/pipeline_router.py +1437 -0
  94. contextsynapse/api/playground_conn.py +116 -0
  95. contextsynapse/api/projects_router.py +131 -0
  96. contextsynapse/api/queries_router.py +282 -0
  97. contextsynapse/api/quota.py +95 -0
  98. contextsynapse/api/rate_limit.py +64 -0
  99. contextsynapse/api/search_router.py +261 -0
  100. contextsynapse/api/shield_router.py +447 -0
  101. contextsynapse/api/tenant_guard.py +103 -0
  102. contextsynapse/api/tenant_quotas.py +231 -0
  103. contextsynapse/api/tenants.py +305 -0
  104. contextsynapse/api/users.py +646 -0
  105. contextsynapse/api/v1.py +167 -0
  106. contextsynapse/api/vertical_builder_router.py +387 -0
  107. contextsynapse/benchmark/__init__.py +15 -0
  108. contextsynapse/benchmark/__main__.py +64 -0
  109. contextsynapse/benchmark/agents.py +338 -0
  110. contextsynapse/benchmark/live_ab.py +338 -0
  111. contextsynapse/benchmark/reporter.py +149 -0
  112. contextsynapse/benchmark/runner.py +264 -0
  113. contextsynapse/benchmark/savings.py +266 -0
  114. contextsynapse/benchmark/token_counter.py +212 -0
  115. contextsynapse/benchmark/workloads.py +289 -0
  116. contextsynapse/buffer/__init__.py +154 -0
  117. contextsynapse/buffer/base.py +257 -0
  118. contextsynapse/buffer/duckdb_buffer.py +246 -0
  119. contextsynapse/buffer/file_buffer.py +229 -0
  120. contextsynapse/buffer/lmdb_buffer.py +286 -0
  121. contextsynapse/buffer/local_buffer.py +154 -0
  122. contextsynapse/buffer/redis_buffer.py +211 -0
  123. contextsynapse/buffer/sqlite_buffer.py +266 -0
  124. contextsynapse/cache.py +2 -0
  125. contextsynapse/cli/__init__.py +40 -0
  126. contextsynapse/cli/cli.py +475 -0
  127. contextsynapse/cli/config_cli.py +177 -0
  128. contextsynapse/cli/scaffold.py +465 -0
  129. contextsynapse/cognition/__init__.py +31 -0
  130. contextsynapse/cognition/derivation.py +55 -0
  131. contextsynapse/cognition/events.py +135 -0
  132. contextsynapse/cognition/feedback.py +52 -0
  133. contextsynapse/cognition/hallucination.py +90 -0
  134. contextsynapse/cognition/invalidation.py +70 -0
  135. contextsynapse/cognition/read_tracker.py +50 -0
  136. contextsynapse/collaboration.py +2 -0
  137. contextsynapse/comparison.py +2 -0
  138. contextsynapse/config/__init__.py +10 -0
  139. contextsynapse/config/connectors/amfi_nav.yaml +18 -0
  140. contextsynapse/config/connectors/auto_sector.yaml +20 -0
  141. contextsynapse/config/connectors/banking_sector.yaml +20 -0
  142. contextsynapse/config/connectors/energy_sector.yaml +20 -0
  143. contextsynapse/config/connectors/fmcg_sector.yaml +20 -0
  144. contextsynapse/config/connectors/nsdl_fpi.yaml +9 -0
  145. contextsynapse/config/connectors/nse_price.yaml +61 -0
  146. contextsynapse/config/connectors/pharma_sector.yaml +20 -0
  147. contextsynapse/config/connectors/rbi_dbie.yaml +12 -0
  148. contextsynapse/config/context_presets/ai_research.json +25 -0
  149. contextsynapse/config/context_presets/chatgpt_conversations.json +19 -0
  150. contextsynapse/config/context_presets/claude_conversations.json +19 -0
  151. contextsynapse/config/context_presets/legal_compliance.json +21 -0
  152. contextsynapse/config/context_presets/meeting_notes.json +21 -0
  153. contextsynapse/config/context_presets/slack_engineering.json +21 -0
  154. contextsynapse/config/context_presets/tech_climate.json +24 -0
  155. contextsynapse/config/context_presets/toi_politics.json +25 -0
  156. contextsynapse/config/database_config.py +453 -0
  157. contextsynapse/config/deployment.py +221 -0
  158. contextsynapse/config/gpt_action_schema.json +164 -0
  159. contextsynapse/config/indicators/india_macro.yaml +213 -0
  160. contextsynapse/config/logging_config.py +88 -0
  161. contextsynapse/config/mode_config.py +223 -0
  162. contextsynapse/config/schemas/api_spec.yaml +48 -0
  163. contextsynapse/config/schemas/architecture_doc.yaml +51 -0
  164. contextsynapse/config/schemas/chat_history.yaml +90 -0
  165. contextsynapse/config/schemas/chatgpt_conversation.yaml +101 -0
  166. contextsynapse/config/schemas/claude_conversation.yaml +148 -0
  167. contextsynapse/config/schemas/contract.yaml +48 -0
  168. contextsynapse/config/schemas/conversation.yaml +167 -0
  169. contextsynapse/config/schemas/database.yaml +74 -0
  170. contextsynapse/config/schemas/decision.yaml +72 -0
  171. contextsynapse/config/schemas/ecommerce.yaml +56 -0
  172. contextsynapse/config/schemas/education.yaml +42 -0
  173. contextsynapse/config/schemas/healthcare.yaml +133 -0
  174. contextsynapse/config/schemas/hr_document.yaml +48 -0
  175. contextsynapse/config/schemas/invoice.yaml +40 -0
  176. contextsynapse/config/schemas/knowledge_base.yaml +75 -0
  177. contextsynapse/config/schemas/legal.yaml +42 -0
  178. contextsynapse/config/schemas/marketing_content.yaml +48 -0
  179. contextsynapse/config/schemas/meeting_notes.yaml +51 -0
  180. contextsynapse/config/schemas/news_article.yaml +61 -0
  181. contextsynapse/config/schemas/real_estate.yaml +49 -0
  182. contextsynapse/config/schemas/requirements_doc.yaml +48 -0
  183. contextsynapse/config/schemas/research_paper.yaml +56 -0
  184. contextsynapse/config/schemas/resume.yaml +49 -0
  185. contextsynapse/config/schemas/retail.yaml +42 -0
  186. contextsynapse/config/schemas/rule_meta.yaml +173 -0
  187. contextsynapse/config/schemas/rules.yaml +72 -0
  188. contextsynapse/config/schemas/screener_meta.yaml +236 -0
  189. contextsynapse/config/schemas/sdlc.yaml +103 -0
  190. contextsynapse/config/schemas/session_graph.yaml +172 -0
  191. contextsynapse/config/schemas/support_ticket.yaml +48 -0
  192. contextsynapse/config/schemas/web.yaml +64 -0
  193. contextsynapse/config/sensors/earnings_calendar.yaml +52 -0
  194. contextsynapse/config/sensors/global_risk.yaml +53 -0
  195. contextsynapse/config/sensors/government_policy.yaml +54 -0
  196. contextsynapse/config/sensors/supply_chain.yaml +50 -0
  197. contextsynapse/config/sensors/templates/behavioral.yaml +56 -0
  198. contextsynapse/config/sensors/templates/equity_sentiment.yaml +48 -0
  199. contextsynapse/config/sensors/templates/global_macro.yaml +47 -0
  200. contextsynapse/config/sensors/templates/market_data.yaml +118 -0
  201. contextsynapse/config/sensors/templates/sector_technical.yaml +47 -0
  202. contextsynapse/config/sensors/weather.yaml +52 -0
  203. contextsynapse/config/storage_strategy.py +202 -0
  204. contextsynapse/config/templates/competitive_intel.yaml +39 -0
  205. contextsynapse/config/templates/geopolitical_risk.yaml +46 -0
  206. contextsynapse/config/templates/market_analysis.yaml +58 -0
  207. contextsynapse/connectors/__init__.py +15 -0
  208. contextsynapse/connectors/api_router.py +121 -0
  209. contextsynapse/connectors/base.py +357 -0
  210. contextsynapse/connectors/financial/audio.py +170 -0
  211. contextsynapse/connectors/financial/youtube.py +272 -0
  212. contextsynapse/connectors/github.py +164 -0
  213. contextsynapse/connectors/jira.py +120 -0
  214. contextsynapse/connectors/kafka_connector.py +124 -0
  215. contextsynapse/connectors/pipeline_connector.py +747 -0
  216. contextsynapse/connectors/registry.py +116 -0
  217. contextsynapse/connectors/scheduler.py +87 -0
  218. contextsynapse/connectors/universal.py +360 -0
  219. contextsynapse/context/__init__.py +96 -0
  220. contextsynapse/context/__main__.py +4 -0
  221. contextsynapse/context/acl.py +125 -0
  222. contextsynapse/context/agent_card.py +201 -0
  223. contextsynapse/context/agent_memory.py +814 -0
  224. contextsynapse/context/agents.py +472 -0
  225. contextsynapse/context/agents_redis.py +391 -0
  226. contextsynapse/context/assembled.py +204 -0
  227. contextsynapse/context/attribution.py +113 -0
  228. contextsynapse/context/blob.py +184 -0
  229. contextsynapse/context/boundaries.py +187 -0
  230. contextsynapse/context/boundary.py +507 -0
  231. contextsynapse/context/bundle.py +155 -0
  232. contextsynapse/context/cli.py +519 -0
  233. contextsynapse/context/collaboration.py +165 -0
  234. contextsynapse/context/compiler.py +604 -0
  235. contextsynapse/context/composite.py +536 -0
  236. contextsynapse/context/compression.py +147 -0
  237. contextsynapse/context/context_manager.py +888 -0
  238. contextsynapse/context/context_schema.py +90 -0
  239. contextsynapse/context/context_schema.yaml +105 -0
  240. contextsynapse/context/context_units.py +891 -0
  241. contextsynapse/context/conversation.py +488 -0
  242. contextsynapse/context/dedup.py +207 -0
  243. contextsynapse/context/document_processor.py +698 -0
  244. contextsynapse/context/embedding_hooks.py +338 -0
  245. contextsynapse/context/execution_schema.py +274 -0
  246. contextsynapse/context/fan_out.py +138 -0
  247. contextsynapse/context/frozen.py +80 -0
  248. contextsynapse/context/gravity.py +227 -0
  249. contextsynapse/context/hub.py +1401 -0
  250. contextsynapse/context/ingest.py +391 -0
  251. contextsynapse/context/injection.py +226 -0
  252. contextsynapse/context/interaction_graphifier.py +562 -0
  253. contextsynapse/context/layers.py +574 -0
  254. contextsynapse/context/projection.py +1195 -0
  255. contextsynapse/context/promotion.py +634 -0
  256. contextsynapse/context/proof.py +131 -0
  257. contextsynapse/context/propagation.py +577 -0
  258. contextsynapse/context/pubsub.py +135 -0
  259. contextsynapse/context/quality.py +1054 -0
  260. contextsynapse/context/quality_tracker.py +270 -0
  261. contextsynapse/context/scoping.py +370 -0
  262. contextsynapse/context/seed.py +992 -0
  263. contextsynapse/context/session.py +1142 -0
  264. contextsynapse/context/session_graph.py +593 -0
  265. contextsynapse/context/session_memory.py +309 -0
  266. contextsynapse/context/session_resolver.py +111 -0
  267. contextsynapse/context/skill_learning.py +237 -0
  268. contextsynapse/context/smart_budget.py +129 -0
  269. contextsynapse/context/source_policy.py +242 -0
  270. contextsynapse/context/store_factory.py +53 -0
  271. contextsynapse/context/sync.py +648 -0
  272. contextsynapse/context/token.py +236 -0
  273. contextsynapse/context/vector_integration.py +297 -0
  274. contextsynapse/context/webhooks.py +194 -0
  275. contextsynapse/context/working_memory.py +224 -0
  276. contextsynapse/core/__init__.py +18 -0
  277. contextsynapse/core/archiver.py +237 -0
  278. contextsynapse/core/change_stream.py +282 -0
  279. contextsynapse/core/checkpoint.py +310 -0
  280. contextsynapse/core/cloud_storage.py +387 -0
  281. contextsynapse/core/context_state.py +651 -0
  282. contextsynapse/core/cron.py +324 -0
  283. contextsynapse/core/datasource.py +643 -0
  284. contextsynapse/core/db.py +289 -0
  285. contextsynapse/core/federation.py +283 -0
  286. contextsynapse/core/graph_coordinator.py +201 -0
  287. contextsynapse/core/graph_intelligence.py +388 -0
  288. contextsynapse/core/graph_structures.py +106 -0
  289. contextsynapse/core/graph_sync.py +157 -0
  290. contextsynapse/core/hybrid_graph_storage.py +2998 -0
  291. contextsynapse/core/lifecycle.py +691 -0
  292. contextsynapse/core/multiworker.py +378 -0
  293. contextsynapse/core/pruning.py +312 -0
  294. contextsynapse/core/redis_registry.py +524 -0
  295. contextsynapse/core/registry.py +1043 -0
  296. contextsynapse/core/registry_factory.py +51 -0
  297. contextsynapse/core/registry_metadata.py +294 -0
  298. contextsynapse/core/replay.py +354 -0
  299. contextsynapse/core/replication.py +339 -0
  300. contextsynapse/core/time_travel.py +260 -0
  301. contextsynapse/core/write_behind.py +152 -0
  302. contextsynapse/core/write_context.py +39 -0
  303. contextsynapse/db/__init__.py +1 -0
  304. contextsynapse/db/postgres.py +267 -0
  305. contextsynapse/db/profile.py +236 -0
  306. contextsynapse/db/rules.py +328 -0
  307. contextsynapse/demo/__init__.py +0 -0
  308. contextsynapse/demo/report.py +147 -0
  309. contextsynapse/demo/runner.py +520 -0
  310. contextsynapse/demo/scenario.py +165 -0
  311. contextsynapse/demo/session_demo.py +280 -0
  312. contextsynapse/demo/session_report.py +143 -0
  313. contextsynapse/demo/universal_runner.py +257 -0
  314. contextsynapse/engine/__init__.py +15 -0
  315. contextsynapse/engine/core.py +555 -0
  316. contextsynapse/extraction/__init__.py +69 -0
  317. contextsynapse/extraction/cli.py +274 -0
  318. contextsynapse/extraction/config.py +107 -0
  319. contextsynapse/extraction/examples/example_usage.py +118 -0
  320. contextsynapse/extraction/examples/sample_manifest.json +53 -0
  321. contextsynapse/extraction/examples/sample_normalized.json +185 -0
  322. contextsynapse/extraction/extract_engine.py +440 -0
  323. contextsynapse/extraction/extractors/__init__.py +37 -0
  324. contextsynapse/extraction/extractors/chat_export_extractor.py +189 -0
  325. contextsynapse/extraction/extractors/csv_extractor.py +83 -0
  326. contextsynapse/extraction/extractors/docx_extractor.py +129 -0
  327. contextsynapse/extraction/extractors/excel_extractor.py +152 -0
  328. contextsynapse/extraction/extractors/html_extractor.py +146 -0
  329. contextsynapse/extraction/extractors/pdf_extractor.py +635 -0
  330. contextsynapse/extraction/extractors/pdf_extractor_layoutparser.py +408 -0
  331. contextsynapse/extraction/extractors/register_extractors.py +63 -0
  332. contextsynapse/extraction/extractors/text_extractor.py +65 -0
  333. contextsynapse/extraction/extractors/txt_extractor.py +120 -0
  334. contextsynapse/extraction/extractors/website_extractor.py +246 -0
  335. contextsynapse/extraction/fact_extractor.py +236 -0
  336. contextsynapse/extraction/file_manager.py +353 -0
  337. contextsynapse/extraction/hierarchy.py +239 -0
  338. contextsynapse/extraction/id_generator.py +450 -0
  339. contextsynapse/extraction/layout_detector.py +409 -0
  340. contextsynapse/extraction/llm_entity_extractor.py +301 -0
  341. contextsynapse/extraction/normalized_store.py +480 -0
  342. contextsynapse/extraction/normalizer.py +594 -0
  343. contextsynapse/extraction/parquet_writer.py +598 -0
  344. contextsynapse/extraction/post_processors/__init__.py +40 -0
  345. contextsynapse/extraction/post_processors/column_reorganizer.py +507 -0
  346. contextsynapse/extraction/ray_normalization.py +233 -0
  347. contextsynapse/extraction/ray_runner.py +207 -0
  348. contextsynapse/extraction/redis_schema_registry.py +222 -0
  349. contextsynapse/extraction/registry.py +204 -0
  350. contextsynapse/extraction/schema_loader.py +663 -0
  351. contextsynapse/extraction/schema_registry.py +230 -0
  352. contextsynapse/extraction/semantic_normalizer.py +245 -0
  353. contextsynapse/extraction/version_manager.py +314 -0
  354. contextsynapse/gateway/__init__.py +24 -0
  355. contextsynapse/gateway/action_emitter.py +212 -0
  356. contextsynapse/gateway/cost_tracker.py +508 -0
  357. contextsynapse/gateway/gateway.py +377 -0
  358. contextsynapse/gateway/policy.py +303 -0
  359. contextsynapse/governance/__init__.py +13 -0
  360. contextsynapse/governance/layer.py +513 -0
  361. contextsynapse/ingestion/__init__.py +15 -0
  362. contextsynapse/ingestion/amplifier.py +288 -0
  363. contextsynapse/ingestion/async_ingest.py +463 -0
  364. contextsynapse/ingestion/chunker.py +303 -0
  365. contextsynapse/ingestion/chunking.py +58 -0
  366. contextsynapse/ingestion/cleaner.py +190 -0
  367. contextsynapse/ingestion/cleanse.py +252 -0
  368. contextsynapse/ingestion/connectors/__init__.py +38 -0
  369. contextsynapse/ingestion/connectors/base.py +300 -0
  370. contextsynapse/ingestion/connectors/chatgpt_memory.py +247 -0
  371. contextsynapse/ingestion/connectors/claude_memory.py +274 -0
  372. contextsynapse/ingestion/connectors/sap_ingestor.py +208 -0
  373. contextsynapse/ingestion/content_detector.py +80 -0
  374. contextsynapse/ingestion/content_monitor.py +502 -0
  375. contextsynapse/ingestion/dedup.py +106 -0
  376. contextsynapse/ingestion/feed.py +640 -0
  377. contextsynapse/ingestion/filters.py +461 -0
  378. contextsynapse/ingestion/graph_builder.py +705 -0
  379. contextsynapse/ingestion/input_classifier.py +307 -0
  380. contextsynapse/ingestion/job_manager.py +571 -0
  381. contextsynapse/ingestion/llm_extractor.py +610 -0
  382. contextsynapse/ingestion/orchestrator.py +116 -0
  383. contextsynapse/ingestion/parsers/__init__.py +19 -0
  384. contextsynapse/ingestion/parsers/graph_parser_registry.py +103 -0
  385. contextsynapse/ingestion/parsers/graphml_parser.py +141 -0
  386. contextsynapse/ingestion/parsers/jsonld_parser.py +171 -0
  387. contextsynapse/ingestion/parsers/rdf_parser.py +150 -0
  388. contextsynapse/ingestion/pipeline_context.py +228 -0
  389. contextsynapse/ingestion/pipeline_store.py +1195 -0
  390. contextsynapse/ingestion/queue.py +133 -0
  391. contextsynapse/ingestion/reprocess.py +179 -0
  392. contextsynapse/ingestion/scenario_router.py +224 -0
  393. contextsynapse/ingestion/schema_extractor.py +759 -0
  394. contextsynapse/ingestion/schema_validator.py +274 -0
  395. contextsynapse/ingestion/sdlc_ingest.py +87 -0
  396. contextsynapse/ingestion/smart_ingest.py +2451 -0
  397. contextsynapse/ingestion/source_connector.py +171 -0
  398. contextsynapse/ingestion/stage_executor.py +2095 -0
  399. contextsynapse/ingestion/stages/__init__.py +9 -0
  400. contextsynapse/ingestion/stages/tabular_to_graph.py +289 -0
  401. contextsynapse/ingestion/strategies.py +386 -0
  402. contextsynapse/ingestion/universal/__init__.py +4 -0
  403. contextsynapse/ingestion/universal/_operator_registry.py +59 -0
  404. contextsynapse/ingestion/universal/ingest_content.py +32 -0
  405. contextsynapse/ingestion/universal/operators/__init__.py +1 -0
  406. contextsynapse/ingestion/universal/operators/base.py +19 -0
  407. contextsynapse/ingestion/universal/operators/build_edges.py +75 -0
  408. contextsynapse/ingestion/universal/operators/cluster_topics.py +397 -0
  409. contextsynapse/ingestion/universal/operators/deduplicate.py +45 -0
  410. contextsynapse/ingestion/universal/operators/detect_signals.py +90 -0
  411. contextsynapse/ingestion/universal/operators/embed.py +80 -0
  412. contextsynapse/ingestion/universal/operators/extract_entities.py +253 -0
  413. contextsynapse/ingestion/universal/operators/extract_preferences.py +117 -0
  414. contextsynapse/ingestion/universal/operators/filter_content.py +69 -0
  415. contextsynapse/ingestion/universal/operators/index_bm25.py +52 -0
  416. contextsynapse/ingestion/universal/operators/infer_domains.py +69 -0
  417. contextsynapse/ingestion/universal/operators/link_cross_reference.py +210 -0
  418. contextsynapse/ingestion/universal/operators/parse_records.py +66 -0
  419. contextsynapse/ingestion/universal/operators/resolve_entities.py +261 -0
  420. contextsynapse/ingestion/universal/operators/scanners/__init__.py +16 -0
  421. contextsynapse/ingestion/universal/operators/scanners/edges.py +173 -0
  422. contextsynapse/ingestion/universal/operators/scanners/git.py +187 -0
  423. contextsynapse/ingestion/universal/operators/scanners/github.py +135 -0
  424. contextsynapse/ingestion/universal/operators/scanners/github_api.py +509 -0
  425. contextsynapse/ingestion/universal/operators/scanners/helpers.py +67 -0
  426. contextsynapse/ingestion/universal/operators/scanners/jira.py +65 -0
  427. contextsynapse/ingestion/universal/operators/scanners/llm_enrichment.py +82 -0
  428. contextsynapse/ingestion/universal/operators/scanners/repo.py +149 -0
  429. contextsynapse/ingestion/universal/operators/sdlc_scan.py +213 -0
  430. contextsynapse/ingestion/universal/operators/spec_scanner.py +149 -0
  431. contextsynapse/ingestion/universal/operators/store_documents.py +72 -0
  432. contextsynapse/ingestion/universal/operators/synthesize_cu.py +180 -0
  433. contextsynapse/ingestion/universal/operators/validate_gate.py +65 -0
  434. contextsynapse/ingestion/universal/parsers/__init__.py +5 -0
  435. contextsynapse/ingestion/universal/parsers/record_parser.py +203 -0
  436. contextsynapse/ingestion/universal/parsers/turn_parser.py +329 -0
  437. contextsynapse/ingestion/universal/shared_signals.py +161 -0
  438. contextsynapse/ingestion/universal/stage_executor.py +219 -0
  439. contextsynapse/ingestion/web_crawler.py +265 -0
  440. contextsynapse/intelligence/__init__.py +53 -0
  441. contextsynapse/intelligence/alerts.py +198 -0
  442. contextsynapse/intelligence/collector.py +655 -0
  443. contextsynapse/intelligence/comparison.py +198 -0
  444. contextsynapse/intelligence/config.py +56 -0
  445. contextsynapse/intelligence/conflict_detector.py +182 -0
  446. contextsynapse/intelligence/context_gaps.py +192 -0
  447. contextsynapse/intelligence/context_radar.py +135 -0
  448. contextsynapse/intelligence/correlation_engine.py +416 -0
  449. contextsynapse/intelligence/correlation_template.py +120 -0
  450. contextsynapse/intelligence/dedup.py +296 -0
  451. contextsynapse/intelligence/event_bus.py +91 -0
  452. contextsynapse/intelligence/feedback_loop.py +101 -0
  453. contextsynapse/intelligence/freshness.py +307 -0
  454. contextsynapse/intelligence/fundamental_feed.py +551 -0
  455. contextsynapse/intelligence/geo_data.py +170 -0
  456. contextsynapse/intelligence/geo_reference.py +112 -0
  457. contextsynapse/intelligence/graph_fusion.py +465 -0
  458. contextsynapse/intelligence/heuristics.py +176 -0
  459. contextsynapse/intelligence/impact_tracker.py +336 -0
  460. contextsynapse/intelligence/macro_indicators.py +319 -0
  461. contextsynapse/intelligence/ocr.py +104 -0
  462. contextsynapse/intelligence/persistence.py +267 -0
  463. contextsynapse/intelligence/pipeline_health.py +130 -0
  464. contextsynapse/intelligence/price_feed.py +221 -0
  465. contextsynapse/intelligence/rate_limiter.py +130 -0
  466. contextsynapse/intelligence/reactive.py +411 -0
  467. contextsynapse/intelligence/runtime_context.py +390 -0
  468. contextsynapse/intelligence/sentiment_decay.py +426 -0
  469. contextsynapse/intelligence/session.py +277 -0
  470. contextsynapse/intelligence/signal_hierarchy.py +438 -0
  471. contextsynapse/intelligence/signals.py +185 -0
  472. contextsynapse/intelligence/source_watcher.py +132 -0
  473. contextsynapse/intelligence/table_enricher.py +256 -0
  474. contextsynapse/intelligence/tagger.py +319 -0
  475. contextsynapse/intelligence/timeseries.py +209 -0
  476. contextsynapse/intelligence/tracker.py +318 -0
  477. contextsynapse/intelligence/watchdog.py +392 -0
  478. contextsynapse/intelligence/watchdog_manager.py +263 -0
  479. contextsynapse/intelligence/webhook_receiver.py +165 -0
  480. contextsynapse/intelligence/weight_learner.py +234 -0
  481. contextsynapse/llm/__init__.py +8 -0
  482. contextsynapse/llm/client.py +348 -0
  483. contextsynapse/llm/rate_limiter.py +166 -0
  484. contextsynapse/logging_config.py +2 -0
  485. contextsynapse/marketplace/__init__.py +7 -0
  486. contextsynapse/marketplace/registry.py +173 -0
  487. contextsynapse/mcp/__init__.py +11 -0
  488. contextsynapse/mcp/__main__.py +4 -0
  489. contextsynapse/mcp/auth_middleware.py +212 -0
  490. contextsynapse/mcp/connection_pool.py +128 -0
  491. contextsynapse/mcp/server.py +986 -0
  492. contextsynapse/mcp/session_context.py +67 -0
  493. contextsynapse/memory/__init__.py +34 -0
  494. contextsynapse/memory/engine.py +475 -0
  495. contextsynapse/memory/temporal.py +327 -0
  496. contextsynapse/metadata/__init__.py +12 -0
  497. contextsynapse/metadata/metadata_db.py +272 -0
  498. contextsynapse/metadata/metadata_query_tool.py +300 -0
  499. contextsynapse/metadata/metadata_tracker.py +425 -0
  500. contextsynapse/models/__init__.py +27 -0
  501. contextsynapse/models/embedding_service.py +360 -0
  502. contextsynapse/models/model_registry.py +228 -0
  503. contextsynapse/multiworker.py +2 -0
  504. contextsynapse/pipelines/__init__.py +1 -0
  505. contextsynapse/pipelines/executor.py +851 -0
  506. contextsynapse/pipelines/filters.py +189 -0
  507. contextsynapse/pipelines/models.py +176 -0
  508. contextsynapse/pipelines/scheduler.py +268 -0
  509. contextsynapse/playground/__init__.py +1 -0
  510. contextsynapse/playground/templates.py +85 -0
  511. contextsynapse/plugins/__init__.py +43 -0
  512. contextsynapse/plugins/api.py +66 -0
  513. contextsynapse/plugins/base.py +350 -0
  514. contextsynapse/plugins/domain_interface.py +508 -0
  515. contextsynapse/plugins/domain_loader.py +241 -0
  516. contextsynapse/plugins/loader.py +148 -0
  517. contextsynapse/plugins/registry.py +265 -0
  518. contextsynapse/project/__init__.py +11 -0
  519. contextsynapse/project/code_context.py +807 -0
  520. contextsynapse/project/connectors/__init__.py +4 -0
  521. contextsynapse/project/connectors/github_connector.py +113 -0
  522. contextsynapse/project/connectors/jira_connector.py +160 -0
  523. contextsynapse/project/graph.py +1023 -0
  524. contextsynapse/project/project_context.py +545 -0
  525. contextsynapse/project/scanner_registry.py +61 -0
  526. contextsynapse/project/schema_manager.py +322 -0
  527. contextsynapse/project/schemas/knowledge.yaml +59 -0
  528. contextsynapse/project/schemas/sdlc.yaml +369 -0
  529. contextsynapse/project/sdlc_schema.py +172 -0
  530. contextsynapse/project/spec_parser.py +85 -0
  531. contextsynapse/project/task_lock.py +181 -0
  532. contextsynapse/project/task_stream.py +203 -0
  533. contextsynapse/project/team_tools.py +537 -0
  534. contextsynapse/py.typed +0 -0
  535. contextsynapse/realtime/__init__.py +6 -0
  536. contextsynapse/realtime/collaboration.py +147 -0
  537. contextsynapse/rules/__init__.py +10 -0
  538. contextsynapse/rules/engine.py +116 -0
  539. contextsynapse/rules/evaluators/__init__.py +6 -0
  540. contextsynapse/rules/evaluators/aggregate.py +205 -0
  541. contextsynapse/rules/evaluators/custom.py +170 -0
  542. contextsynapse/rules/evaluators/pre_action.py +125 -0
  543. contextsynapse/rules/evaluators/temporal.py +129 -0
  544. contextsynapse/rules/loader.py +52 -0
  545. contextsynapse/rules/models.py +124 -0
  546. contextsynapse/rules/universal.py +393 -0
  547. contextsynapse/scheduler.py +229 -0
  548. contextsynapse/schema/__init__.py +42 -0
  549. contextsynapse/schema/compiler.py +252 -0
  550. contextsynapse/schema/composer.py +158 -0
  551. contextsynapse/schema/dedup.py +73 -0
  552. contextsynapse/schema/derivation.py +172 -0
  553. contextsynapse/schema/sdl.py +326 -0
  554. contextsynapse/schema/validation_gate.py +124 -0
  555. contextsynapse/sdk/__init__.py +63 -0
  556. contextsynapse/sdk/agent.py +214 -0
  557. contextsynapse/sdk/client.py +269 -0
  558. contextsynapse/sdk/exceptions.py +23 -0
  559. contextsynapse/sdk/models.py +78 -0
  560. contextsynapse/sdk/session.py +408 -0
  561. contextsynapse/sdk/tracked.py +218 -0
  562. contextsynapse/sdk/worker.py +266 -0
  563. contextsynapse/sdk/workspace_client.py +98 -0
  564. contextsynapse/sdk/ws.py +41 -0
  565. contextsynapse/search/__init__.py +20 -0
  566. contextsynapse/search/embedding_cache.py +140 -0
  567. contextsynapse/search/enhanced_search.py +152 -0
  568. contextsynapse/search/fulltext.py +130 -0
  569. contextsynapse/search/graph_search.py +634 -0
  570. contextsynapse/search/lmdb_index.py +1218 -0
  571. contextsynapse/search/nl_to_aiql.py +199 -0
  572. contextsynapse/search/rag.py +1346 -0
  573. contextsynapse/search/rag_cache.py +167 -0
  574. contextsynapse/search/reasoning_chain.py +231 -0
  575. contextsynapse/search/redis_search.py +207 -0
  576. contextsynapse/search/retrieval_quality.py +314 -0
  577. contextsynapse/search/semantic_query.py +455 -0
  578. contextsynapse/search/text_resolver.py +159 -0
  579. contextsynapse/search/whoosh_search.py +373 -0
  580. contextsynapse/security/__init__.py +17 -0
  581. contextsynapse/security/audit.py +297 -0
  582. contextsynapse/security/audit_logger.py +97 -0
  583. contextsynapse/security/audit_trail.py +270 -0
  584. contextsynapse/security/auth_unified.py +284 -0
  585. contextsynapse/security/auto_tagger.py +230 -0
  586. contextsynapse/security/data_security.py +218 -0
  587. contextsynapse/security/encryption.py +207 -0
  588. contextsynapse/security/identity.py +447 -0
  589. contextsynapse/security/jwt_identity.py +59 -0
  590. contextsynapse/security/middleware.py +336 -0
  591. contextsynapse/security/pii.py +233 -0
  592. contextsynapse/security/rbac.py +96 -0
  593. contextsynapse/security/rls.py +205 -0
  594. contextsynapse/security/sanitize.py +233 -0
  595. contextsynapse/security/scoped_encryption.py +131 -0
  596. contextsynapse/security/tenant.py +401 -0
  597. contextsynapse/shield/__init__.py +21 -0
  598. contextsynapse/shield/anomaly.py +56 -0
  599. contextsynapse/shield/permissions.py +99 -0
  600. contextsynapse/shield/profile.py +103 -0
  601. contextsynapse/shield/shield.py +108 -0
  602. contextsynapse/shield/trust_engine.py +69 -0
  603. contextsynapse/skills/__init__.py +14 -0
  604. contextsynapse/skills/engine.py +233 -0
  605. contextsynapse/skills/loader.py +59 -0
  606. contextsynapse/skills/models.py +139 -0
  607. contextsynapse/storage/__init__.py +7 -0
  608. contextsynapse/storage/cache.py +208 -0
  609. contextsynapse/storage/columnar_store_v2.py +178 -0
  610. contextsynapse/storage/csr_graph_storage.py +587 -0
  611. contextsynapse/storage/csr_store.py +388 -0
  612. contextsynapse/storage/document_store.py +491 -0
  613. contextsynapse/storage/hnsw_index.py +294 -0
  614. contextsynapse/storage/hybrid_store_v2.py +278 -0
  615. contextsynapse/storage/lmdb_graph_storage.py +366 -0
  616. contextsynapse/storage/migration.py +279 -0
  617. contextsynapse/storage/namespace_store.py +346 -0
  618. contextsynapse/storage/pure_graph_storage.py +185 -0
  619. contextsynapse/storage/redis_graph_adapter.py +186 -0
  620. contextsynapse/storage/redis_graph_storage.py +375 -0
  621. contextsynapse/storage/router/__init__.py +29 -0
  622. contextsynapse/storage/router/duckdb_store.py +398 -0
  623. contextsynapse/storage/router/factory.py +83 -0
  624. contextsynapse/storage/router/interface.py +125 -0
  625. contextsynapse/storage/router/postgres_store.py +339 -0
  626. contextsynapse/storage/router/resolver.py +192 -0
  627. contextsynapse/storage/router/router.py +300 -0
  628. contextsynapse/storage/single_file_storage.py +391 -0
  629. contextsynapse/storage/storage_manager.py +214 -0
  630. contextsynapse/storage/storage_query_optimizer.py +314 -0
  631. contextsynapse/storage/wal.py +359 -0
  632. contextsynapse/temporal/__init__.py +0 -0
  633. contextsynapse/temporal/storage.py +106 -0
  634. contextsynapse/tests/__init__.py +11 -0
  635. contextsynapse/tests/regression/__init__.py +1 -0
  636. contextsynapse/tests/regression/add_metadata.py +395 -0
  637. contextsynapse/tests/regression/add_skip_flags.py +85 -0
  638. contextsynapse/tests/regression/cases/test_basic_create_node.yaml +20 -0
  639. contextsynapse/tests/regression/cases/test_basic_select.yaml +21 -0
  640. contextsynapse/tests/regression/cases/test_blockchain_audit_trail.yaml +37 -0
  641. contextsynapse/tests/regression/cases/test_blockchain_audit_trail_filtered.yaml +39 -0
  642. contextsynapse/tests/regression/cases/test_blockchain_blocks_range.yaml +39 -0
  643. contextsynapse/tests/regression/cases/test_blockchain_get_block.yaml +35 -0
  644. contextsynapse/tests/regression/cases/test_blockchain_latest_block.yaml +35 -0
  645. contextsynapse/tests/regression/cases/test_blockchain_length.yaml +37 -0
  646. contextsynapse/tests/regression/cases/test_blockchain_merkle_root.yaml +37 -0
  647. contextsynapse/tests/regression/cases/test_blockchain_verify.yaml +35 -0
  648. contextsynapse/tests/regression/cases/test_blockchain_verify_block.yaml +35 -0
  649. contextsynapse/tests/regression/cases/test_create_edge.yaml +28 -0
  650. contextsynapse/tests/regression/cases/test_delete_edge_basic.yaml +29 -0
  651. contextsynapse/tests/regression/cases/test_delete_node_basic.yaml +29 -0
  652. contextsynapse/tests/regression/cases/test_evaluation_mrr.yaml +40 -0
  653. contextsynapse/tests/regression/cases/test_evaluation_ndcg.yaml +40 -0
  654. contextsynapse/tests/regression/cases/test_evaluation_precision.yaml +40 -0
  655. contextsynapse/tests/regression/cases/test_evaluation_recall.yaml +40 -0
  656. contextsynapse/tests/regression/cases/test_group_by_basic.yaml +56 -0
  657. contextsynapse/tests/regression/cases/test_match_node_basic.yaml +29 -0
  658. contextsynapse/tests/regression/cases/test_match_node_with_order_limit.yaml +31 -0
  659. contextsynapse/tests/regression/cases/test_namespace_switch.yaml +18 -0
  660. contextsynapse/tests/regression/cases/test_pipeline_cascaded.yaml +61 -0
  661. contextsynapse/tests/regression/cases/test_pipeline_fixed_size_chunking.yaml +48 -0
  662. contextsynapse/tests/regression/cases/test_pipeline_gpt4_chunking.yaml +54 -0
  663. contextsynapse/tests/regression/cases/test_pipeline_hybrid_indexing.yaml +57 -0
  664. contextsynapse/tests/regression/cases/test_pipeline_large_embedding.yaml +52 -0
  665. contextsynapse/tests/regression/cases/test_pipeline_paragraph_chunking.yaml +49 -0
  666. contextsynapse/tests/regression/cases/test_pipeline_recursive_chunking.yaml +47 -0
  667. contextsynapse/tests/regression/cases/test_pipeline_run_pipeline.yaml +56 -0
  668. contextsynapse/tests/regression/cases/test_pipeline_section_based_chunking.yaml +53 -0
  669. contextsynapse/tests/regression/cases/test_pipeline_semantic_chunking.yaml +51 -0
  670. contextsynapse/tests/regression/cases/test_pipeline_with_entity_extraction.yaml +57 -0
  671. contextsynapse/tests/regression/cases/test_pipeline_with_relationships.yaml +62 -0
  672. contextsynapse/tests/regression/cases/test_rag_evaluation.yaml +38 -0
  673. contextsynapse/tests/regression/cases/test_rag_graph_bfs_with_eval.yaml +48 -0
  674. contextsynapse/tests/regression/cases/test_rag_graph_centrality.yaml +40 -0
  675. contextsynapse/tests/regression/cases/test_rag_graph_centrality_eval.yaml +35 -0
  676. contextsynapse/tests/regression/cases/test_rag_graph_community.yaml +40 -0
  677. contextsynapse/tests/regression/cases/test_rag_graph_community_eval.yaml +35 -0
  678. contextsynapse/tests/regression/cases/test_rag_graph_dfs_with_eval.yaml +35 -0
  679. contextsynapse/tests/regression/cases/test_rag_graph_multi_hop_eval.yaml +35 -0
  680. contextsynapse/tests/regression/cases/test_rag_graph_path.yaml +40 -0
  681. contextsynapse/tests/regression/cases/test_rag_graph_search_basic.yaml +38 -0
  682. contextsynapse/tests/regression/cases/test_rag_graph_search_bfs.yaml +52 -0
  683. contextsynapse/tests/regression/cases/test_rag_graph_search_dfs.yaml +41 -0
  684. contextsynapse/tests/regression/cases/test_rag_graph_search_multi_hop.yaml +43 -0
  685. contextsynapse/tests/regression/cases/test_rag_graph_search_shortest_path.yaml +41 -0
  686. contextsynapse/tests/regression/cases/test_rag_graph_search_weighted.yaml +43 -0
  687. contextsynapse/tests/regression/cases/test_rag_graph_shortest_path_eval.yaml +35 -0
  688. contextsynapse/tests/regression/cases/test_rag_graph_traversal.yaml +30 -0
  689. contextsynapse/tests/regression/cases/test_rag_graph_weighted_eval.yaml +35 -0
  690. contextsynapse/tests/regression/cases/test_rag_hybrid_search.yaml +33 -0
  691. contextsynapse/tests/regression/cases/test_rag_hybrid_with_prompt_embedding.yaml +44 -0
  692. contextsynapse/tests/regression/cases/test_rag_keyword_search.yaml +32 -0
  693. contextsynapse/tests/regression/cases/test_rag_multi_hop_graph.yaml +32 -0
  694. contextsynapse/tests/regression/cases/test_rag_path_based.yaml +34 -0
  695. contextsynapse/tests/regression/cases/test_rag_profile_based.yaml +32 -0
  696. contextsynapse/tests/regression/cases/test_rag_vector_search.yaml +32 -0
  697. contextsynapse/tests/regression/cases/test_rag_weighted_graph.yaml +34 -0
  698. contextsynapse/tests/regression/cases/test_rag_weighted_hybrid.yaml +31 -0
  699. contextsynapse/tests/regression/cases/test_semantic_boundary_chunking.yaml +57 -0
  700. contextsynapse/tests/regression/cases/test_semantic_boundary_detection.yaml +33 -0
  701. contextsynapse/tests/regression/cases/test_semantic_hash_function.yaml +30 -0
  702. contextsynapse/tests/regression/cases/test_semantic_hash_with_model.yaml +31 -0
  703. contextsynapse/tests/regression/cases/test_tcs_ingestion_fixed_size.yaml +44 -0
  704. contextsynapse/tests/regression/cases/test_tcs_ingestion_recursive.yaml +46 -0
  705. contextsynapse/tests/regression/cases/test_tcs_ingestion_section.yaml +46 -0
  706. contextsynapse/tests/regression/cases/test_tcs_ingestion_semantic.yaml +44 -0
  707. contextsynapse/tests/regression/cases/test_time_travel_as_of.yaml +35 -0
  708. contextsynapse/tests/regression/cases/test_time_travel_at_commit.yaml +33 -0
  709. contextsynapse/tests/regression/cases/test_time_travel_at_timestamp.yaml +33 -0
  710. contextsynapse/tests/regression/cases/test_time_travel_for_system_time_all.yaml +37 -0
  711. contextsynapse/tests/regression/cases/test_time_travel_versions_between.yaml +39 -0
  712. contextsynapse/tests/regression/cases/test_traverse_basic.yaml +31 -0
  713. contextsynapse/tests/regression/cases/test_traverse_max_depth.yaml +34 -0
  714. contextsynapse/tests/regression/cases/test_update_edge_basic.yaml +29 -0
  715. contextsynapse/tests/regression/evaluation_metrics.py +227 -0
  716. contextsynapse/tests/regression/regression_runner.py +1081 -0
  717. contextsynapse/tests/regression/setup_regression_data.py +212 -0
  718. contextsynapse/tests/regression/test_basic_only.py +84 -0
  719. contextsynapse/tests/regression/test_namespace_creation.py +29 -0
  720. contextsynapse/tests/regression/test_single_fix.py +61 -0
  721. contextsynapse/tests/regression/test_single_query.py +20 -0
  722. contextsynapse/tests/regression/validate_test_syntax.py +179 -0
  723. contextsynapse/tests/test_advanced_features.py +347 -0
  724. contextsynapse/tests/test_aiql_summary.py +283 -0
  725. contextsynapse/tests/test_all_aiql_queries.py +948 -0
  726. contextsynapse/tests/test_all_create_queries.py +213 -0
  727. contextsynapse/tests/test_api_vs_native.py +280 -0
  728. contextsynapse/tests/test_chunking_dedup.py +199 -0
  729. contextsynapse/tests/test_connectors.py +171 -0
  730. contextsynapse/tests/test_edges_and_match.py +202 -0
  731. contextsynapse/tests/test_extraction_pipeline_integration.py +581 -0
  732. contextsynapse/tests/test_name_property.py +224 -0
  733. contextsynapse/tests/test_node_edge_metadata.py +250 -0
  734. contextsynapse/tests/test_playground_conn.py +114 -0
  735. contextsynapse/tests/test_query_cache.py +213 -0
  736. contextsynapse/tests/test_source_policy_auth.py +348 -0
  737. contextsynapse/tests/test_temporal_and_domain.py +224 -0
  738. contextsynapse/tests/test_time_travel_e2e.py +571 -0
  739. contextsynapse/tests/test_traversal_and_aggregation.py +275 -0
  740. contextsynapse/tools/__init__.py +36 -0
  741. contextsynapse/tools/a2a_tools.py +106 -0
  742. contextsynapse/tools/cognition_tools.py +118 -0
  743. contextsynapse/tools/context.py +905 -0
  744. contextsynapse/tools/formatting.py +92 -0
  745. contextsynapse/tools/gateway_tools.py +186 -0
  746. contextsynapse/tools/graph.py +1700 -0
  747. contextsynapse/tools/intelligence.py +235 -0
  748. contextsynapse/tools/memory.py +118 -0
  749. contextsynapse/tools/registry.py +373 -0
  750. contextsynapse/tools/shield_tools.py +159 -0
  751. contextsynapse/tools/tasks.py +341 -0
  752. contextsynapse/tools/workspace_tools.py +124 -0
  753. contextsynapse/utils/__init__.py +15 -0
  754. contextsynapse/utils/config_loader.py +198 -0
  755. contextsynapse/utils/env_loader.py +156 -0
  756. contextsynapse/utils/file_utils.py +288 -0
  757. contextsynapse/utils/logging_optimizer.py +119 -0
  758. contextsynapse/vector/__init__.py +30 -0
  759. contextsynapse/vector/chroma_vector_store.py +152 -0
  760. contextsynapse/vector/custom_vector_store.py +537 -0
  761. contextsynapse/vector/faiss_vector_store.py +296 -0
  762. contextsynapse/vector/qdrant_vector_store.py +163 -0
  763. contextsynapse/vector/vector_db.py +953 -0
  764. contextsynapse/vector/vector_db_manager.py +197 -0
  765. contextsynapse/workspace/__init__.py +27 -0
  766. contextsynapse/workspace/base.py +94 -0
  767. contextsynapse/workspace/git.py +228 -0
  768. contextsynapse/workspace/github.py +78 -0
  769. contextsynapse/workspace/local.py +73 -0
  770. contextsynapse/workspace/tools.py +222 -0
  771. contextsynapse-1.0.0.dist-info/METADATA +847 -0
  772. contextsynapse-1.0.0.dist-info/RECORD +776 -0
  773. contextsynapse-1.0.0.dist-info/WHEEL +5 -0
  774. contextsynapse-1.0.0.dist-info/entry_points.txt +4 -0
  775. contextsynapse-1.0.0.dist-info/licenses/LICENSE +200 -0
  776. contextsynapse-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,691 @@
1
+ """
2
+ Graph Lifecycle Manager
3
+ ========================
4
+ Safe-mode graph lifecycle: no auto-delete. Graphs get smarter over time.
5
+
6
+ States:
7
+ active — being used (context, session, running experiment)
8
+ archived — run ended, intelligence merged back, kept for replay/audit
9
+ promoted — sandbox results merged into source
10
+ discarded — user explicitly deleted (manual only)
11
+
12
+ Intelligence merge: after each experiment run, Decisions/Findings/Insights/Memories
13
+ from the sandbox are merged back into the source graph, making it smarter.
14
+
15
+ Usage::
16
+
17
+ lcm = GraphLifecycleManager(graph_registry)
18
+
19
+ # After experiment run ends
20
+ result = lcm.merge_intelligence("exp_sandbox_123", "source_graph")
21
+ # → {merged_decisions: 2, merged_findings: 3, merged_insights: 1}
22
+
23
+ # Archive sandbox (don't delete)
24
+ lcm.archive("exp_sandbox_123")
25
+
26
+ # List all graphs with status
27
+ graphs = lcm.list_graphs()
28
+
29
+ # User manually cleans up
30
+ lcm.discard("exp_sandbox_123") # only on explicit user action
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import gzip
36
+ import json
37
+ import logging
38
+ import os
39
+ import shutil
40
+ import time
41
+ import uuid
42
+ from dataclasses import dataclass, field
43
+ from datetime import datetime, timezone, timedelta
44
+ from pathlib import Path
45
+ from typing import Any, Callable, Dict, List, Optional
46
+
47
+ logger = logging.getLogger(__name__)
48
+
49
+ # Node types that represent accumulated team intelligence
50
+ INTELLIGENCE_LABELS = frozenset({
51
+ "Decision", "Finding", "Insight", "Memory",
52
+ })
53
+
54
+ # Node types that are ephemeral — not merged back
55
+ NOISE_LABELS = frozenset({
56
+ "AgentThought", "AgentAction", "AgentMessage",
57
+ "AgentPresence", "ExperimentRun", "PipelineRun",
58
+ })
59
+
60
+
61
+ class StorageTier:
62
+ """Storage tier for graph data."""
63
+ HOT = "hot" # Redis + Qdrant — fast access, running/recent
64
+ WARM = "warm" # Disk only — queryable but slower
65
+ COLD = "cold" # Compressed archive — compliance storage
66
+ TOMBSTONE = "tombstone" # Metadata only — data deleted, audit record kept
67
+
68
+
69
+ @dataclass
70
+ class LifecycleConfig:
71
+ """Production lifecycle configuration."""
72
+ safe_mode: bool = True # never auto-delete (default)
73
+ auto_merge: bool = True # merge intelligence on experiment end
74
+ auto_tier: bool = False # automatically move between tiers
75
+ hot_retention_hours: int = 24 # keep in Redis after run ends
76
+ warm_retention_days: int = 30 # keep on disk after eviction from Redis
77
+ cold_retention_days: int = 365 # keep compressed archive
78
+ require_manual_delete: bool = True # user must explicitly delete
79
+ archive_dir: str = "contextcore_data/archive"
80
+
81
+ def to_dict(self) -> Dict[str, Any]:
82
+ return {
83
+ "safe_mode": self.safe_mode,
84
+ "auto_merge": self.auto_merge,
85
+ "auto_tier": self.auto_tier,
86
+ "hot_retention_hours": self.hot_retention_hours,
87
+ "warm_retention_days": self.warm_retention_days,
88
+ "cold_retention_days": self.cold_retention_days,
89
+ "require_manual_delete": self.require_manual_delete,
90
+ }
91
+
92
+ @classmethod
93
+ def development(cls) -> "LifecycleConfig":
94
+ """Dev config — keep everything in hot, no tiering."""
95
+ return cls(safe_mode=True, auto_tier=False)
96
+
97
+ @classmethod
98
+ def production(cls) -> "LifecycleConfig":
99
+ """Production config — auto-tier with safe defaults."""
100
+ return cls(safe_mode=True, auto_tier=True,
101
+ hot_retention_hours=24, warm_retention_days=30, cold_retention_days=365)
102
+
103
+
104
+ @dataclass
105
+ class GraphInfo:
106
+ """Information about a managed graph."""
107
+ namespace: str
108
+ status: str = "active" # active | archived | promoted | discarded
109
+ tier: str = "hot" # hot | warm | cold | tombstone
110
+ graph_type: str = "unknown" # context | session | sandbox | test | system
111
+ source_namespace: str = "" # for sandboxes: the source graph
112
+ experiment_id: str = ""
113
+ run_id: str = ""
114
+ node_count: int = 0
115
+ edge_count: int = 0
116
+ intelligence_count: int = 0 # Decisions + Findings + Insights + Memories
117
+ created_at: str = ""
118
+ last_accessed: str = ""
119
+ merged_at: str = "" # when intelligence was merged back
120
+ size_estimate_kb: int = 0
121
+
122
+ def to_dict(self) -> Dict[str, Any]:
123
+ return {
124
+ "namespace": self.namespace,
125
+ "status": self.status,
126
+ "tier": self.tier,
127
+ "graph_type": self.graph_type,
128
+ "source_namespace": self.source_namespace,
129
+ "experiment_id": self.experiment_id,
130
+ "run_id": self.run_id,
131
+ "node_count": self.node_count,
132
+ "edge_count": self.edge_count,
133
+ "intelligence_count": self.intelligence_count,
134
+ "created_at": self.created_at,
135
+ "last_accessed": self.last_accessed,
136
+ "merged_at": self.merged_at,
137
+ "size_estimate_kb": self.size_estimate_kb,
138
+ }
139
+
140
+
141
+ @dataclass
142
+ class MergeResult:
143
+ """Result of merging intelligence from sandbox → source."""
144
+ source_namespace: str
145
+ target_namespace: str
146
+ merged_decisions: int = 0
147
+ merged_findings: int = 0
148
+ merged_insights: int = 0
149
+ merged_memories: int = 0
150
+ merged_tasks: int = 0 # completed tasks only
151
+ skipped_noise: int = 0
152
+ skipped_duplicates: int = 0
153
+ total_merged: int = 0
154
+
155
+ def to_dict(self) -> Dict[str, Any]:
156
+ return {
157
+ "source_namespace": self.source_namespace,
158
+ "target_namespace": self.target_namespace,
159
+ "merged_decisions": self.merged_decisions,
160
+ "merged_findings": self.merged_findings,
161
+ "merged_insights": self.merged_insights,
162
+ "merged_memories": self.merged_memories,
163
+ "merged_tasks": self.merged_tasks,
164
+ "skipped_noise": self.skipped_noise,
165
+ "skipped_duplicates": self.skipped_duplicates,
166
+ "total_merged": self.total_merged,
167
+ }
168
+
169
+
170
+ class GraphLifecycleManager:
171
+ """Manages graph lifecycle with safe mode (no auto-delete).
172
+
173
+ Core principle: graphs get smarter over time. Intelligence from
174
+ experiment runs is merged back into the source graph. Nothing is
175
+ auto-deleted — only archived or explicitly discarded by the user.
176
+ """
177
+
178
+ def __init__(self, graph_registry=None, config: Optional[LifecycleConfig] = None):
179
+ self._registry = graph_registry
180
+ self._config = config or LifecycleConfig()
181
+ self._graph_info: Dict[str, GraphInfo] = {}
182
+
183
+ @property
184
+ def config(self) -> LifecycleConfig:
185
+ return self._config
186
+
187
+ # ── Intelligence Merge ────────────────────────
188
+
189
+ def merge_intelligence(
190
+ self,
191
+ sandbox_namespace: str,
192
+ source_namespace: str,
193
+ include_tasks: bool = True,
194
+ ) -> MergeResult:
195
+ """Merge intelligence nodes from sandbox back into the source graph.
196
+
197
+ After an experiment run, the sandbox has new Decisions, Findings,
198
+ Insights, and Memories created by agents. This merges them into
199
+ the source graph so future runs start with accumulated intelligence.
200
+
201
+ Only merges INTELLIGENCE nodes. Skips noise (AgentThought, etc.)
202
+ and duplicates (same content already in source).
203
+ """
204
+ if not self._registry:
205
+ return MergeResult(sandbox_namespace, source_namespace)
206
+
207
+ sandbox_db = self._registry.get_graph(sandbox_namespace, load_if_missing=True)
208
+ source_db = self._registry.get_graph(source_namespace, load_if_missing=True)
209
+
210
+ if not sandbox_db or not source_db:
211
+ logger.warning("[LIFECYCLE] Cannot merge: sandbox=%s source=%s",
212
+ sandbox_namespace, source_namespace)
213
+ return MergeResult(sandbox_namespace, source_namespace)
214
+
215
+ from .graph_structures import GraphNode, GraphEdge
216
+
217
+ result = MergeResult(sandbox_namespace, source_namespace)
218
+ now = datetime.now(timezone.utc).isoformat()
219
+
220
+ # Get existing content hashes in source (for dedup)
221
+ source_hashes = set()
222
+ for n in source_db.get_all_nodes():
223
+ props = getattr(n, "properties", {}) or {}
224
+ for key in ("content", "title", "name"):
225
+ val = props.get(key, "")
226
+ if val:
227
+ source_hashes.add(val[:200].lower().strip())
228
+
229
+ # Scan sandbox for intelligence nodes
230
+ for node in sandbox_db.get_all_nodes():
231
+ label = getattr(node, "label", "")
232
+ props = (getattr(node, "properties", {}) or {}).copy()
233
+
234
+ # Skip noise
235
+ if label in NOISE_LABELS:
236
+ result.skipped_noise += 1
237
+ continue
238
+
239
+ # Skip non-intelligence
240
+ if label not in INTELLIGENCE_LABELS and label != "Task":
241
+ continue
242
+
243
+ # For Tasks: only merge completed ones
244
+ if label == "Task":
245
+ if not include_tasks:
246
+ continue
247
+ if props.get("status") != "completed":
248
+ continue
249
+
250
+ # Skip garbage auto-write findings
251
+ content = props.get("content", "")
252
+ if "Found 20 node(s) via aiql" in content or "Analysis: Found" in content:
253
+ result.skipped_noise += 1
254
+ continue
255
+
256
+ # Dedup: skip if same content or title already in source
257
+ dedup_keys = []
258
+ for key in ("content", "title", "name"):
259
+ val = props.get(key, "")
260
+ if val:
261
+ dedup_keys.append(val[:200].lower().strip())
262
+ if any(k in source_hashes for k in dedup_keys):
263
+ result.skipped_duplicates += 1
264
+ continue
265
+ content_key = dedup_keys[0] if dedup_keys else ""
266
+
267
+ # Merge: create node in source graph
268
+ props["_merged_from"] = sandbox_namespace
269
+ props["_merged_at"] = now
270
+ merge_id = f"merged_{uuid.uuid4().hex[:10]}"
271
+
272
+ try:
273
+ source_db.add_node(GraphNode(
274
+ id=merge_id, label=label, properties=props,
275
+ ), write_through=True)
276
+
277
+ if content_key:
278
+ source_hashes.add(content_key)
279
+
280
+ if label == "Decision":
281
+ result.merged_decisions += 1
282
+ elif label == "Finding":
283
+ result.merged_findings += 1
284
+ elif label == "Insight":
285
+ result.merged_insights += 1
286
+ elif label == "Memory":
287
+ result.merged_memories += 1
288
+ elif label == "Task":
289
+ result.merged_tasks += 1
290
+
291
+ result.total_merged += 1
292
+ except Exception as e:
293
+ logger.debug("[LIFECYCLE] Merge node failed: %s", e)
294
+
295
+ logger.info(
296
+ "[LIFECYCLE] Merged intelligence %s → %s: "
297
+ "%d decisions, %d findings, %d insights, %d memories, %d tasks "
298
+ "(%d noise skipped, %d duplicates skipped)",
299
+ sandbox_namespace, source_namespace,
300
+ result.merged_decisions, result.merged_findings,
301
+ result.merged_insights, result.merged_memories, result.merged_tasks,
302
+ result.skipped_noise, result.skipped_duplicates,
303
+ )
304
+
305
+ return result
306
+
307
+ # ── Graph State Management ────────────────────
308
+
309
+ def register_graph(self, namespace: str, graph_type: str = "unknown",
310
+ source_namespace: str = "", experiment_id: str = "",
311
+ run_id: str = "") -> GraphInfo:
312
+ """Register a graph in the lifecycle tracker."""
313
+ info = GraphInfo(
314
+ namespace=namespace,
315
+ status="active",
316
+ graph_type=graph_type,
317
+ source_namespace=source_namespace,
318
+ experiment_id=experiment_id,
319
+ run_id=run_id,
320
+ created_at=datetime.now(timezone.utc).isoformat(),
321
+ last_accessed=datetime.now(timezone.utc).isoformat(),
322
+ )
323
+ self._graph_info[namespace] = info
324
+ return info
325
+
326
+ def archive(self, namespace: str) -> bool:
327
+ """Archive a graph (safe — no deletion). Marked for potential cleanup."""
328
+ info = self._graph_info.get(namespace)
329
+ if info:
330
+ info.status = "archived"
331
+ return True
332
+ # Create info if not tracked
333
+ self._graph_info[namespace] = GraphInfo(
334
+ namespace=namespace, status="archived",
335
+ created_at=datetime.now(timezone.utc).isoformat(),
336
+ )
337
+ return True
338
+
339
+ def promote(self, namespace: str) -> bool:
340
+ """Mark sandbox as promoted (intelligence merged into source)."""
341
+ info = self._graph_info.get(namespace)
342
+ if info:
343
+ info.status = "promoted"
344
+ info.merged_at = datetime.now(timezone.utc).isoformat()
345
+ return True
346
+ return False
347
+
348
+ def discard(self, namespace: str, force: bool = False) -> Dict[str, Any]:
349
+ """Explicitly discard a graph (USER ACTION ONLY).
350
+
351
+ Safe mode: requires force=True for active graphs.
352
+ Only archived/promoted graphs can be discarded without force.
353
+ """
354
+ info = self._graph_info.get(namespace)
355
+
356
+ if info and info.status == "active" and not force:
357
+ return {
358
+ "error": "Cannot discard active graph. Archive it first or use force=True.",
359
+ "status": info.status,
360
+ }
361
+
362
+ # Actually delete from registry
363
+ deleted_registry = False
364
+ if self._registry:
365
+ try:
366
+ self._registry.delete_graph(namespace)
367
+ deleted_registry = True
368
+ except Exception as e:
369
+ logger.warning("[LIFECYCLE] Registry delete failed for %s: %s", namespace, e)
370
+
371
+ if info:
372
+ info.status = "discarded"
373
+
374
+ logger.info("[LIFECYCLE] Discarded graph '%s' (registry_deleted=%s)",
375
+ namespace, deleted_registry)
376
+ return {"discarded": True, "namespace": namespace, "registry_deleted": deleted_registry}
377
+
378
+ # ── Listing + Visibility ──────────────────────
379
+
380
+ def list_graphs(self, status: str = "", graph_type: str = "") -> List[GraphInfo]:
381
+ """List all tracked graphs with optional filters."""
382
+ # Refresh counts from registry
383
+ self._refresh_counts()
384
+
385
+ graphs = list(self._graph_info.values())
386
+ if status:
387
+ graphs = [g for g in graphs if g.status == status]
388
+ if graph_type:
389
+ graphs = [g for g in graphs if g.graph_type == graph_type]
390
+ return sorted(graphs, key=lambda g: g.created_at or "", reverse=True)
391
+
392
+ def get_summary(self) -> Dict[str, Any]:
393
+ """Get summary of all graphs by status and type."""
394
+ self._refresh_counts()
395
+ by_status = {}
396
+ by_type = {}
397
+ total_nodes = 0
398
+ total_intelligence = 0
399
+
400
+ for info in self._graph_info.values():
401
+ by_status[info.status] = by_status.get(info.status, 0) + 1
402
+ by_type[info.graph_type] = by_type.get(info.graph_type, 0) + 1
403
+ total_nodes += info.node_count
404
+ total_intelligence += info.intelligence_count
405
+
406
+ return {
407
+ "total_graphs": len(self._graph_info),
408
+ "by_status": by_status,
409
+ "by_type": by_type,
410
+ "total_nodes": total_nodes,
411
+ "total_intelligence_nodes": total_intelligence,
412
+ }
413
+
414
+ def _refresh_counts(self):
415
+ """Update node/edge counts from registry."""
416
+ if not self._registry:
417
+ return
418
+ for ns, info in self._graph_info.items():
419
+ if info.status == "discarded":
420
+ continue
421
+ try:
422
+ db = self._registry.get_graph(ns)
423
+ if db:
424
+ nodes = db.get_all_nodes()
425
+ info.node_count = len(nodes)
426
+ info.edge_count = len(db.get_all_edges())
427
+ info.intelligence_count = sum(
428
+ 1 for n in nodes
429
+ if getattr(n, "label", "") in INTELLIGENCE_LABELS
430
+ )
431
+ except Exception:
432
+ pass
433
+
434
+ # ── Experiment Run Integration ────────────────
435
+
436
+ def on_experiment_start(self, sandbox_namespace: str, source_namespace: str,
437
+ experiment_id: str, run_id: str) -> GraphInfo:
438
+ """Called when an experiment creates a sandbox."""
439
+ return self.register_graph(
440
+ sandbox_namespace, graph_type="sandbox",
441
+ source_namespace=source_namespace,
442
+ experiment_id=experiment_id, run_id=run_id,
443
+ )
444
+
445
+ def on_experiment_end(self, sandbox_namespace: str, source_namespace: str,
446
+ auto_merge: bool = True) -> Dict[str, Any]:
447
+ """Called when an experiment run completes.
448
+
449
+ Default behavior: merge intelligence back, then archive sandbox.
450
+ Safe mode: NEVER deletes. Archives only.
451
+ """
452
+ result = {"sandbox": sandbox_namespace, "source": source_namespace}
453
+
454
+ # Merge intelligence back to source
455
+ if auto_merge and source_namespace:
456
+ merge = self.merge_intelligence(sandbox_namespace, source_namespace)
457
+ result["merge"] = merge.to_dict()
458
+ self.promote(sandbox_namespace)
459
+ else:
460
+ self.archive(sandbox_namespace)
461
+
462
+ return result
463
+
464
+ # ── Tiered Storage ────────────────────────────
465
+
466
+ def migrate_to_warm(self, namespace: str) -> Dict[str, Any]:
467
+ """Move graph from HOT (Redis) to WARM (disk only).
468
+
469
+ Evicts from Redis but keeps HDF5/JSON files on disk.
470
+ Graph can still be loaded on-demand (slower).
471
+ """
472
+ info = self._graph_info.get(namespace)
473
+ if not info:
474
+ return {"error": f"Graph '{namespace}' not tracked"}
475
+ if info.tier != StorageTier.HOT:
476
+ return {"error": f"Graph is already in {info.tier} tier"}
477
+
478
+ # Save to disk first
479
+ if self._registry:
480
+ try:
481
+ self._registry.save_graph(namespace, create_checkpoint=False)
482
+ except Exception as e:
483
+ logger.warning("[LIFECYCLE] Save before warm migration failed: %s", e)
484
+
485
+ # Evict from Redis (keep disk files)
486
+ evicted_keys = 0
487
+ try:
488
+ import redis as _redis_mod
489
+ redis_url = os.environ.get("CONTEXTSYNAPSE_REDIS_URL") or os.environ.get("AICONTEXTDB_REDIS_URL", "")
490
+ if redis_url:
491
+ r = _redis_mod.from_url(redis_url, decode_responses=True)
492
+ safe_ns = namespace.replace(":", "_")
493
+ # Delete graph-specific Redis keys
494
+ for pattern in [f"context:{safe_ns}:*", f"ntype:{safe_ns}:*",
495
+ f"etype:{safe_ns}:*", f"adj:out:{safe_ns}:*",
496
+ f"adj:in:{safe_ns}:*"]:
497
+ for key in r.scan_iter(pattern, count=100):
498
+ r.delete(key)
499
+ evicted_keys += 1
500
+ # Remove from context registry in Redis
501
+ r.srem("context:names", namespace)
502
+ r.hdel(f"context:{namespace}", *r.hkeys(f"context:{namespace}") or ["_"])
503
+ except Exception as e:
504
+ logger.debug("[LIFECYCLE] Redis eviction partial: %s", e)
505
+
506
+ info.tier = StorageTier.WARM
507
+ logger.info("[LIFECYCLE] %s: HOT → WARM (evicted %d Redis keys)", namespace, evicted_keys)
508
+ return {"namespace": namespace, "tier": "warm", "evicted_keys": evicted_keys}
509
+
510
+ def migrate_to_cold(self, namespace: str) -> Dict[str, Any]:
511
+ """Move graph from WARM (disk) to COLD (compressed archive).
512
+
513
+ Compresses graph data into a gzipped bundle and removes original files.
514
+ """
515
+ info = self._graph_info.get(namespace)
516
+ if not info:
517
+ return {"error": f"Graph '{namespace}' not tracked"}
518
+ if info.tier not in (StorageTier.WARM, StorageTier.HOT):
519
+ return {"error": f"Graph is already in {info.tier} tier"}
520
+
521
+ archive_dir = Path(self._config.archive_dir)
522
+ archive_dir.mkdir(parents=True, exist_ok=True)
523
+
524
+ safe_ns = namespace.replace(":", "_")
525
+ ns_dir = Path("contextcore_data/namespaces") / safe_ns
526
+
527
+ if not ns_dir.exists():
528
+ return {"error": f"Namespace directory not found: {ns_dir}"}
529
+
530
+ # Compress to gzip bundle
531
+ archive_path = archive_dir / f"{safe_ns}_{datetime.now(timezone.utc).strftime('%Y%m%d')}.tar.gz"
532
+ try:
533
+ import tarfile
534
+ with tarfile.open(str(archive_path), "w:gz") as tar:
535
+ tar.add(str(ns_dir), arcname=safe_ns)
536
+ archive_size = archive_path.stat().st_size
537
+
538
+ # Remove original files (keep archive)
539
+ shutil.rmtree(str(ns_dir), ignore_errors=True)
540
+
541
+ # Also delete Qdrant collection
542
+ try:
543
+ from qdrant_client import QdrantClient
544
+ qc = QdrantClient(url=os.environ.get("QDRANT_URL", "http://localhost:6333"))
545
+ collection = f"{safe_ns}_passages"
546
+ if qc.collection_exists(collection):
547
+ qc.delete_collection(collection)
548
+ except Exception:
549
+ pass
550
+
551
+ # Delete BM25 index
552
+ bm25_dir = Path(f"contextcore_data/bm25_index/{safe_ns}")
553
+ if bm25_dir.exists():
554
+ shutil.rmtree(str(bm25_dir), ignore_errors=True)
555
+
556
+ info.tier = StorageTier.COLD
557
+ info.size_estimate_kb = archive_size // 1024
558
+ logger.info("[LIFECYCLE] %s: WARM → COLD (archive: %s, %dKB)",
559
+ namespace, archive_path, archive_size // 1024)
560
+ return {"namespace": namespace, "tier": "cold",
561
+ "archive_path": str(archive_path), "size_kb": archive_size // 1024}
562
+
563
+ except Exception as e:
564
+ return {"error": f"Archive failed: {e}"}
565
+
566
+ def tombstone(self, namespace: str) -> Dict[str, Any]:
567
+ """Delete all data, keep only metadata record (for audit).
568
+
569
+ FINAL STATE — cannot be undone.
570
+ """
571
+ info = self._graph_info.get(namespace)
572
+ if not info:
573
+ return {"error": f"Graph '{namespace}' not tracked"}
574
+
575
+ if info.status == "active" and self._config.require_manual_delete:
576
+ return {"error": "Cannot tombstone active graph. Archive or discard first."}
577
+
578
+ # Delete archive if in cold
579
+ safe_ns = namespace.replace(":", "_")
580
+ archive_dir = Path(self._config.archive_dir)
581
+ for f in archive_dir.glob(f"{safe_ns}_*"):
582
+ f.unlink(missing_ok=True)
583
+
584
+ # Delete any remaining disk files
585
+ ns_dir = Path("contextcore_data/namespaces") / safe_ns
586
+ if ns_dir.exists():
587
+ shutil.rmtree(str(ns_dir), ignore_errors=True)
588
+
589
+ # Delete from registry
590
+ if self._registry:
591
+ try:
592
+ self._registry.delete_graph(namespace)
593
+ except Exception:
594
+ pass
595
+
596
+ info.tier = StorageTier.TOMBSTONE
597
+ info.status = "discarded"
598
+ info.node_count = 0
599
+ info.edge_count = 0
600
+ info.intelligence_count = 0
601
+ info.size_estimate_kb = 0
602
+
603
+ logger.info("[LIFECYCLE] %s: → TOMBSTONE (metadata record kept for audit)", namespace)
604
+ return {"namespace": namespace, "tier": "tombstone", "data_deleted": True}
605
+
606
+ # ── Automatic Tier Migration ──────────────────
607
+
608
+ def run_tier_migration(self) -> Dict[str, Any]:
609
+ """Run automatic tier migration based on lifecycle config.
610
+
611
+ Called by scheduler (e.g., every hour). Moves graphs between tiers
612
+ based on age and retention policy.
613
+
614
+ Safe mode: NEVER deletes. Only moves between tiers.
615
+ """
616
+ if not self._config.auto_tier:
617
+ return {"skipped": True, "reason": "auto_tier disabled"}
618
+
619
+ now = datetime.now(timezone.utc)
620
+ migrated = {"hot_to_warm": 0, "warm_to_cold": 0}
621
+
622
+ for ns, info in list(self._graph_info.items()):
623
+ if info.status == "active":
624
+ continue # never touch active graphs
625
+ if info.tier == StorageTier.TOMBSTONE:
626
+ continue
627
+
628
+ if not info.created_at:
629
+ continue
630
+
631
+ try:
632
+ created = datetime.fromisoformat(info.created_at.replace("Z", "+00:00"))
633
+ age = now - created
634
+ except Exception:
635
+ continue
636
+
637
+ # HOT → WARM: after hot_retention_hours
638
+ if (info.tier == StorageTier.HOT and
639
+ info.status in ("archived", "promoted") and
640
+ age > timedelta(hours=self._config.hot_retention_hours)):
641
+ result = self.migrate_to_warm(ns)
642
+ if "error" not in result:
643
+ migrated["hot_to_warm"] += 1
644
+
645
+ # WARM → COLD: after warm_retention_days
646
+ elif (info.tier == StorageTier.WARM and
647
+ age > timedelta(days=self._config.warm_retention_days)):
648
+ result = self.migrate_to_cold(ns)
649
+ if "error" not in result:
650
+ migrated["warm_to_cold"] += 1
651
+
652
+ logger.info("[LIFECYCLE] Tier migration: %d HOT→WARM, %d WARM→COLD",
653
+ migrated["hot_to_warm"], migrated["warm_to_cold"])
654
+ return migrated
655
+
656
+ def get_tier_summary(self) -> Dict[str, Any]:
657
+ """Summary of graphs by storage tier."""
658
+ by_tier = {StorageTier.HOT: 0, StorageTier.WARM: 0,
659
+ StorageTier.COLD: 0, StorageTier.TOMBSTONE: 0}
660
+ size_by_tier = {StorageTier.HOT: 0, StorageTier.WARM: 0,
661
+ StorageTier.COLD: 0}
662
+
663
+ for info in self._graph_info.values():
664
+ tier = info.tier
665
+ by_tier[tier] = by_tier.get(tier, 0) + 1
666
+ if tier != StorageTier.TOMBSTONE:
667
+ size_by_tier[tier] = size_by_tier.get(tier, 0) + info.size_estimate_kb
668
+
669
+ return {
670
+ "by_tier": by_tier,
671
+ "size_kb_by_tier": size_by_tier,
672
+ "total_graphs": len(self._graph_info),
673
+ "config": self._config.to_dict(),
674
+ }
675
+
676
+
677
+ # Global singleton
678
+ _lifecycle: Optional[GraphLifecycleManager] = None
679
+
680
+
681
+ def get_lifecycle_manager(graph_registry=None,
682
+ config: Optional[LifecycleConfig] = None) -> GraphLifecycleManager:
683
+ global _lifecycle
684
+ if _lifecycle is None:
685
+ _lifecycle = GraphLifecycleManager(graph_registry, config)
686
+ else:
687
+ if graph_registry and not _lifecycle._registry:
688
+ _lifecycle._registry = graph_registry
689
+ if config:
690
+ _lifecycle._config = config
691
+ return _lifecycle