contextsynapse 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (776) hide show
  1. contextsynapse/__init__.py +107 -0
  2. contextsynapse/__main__.py +9 -0
  3. contextsynapse/a2a/__init__.py +28 -0
  4. contextsynapse/a2a/client.py +120 -0
  5. contextsynapse/a2a/discovery.py +128 -0
  6. contextsynapse/a2a/handlers.py +164 -0
  7. contextsynapse/a2a/models.py +246 -0
  8. contextsynapse/a2a/streaming.py +92 -0
  9. contextsynapse/a2a/task_manager.py +275 -0
  10. contextsynapse/adapters/__init__.py +98 -0
  11. contextsynapse/adapters/_base.py +298 -0
  12. contextsynapse/adapters/autogen/__init__.py +4 -0
  13. contextsynapse/adapters/autogen/tools.py +34 -0
  14. contextsynapse/adapters/crewai/__init__.py +4 -0
  15. contextsynapse/adapters/crewai/tools.py +63 -0
  16. contextsynapse/adapters/langchain/__init__.py +19 -0
  17. contextsynapse/adapters/langchain/message_history.py +17 -0
  18. contextsynapse/adapters/langchain/retriever.py +158 -0
  19. contextsynapse/adapters/langchain/tools.py +43 -0
  20. contextsynapse/adapters/langgraph/__init__.py +21 -0
  21. contextsynapse/adapters/langgraph/checkpoint.py +223 -0
  22. contextsynapse/adapters/langgraph/context_tools.py +341 -0
  23. contextsynapse/adapters/langgraph/message_history.py +138 -0
  24. contextsynapse/adapters/langgraph/tools.py +43 -0
  25. contextsynapse/adapters/llamaindex/__init__.py +4 -0
  26. contextsynapse/adapters/llamaindex/tools.py +64 -0
  27. contextsynapse/adapters/openai/__init__.py +24 -0
  28. contextsynapse/adapters/openai/tools.py +206 -0
  29. contextsynapse/adapters/pydantic_ai/__init__.py +4 -0
  30. contextsynapse/adapters/pydantic_ai/tools.py +36 -0
  31. contextsynapse/adapters/swarm/__init__.py +4 -0
  32. contextsynapse/adapters/swarm/tools.py +44 -0
  33. contextsynapse/agents/__init__.py +6 -0
  34. contextsynapse/agents/auto_capture.py +80 -0
  35. contextsynapse/agents/dispatcher.py +299 -0
  36. contextsynapse/agents/notifications.py +112 -0
  37. contextsynapse/aiql/__init__.py +71 -0
  38. contextsynapse/aiql/cache.py +341 -0
  39. contextsynapse/aiql/compiler/__init__.py +62 -0
  40. contextsynapse/aiql/compiler/optimizer.py +126 -0
  41. contextsynapse/aiql/compiler/planner.py +366 -0
  42. contextsynapse/aiql/compiler/statistics.py +235 -0
  43. contextsynapse/aiql/compiler/validator.py +425 -0
  44. contextsynapse/aiql/engine/__init__.py +11 -0
  45. contextsynapse/aiql/engine/chunk_storage.py +2584 -0
  46. contextsynapse/aiql/engine/executor.py +7737 -0
  47. contextsynapse/aiql/engine/extraction_integration.py +442 -0
  48. contextsynapse/aiql/engine/graph_builder.py +405 -0
  49. contextsynapse/aiql/engine/node_creation_from_normalized.py +691 -0
  50. contextsynapse/aiql/engine/performance_monitor.py +289 -0
  51. contextsynapse/aiql/engine/postprocessing.py +365 -0
  52. contextsynapse/aiql/engine/ray_stage_executors.py +918 -0
  53. contextsynapse/aiql/engine/redis_cache.py +106 -0
  54. contextsynapse/aiql/engine/streaming_json.py +366 -0
  55. contextsynapse/aiql/engine/streaming_parquet.py +568 -0
  56. contextsynapse/aiql/engine/streaming_ray_chunking.py +419 -0
  57. contextsynapse/aiql/grammar/__init__.py +14 -0
  58. contextsynapse/aiql/grammar/aiql_grammar.py +1822 -0
  59. contextsynapse/aiql/grammar/base.py +71 -0
  60. contextsynapse/aiql/parser/__init__.py +46 -0
  61. contextsynapse/aiql/parser/aiql_parser.py +11330 -0
  62. contextsynapse/aiql/parser/base_parser.py +47 -0
  63. contextsynapse/api/__init__.py +5 -0
  64. contextsynapse/api/a2a_router.py +233 -0
  65. contextsynapse/api/admin_router.py +140 -0
  66. contextsynapse/api/agent_worker_router.py +978 -0
  67. contextsynapse/api/algorithms_router.py +317 -0
  68. contextsynapse/api/api.py +4144 -0
  69. contextsynapse/api/audit.py +204 -0
  70. contextsynapse/api/auth.py +441 -0
  71. contextsynapse/api/auth_router.py +537 -0
  72. contextsynapse/api/billing.py +261 -0
  73. contextsynapse/api/billing_router.py +275 -0
  74. contextsynapse/api/boundary_router.py +452 -0
  75. contextsynapse/api/cognition_router.py +116 -0
  76. contextsynapse/api/connector_router.py +237 -0
  77. contextsynapse/api/context_router.py +1619 -0
  78. contextsynapse/api/dashboard_router.py +11367 -0
  79. contextsynapse/api/errors.py +183 -0
  80. contextsynapse/api/events.py +88 -0
  81. contextsynapse/api/experiment_bootstrap.py +321 -0
  82. contextsynapse/api/experiment_orchestrator.py +1078 -0
  83. contextsynapse/api/experiment_router.py +2089 -0
  84. contextsynapse/api/feature_gate.py +89 -0
  85. contextsynapse/api/federation_router.py +108 -0
  86. contextsynapse/api/integrations.py +567 -0
  87. contextsynapse/api/integrations_router.py +461 -0
  88. contextsynapse/api/intelligence_router.py +1028 -0
  89. contextsynapse/api/metering.py +155 -0
  90. contextsynapse/api/metrics.py +112 -0
  91. contextsynapse/api/models.py +38 -0
  92. contextsynapse/api/monitoring_router.py +174 -0
  93. contextsynapse/api/pipeline_router.py +1437 -0
  94. contextsynapse/api/playground_conn.py +116 -0
  95. contextsynapse/api/projects_router.py +131 -0
  96. contextsynapse/api/queries_router.py +282 -0
  97. contextsynapse/api/quota.py +95 -0
  98. contextsynapse/api/rate_limit.py +64 -0
  99. contextsynapse/api/search_router.py +261 -0
  100. contextsynapse/api/shield_router.py +447 -0
  101. contextsynapse/api/tenant_guard.py +103 -0
  102. contextsynapse/api/tenant_quotas.py +231 -0
  103. contextsynapse/api/tenants.py +305 -0
  104. contextsynapse/api/users.py +646 -0
  105. contextsynapse/api/v1.py +167 -0
  106. contextsynapse/api/vertical_builder_router.py +387 -0
  107. contextsynapse/benchmark/__init__.py +15 -0
  108. contextsynapse/benchmark/__main__.py +64 -0
  109. contextsynapse/benchmark/agents.py +338 -0
  110. contextsynapse/benchmark/live_ab.py +338 -0
  111. contextsynapse/benchmark/reporter.py +149 -0
  112. contextsynapse/benchmark/runner.py +264 -0
  113. contextsynapse/benchmark/savings.py +266 -0
  114. contextsynapse/benchmark/token_counter.py +212 -0
  115. contextsynapse/benchmark/workloads.py +289 -0
  116. contextsynapse/buffer/__init__.py +154 -0
  117. contextsynapse/buffer/base.py +257 -0
  118. contextsynapse/buffer/duckdb_buffer.py +246 -0
  119. contextsynapse/buffer/file_buffer.py +229 -0
  120. contextsynapse/buffer/lmdb_buffer.py +286 -0
  121. contextsynapse/buffer/local_buffer.py +154 -0
  122. contextsynapse/buffer/redis_buffer.py +211 -0
  123. contextsynapse/buffer/sqlite_buffer.py +266 -0
  124. contextsynapse/cache.py +2 -0
  125. contextsynapse/cli/__init__.py +40 -0
  126. contextsynapse/cli/cli.py +475 -0
  127. contextsynapse/cli/config_cli.py +177 -0
  128. contextsynapse/cli/scaffold.py +465 -0
  129. contextsynapse/cognition/__init__.py +31 -0
  130. contextsynapse/cognition/derivation.py +55 -0
  131. contextsynapse/cognition/events.py +135 -0
  132. contextsynapse/cognition/feedback.py +52 -0
  133. contextsynapse/cognition/hallucination.py +90 -0
  134. contextsynapse/cognition/invalidation.py +70 -0
  135. contextsynapse/cognition/read_tracker.py +50 -0
  136. contextsynapse/collaboration.py +2 -0
  137. contextsynapse/comparison.py +2 -0
  138. contextsynapse/config/__init__.py +10 -0
  139. contextsynapse/config/connectors/amfi_nav.yaml +18 -0
  140. contextsynapse/config/connectors/auto_sector.yaml +20 -0
  141. contextsynapse/config/connectors/banking_sector.yaml +20 -0
  142. contextsynapse/config/connectors/energy_sector.yaml +20 -0
  143. contextsynapse/config/connectors/fmcg_sector.yaml +20 -0
  144. contextsynapse/config/connectors/nsdl_fpi.yaml +9 -0
  145. contextsynapse/config/connectors/nse_price.yaml +61 -0
  146. contextsynapse/config/connectors/pharma_sector.yaml +20 -0
  147. contextsynapse/config/connectors/rbi_dbie.yaml +12 -0
  148. contextsynapse/config/context_presets/ai_research.json +25 -0
  149. contextsynapse/config/context_presets/chatgpt_conversations.json +19 -0
  150. contextsynapse/config/context_presets/claude_conversations.json +19 -0
  151. contextsynapse/config/context_presets/legal_compliance.json +21 -0
  152. contextsynapse/config/context_presets/meeting_notes.json +21 -0
  153. contextsynapse/config/context_presets/slack_engineering.json +21 -0
  154. contextsynapse/config/context_presets/tech_climate.json +24 -0
  155. contextsynapse/config/context_presets/toi_politics.json +25 -0
  156. contextsynapse/config/database_config.py +453 -0
  157. contextsynapse/config/deployment.py +221 -0
  158. contextsynapse/config/gpt_action_schema.json +164 -0
  159. contextsynapse/config/indicators/india_macro.yaml +213 -0
  160. contextsynapse/config/logging_config.py +88 -0
  161. contextsynapse/config/mode_config.py +223 -0
  162. contextsynapse/config/schemas/api_spec.yaml +48 -0
  163. contextsynapse/config/schemas/architecture_doc.yaml +51 -0
  164. contextsynapse/config/schemas/chat_history.yaml +90 -0
  165. contextsynapse/config/schemas/chatgpt_conversation.yaml +101 -0
  166. contextsynapse/config/schemas/claude_conversation.yaml +148 -0
  167. contextsynapse/config/schemas/contract.yaml +48 -0
  168. contextsynapse/config/schemas/conversation.yaml +167 -0
  169. contextsynapse/config/schemas/database.yaml +74 -0
  170. contextsynapse/config/schemas/decision.yaml +72 -0
  171. contextsynapse/config/schemas/ecommerce.yaml +56 -0
  172. contextsynapse/config/schemas/education.yaml +42 -0
  173. contextsynapse/config/schemas/healthcare.yaml +133 -0
  174. contextsynapse/config/schemas/hr_document.yaml +48 -0
  175. contextsynapse/config/schemas/invoice.yaml +40 -0
  176. contextsynapse/config/schemas/knowledge_base.yaml +75 -0
  177. contextsynapse/config/schemas/legal.yaml +42 -0
  178. contextsynapse/config/schemas/marketing_content.yaml +48 -0
  179. contextsynapse/config/schemas/meeting_notes.yaml +51 -0
  180. contextsynapse/config/schemas/news_article.yaml +61 -0
  181. contextsynapse/config/schemas/real_estate.yaml +49 -0
  182. contextsynapse/config/schemas/requirements_doc.yaml +48 -0
  183. contextsynapse/config/schemas/research_paper.yaml +56 -0
  184. contextsynapse/config/schemas/resume.yaml +49 -0
  185. contextsynapse/config/schemas/retail.yaml +42 -0
  186. contextsynapse/config/schemas/rule_meta.yaml +173 -0
  187. contextsynapse/config/schemas/rules.yaml +72 -0
  188. contextsynapse/config/schemas/screener_meta.yaml +236 -0
  189. contextsynapse/config/schemas/sdlc.yaml +103 -0
  190. contextsynapse/config/schemas/session_graph.yaml +172 -0
  191. contextsynapse/config/schemas/support_ticket.yaml +48 -0
  192. contextsynapse/config/schemas/web.yaml +64 -0
  193. contextsynapse/config/sensors/earnings_calendar.yaml +52 -0
  194. contextsynapse/config/sensors/global_risk.yaml +53 -0
  195. contextsynapse/config/sensors/government_policy.yaml +54 -0
  196. contextsynapse/config/sensors/supply_chain.yaml +50 -0
  197. contextsynapse/config/sensors/templates/behavioral.yaml +56 -0
  198. contextsynapse/config/sensors/templates/equity_sentiment.yaml +48 -0
  199. contextsynapse/config/sensors/templates/global_macro.yaml +47 -0
  200. contextsynapse/config/sensors/templates/market_data.yaml +118 -0
  201. contextsynapse/config/sensors/templates/sector_technical.yaml +47 -0
  202. contextsynapse/config/sensors/weather.yaml +52 -0
  203. contextsynapse/config/storage_strategy.py +202 -0
  204. contextsynapse/config/templates/competitive_intel.yaml +39 -0
  205. contextsynapse/config/templates/geopolitical_risk.yaml +46 -0
  206. contextsynapse/config/templates/market_analysis.yaml +58 -0
  207. contextsynapse/connectors/__init__.py +15 -0
  208. contextsynapse/connectors/api_router.py +121 -0
  209. contextsynapse/connectors/base.py +357 -0
  210. contextsynapse/connectors/financial/audio.py +170 -0
  211. contextsynapse/connectors/financial/youtube.py +272 -0
  212. contextsynapse/connectors/github.py +164 -0
  213. contextsynapse/connectors/jira.py +120 -0
  214. contextsynapse/connectors/kafka_connector.py +124 -0
  215. contextsynapse/connectors/pipeline_connector.py +747 -0
  216. contextsynapse/connectors/registry.py +116 -0
  217. contextsynapse/connectors/scheduler.py +87 -0
  218. contextsynapse/connectors/universal.py +360 -0
  219. contextsynapse/context/__init__.py +96 -0
  220. contextsynapse/context/__main__.py +4 -0
  221. contextsynapse/context/acl.py +125 -0
  222. contextsynapse/context/agent_card.py +201 -0
  223. contextsynapse/context/agent_memory.py +814 -0
  224. contextsynapse/context/agents.py +472 -0
  225. contextsynapse/context/agents_redis.py +391 -0
  226. contextsynapse/context/assembled.py +204 -0
  227. contextsynapse/context/attribution.py +113 -0
  228. contextsynapse/context/blob.py +184 -0
  229. contextsynapse/context/boundaries.py +187 -0
  230. contextsynapse/context/boundary.py +507 -0
  231. contextsynapse/context/bundle.py +155 -0
  232. contextsynapse/context/cli.py +519 -0
  233. contextsynapse/context/collaboration.py +165 -0
  234. contextsynapse/context/compiler.py +604 -0
  235. contextsynapse/context/composite.py +536 -0
  236. contextsynapse/context/compression.py +147 -0
  237. contextsynapse/context/context_manager.py +888 -0
  238. contextsynapse/context/context_schema.py +90 -0
  239. contextsynapse/context/context_schema.yaml +105 -0
  240. contextsynapse/context/context_units.py +891 -0
  241. contextsynapse/context/conversation.py +488 -0
  242. contextsynapse/context/dedup.py +207 -0
  243. contextsynapse/context/document_processor.py +698 -0
  244. contextsynapse/context/embedding_hooks.py +338 -0
  245. contextsynapse/context/execution_schema.py +274 -0
  246. contextsynapse/context/fan_out.py +138 -0
  247. contextsynapse/context/frozen.py +80 -0
  248. contextsynapse/context/gravity.py +227 -0
  249. contextsynapse/context/hub.py +1401 -0
  250. contextsynapse/context/ingest.py +391 -0
  251. contextsynapse/context/injection.py +226 -0
  252. contextsynapse/context/interaction_graphifier.py +562 -0
  253. contextsynapse/context/layers.py +574 -0
  254. contextsynapse/context/projection.py +1195 -0
  255. contextsynapse/context/promotion.py +634 -0
  256. contextsynapse/context/proof.py +131 -0
  257. contextsynapse/context/propagation.py +577 -0
  258. contextsynapse/context/pubsub.py +135 -0
  259. contextsynapse/context/quality.py +1054 -0
  260. contextsynapse/context/quality_tracker.py +270 -0
  261. contextsynapse/context/scoping.py +370 -0
  262. contextsynapse/context/seed.py +992 -0
  263. contextsynapse/context/session.py +1142 -0
  264. contextsynapse/context/session_graph.py +593 -0
  265. contextsynapse/context/session_memory.py +309 -0
  266. contextsynapse/context/session_resolver.py +111 -0
  267. contextsynapse/context/skill_learning.py +237 -0
  268. contextsynapse/context/smart_budget.py +129 -0
  269. contextsynapse/context/source_policy.py +242 -0
  270. contextsynapse/context/store_factory.py +53 -0
  271. contextsynapse/context/sync.py +648 -0
  272. contextsynapse/context/token.py +236 -0
  273. contextsynapse/context/vector_integration.py +297 -0
  274. contextsynapse/context/webhooks.py +194 -0
  275. contextsynapse/context/working_memory.py +224 -0
  276. contextsynapse/core/__init__.py +18 -0
  277. contextsynapse/core/archiver.py +237 -0
  278. contextsynapse/core/change_stream.py +282 -0
  279. contextsynapse/core/checkpoint.py +310 -0
  280. contextsynapse/core/cloud_storage.py +387 -0
  281. contextsynapse/core/context_state.py +651 -0
  282. contextsynapse/core/cron.py +324 -0
  283. contextsynapse/core/datasource.py +643 -0
  284. contextsynapse/core/db.py +289 -0
  285. contextsynapse/core/federation.py +283 -0
  286. contextsynapse/core/graph_coordinator.py +201 -0
  287. contextsynapse/core/graph_intelligence.py +388 -0
  288. contextsynapse/core/graph_structures.py +106 -0
  289. contextsynapse/core/graph_sync.py +157 -0
  290. contextsynapse/core/hybrid_graph_storage.py +2998 -0
  291. contextsynapse/core/lifecycle.py +691 -0
  292. contextsynapse/core/multiworker.py +378 -0
  293. contextsynapse/core/pruning.py +312 -0
  294. contextsynapse/core/redis_registry.py +524 -0
  295. contextsynapse/core/registry.py +1043 -0
  296. contextsynapse/core/registry_factory.py +51 -0
  297. contextsynapse/core/registry_metadata.py +294 -0
  298. contextsynapse/core/replay.py +354 -0
  299. contextsynapse/core/replication.py +339 -0
  300. contextsynapse/core/time_travel.py +260 -0
  301. contextsynapse/core/write_behind.py +152 -0
  302. contextsynapse/core/write_context.py +39 -0
  303. contextsynapse/db/__init__.py +1 -0
  304. contextsynapse/db/postgres.py +267 -0
  305. contextsynapse/db/profile.py +236 -0
  306. contextsynapse/db/rules.py +328 -0
  307. contextsynapse/demo/__init__.py +0 -0
  308. contextsynapse/demo/report.py +147 -0
  309. contextsynapse/demo/runner.py +520 -0
  310. contextsynapse/demo/scenario.py +165 -0
  311. contextsynapse/demo/session_demo.py +280 -0
  312. contextsynapse/demo/session_report.py +143 -0
  313. contextsynapse/demo/universal_runner.py +257 -0
  314. contextsynapse/engine/__init__.py +15 -0
  315. contextsynapse/engine/core.py +555 -0
  316. contextsynapse/extraction/__init__.py +69 -0
  317. contextsynapse/extraction/cli.py +274 -0
  318. contextsynapse/extraction/config.py +107 -0
  319. contextsynapse/extraction/examples/example_usage.py +118 -0
  320. contextsynapse/extraction/examples/sample_manifest.json +53 -0
  321. contextsynapse/extraction/examples/sample_normalized.json +185 -0
  322. contextsynapse/extraction/extract_engine.py +440 -0
  323. contextsynapse/extraction/extractors/__init__.py +37 -0
  324. contextsynapse/extraction/extractors/chat_export_extractor.py +189 -0
  325. contextsynapse/extraction/extractors/csv_extractor.py +83 -0
  326. contextsynapse/extraction/extractors/docx_extractor.py +129 -0
  327. contextsynapse/extraction/extractors/excel_extractor.py +152 -0
  328. contextsynapse/extraction/extractors/html_extractor.py +146 -0
  329. contextsynapse/extraction/extractors/pdf_extractor.py +635 -0
  330. contextsynapse/extraction/extractors/pdf_extractor_layoutparser.py +408 -0
  331. contextsynapse/extraction/extractors/register_extractors.py +63 -0
  332. contextsynapse/extraction/extractors/text_extractor.py +65 -0
  333. contextsynapse/extraction/extractors/txt_extractor.py +120 -0
  334. contextsynapse/extraction/extractors/website_extractor.py +246 -0
  335. contextsynapse/extraction/fact_extractor.py +236 -0
  336. contextsynapse/extraction/file_manager.py +353 -0
  337. contextsynapse/extraction/hierarchy.py +239 -0
  338. contextsynapse/extraction/id_generator.py +450 -0
  339. contextsynapse/extraction/layout_detector.py +409 -0
  340. contextsynapse/extraction/llm_entity_extractor.py +301 -0
  341. contextsynapse/extraction/normalized_store.py +480 -0
  342. contextsynapse/extraction/normalizer.py +594 -0
  343. contextsynapse/extraction/parquet_writer.py +598 -0
  344. contextsynapse/extraction/post_processors/__init__.py +40 -0
  345. contextsynapse/extraction/post_processors/column_reorganizer.py +507 -0
  346. contextsynapse/extraction/ray_normalization.py +233 -0
  347. contextsynapse/extraction/ray_runner.py +207 -0
  348. contextsynapse/extraction/redis_schema_registry.py +222 -0
  349. contextsynapse/extraction/registry.py +204 -0
  350. contextsynapse/extraction/schema_loader.py +663 -0
  351. contextsynapse/extraction/schema_registry.py +230 -0
  352. contextsynapse/extraction/semantic_normalizer.py +245 -0
  353. contextsynapse/extraction/version_manager.py +314 -0
  354. contextsynapse/gateway/__init__.py +24 -0
  355. contextsynapse/gateway/action_emitter.py +212 -0
  356. contextsynapse/gateway/cost_tracker.py +508 -0
  357. contextsynapse/gateway/gateway.py +377 -0
  358. contextsynapse/gateway/policy.py +303 -0
  359. contextsynapse/governance/__init__.py +13 -0
  360. contextsynapse/governance/layer.py +513 -0
  361. contextsynapse/ingestion/__init__.py +15 -0
  362. contextsynapse/ingestion/amplifier.py +288 -0
  363. contextsynapse/ingestion/async_ingest.py +463 -0
  364. contextsynapse/ingestion/chunker.py +303 -0
  365. contextsynapse/ingestion/chunking.py +58 -0
  366. contextsynapse/ingestion/cleaner.py +190 -0
  367. contextsynapse/ingestion/cleanse.py +252 -0
  368. contextsynapse/ingestion/connectors/__init__.py +38 -0
  369. contextsynapse/ingestion/connectors/base.py +300 -0
  370. contextsynapse/ingestion/connectors/chatgpt_memory.py +247 -0
  371. contextsynapse/ingestion/connectors/claude_memory.py +274 -0
  372. contextsynapse/ingestion/connectors/sap_ingestor.py +208 -0
  373. contextsynapse/ingestion/content_detector.py +80 -0
  374. contextsynapse/ingestion/content_monitor.py +502 -0
  375. contextsynapse/ingestion/dedup.py +106 -0
  376. contextsynapse/ingestion/feed.py +640 -0
  377. contextsynapse/ingestion/filters.py +461 -0
  378. contextsynapse/ingestion/graph_builder.py +705 -0
  379. contextsynapse/ingestion/input_classifier.py +307 -0
  380. contextsynapse/ingestion/job_manager.py +571 -0
  381. contextsynapse/ingestion/llm_extractor.py +610 -0
  382. contextsynapse/ingestion/orchestrator.py +116 -0
  383. contextsynapse/ingestion/parsers/__init__.py +19 -0
  384. contextsynapse/ingestion/parsers/graph_parser_registry.py +103 -0
  385. contextsynapse/ingestion/parsers/graphml_parser.py +141 -0
  386. contextsynapse/ingestion/parsers/jsonld_parser.py +171 -0
  387. contextsynapse/ingestion/parsers/rdf_parser.py +150 -0
  388. contextsynapse/ingestion/pipeline_context.py +228 -0
  389. contextsynapse/ingestion/pipeline_store.py +1195 -0
  390. contextsynapse/ingestion/queue.py +133 -0
  391. contextsynapse/ingestion/reprocess.py +179 -0
  392. contextsynapse/ingestion/scenario_router.py +224 -0
  393. contextsynapse/ingestion/schema_extractor.py +759 -0
  394. contextsynapse/ingestion/schema_validator.py +274 -0
  395. contextsynapse/ingestion/sdlc_ingest.py +87 -0
  396. contextsynapse/ingestion/smart_ingest.py +2451 -0
  397. contextsynapse/ingestion/source_connector.py +171 -0
  398. contextsynapse/ingestion/stage_executor.py +2095 -0
  399. contextsynapse/ingestion/stages/__init__.py +9 -0
  400. contextsynapse/ingestion/stages/tabular_to_graph.py +289 -0
  401. contextsynapse/ingestion/strategies.py +386 -0
  402. contextsynapse/ingestion/universal/__init__.py +4 -0
  403. contextsynapse/ingestion/universal/_operator_registry.py +59 -0
  404. contextsynapse/ingestion/universal/ingest_content.py +32 -0
  405. contextsynapse/ingestion/universal/operators/__init__.py +1 -0
  406. contextsynapse/ingestion/universal/operators/base.py +19 -0
  407. contextsynapse/ingestion/universal/operators/build_edges.py +75 -0
  408. contextsynapse/ingestion/universal/operators/cluster_topics.py +397 -0
  409. contextsynapse/ingestion/universal/operators/deduplicate.py +45 -0
  410. contextsynapse/ingestion/universal/operators/detect_signals.py +90 -0
  411. contextsynapse/ingestion/universal/operators/embed.py +80 -0
  412. contextsynapse/ingestion/universal/operators/extract_entities.py +253 -0
  413. contextsynapse/ingestion/universal/operators/extract_preferences.py +117 -0
  414. contextsynapse/ingestion/universal/operators/filter_content.py +69 -0
  415. contextsynapse/ingestion/universal/operators/index_bm25.py +52 -0
  416. contextsynapse/ingestion/universal/operators/infer_domains.py +69 -0
  417. contextsynapse/ingestion/universal/operators/link_cross_reference.py +210 -0
  418. contextsynapse/ingestion/universal/operators/parse_records.py +66 -0
  419. contextsynapse/ingestion/universal/operators/resolve_entities.py +261 -0
  420. contextsynapse/ingestion/universal/operators/scanners/__init__.py +16 -0
  421. contextsynapse/ingestion/universal/operators/scanners/edges.py +173 -0
  422. contextsynapse/ingestion/universal/operators/scanners/git.py +187 -0
  423. contextsynapse/ingestion/universal/operators/scanners/github.py +135 -0
  424. contextsynapse/ingestion/universal/operators/scanners/github_api.py +509 -0
  425. contextsynapse/ingestion/universal/operators/scanners/helpers.py +67 -0
  426. contextsynapse/ingestion/universal/operators/scanners/jira.py +65 -0
  427. contextsynapse/ingestion/universal/operators/scanners/llm_enrichment.py +82 -0
  428. contextsynapse/ingestion/universal/operators/scanners/repo.py +149 -0
  429. contextsynapse/ingestion/universal/operators/sdlc_scan.py +213 -0
  430. contextsynapse/ingestion/universal/operators/spec_scanner.py +149 -0
  431. contextsynapse/ingestion/universal/operators/store_documents.py +72 -0
  432. contextsynapse/ingestion/universal/operators/synthesize_cu.py +180 -0
  433. contextsynapse/ingestion/universal/operators/validate_gate.py +65 -0
  434. contextsynapse/ingestion/universal/parsers/__init__.py +5 -0
  435. contextsynapse/ingestion/universal/parsers/record_parser.py +203 -0
  436. contextsynapse/ingestion/universal/parsers/turn_parser.py +329 -0
  437. contextsynapse/ingestion/universal/shared_signals.py +161 -0
  438. contextsynapse/ingestion/universal/stage_executor.py +219 -0
  439. contextsynapse/ingestion/web_crawler.py +265 -0
  440. contextsynapse/intelligence/__init__.py +53 -0
  441. contextsynapse/intelligence/alerts.py +198 -0
  442. contextsynapse/intelligence/collector.py +655 -0
  443. contextsynapse/intelligence/comparison.py +198 -0
  444. contextsynapse/intelligence/config.py +56 -0
  445. contextsynapse/intelligence/conflict_detector.py +182 -0
  446. contextsynapse/intelligence/context_gaps.py +192 -0
  447. contextsynapse/intelligence/context_radar.py +135 -0
  448. contextsynapse/intelligence/correlation_engine.py +416 -0
  449. contextsynapse/intelligence/correlation_template.py +120 -0
  450. contextsynapse/intelligence/dedup.py +296 -0
  451. contextsynapse/intelligence/event_bus.py +91 -0
  452. contextsynapse/intelligence/feedback_loop.py +101 -0
  453. contextsynapse/intelligence/freshness.py +307 -0
  454. contextsynapse/intelligence/fundamental_feed.py +551 -0
  455. contextsynapse/intelligence/geo_data.py +170 -0
  456. contextsynapse/intelligence/geo_reference.py +112 -0
  457. contextsynapse/intelligence/graph_fusion.py +465 -0
  458. contextsynapse/intelligence/heuristics.py +176 -0
  459. contextsynapse/intelligence/impact_tracker.py +336 -0
  460. contextsynapse/intelligence/macro_indicators.py +319 -0
  461. contextsynapse/intelligence/ocr.py +104 -0
  462. contextsynapse/intelligence/persistence.py +267 -0
  463. contextsynapse/intelligence/pipeline_health.py +130 -0
  464. contextsynapse/intelligence/price_feed.py +221 -0
  465. contextsynapse/intelligence/rate_limiter.py +130 -0
  466. contextsynapse/intelligence/reactive.py +411 -0
  467. contextsynapse/intelligence/runtime_context.py +390 -0
  468. contextsynapse/intelligence/sentiment_decay.py +426 -0
  469. contextsynapse/intelligence/session.py +277 -0
  470. contextsynapse/intelligence/signal_hierarchy.py +438 -0
  471. contextsynapse/intelligence/signals.py +185 -0
  472. contextsynapse/intelligence/source_watcher.py +132 -0
  473. contextsynapse/intelligence/table_enricher.py +256 -0
  474. contextsynapse/intelligence/tagger.py +319 -0
  475. contextsynapse/intelligence/timeseries.py +209 -0
  476. contextsynapse/intelligence/tracker.py +318 -0
  477. contextsynapse/intelligence/watchdog.py +392 -0
  478. contextsynapse/intelligence/watchdog_manager.py +263 -0
  479. contextsynapse/intelligence/webhook_receiver.py +165 -0
  480. contextsynapse/intelligence/weight_learner.py +234 -0
  481. contextsynapse/llm/__init__.py +8 -0
  482. contextsynapse/llm/client.py +348 -0
  483. contextsynapse/llm/rate_limiter.py +166 -0
  484. contextsynapse/logging_config.py +2 -0
  485. contextsynapse/marketplace/__init__.py +7 -0
  486. contextsynapse/marketplace/registry.py +173 -0
  487. contextsynapse/mcp/__init__.py +11 -0
  488. contextsynapse/mcp/__main__.py +4 -0
  489. contextsynapse/mcp/auth_middleware.py +212 -0
  490. contextsynapse/mcp/connection_pool.py +128 -0
  491. contextsynapse/mcp/server.py +986 -0
  492. contextsynapse/mcp/session_context.py +67 -0
  493. contextsynapse/memory/__init__.py +34 -0
  494. contextsynapse/memory/engine.py +475 -0
  495. contextsynapse/memory/temporal.py +327 -0
  496. contextsynapse/metadata/__init__.py +12 -0
  497. contextsynapse/metadata/metadata_db.py +272 -0
  498. contextsynapse/metadata/metadata_query_tool.py +300 -0
  499. contextsynapse/metadata/metadata_tracker.py +425 -0
  500. contextsynapse/models/__init__.py +27 -0
  501. contextsynapse/models/embedding_service.py +360 -0
  502. contextsynapse/models/model_registry.py +228 -0
  503. contextsynapse/multiworker.py +2 -0
  504. contextsynapse/pipelines/__init__.py +1 -0
  505. contextsynapse/pipelines/executor.py +851 -0
  506. contextsynapse/pipelines/filters.py +189 -0
  507. contextsynapse/pipelines/models.py +176 -0
  508. contextsynapse/pipelines/scheduler.py +268 -0
  509. contextsynapse/playground/__init__.py +1 -0
  510. contextsynapse/playground/templates.py +85 -0
  511. contextsynapse/plugins/__init__.py +43 -0
  512. contextsynapse/plugins/api.py +66 -0
  513. contextsynapse/plugins/base.py +350 -0
  514. contextsynapse/plugins/domain_interface.py +508 -0
  515. contextsynapse/plugins/domain_loader.py +241 -0
  516. contextsynapse/plugins/loader.py +148 -0
  517. contextsynapse/plugins/registry.py +265 -0
  518. contextsynapse/project/__init__.py +11 -0
  519. contextsynapse/project/code_context.py +807 -0
  520. contextsynapse/project/connectors/__init__.py +4 -0
  521. contextsynapse/project/connectors/github_connector.py +113 -0
  522. contextsynapse/project/connectors/jira_connector.py +160 -0
  523. contextsynapse/project/graph.py +1023 -0
  524. contextsynapse/project/project_context.py +545 -0
  525. contextsynapse/project/scanner_registry.py +61 -0
  526. contextsynapse/project/schema_manager.py +322 -0
  527. contextsynapse/project/schemas/knowledge.yaml +59 -0
  528. contextsynapse/project/schemas/sdlc.yaml +369 -0
  529. contextsynapse/project/sdlc_schema.py +172 -0
  530. contextsynapse/project/spec_parser.py +85 -0
  531. contextsynapse/project/task_lock.py +181 -0
  532. contextsynapse/project/task_stream.py +203 -0
  533. contextsynapse/project/team_tools.py +537 -0
  534. contextsynapse/py.typed +0 -0
  535. contextsynapse/realtime/__init__.py +6 -0
  536. contextsynapse/realtime/collaboration.py +147 -0
  537. contextsynapse/rules/__init__.py +10 -0
  538. contextsynapse/rules/engine.py +116 -0
  539. contextsynapse/rules/evaluators/__init__.py +6 -0
  540. contextsynapse/rules/evaluators/aggregate.py +205 -0
  541. contextsynapse/rules/evaluators/custom.py +170 -0
  542. contextsynapse/rules/evaluators/pre_action.py +125 -0
  543. contextsynapse/rules/evaluators/temporal.py +129 -0
  544. contextsynapse/rules/loader.py +52 -0
  545. contextsynapse/rules/models.py +124 -0
  546. contextsynapse/rules/universal.py +393 -0
  547. contextsynapse/scheduler.py +229 -0
  548. contextsynapse/schema/__init__.py +42 -0
  549. contextsynapse/schema/compiler.py +252 -0
  550. contextsynapse/schema/composer.py +158 -0
  551. contextsynapse/schema/dedup.py +73 -0
  552. contextsynapse/schema/derivation.py +172 -0
  553. contextsynapse/schema/sdl.py +326 -0
  554. contextsynapse/schema/validation_gate.py +124 -0
  555. contextsynapse/sdk/__init__.py +63 -0
  556. contextsynapse/sdk/agent.py +214 -0
  557. contextsynapse/sdk/client.py +269 -0
  558. contextsynapse/sdk/exceptions.py +23 -0
  559. contextsynapse/sdk/models.py +78 -0
  560. contextsynapse/sdk/session.py +408 -0
  561. contextsynapse/sdk/tracked.py +218 -0
  562. contextsynapse/sdk/worker.py +266 -0
  563. contextsynapse/sdk/workspace_client.py +98 -0
  564. contextsynapse/sdk/ws.py +41 -0
  565. contextsynapse/search/__init__.py +20 -0
  566. contextsynapse/search/embedding_cache.py +140 -0
  567. contextsynapse/search/enhanced_search.py +152 -0
  568. contextsynapse/search/fulltext.py +130 -0
  569. contextsynapse/search/graph_search.py +634 -0
  570. contextsynapse/search/lmdb_index.py +1218 -0
  571. contextsynapse/search/nl_to_aiql.py +199 -0
  572. contextsynapse/search/rag.py +1346 -0
  573. contextsynapse/search/rag_cache.py +167 -0
  574. contextsynapse/search/reasoning_chain.py +231 -0
  575. contextsynapse/search/redis_search.py +207 -0
  576. contextsynapse/search/retrieval_quality.py +314 -0
  577. contextsynapse/search/semantic_query.py +455 -0
  578. contextsynapse/search/text_resolver.py +159 -0
  579. contextsynapse/search/whoosh_search.py +373 -0
  580. contextsynapse/security/__init__.py +17 -0
  581. contextsynapse/security/audit.py +297 -0
  582. contextsynapse/security/audit_logger.py +97 -0
  583. contextsynapse/security/audit_trail.py +270 -0
  584. contextsynapse/security/auth_unified.py +284 -0
  585. contextsynapse/security/auto_tagger.py +230 -0
  586. contextsynapse/security/data_security.py +218 -0
  587. contextsynapse/security/encryption.py +207 -0
  588. contextsynapse/security/identity.py +447 -0
  589. contextsynapse/security/jwt_identity.py +59 -0
  590. contextsynapse/security/middleware.py +336 -0
  591. contextsynapse/security/pii.py +233 -0
  592. contextsynapse/security/rbac.py +96 -0
  593. contextsynapse/security/rls.py +205 -0
  594. contextsynapse/security/sanitize.py +233 -0
  595. contextsynapse/security/scoped_encryption.py +131 -0
  596. contextsynapse/security/tenant.py +401 -0
  597. contextsynapse/shield/__init__.py +21 -0
  598. contextsynapse/shield/anomaly.py +56 -0
  599. contextsynapse/shield/permissions.py +99 -0
  600. contextsynapse/shield/profile.py +103 -0
  601. contextsynapse/shield/shield.py +108 -0
  602. contextsynapse/shield/trust_engine.py +69 -0
  603. contextsynapse/skills/__init__.py +14 -0
  604. contextsynapse/skills/engine.py +233 -0
  605. contextsynapse/skills/loader.py +59 -0
  606. contextsynapse/skills/models.py +139 -0
  607. contextsynapse/storage/__init__.py +7 -0
  608. contextsynapse/storage/cache.py +208 -0
  609. contextsynapse/storage/columnar_store_v2.py +178 -0
  610. contextsynapse/storage/csr_graph_storage.py +587 -0
  611. contextsynapse/storage/csr_store.py +388 -0
  612. contextsynapse/storage/document_store.py +491 -0
  613. contextsynapse/storage/hnsw_index.py +294 -0
  614. contextsynapse/storage/hybrid_store_v2.py +278 -0
  615. contextsynapse/storage/lmdb_graph_storage.py +366 -0
  616. contextsynapse/storage/migration.py +279 -0
  617. contextsynapse/storage/namespace_store.py +346 -0
  618. contextsynapse/storage/pure_graph_storage.py +185 -0
  619. contextsynapse/storage/redis_graph_adapter.py +186 -0
  620. contextsynapse/storage/redis_graph_storage.py +375 -0
  621. contextsynapse/storage/router/__init__.py +29 -0
  622. contextsynapse/storage/router/duckdb_store.py +398 -0
  623. contextsynapse/storage/router/factory.py +83 -0
  624. contextsynapse/storage/router/interface.py +125 -0
  625. contextsynapse/storage/router/postgres_store.py +339 -0
  626. contextsynapse/storage/router/resolver.py +192 -0
  627. contextsynapse/storage/router/router.py +300 -0
  628. contextsynapse/storage/single_file_storage.py +391 -0
  629. contextsynapse/storage/storage_manager.py +214 -0
  630. contextsynapse/storage/storage_query_optimizer.py +314 -0
  631. contextsynapse/storage/wal.py +359 -0
  632. contextsynapse/temporal/__init__.py +0 -0
  633. contextsynapse/temporal/storage.py +106 -0
  634. contextsynapse/tests/__init__.py +11 -0
  635. contextsynapse/tests/regression/__init__.py +1 -0
  636. contextsynapse/tests/regression/add_metadata.py +395 -0
  637. contextsynapse/tests/regression/add_skip_flags.py +85 -0
  638. contextsynapse/tests/regression/cases/test_basic_create_node.yaml +20 -0
  639. contextsynapse/tests/regression/cases/test_basic_select.yaml +21 -0
  640. contextsynapse/tests/regression/cases/test_blockchain_audit_trail.yaml +37 -0
  641. contextsynapse/tests/regression/cases/test_blockchain_audit_trail_filtered.yaml +39 -0
  642. contextsynapse/tests/regression/cases/test_blockchain_blocks_range.yaml +39 -0
  643. contextsynapse/tests/regression/cases/test_blockchain_get_block.yaml +35 -0
  644. contextsynapse/tests/regression/cases/test_blockchain_latest_block.yaml +35 -0
  645. contextsynapse/tests/regression/cases/test_blockchain_length.yaml +37 -0
  646. contextsynapse/tests/regression/cases/test_blockchain_merkle_root.yaml +37 -0
  647. contextsynapse/tests/regression/cases/test_blockchain_verify.yaml +35 -0
  648. contextsynapse/tests/regression/cases/test_blockchain_verify_block.yaml +35 -0
  649. contextsynapse/tests/regression/cases/test_create_edge.yaml +28 -0
  650. contextsynapse/tests/regression/cases/test_delete_edge_basic.yaml +29 -0
  651. contextsynapse/tests/regression/cases/test_delete_node_basic.yaml +29 -0
  652. contextsynapse/tests/regression/cases/test_evaluation_mrr.yaml +40 -0
  653. contextsynapse/tests/regression/cases/test_evaluation_ndcg.yaml +40 -0
  654. contextsynapse/tests/regression/cases/test_evaluation_precision.yaml +40 -0
  655. contextsynapse/tests/regression/cases/test_evaluation_recall.yaml +40 -0
  656. contextsynapse/tests/regression/cases/test_group_by_basic.yaml +56 -0
  657. contextsynapse/tests/regression/cases/test_match_node_basic.yaml +29 -0
  658. contextsynapse/tests/regression/cases/test_match_node_with_order_limit.yaml +31 -0
  659. contextsynapse/tests/regression/cases/test_namespace_switch.yaml +18 -0
  660. contextsynapse/tests/regression/cases/test_pipeline_cascaded.yaml +61 -0
  661. contextsynapse/tests/regression/cases/test_pipeline_fixed_size_chunking.yaml +48 -0
  662. contextsynapse/tests/regression/cases/test_pipeline_gpt4_chunking.yaml +54 -0
  663. contextsynapse/tests/regression/cases/test_pipeline_hybrid_indexing.yaml +57 -0
  664. contextsynapse/tests/regression/cases/test_pipeline_large_embedding.yaml +52 -0
  665. contextsynapse/tests/regression/cases/test_pipeline_paragraph_chunking.yaml +49 -0
  666. contextsynapse/tests/regression/cases/test_pipeline_recursive_chunking.yaml +47 -0
  667. contextsynapse/tests/regression/cases/test_pipeline_run_pipeline.yaml +56 -0
  668. contextsynapse/tests/regression/cases/test_pipeline_section_based_chunking.yaml +53 -0
  669. contextsynapse/tests/regression/cases/test_pipeline_semantic_chunking.yaml +51 -0
  670. contextsynapse/tests/regression/cases/test_pipeline_with_entity_extraction.yaml +57 -0
  671. contextsynapse/tests/regression/cases/test_pipeline_with_relationships.yaml +62 -0
  672. contextsynapse/tests/regression/cases/test_rag_evaluation.yaml +38 -0
  673. contextsynapse/tests/regression/cases/test_rag_graph_bfs_with_eval.yaml +48 -0
  674. contextsynapse/tests/regression/cases/test_rag_graph_centrality.yaml +40 -0
  675. contextsynapse/tests/regression/cases/test_rag_graph_centrality_eval.yaml +35 -0
  676. contextsynapse/tests/regression/cases/test_rag_graph_community.yaml +40 -0
  677. contextsynapse/tests/regression/cases/test_rag_graph_community_eval.yaml +35 -0
  678. contextsynapse/tests/regression/cases/test_rag_graph_dfs_with_eval.yaml +35 -0
  679. contextsynapse/tests/regression/cases/test_rag_graph_multi_hop_eval.yaml +35 -0
  680. contextsynapse/tests/regression/cases/test_rag_graph_path.yaml +40 -0
  681. contextsynapse/tests/regression/cases/test_rag_graph_search_basic.yaml +38 -0
  682. contextsynapse/tests/regression/cases/test_rag_graph_search_bfs.yaml +52 -0
  683. contextsynapse/tests/regression/cases/test_rag_graph_search_dfs.yaml +41 -0
  684. contextsynapse/tests/regression/cases/test_rag_graph_search_multi_hop.yaml +43 -0
  685. contextsynapse/tests/regression/cases/test_rag_graph_search_shortest_path.yaml +41 -0
  686. contextsynapse/tests/regression/cases/test_rag_graph_search_weighted.yaml +43 -0
  687. contextsynapse/tests/regression/cases/test_rag_graph_shortest_path_eval.yaml +35 -0
  688. contextsynapse/tests/regression/cases/test_rag_graph_traversal.yaml +30 -0
  689. contextsynapse/tests/regression/cases/test_rag_graph_weighted_eval.yaml +35 -0
  690. contextsynapse/tests/regression/cases/test_rag_hybrid_search.yaml +33 -0
  691. contextsynapse/tests/regression/cases/test_rag_hybrid_with_prompt_embedding.yaml +44 -0
  692. contextsynapse/tests/regression/cases/test_rag_keyword_search.yaml +32 -0
  693. contextsynapse/tests/regression/cases/test_rag_multi_hop_graph.yaml +32 -0
  694. contextsynapse/tests/regression/cases/test_rag_path_based.yaml +34 -0
  695. contextsynapse/tests/regression/cases/test_rag_profile_based.yaml +32 -0
  696. contextsynapse/tests/regression/cases/test_rag_vector_search.yaml +32 -0
  697. contextsynapse/tests/regression/cases/test_rag_weighted_graph.yaml +34 -0
  698. contextsynapse/tests/regression/cases/test_rag_weighted_hybrid.yaml +31 -0
  699. contextsynapse/tests/regression/cases/test_semantic_boundary_chunking.yaml +57 -0
  700. contextsynapse/tests/regression/cases/test_semantic_boundary_detection.yaml +33 -0
  701. contextsynapse/tests/regression/cases/test_semantic_hash_function.yaml +30 -0
  702. contextsynapse/tests/regression/cases/test_semantic_hash_with_model.yaml +31 -0
  703. contextsynapse/tests/regression/cases/test_tcs_ingestion_fixed_size.yaml +44 -0
  704. contextsynapse/tests/regression/cases/test_tcs_ingestion_recursive.yaml +46 -0
  705. contextsynapse/tests/regression/cases/test_tcs_ingestion_section.yaml +46 -0
  706. contextsynapse/tests/regression/cases/test_tcs_ingestion_semantic.yaml +44 -0
  707. contextsynapse/tests/regression/cases/test_time_travel_as_of.yaml +35 -0
  708. contextsynapse/tests/regression/cases/test_time_travel_at_commit.yaml +33 -0
  709. contextsynapse/tests/regression/cases/test_time_travel_at_timestamp.yaml +33 -0
  710. contextsynapse/tests/regression/cases/test_time_travel_for_system_time_all.yaml +37 -0
  711. contextsynapse/tests/regression/cases/test_time_travel_versions_between.yaml +39 -0
  712. contextsynapse/tests/regression/cases/test_traverse_basic.yaml +31 -0
  713. contextsynapse/tests/regression/cases/test_traverse_max_depth.yaml +34 -0
  714. contextsynapse/tests/regression/cases/test_update_edge_basic.yaml +29 -0
  715. contextsynapse/tests/regression/evaluation_metrics.py +227 -0
  716. contextsynapse/tests/regression/regression_runner.py +1081 -0
  717. contextsynapse/tests/regression/setup_regression_data.py +212 -0
  718. contextsynapse/tests/regression/test_basic_only.py +84 -0
  719. contextsynapse/tests/regression/test_namespace_creation.py +29 -0
  720. contextsynapse/tests/regression/test_single_fix.py +61 -0
  721. contextsynapse/tests/regression/test_single_query.py +20 -0
  722. contextsynapse/tests/regression/validate_test_syntax.py +179 -0
  723. contextsynapse/tests/test_advanced_features.py +347 -0
  724. contextsynapse/tests/test_aiql_summary.py +283 -0
  725. contextsynapse/tests/test_all_aiql_queries.py +948 -0
  726. contextsynapse/tests/test_all_create_queries.py +213 -0
  727. contextsynapse/tests/test_api_vs_native.py +280 -0
  728. contextsynapse/tests/test_chunking_dedup.py +199 -0
  729. contextsynapse/tests/test_connectors.py +171 -0
  730. contextsynapse/tests/test_edges_and_match.py +202 -0
  731. contextsynapse/tests/test_extraction_pipeline_integration.py +581 -0
  732. contextsynapse/tests/test_name_property.py +224 -0
  733. contextsynapse/tests/test_node_edge_metadata.py +250 -0
  734. contextsynapse/tests/test_playground_conn.py +114 -0
  735. contextsynapse/tests/test_query_cache.py +213 -0
  736. contextsynapse/tests/test_source_policy_auth.py +348 -0
  737. contextsynapse/tests/test_temporal_and_domain.py +224 -0
  738. contextsynapse/tests/test_time_travel_e2e.py +571 -0
  739. contextsynapse/tests/test_traversal_and_aggregation.py +275 -0
  740. contextsynapse/tools/__init__.py +36 -0
  741. contextsynapse/tools/a2a_tools.py +106 -0
  742. contextsynapse/tools/cognition_tools.py +118 -0
  743. contextsynapse/tools/context.py +905 -0
  744. contextsynapse/tools/formatting.py +92 -0
  745. contextsynapse/tools/gateway_tools.py +186 -0
  746. contextsynapse/tools/graph.py +1700 -0
  747. contextsynapse/tools/intelligence.py +235 -0
  748. contextsynapse/tools/memory.py +118 -0
  749. contextsynapse/tools/registry.py +373 -0
  750. contextsynapse/tools/shield_tools.py +159 -0
  751. contextsynapse/tools/tasks.py +341 -0
  752. contextsynapse/tools/workspace_tools.py +124 -0
  753. contextsynapse/utils/__init__.py +15 -0
  754. contextsynapse/utils/config_loader.py +198 -0
  755. contextsynapse/utils/env_loader.py +156 -0
  756. contextsynapse/utils/file_utils.py +288 -0
  757. contextsynapse/utils/logging_optimizer.py +119 -0
  758. contextsynapse/vector/__init__.py +30 -0
  759. contextsynapse/vector/chroma_vector_store.py +152 -0
  760. contextsynapse/vector/custom_vector_store.py +537 -0
  761. contextsynapse/vector/faiss_vector_store.py +296 -0
  762. contextsynapse/vector/qdrant_vector_store.py +163 -0
  763. contextsynapse/vector/vector_db.py +953 -0
  764. contextsynapse/vector/vector_db_manager.py +197 -0
  765. contextsynapse/workspace/__init__.py +27 -0
  766. contextsynapse/workspace/base.py +94 -0
  767. contextsynapse/workspace/git.py +228 -0
  768. contextsynapse/workspace/github.py +78 -0
  769. contextsynapse/workspace/local.py +73 -0
  770. contextsynapse/workspace/tools.py +222 -0
  771. contextsynapse-1.0.0.dist-info/METADATA +847 -0
  772. contextsynapse-1.0.0.dist-info/RECORD +776 -0
  773. contextsynapse-1.0.0.dist-info/WHEEL +5 -0
  774. contextsynapse-1.0.0.dist-info/entry_points.txt +4 -0
  775. contextsynapse-1.0.0.dist-info/licenses/LICENSE +200 -0
  776. contextsynapse-1.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1401 @@
1
+ """
2
+ ContextHub
3
+ ==========
4
+ A structured context builder for agentic LLM workflows.
5
+
6
+ Nodes, edges, query results, and free-form text can all be collected into
7
+ a single hub and then exported in the format that any downstream model
8
+ expects (OpenAI/Anthropic messages, raw prompt string, JSON dict, …).
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import copy
14
+ import json
15
+ import textwrap
16
+ from dataclasses import dataclass, field
17
+ from datetime import datetime, timezone
18
+ from enum import Enum
19
+ from typing import Any, Dict, List, Optional, Union
20
+
21
+
22
+ class ContextRole(str, Enum):
23
+ """Role of a context item – mirrors the standard LLM roles."""
24
+ SYSTEM = "system"
25
+ USER = "user"
26
+ ASSISTANT = "assistant"
27
+ # Semantic roles understood by the export layer
28
+ BACKGROUND = "background" # general knowledge / graph facts
29
+ RETRIEVED = "retrieved" # RAG-retrieved chunks / nodes
30
+ INSTRUCTION = "instruction" # task-specific instructions
31
+ EXAMPLE = "example" # few-shot examples
32
+ # Agent-generated context (output from LLM calls / reasoning)
33
+ GENERATED = "generated" # raw LLM output
34
+ SYNTHESIS = "synthesis" # summary / analysis derived from other items
35
+ DECISION = "decision" # conclusion or action decided by an agent
36
+ # Phase 3: Extended context types
37
+ DOCUMENT = "document" # full source documents (vs RETRIEVED snippets)
38
+ INTERACTION = "interaction" # multi-turn dialogue transcripts between agents
39
+ HUMAN = "human" # explicitly human-provided context
40
+ TOOL = "tool" # tool call results
41
+ FEEDBACK = "feedback" # human corrections, validations, RLHF signals
42
+
43
+
44
+ class ContextFormat(str, Enum):
45
+ """Target export format."""
46
+ MESSAGES = "messages" # List[{"role": ..., "content": ...}]
47
+ PROMPT = "prompt" # Single concatenated string
48
+ DICT = "dict" # Serialisable dict (for custom pipelines)
49
+ MARKDOWN = "markdown" # Human-readable markdown
50
+
51
+
52
+ class Sensitivity(str, Enum):
53
+ """Classification level for context items."""
54
+ PUBLIC = "public" # visible to any agent with session access
55
+ INTERNAL = "internal" # visible to agents with write+ access
56
+ CONFIDENTIAL = "confidential" # visible only to agents with explicit tag grant
57
+ RESTRICTED = "restricted" # visible only to admins
58
+
59
+
60
+ @dataclass
61
+ class ContextItem:
62
+ """A single piece of context with metadata."""
63
+ content: str
64
+ role: ContextRole = ContextRole.USER
65
+ label: Optional[str] = None # human-readable label / section heading
66
+ source: Optional[str] = None # where the context came from
67
+ metadata: Dict[str, Any] = field(default_factory=dict)
68
+ created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
69
+ agent_id: Optional[str] = None # which agent wrote this item
70
+ provenance_id: Optional[str] = None # link to provenance record
71
+ # ── Tagging & authorization ──────────────────────────────
72
+ tags: List[str] = field(default_factory=list) # e.g. ["financial", "q4"]
73
+ sensitivity: str = "public" # public|internal|confidential|restricted
74
+ pii_detected: bool = False # set by auto-tagger on ingest
75
+ # ── Phase 3: Confidence, lifecycle, slots ─────────────
76
+ confidence: float = 1.0 # [0.0, 1.0] trust score
77
+ state: str = "active" # active | stale | archived
78
+ _slot: Optional[str] = None # assigned context slot name
79
+
80
+ def to_dict(self) -> Dict[str, Any]:
81
+ return {
82
+ "role": self.role.value,
83
+ "label": self.label,
84
+ "source": self.source,
85
+ "content": self.content,
86
+ "metadata": self.metadata,
87
+ "created_at": self.created_at,
88
+ "agent_id": self.agent_id,
89
+ "provenance_id": self.provenance_id,
90
+ "tags": self.tags,
91
+ "sensitivity": self.sensitivity,
92
+ "pii_detected": self.pii_detected,
93
+ "confidence": self.confidence,
94
+ "state": self.state,
95
+ "_slot": self._slot,
96
+ }
97
+
98
+
99
+ @dataclass
100
+ class SlotConfig:
101
+ """A named token bucket within the context hub."""
102
+ name: str
103
+ max_tokens: int
104
+ roles: Optional[List[ContextRole]] = None # auto-assign items with these roles
105
+ pinned: bool = False # pinned slots survive any scoping
106
+ priority: int = 5 # export ordering (higher = earlier)
107
+
108
+
109
+ # ---------------------------------------------------------------------------
110
+ # Helpers
111
+ # ---------------------------------------------------------------------------
112
+
113
+ def _resolve_thin_node(node: Any) -> dict:
114
+ """If node has a _ref pointer, resolve full content from DuckDB."""
115
+ props = getattr(node, "properties", None) or (node if isinstance(node, dict) else {})
116
+ if isinstance(props, dict) and props.get("_ref", "").startswith("duckdb:"):
117
+ try:
118
+ from contextsynapse.storage.router import get_content_resolver
119
+ return get_content_resolver().resolve(node)
120
+ except Exception:
121
+ pass
122
+ return {}
123
+
124
+
125
+ def _node_to_text(node: Any) -> str:
126
+ """Convert a GraphNode (or dict) to a compact text representation.
127
+
128
+ If the node has a _ref pointer (thin graph node), auto-resolves
129
+ full content from DuckDB before rendering.
130
+ """
131
+ _PRIORITY_KEYS = ("name", "title", "text", "content", "statement", "description", "status", "type")
132
+ _SKIP_KEYS = {"uuid", "domain", "_ref", "_created_at", "_agent_id", "_origin",
133
+ "_verified", "_region", "source_type", "valid_from", "valid_to", "version"}
134
+
135
+ # Auto-resolve thin nodes
136
+ resolved = _resolve_thin_node(node)
137
+
138
+ if isinstance(node, dict):
139
+ props = {**node, **resolved}
140
+ props = {k: v for k, v in props.items() if k not in _SKIP_KEYS}
141
+ label = props.pop("label", None) or props.pop("node_type", None) or "Node"
142
+ nid = props.pop("uuid", "") or props.pop("id", "")
143
+ lines = [f"[{label}]" + (f" (id: {nid})" if nid else "")]
144
+ sorted_keys = [k for k in _PRIORITY_KEYS if k in props]
145
+ sorted_keys += [k for k in props if k not in _PRIORITY_KEYS]
146
+ for k in sorted_keys:
147
+ lines.append(f" {k}: {props[k]}")
148
+ return "\n".join(lines)
149
+
150
+ # GraphNode object
151
+ label = getattr(node, "label", "Node")
152
+ nid = getattr(node, "id", "")
153
+ props = {**(getattr(node, "properties", {}) or {}), **resolved}
154
+ props = {k: v for k, v in props.items() if k not in _SKIP_KEYS}
155
+ lines = [f"[{label}]" + (f" (id: {nid})" if nid else "")]
156
+ sorted_keys = [k for k in _PRIORITY_KEYS if k in props]
157
+ sorted_keys += [k for k in props if k not in _PRIORITY_KEYS]
158
+ for k in sorted_keys:
159
+ lines.append(f" {k}: {props[k]}")
160
+ return "\n".join(lines)
161
+
162
+
163
+ def _edge_to_text(edge: Any, node_names: Optional[Dict[str, str]] = None) -> str:
164
+ """Convert a GraphEdge (or dict) to human-readable text.
165
+
166
+ When *node_names* is provided (id → display name), the edge is rendered
167
+ with readable names instead of raw UUIDs, e.g.:
168
+ "Alice" -[WORKS_ON]-> "Project X"
169
+ """
170
+ names = node_names or {}
171
+
172
+ if isinstance(edge, dict):
173
+ etype = edge.get("label") or edge.get("edge_type") or "EDGE"
174
+ src = str(edge.get("source") or edge.get("from_id", "?"))
175
+ tgt = str(edge.get("target") or edge.get("to_id", "?"))
176
+ props = {k: v for k, v in edge.items()
177
+ if k not in ("label", "edge_type", "source", "from_id", "target", "to_id", "uuid")}
178
+ else:
179
+ etype = getattr(edge, "label", getattr(edge, "edge_type", "EDGE"))
180
+ src = str(getattr(edge, "source_id", getattr(edge, "from_id", "?")))
181
+ tgt = str(getattr(edge, "target_id", getattr(edge, "to_id", "?")))
182
+ props = {}
183
+
184
+ src_display = names.get(src) or src[:12]
185
+ tgt_display = names.get(tgt) or tgt[:12]
186
+ line = f"{src_display} -[{etype}]-> {tgt_display}"
187
+ if props:
188
+ line += " " + ", ".join(f"{k}={v}" for k, v in props.items())
189
+ return line
190
+
191
+
192
+ # ---------------------------------------------------------------------------
193
+ # ContextHub
194
+ # ---------------------------------------------------------------------------
195
+
196
+ class ContextHub:
197
+ """
198
+ Build, collect, and export structured context for LLMs.
199
+
200
+ Quick-start
201
+ -----------
202
+ >>> hub = ContextHub(system_prompt="You are a helpful graph analyst.")
203
+ >>> hub.add_text("Company overview …", role="background")
204
+ >>> hub.add_nodes(graph.get_all_nodes())
205
+ >>> messages = hub.to_messages() # ready for openai.chat.completions.create(messages=…)
206
+ """
207
+
208
+ def __init__(
209
+ self,
210
+ system_prompt: Optional[str] = None,
211
+ max_tokens: int = 8000,
212
+ token_estimate_ratio: float = 4.0, # chars per token (rough)
213
+ ):
214
+ self._items: List[ContextItem] = []
215
+ self.max_tokens = max_tokens
216
+ self._token_ratio = token_estimate_ratio
217
+ self._scoper = None # Optional ContextScoper
218
+ self._compression_config = None # Optional CompressionConfig
219
+ self._slots: Dict[str, SlotConfig] = {} # name -> slot config
220
+ self._db = None # Optional graph ref for ContextMeta + lineage + contradiction detection
221
+ self._memory = None # Optional AgentMemory for memory subgraph
222
+ self._memory_agent_id = None
223
+ self._memory_query = None
224
+ self._last_scored_items = [] # For explainability
225
+
226
+ if system_prompt:
227
+ self.add_text(system_prompt, role=ContextRole.SYSTEM, label="System")
228
+
229
+ def set_scoping(self, config) -> "ContextHub":
230
+ """
231
+ Enable token-budget scoping on exports.
232
+
233
+ Args:
234
+ config: A ``ScopingConfig`` instance (from ``contextsynapse.context.scoping``).
235
+ """
236
+ from .scoping import ContextScoper
237
+ self._scoper = ContextScoper(config)
238
+ return self
239
+
240
+ # ------------------------------------------------------------------
241
+ # Fluent add-methods
242
+ # ------------------------------------------------------------------
243
+
244
+ def add_text(
245
+ self,
246
+ text: str,
247
+ role: Union[ContextRole, str] = ContextRole.USER,
248
+ label: Optional[str] = None,
249
+ source: Optional[str] = None,
250
+ metadata: Optional[Dict] = None,
251
+ tags: Optional[List[str]] = None,
252
+ sensitivity: str = "public",
253
+ ) -> "ContextHub":
254
+ """Add free-form text."""
255
+ self._items.append(ContextItem(
256
+ content=text.strip(),
257
+ role=ContextRole(role) if isinstance(role, str) else role,
258
+ label=label,
259
+ source=source,
260
+ metadata=metadata or {},
261
+ tags=tags or [],
262
+ sensitivity=sensitivity,
263
+ ))
264
+ return self
265
+
266
+ def add_nodes(
267
+ self,
268
+ nodes: List[Any],
269
+ role: Union[ContextRole, str] = ContextRole.RETRIEVED,
270
+ label: Optional[str] = None,
271
+ source: Optional[str] = None,
272
+ ) -> "ContextHub":
273
+ """Add a list of GraphNode objects or dicts as context."""
274
+ if not nodes:
275
+ return self
276
+ lines = [f"## Graph Nodes ({len(nodes)} total)"]
277
+ for node in nodes:
278
+ lines.append(_node_to_text(node))
279
+ self._items.append(ContextItem(
280
+ content="\n".join(lines),
281
+ role=ContextRole(role) if isinstance(role, str) else role,
282
+ label=label or "Graph Nodes",
283
+ source=source or "graph",
284
+ metadata={"count": len(nodes)},
285
+ ))
286
+ return self
287
+
288
+ def add_edges(
289
+ self,
290
+ edges: List[Any],
291
+ role: Union[ContextRole, str] = ContextRole.RETRIEVED,
292
+ label: Optional[str] = None,
293
+ source: Optional[str] = None,
294
+ node_names: Optional[Dict[str, str]] = None,
295
+ ) -> "ContextHub":
296
+ """Add a list of GraphEdge objects or dicts as context.
297
+
298
+ Args:
299
+ node_names: Optional mapping of node ID → display name for
300
+ readable edge rendering.
301
+ """
302
+ if not edges:
303
+ return self
304
+ lines = [f"## Graph Relationships ({len(edges)} total)"]
305
+ for edge in edges:
306
+ lines.append(_edge_to_text(edge, node_names=node_names))
307
+ self._items.append(ContextItem(
308
+ content="\n".join(lines),
309
+ role=ContextRole(role) if isinstance(role, str) else role,
310
+ label=label or "Graph Relationships",
311
+ source=source or "graph",
312
+ metadata={"count": len(edges)},
313
+ ))
314
+ return self
315
+
316
+ def add_query_result(
317
+ self,
318
+ result: Any,
319
+ query: Optional[str] = None,
320
+ role: Union[ContextRole, str] = ContextRole.RETRIEVED,
321
+ label: Optional[str] = None,
322
+ ) -> "ContextHub":
323
+ """Add the result of an AIQL query (list, dict, or raw string)."""
324
+ if result is None:
325
+ return self
326
+ if isinstance(result, list):
327
+ # List of nodes / dicts
328
+ if result and isinstance(result[0], dict):
329
+ self.add_nodes(result, role=role, label=label or (f"Query: {query}" if query else "Query Result"))
330
+ else:
331
+ text = "\n".join(str(r) for r in result)
332
+ self.add_text(text, role=role, label=label or "Query Result")
333
+ elif isinstance(result, dict):
334
+ nodes = result.get("nodes") or result.get("data", {}).get("nodes", [])
335
+ if nodes:
336
+ self.add_nodes(nodes, role=role, label=label or (f"Query: {query}" if query else "Query Result"))
337
+ else:
338
+ self.add_text(json.dumps(result, indent=2), role=role, label=label or "Query Result")
339
+ else:
340
+ self.add_text(str(result), role=role, label=label)
341
+ return self
342
+
343
+ def add_graph_summary(
344
+ self, db: Any, namespace: str = "default", categories: Optional[List[str]] = None,
345
+ ) -> "ContextHub":
346
+ """
347
+ Pull a summary (node/edge counts by type) from a live AIContextDB instance
348
+ and add it as background context.
349
+
350
+ Args:
351
+ db: AIContextDB graph instance.
352
+ namespace: Graph namespace name.
353
+ categories: If provided, only include nodes matching these categories.
354
+ """
355
+ from .boundaries import BOUNDARY_NODE_LABELS
356
+
357
+ try:
358
+ all_nodes = db.get_all_nodes()
359
+ type_counts: Dict[str, int] = {}
360
+ filtered_count = 0
361
+ for n in all_nodes:
362
+ if isinstance(n, dict):
363
+ lbl = n.get("label", "?")
364
+ cat = n.get("properties", {}).get("category") or n.get("category")
365
+ else:
366
+ lbl = getattr(n, "label", "?")
367
+ props = getattr(n, "properties", {}) or {}
368
+ cat = props.get("category")
369
+
370
+ # Skip boundary nodes from summary
371
+ if lbl in BOUNDARY_NODE_LABELS:
372
+ continue
373
+
374
+ # Category filter
375
+ if categories and cat not in categories:
376
+ continue
377
+
378
+ filtered_count += 1
379
+ type_counts[lbl] = type_counts.get(lbl, 0) + 1
380
+
381
+ lines = [f"## Graph Summary (namespace: {namespace})"]
382
+ if categories:
383
+ lines.append(f"Categories: {', '.join(categories)}")
384
+ lines.append(f"Total nodes: {filtered_count}")
385
+ for t, c in sorted(type_counts.items()):
386
+ lines.append(f" {t}: {c}")
387
+ self.add_text("\n".join(lines), role=ContextRole.BACKGROUND, label="Graph Summary", source="graph")
388
+ except Exception as exc:
389
+ self.add_text(f"(Graph summary unavailable: {exc})", role=ContextRole.BACKGROUND)
390
+ return self
391
+
392
+ def add_graph_category(
393
+ self, db: Any, namespace: str, category: str, role: Union["ContextRole", str] = "background",
394
+ ) -> "ContextHub":
395
+ """Pull nodes from one specific category and add their content.
396
+
397
+ Args:
398
+ db: AIContextDB graph instance.
399
+ namespace: Graph namespace name.
400
+ category: Category to pull (e.g. "knowledge_base", "user_message").
401
+ role: ContextRole for the added items.
402
+ """
403
+ from .boundaries import BOUNDARY_NODE_LABELS
404
+
405
+ try:
406
+ all_nodes = db.get_all_nodes()
407
+ items_added = 0
408
+ for n in all_nodes:
409
+ if isinstance(n, dict):
410
+ lbl = n.get("label", "")
411
+ props = n.get("properties", {})
412
+ else:
413
+ lbl = getattr(n, "label", "")
414
+ props = getattr(n, "properties", {}) or {}
415
+
416
+ if lbl in BOUNDARY_NODE_LABELS:
417
+ continue
418
+
419
+ if props.get("category") != category:
420
+ continue
421
+
422
+ content = props.get("content", "")
423
+ if not content:
424
+ continue
425
+
426
+ ctx_role = ContextRole(role) if isinstance(role, str) else role
427
+ node_label = props.get("label") or props.get("name") or f"{lbl} item"
428
+ self.add_text(content, role=ctx_role, label=node_label, source=f"context:{namespace}")
429
+ items_added += 1
430
+
431
+ if items_added == 0:
432
+ self.add_text(
433
+ f"(No items in category '{category}' for namespace '{namespace}')",
434
+ role=ContextRole.BACKGROUND,
435
+ )
436
+ except Exception as exc:
437
+ self.add_text(f"(Category pull failed: {exc})", role=ContextRole.BACKGROUND)
438
+ return self
439
+
440
+ def contribute(
441
+ self,
442
+ content: str,
443
+ agent_id: str,
444
+ role: Union[ContextRole, str] = ContextRole.GENERATED,
445
+ label: Optional[str] = None,
446
+ source_ids: Optional[List[str]] = None,
447
+ metadata: Optional[Dict] = None,
448
+ tags: Optional[List[str]] = None,
449
+ sensitivity: str = "internal",
450
+ confidence: float = 0.8,
451
+ ) -> "ContextHub":
452
+ """
453
+ Write agent-generated output back into the context.
454
+
455
+ This is the primary method agents use to share their work with other
456
+ agents. The ``source_ids`` field records which context items or graph
457
+ nodes this output was derived from (lineage).
458
+
459
+ Args:
460
+ content: The agent's output text (summary, analysis, decision, …)
461
+ agent_id: Who produced this
462
+ role: generated | synthesis | decision
463
+ label: Human-readable heading
464
+ source_ids: IDs of items/nodes this was derived from
465
+ metadata: Extra info (model used, confidence, etc.)
466
+ tags: Classification tags for this item
467
+ sensitivity: public|internal|confidential|restricted (default: internal)
468
+ confidence: Trust score [0.0, 1.0]. When source_ids are provided,
469
+ auto-computed as min(parent_confidences) * 0.95 if lower.
470
+ """
471
+ meta = metadata or {}
472
+ if source_ids:
473
+ meta["derived_from"] = source_ids
474
+ # Auto-compute derived confidence from parents
475
+ parent_confs = []
476
+ for item in self._items:
477
+ pid = item.provenance_id or item.label
478
+ if pid and pid in source_ids:
479
+ parent_confs.append(item.confidence)
480
+ if parent_confs:
481
+ derived_conf = min(parent_confs) * 0.95
482
+ confidence = min(confidence, derived_conf)
483
+ # Gracefully handle unknown role strings (e.g., "finding", "observation")
484
+ if isinstance(role, str):
485
+ try:
486
+ resolved_role = ContextRole(role)
487
+ except ValueError:
488
+ resolved_role = ContextRole.GENERATED
489
+ else:
490
+ resolved_role = role
491
+ self._items.append(ContextItem(
492
+ content=content.strip(),
493
+ role=resolved_role,
494
+ label=label or f"Agent output ({agent_id[:8]})",
495
+ source="agent",
496
+ metadata=meta,
497
+ agent_id=agent_id,
498
+ tags=tags or [],
499
+ sensitivity=sensitivity,
500
+ confidence=confidence,
501
+ ))
502
+ return self
503
+
504
+ def add_instruction(self, instruction: str) -> "ContextHub":
505
+ """Add a task instruction (maps to USER role in most LLMs)."""
506
+ return self.add_text(instruction, role=ContextRole.INSTRUCTION, label="Instruction")
507
+
508
+ def add_example(self, example: str, label: Optional[str] = None) -> "ContextHub":
509
+ """Add a few-shot example."""
510
+ return self.add_text(example, role=ContextRole.EXAMPLE, label=label or "Example")
511
+
512
+ @classmethod
513
+ def rag_query(
514
+ cls,
515
+ question: str,
516
+ retrieved_nodes: list,
517
+ llm,
518
+ model_name: str = None,
519
+ ) -> Dict[str, Any]:
520
+ """Build a RAG prompt from retrieved nodes and generate an answer.
521
+
522
+ Args:
523
+ question: The user's question.
524
+ retrieved_nodes: List of node dicts with 'properties' or raw props.
525
+ llm: AIContextDBUniversalLLM instance.
526
+ model_name: LLM model to use (defaults to first available).
527
+
528
+ Returns:
529
+ ``{"answer": str, "sources": [...], "hub": ContextHub}``
530
+ """
531
+ hub = cls(system_prompt=(
532
+ "Answer the question based ONLY on the provided context. "
533
+ "Cite source names. If context is insufficient, say so."
534
+ ))
535
+ sources = []
536
+ for node in retrieved_nodes:
537
+ props = getattr(node, "properties", node) if not isinstance(node, dict) else node.get("properties", node)
538
+ content = props.get("content") or props.get("statement") or props.get("name", "")
539
+ label = props.get("name") or props.get("label", "Source")
540
+ hub.add_text(str(content), role=ContextRole.RETRIEVED, label=label)
541
+ sources.append({"label": label, "node_id": str(props.get("node_id", ""))})
542
+
543
+ hub.add_text(question, role=ContextRole.INSTRUCTION, label="Question")
544
+
545
+ model = model_name or (llm.list_models()[0] if llm.list_models() else None)
546
+ if not model:
547
+ return {"answer": hub.to_prompt(), "sources": sources, "hub": hub}
548
+
549
+ result = llm.generate(model, hub.to_prompt())
550
+ return {
551
+ "answer": result.get("response", "Could not generate answer."),
552
+ "sources": sources,
553
+ "hub": hub,
554
+ }
555
+
556
+ # ------------------------------------------------------------------
557
+ # Phase 3: Extended context type methods
558
+ # ------------------------------------------------------------------
559
+
560
+ def add_document(
561
+ self,
562
+ content: str,
563
+ label: Optional[str] = None,
564
+ source: Optional[str] = None,
565
+ metadata: Optional[Dict] = None,
566
+ tags: Optional[List[str]] = None,
567
+ sensitivity: str = "public",
568
+ ) -> "ContextHub":
569
+ """Add a full source document (distinct from retrieved snippets)."""
570
+ return self.add_text(
571
+ content, role=ContextRole.DOCUMENT,
572
+ label=label or "Document", source=source,
573
+ metadata=metadata, tags=tags, sensitivity=sensitivity,
574
+ )
575
+
576
+ def add_interaction(
577
+ self,
578
+ content: str,
579
+ label: Optional[str] = None,
580
+ source: Optional[str] = None,
581
+ metadata: Optional[Dict] = None,
582
+ ) -> "ContextHub":
583
+ """Add a multi-turn dialogue transcript."""
584
+ return self.add_text(
585
+ content, role=ContextRole.INTERACTION,
586
+ label=label or "Interaction", source=source, metadata=metadata,
587
+ )
588
+
589
+ def add_feedback(
590
+ self,
591
+ content: str,
592
+ agent_id: Optional[str] = None,
593
+ label: Optional[str] = None,
594
+ metadata: Optional[Dict] = None,
595
+ ) -> "ContextHub":
596
+ """Add human feedback / correction. Confidence pinned to 1.0."""
597
+ self._items.append(ContextItem(
598
+ content=content.strip(),
599
+ role=ContextRole.FEEDBACK,
600
+ label=label or "Feedback",
601
+ source="human",
602
+ metadata=metadata or {},
603
+ agent_id=agent_id,
604
+ confidence=1.0,
605
+ ))
606
+ return self
607
+
608
+ def add_tool_result(
609
+ self,
610
+ content: str,
611
+ tool_name: str,
612
+ label: Optional[str] = None,
613
+ metadata: Optional[Dict] = None,
614
+ confidence: float = 0.9,
615
+ ) -> "ContextHub":
616
+ """Add a tool call result with optional confidence."""
617
+ meta = metadata or {}
618
+ meta["tool_name"] = tool_name
619
+ self._items.append(ContextItem(
620
+ content=content.strip(),
621
+ role=ContextRole.TOOL,
622
+ label=label or f"Tool: {tool_name}",
623
+ source="tool",
624
+ metadata=meta,
625
+ confidence=confidence,
626
+ ))
627
+ return self
628
+
629
+ # ------------------------------------------------------------------
630
+ # Phase 3: Confidence & lifecycle management
631
+ # ------------------------------------------------------------------
632
+
633
+ def set_confidence(self, index: int, confidence: float) -> "ContextHub":
634
+ """Set confidence on a specific item by index."""
635
+ self._items[index].confidence = max(0.0, min(1.0, confidence))
636
+ return self
637
+
638
+ def mark_stale(self, index_or_label: Union[int, str]) -> "ContextHub":
639
+ """Mark an item stale by index or label. Triggers propagate_staleness()."""
640
+ if isinstance(index_or_label, int):
641
+ self._items[index_or_label].state = "stale"
642
+ else:
643
+ for item in self._items:
644
+ if item.label == index_or_label:
645
+ item.state = "stale"
646
+ self.propagate_staleness()
647
+ return self
648
+
649
+ def propagate_staleness(self) -> int:
650
+ """Mark derived items as stale when their sources are stale.
651
+
652
+ Returns the count of newly stale items.
653
+ """
654
+ # Build set of identifiers for stale items
655
+ stale_ids: set = set()
656
+ for i, item in enumerate(self._items):
657
+ if item.state == "stale":
658
+ if item.provenance_id:
659
+ stale_ids.add(item.provenance_id)
660
+ if item.label:
661
+ stale_ids.add(item.label)
662
+ stale_ids.add(str(i))
663
+
664
+ newly_stale = 0
665
+ changed = True
666
+ while changed:
667
+ changed = False
668
+ for i, item in enumerate(self._items):
669
+ if item.state != "active":
670
+ continue
671
+ derived = item.metadata.get("derived_from", [])
672
+ if not derived:
673
+ continue
674
+ if any(src in stale_ids for src in derived):
675
+ item.state = "stale"
676
+ newly_stale += 1
677
+ changed = True
678
+ if item.provenance_id:
679
+ stale_ids.add(item.provenance_id)
680
+ if item.label:
681
+ stale_ids.add(item.label)
682
+ stale_ids.add(str(i))
683
+
684
+ return newly_stale
685
+
686
+ def refresh(self, index: int) -> "ContextHub":
687
+ """Restore an item to active state."""
688
+ self._items[index].state = "active"
689
+ return self
690
+
691
+ def archive(self, index: int) -> "ContextHub":
692
+ """Archive an item — excluded from all exports."""
693
+ self._items[index].state = "archived"
694
+ return self
695
+
696
+ # ------------------------------------------------------------------
697
+ # Phase 3: Context Slots
698
+ # ------------------------------------------------------------------
699
+
700
+ def define_slot(
701
+ self,
702
+ name: str,
703
+ max_tokens: int,
704
+ roles: Optional[List[ContextRole]] = None,
705
+ pinned: bool = False,
706
+ priority: int = 5,
707
+ ) -> "ContextHub":
708
+ """Define a named token bucket."""
709
+ self._slots[name] = SlotConfig(
710
+ name=name, max_tokens=max_tokens,
711
+ roles=roles, pinned=pinned, priority=priority,
712
+ )
713
+ return self
714
+
715
+ def assign_to_slot(self, index: int, slot_name: str) -> "ContextHub":
716
+ """Manually assign an item to a named slot."""
717
+ if slot_name not in self._slots:
718
+ raise ValueError(f"Slot '{slot_name}' not defined")
719
+ self._items[index]._slot = slot_name
720
+ return self
721
+
722
+ def set_slot(
723
+ self,
724
+ name: str,
725
+ content: str,
726
+ role: Union[ContextRole, str] = ContextRole.USER,
727
+ label: Optional[str] = None,
728
+ **kwargs,
729
+ ) -> "ContextHub":
730
+ """Overwrite a slot's content (removes old items in this slot, adds new one)."""
731
+ if name not in self._slots:
732
+ raise ValueError(f"Slot '{name}' not defined")
733
+ self._items = [i for i in self._items if i._slot != name]
734
+ item = ContextItem(
735
+ content=content.strip(),
736
+ role=ContextRole(role) if isinstance(role, str) else role,
737
+ label=label or name,
738
+ _slot=name,
739
+ **kwargs,
740
+ )
741
+ self._items.append(item)
742
+ return self
743
+
744
+ def _auto_assign_slots(self) -> None:
745
+ """Match unassigned items to slots by role."""
746
+ for item in self._items:
747
+ if item._slot is not None:
748
+ continue
749
+ for slot in self._slots.values():
750
+ if slot.roles and item.role in slot.roles:
751
+ item._slot = slot.name
752
+ break
753
+
754
+ # ------------------------------------------------------------------
755
+ # Phase 3: Compression
756
+ # ------------------------------------------------------------------
757
+
758
+ def set_compression(self, config) -> "ContextHub":
759
+ """Enable compression tiers on exports."""
760
+ self._compression_config = config
761
+ return self
762
+
763
+ def _get_rendered_content(self, item: ContextItem) -> str:
764
+ """Return content based on the item's compression tier."""
765
+ tier = item.metadata.get("_compression_tier", 1)
766
+ if tier <= 1:
767
+ return item.content
768
+ if tier == 2:
769
+ cached = item.metadata.get("_summary")
770
+ if cached:
771
+ return cached
772
+ if self._compression_config is not None:
773
+ from .compression import generate_summary
774
+ summary = generate_summary(item.content, self._compression_config)
775
+ item.metadata["_summary"] = summary
776
+ return summary
777
+ return item.content[:300] + "..."
778
+ if tier == 3:
779
+ lbl = item.label or item.role.value
780
+ tokens = max(1, int(len(item.content) / self._token_ratio))
781
+ return f"[{lbl}] ~{tokens} tokens, confidence: {item.confidence:.2f}"
782
+ return ""
783
+
784
+ # ------------------------------------------------------------------
785
+ # Inspection
786
+ # ------------------------------------------------------------------
787
+
788
+ def __len__(self) -> int:
789
+ return len(self._items)
790
+
791
+ def estimate_tokens(self) -> int:
792
+ total_chars = sum(len(item.content) for item in self._items)
793
+ return int(total_chars / self._token_ratio)
794
+
795
+ def is_within_limit(self) -> bool:
796
+ return self.estimate_tokens() <= self.max_tokens
797
+
798
+ def clear(self) -> "ContextHub":
799
+ """Remove all context items."""
800
+ self._items.clear()
801
+ return self
802
+
803
+ def items(self) -> List[ContextItem]:
804
+ return list(self._items)
805
+
806
+ def filter(
807
+ self,
808
+ role: Optional[Union[ContextRole, str]] = None,
809
+ source: Optional[str] = None,
810
+ agent_id: Optional[str] = None,
811
+ time_after: Optional[str] = None,
812
+ time_before: Optional[str] = None,
813
+ label: Optional[str] = None,
814
+ tags: Optional[List[str]] = None,
815
+ sensitivity: Optional[str] = None,
816
+ max_sensitivity: Optional[str] = None,
817
+ ) -> List[ContextItem]:
818
+ """Return items matching ALL supplied criteria.
819
+
820
+ Args:
821
+ tags: If provided, item must contain at least one of these tags.
822
+ sensitivity: Exact sensitivity match.
823
+ max_sensitivity: Return items at or below this level
824
+ (public < internal < confidential < restricted).
825
+ """
826
+ _SENS_ORDER = {"public": 0, "internal": 1, "confidential": 2, "restricted": 3}
827
+
828
+ result = list(self._items)
829
+ if role is not None:
830
+ r = ContextRole(role) if isinstance(role, str) else role
831
+ result = [i for i in result if i.role == r]
832
+ if source is not None:
833
+ result = [i for i in result if i.source == source]
834
+ if agent_id is not None:
835
+ result = [i for i in result if i.agent_id == agent_id]
836
+ if time_after is not None:
837
+ result = [i for i in result if i.created_at >= time_after]
838
+ if time_before is not None:
839
+ result = [i for i in result if i.created_at <= time_before]
840
+ if label is not None:
841
+ lbl = label.lower()
842
+ result = [i for i in result if i.label and lbl in i.label.lower()]
843
+ if tags is not None:
844
+ tag_set = set(tags)
845
+ result = [i for i in result if tag_set & set(i.tags)]
846
+ if sensitivity is not None:
847
+ result = [i for i in result if i.sensitivity == sensitivity]
848
+ if max_sensitivity is not None:
849
+ ceiling = _SENS_ORDER.get(max_sensitivity, 0)
850
+ result = [i for i in result if _SENS_ORDER.get(i.sensitivity, 0) <= ceiling]
851
+ return result
852
+
853
+ def filter_for_agent(
854
+ self,
855
+ access_level: str,
856
+ allowed_tags: Optional[List[str]] = None,
857
+ pii_mode: str = "block",
858
+ ) -> List[ContextItem]:
859
+ """Return only items this agent is authorized to see.
860
+
861
+ Policy:
862
+ - ``read`` → public items only (+ items matching allowed_tags)
863
+ - ``write`` → public + internal (+ allowed_tags)
864
+ - ``admin`` → everything
865
+
866
+ Args:
867
+ pii_mode: How to handle PII-containing items that the agent
868
+ *is* authorized to see:
869
+ - ``"block"`` — drop items with ``pii_detected=True`` unless
870
+ agent has ``"pii"`` in allowed_tags (current default).
871
+ - ``"mask"`` — keep the item but replace PII patterns with
872
+ ``***`` using ``PIIDetector.mask_pii()``.
873
+ - ``"redact"`` — keep the item structure but replace content
874
+ with ``"[REDACTED — contains PII]"``.
875
+ - ``"allow"`` — return items unmodified (for agents with
876
+ explicit PII access).
877
+ """
878
+ _LEVEL_CEILING = {"read": "public", "write": "internal", "admin": "restricted"}
879
+ ceiling = _LEVEL_CEILING.get(access_level, "public")
880
+
881
+ base = self.filter(max_sensitivity=ceiling)
882
+
883
+ # Add higher-sensitivity items that match allowed_tags
884
+ if allowed_tags:
885
+ tag_items = self.filter(tags=allowed_tags)
886
+ base_ids = {id(i) for i in base}
887
+ for item in tag_items:
888
+ if id(item) not in base_ids:
889
+ base.append(item)
890
+
891
+ # Apply PII mode
892
+ has_pii_access = allowed_tags and "pii" in allowed_tags
893
+ if pii_mode == "allow" or has_pii_access:
894
+ return base
895
+
896
+ if pii_mode == "block":
897
+ return [i for i in base if not i.pii_detected]
898
+
899
+ if pii_mode in ("mask", "redact"):
900
+ from ..security.data_security import PIIDetector
901
+ result = []
902
+ for item in base:
903
+ if not item.pii_detected:
904
+ result.append(item)
905
+ else:
906
+ masked = copy.copy(item)
907
+ if pii_mode == "mask":
908
+ masked.content = PIIDetector.mask_pii(item.content)
909
+ else: # redact
910
+ masked.content = "[REDACTED — contains PII]"
911
+ result.append(masked)
912
+ return result
913
+
914
+ return base
915
+
916
+ # ------------------------------------------------------------------
917
+ # Scoping & unified pipeline
918
+ # ------------------------------------------------------------------
919
+
920
+ def _apply_scoping(self, budget: Optional[int] = None) -> List[ContextItem]:
921
+ """Apply token-budget scoping if a scoper is configured or budget is given."""
922
+ if budget is not None and self._scoper is not None:
923
+ return self._scoper.select_within_budget(self._items, max_tokens=budget)
924
+ if budget is not None:
925
+ from .scoping import ContextScoper
926
+ return ContextScoper().select_within_budget(self._items, max_tokens=budget)
927
+ if self._scoper is not None:
928
+ return self._scoper.select_within_budget(self._items)
929
+ return self._items
930
+
931
+ def get_last_scored_items(self):
932
+ """Return scored items from last assembly, with breakdowns."""
933
+ return self._last_scored_items
934
+
935
+ def _prepare_items(self, budget: Optional[int] = None) -> List[ContextItem]:
936
+ """Unified export pipeline: filter → slots → score → compress → budget.
937
+
938
+ When no slots or compression are configured, falls back to _apply_scoping.
939
+ """
940
+ # C1: Prepend ContextMeta if graph is attached
941
+ if self._db and not any(getattr(i, 'label', '') == "ContextMeta" for i in self._items):
942
+ try:
943
+ meta_node = self._db.csr_adapter.get_node("_context_meta")
944
+ if meta_node:
945
+ mp = meta_node.properties or {}
946
+ meta_text = (
947
+ f"This context is from: {mp.get('name', 'unknown')}. "
948
+ f"It contains: {mp.get('schema_summary', 'various nodes')}. "
949
+ f"Purpose: {mp.get('purpose', 'knowledge_base')}. "
950
+ f"Nodes: {mp.get('node_count', '?')}, Edges: {mp.get('edge_count', '?')}."
951
+ )
952
+ self._items.insert(0, ContextItem(
953
+ content=meta_text, role=ContextRole.BACKGROUND,
954
+ label="ContextMeta", source="context_meta",
955
+ ))
956
+ except Exception:
957
+ pass
958
+
959
+ # C2: Pull from memory subgraph if available
960
+ if self._memory and self._memory_agent_id:
961
+ try:
962
+ query = self._memory_query or ""
963
+ memories = self._memory.recall(self._memory_agent_id, query=query, limit=10)
964
+ for m in memories:
965
+ if not any(i.content == m["content"] for i in self._items):
966
+ self._items.append(ContextItem(
967
+ content=m["content"], role=ContextRole.BACKGROUND,
968
+ label=f"Memory ({m.get('tags', [''])[0] if m.get('tags') else 'fact'})",
969
+ source="agent_memory",
970
+ confidence=m.get("effective_confidence", m.get("confidence", 1.0)),
971
+ metadata={"memory_id": m["id"]},
972
+ ))
973
+ except Exception:
974
+ pass
975
+
976
+ # C5: Contradiction detection
977
+ if self._db:
978
+ try:
979
+ node_ids = [i.metadata.get("node_id") for i in self._items if hasattr(i, 'metadata') and i.metadata and i.metadata.get("node_id")]
980
+ if len(node_ids) >= 2:
981
+ node_id_set = set(node_ids)
982
+ edges = self._db.csr_adapter.get_all_edges() if hasattr(self._db.csr_adapter, 'get_all_edges') else []
983
+ for edge in edges:
984
+ elabel = getattr(edge, 'edge_type', getattr(edge, 'label', ''))
985
+ src = getattr(edge, 'source_id', getattr(edge, 'source', ''))
986
+ tgt = getattr(edge, 'target_id', getattr(edge, 'target', ''))
987
+ if elabel == "CONTRADICTS" and src in node_id_set and tgt in node_id_set:
988
+ src_item = next((i for i in self._items if i.metadata and i.metadata.get("node_id") == src), None)
989
+ tgt_item = next((i for i in self._items if i.metadata and i.metadata.get("node_id") == tgt), None)
990
+ if src_item and tgt_item:
991
+ warning = f"Conflict: '{src_item.content[:60]}' contradicts '{tgt_item.content[:60]}'"
992
+ self._items.append(ContextItem(
993
+ content=warning, role=ContextRole.SYSTEM,
994
+ label="Contradiction Warning", source="contradiction_detection",
995
+ tags=["conflict"],
996
+ ))
997
+ except Exception:
998
+ pass
999
+
1000
+ # Fast path: no Phase 3 features active
1001
+ has_phase3 = self._slots or self._compression_config or any(
1002
+ i.state != "active" for i in self._items
1003
+ )
1004
+ if not has_phase3:
1005
+ result = self._apply_scoping(budget)
1006
+ # Store scored items for explainability (C6)
1007
+ if self._scoper:
1008
+ try:
1009
+ self._last_scored_items = self._scoper.score_items(self._items)
1010
+ except Exception:
1011
+ pass
1012
+ return result
1013
+
1014
+ # 1. Filter out archived items
1015
+ active = [i for i in self._items if i.state != "archived"]
1016
+
1017
+ # 2. Auto-assign items to slots
1018
+ if self._slots:
1019
+ for item in active:
1020
+ if item._slot is None:
1021
+ for slot in self._slots.values():
1022
+ if slot.roles and item.role in slot.roles:
1023
+ item._slot = slot.name
1024
+ break
1025
+
1026
+ # 3. Separate pinned-slot items
1027
+ pinned: List[ContextItem] = []
1028
+ remaining: List[ContextItem] = []
1029
+ for item in active:
1030
+ if item._slot and self._slots.get(item._slot, SlotConfig("", 0)).pinned:
1031
+ pinned.append(item)
1032
+ else:
1033
+ remaining.append(item)
1034
+
1035
+ # 4. Determine budget
1036
+ effective_budget = budget or (self._scoper.config.max_tokens if self._scoper else self.max_tokens)
1037
+ pinned_tokens = sum(max(1, int(len(i.content) / self._token_ratio)) for i in pinned)
1038
+ remaining_budget = max(0, effective_budget - pinned_tokens)
1039
+
1040
+ # 5. Score remaining items
1041
+ from .scoping import ContextScoper
1042
+ scoper = self._scoper or ContextScoper()
1043
+ scored = scoper.score_items(remaining)
1044
+ self._last_scored_items = scored # Store for explainability (C6)
1045
+
1046
+ # 6. Apply compression tiers if configured
1047
+ if self._compression_config is not None:
1048
+ from .compression import assign_compression_tiers, CompressionTier
1049
+ tier_map = assign_compression_tiers(scored, self._compression_config)
1050
+ for si in scored:
1051
+ tier = tier_map.get(si.index, CompressionTier.FULL)
1052
+ si.item.metadata["_compression_tier"] = tier
1053
+ # Recalculate token estimate for compressed content
1054
+ rendered = self._get_rendered_content(si.item)
1055
+ si.token_estimate = max(1, int(len(rendered) / self._token_ratio))
1056
+
1057
+ # 7. Per-slot budget enforcement
1058
+ slot_selected: Dict[str, set] = {name: set() for name in self._slots}
1059
+ slot_used: Dict[str, int] = {name: 0 for name in self._slots}
1060
+
1061
+ # Sort by score descending for greedy selection
1062
+ scored.sort(key=lambda s: s.score, reverse=True)
1063
+
1064
+ selected_indices: set = set()
1065
+ used_tokens = 0
1066
+
1067
+ for si in scored:
1068
+ # Skip dropped items
1069
+ if si.item.metadata.get("_compression_tier", 1) >= 4:
1070
+ continue
1071
+
1072
+ slot_name = si.item._slot
1073
+ if slot_name and slot_name in self._slots:
1074
+ slot_cfg = self._slots[slot_name]
1075
+ if slot_used[slot_name] + si.token_estimate <= slot_cfg.max_tokens:
1076
+ if used_tokens + si.token_estimate <= remaining_budget:
1077
+ selected_indices.add(si.index)
1078
+ slot_used[slot_name] += si.token_estimate
1079
+ used_tokens += si.token_estimate
1080
+ else:
1081
+ if used_tokens + si.token_estimate <= remaining_budget:
1082
+ selected_indices.add(si.index)
1083
+ used_tokens += si.token_estimate
1084
+
1085
+ # 8. Reconstruct in original order
1086
+ selected_remaining = [remaining[i] for i in sorted(selected_indices)]
1087
+
1088
+ # Merge pinned + selected, preserving original order from active list
1089
+ result_set = set(id(i) for i in pinned) | set(id(i) for i in selected_remaining)
1090
+ return [i for i in active if id(i) in result_set]
1091
+
1092
+ # ------------------------------------------------------------------
1093
+ # Export
1094
+ # ------------------------------------------------------------------
1095
+
1096
+ def _role_to_llm_role(self, role: ContextRole) -> str:
1097
+ """Map semantic roles to the standard system/user/assistant triad."""
1098
+ mapping = {
1099
+ ContextRole.SYSTEM: "system",
1100
+ ContextRole.USER: "user",
1101
+ ContextRole.ASSISTANT: "assistant",
1102
+ ContextRole.BACKGROUND: "user",
1103
+ ContextRole.RETRIEVED: "user",
1104
+ ContextRole.INSTRUCTION: "user",
1105
+ ContextRole.EXAMPLE: "user",
1106
+ # Agent-generated outputs map to assistant (they came from an LLM)
1107
+ ContextRole.GENERATED: "assistant",
1108
+ ContextRole.SYNTHESIS: "assistant",
1109
+ ContextRole.DECISION: "assistant",
1110
+ # Phase 3: Extended context types → user side
1111
+ ContextRole.DOCUMENT: "user",
1112
+ ContextRole.INTERACTION: "user",
1113
+ ContextRole.HUMAN: "user",
1114
+ ContextRole.TOOL: "user",
1115
+ ContextRole.FEEDBACK: "user",
1116
+ }
1117
+ return mapping.get(role, "user")
1118
+
1119
+ def to_messages(self, budget: Optional[int] = None) -> List[Dict[str, str]]:
1120
+ """
1121
+ Export as a list of OpenAI / Anthropic-style messages.
1122
+
1123
+ System items become {"role": "system", "content": "…"}.
1124
+ Everything else is merged into a single "user" message to avoid
1125
+ alternating-role constraints, unless you have explicit assistant turns.
1126
+
1127
+ Args:
1128
+ budget: Optional token budget. When provided (or when a scoper is
1129
+ configured), items are pruned to fit within the budget.
1130
+ """
1131
+ items = self._prepare_items(budget)
1132
+ messages: List[Dict[str, str]] = []
1133
+ user_parts: List[str] = []
1134
+
1135
+ for item in items:
1136
+ content = self._get_rendered_content(item)
1137
+ if not content:
1138
+ continue
1139
+ llm_role = self._role_to_llm_role(item.role)
1140
+ if llm_role == "system":
1141
+ if user_parts:
1142
+ messages.append({"role": "user", "content": "\n\n".join(user_parts)})
1143
+ user_parts = []
1144
+ messages.append({"role": "system", "content": content})
1145
+ elif llm_role == "assistant":
1146
+ if user_parts:
1147
+ messages.append({"role": "user", "content": "\n\n".join(user_parts)})
1148
+ user_parts = []
1149
+ messages.append({"role": "assistant", "content": content})
1150
+ else:
1151
+ heading = f"### {item.label}\n" if item.label else ""
1152
+ user_parts.append(f"{heading}{content}")
1153
+
1154
+ if user_parts:
1155
+ messages.append({"role": "user", "content": "\n\n".join(user_parts)})
1156
+
1157
+ return messages
1158
+
1159
+ def to_prompt(self, separator: str = "\n\n---\n\n", budget: Optional[int] = None,
1160
+ agent_id: Optional[str] = None) -> str:
1161
+ """Export as a single concatenated prompt string.
1162
+
1163
+ Args:
1164
+ budget: Optional token budget for scoping.
1165
+ agent_id: If provided, records delivery to the quality tracker.
1166
+ """
1167
+ items = self._prepare_items(budget)
1168
+ parts = []
1169
+ node_ids = []
1170
+ node_labels = []
1171
+ for item in items:
1172
+ content = self._get_rendered_content(item)
1173
+ if not content:
1174
+ continue
1175
+ heading = f"### {item.label}\n" if item.label else ""
1176
+ parts.append(f"{heading}{content}")
1177
+ if item.source == "graph":
1178
+ node_labels.append(item.label or "")
1179
+ # Extract node count from metadata
1180
+ count = item.metadata.get("count", 0) if item.metadata else 0
1181
+ for _ in range(min(count, 50)):
1182
+ node_ids.append(item.label or "")
1183
+
1184
+ result = separator.join(parts)
1185
+
1186
+ # Record delivery for quality tracking
1187
+ if agent_id:
1188
+ try:
1189
+ from .quality_tracker import get_quality_tracker
1190
+ tracker = get_quality_tracker()
1191
+ token_est = max(1, len(result) // 4)
1192
+ tracker.record_delivery(
1193
+ agent_id, node_ids=node_ids, token_count=token_est,
1194
+ node_labels=list(set(node_labels)),
1195
+ total_graph_nodes=self.max_tokens,
1196
+ )
1197
+ except Exception:
1198
+ pass
1199
+
1200
+ return result
1201
+
1202
+ def to_markdown(self) -> str:
1203
+ """Export as formatted Markdown (useful for display / debugging)."""
1204
+ sections = []
1205
+ for item in self._items:
1206
+ heading = item.label or item.role.value.title()
1207
+ meta = f"_role: {item.role.value}, source: {item.source or 'manual'}_"
1208
+ sections.append(f"## {heading}\n{meta}\n\n{item.content}")
1209
+ return "\n\n---\n\n".join(sections)
1210
+
1211
+ def to_dict(self) -> Dict[str, Any]:
1212
+ """Export as a serialisable dict."""
1213
+ return {
1214
+ "items": [i.to_dict() for i in self._items],
1215
+ "estimated_tokens": self.estimate_tokens(),
1216
+ "max_tokens": self.max_tokens,
1217
+ "within_limit": self.is_within_limit(),
1218
+ }
1219
+
1220
+ def export(self, fmt: Union[ContextFormat, str] = ContextFormat.MESSAGES) -> Any:
1221
+ """Generic export — choose format by enum or string name."""
1222
+ fmt = ContextFormat(fmt) if isinstance(fmt, str) else fmt
1223
+ if fmt == ContextFormat.MESSAGES:
1224
+ return self.to_messages()
1225
+ elif fmt == ContextFormat.PROMPT:
1226
+ return self.to_prompt()
1227
+ elif fmt == ContextFormat.MARKDOWN:
1228
+ return self.to_markdown()
1229
+ elif fmt == ContextFormat.DICT:
1230
+ return self.to_dict()
1231
+ raise ValueError(f"Unknown format: {fmt}")
1232
+
1233
+ # ------------------------------------------------------------------
1234
+ # Serialisation
1235
+ # ------------------------------------------------------------------
1236
+
1237
+ def save(self, path: str) -> None:
1238
+ """Persist the hub to a JSON file."""
1239
+ with open(path, "w", encoding="utf-8") as f:
1240
+ json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
1241
+
1242
+ @classmethod
1243
+ def load(cls, path: str) -> "ContextHub":
1244
+ """Load a previously saved hub."""
1245
+ with open(path, encoding="utf-8") as f:
1246
+ data = json.load(f)
1247
+ hub = cls(max_tokens=data.get("max_tokens", 8000))
1248
+ for item_data in data.get("items", []):
1249
+ hub._items.append(ContextItem(
1250
+ content=item_data["content"],
1251
+ role=ContextRole(item_data.get("role", "user")),
1252
+ label=item_data.get("label"),
1253
+ source=item_data.get("source"),
1254
+ metadata=item_data.get("metadata", {}),
1255
+ created_at=item_data.get("created_at", ""),
1256
+ agent_id=item_data.get("agent_id"),
1257
+ provenance_id=item_data.get("provenance_id"),
1258
+ tags=item_data.get("tags", []),
1259
+ sensitivity=item_data.get("sensitivity", "public"),
1260
+ pii_detected=item_data.get("pii_detected", False),
1261
+ confidence=item_data.get("confidence", 1.0),
1262
+ state=item_data.get("state", "active"),
1263
+ _slot=item_data.get("_slot"),
1264
+ ))
1265
+ return hub
1266
+
1267
+ @classmethod
1268
+ def from_session_graph(
1269
+ cls,
1270
+ db,
1271
+ system_prompt: Optional[str] = None,
1272
+ max_tokens: int = 8000,
1273
+ recent_turns: int = 20,
1274
+ agent_id: Optional[str] = None,
1275
+ ) -> "ContextHub":
1276
+ """
1277
+ Build a ContextHub from a unified session graph.
1278
+
1279
+ Follows REFERENCES edges from recent Turn nodes to pull in
1280
+ contextually relevant knowledge nodes rather than blindly
1281
+ grabbing the most recent chunks.
1282
+
1283
+ Slot allocation:
1284
+ - decisions + actions: 15%
1285
+ - recent turns: 40%
1286
+ - referenced knowledge: 35%
1287
+ - topics: 10%
1288
+
1289
+ Args:
1290
+ db: AIContextDB instance for the session namespace.
1291
+ system_prompt: System prompt to pin at the top.
1292
+ max_tokens: Total token budget.
1293
+ recent_turns: Number of recent Turn nodes to include.
1294
+ agent_id: Optional agent ID for agent-scoped context.
1295
+ """
1296
+ from .session_graph import SessionGraphBuilder as _SGB
1297
+
1298
+ # Delegate to SessionGraphBuilder's build_context logic
1299
+ # which uses the same ContextHub but with graph-aware assembly.
1300
+ # This classmethod is a convenience for direct use without a builder.
1301
+ hub = cls(system_prompt=system_prompt, max_tokens=max_tokens)
1302
+
1303
+ decisions_budget = int(max_tokens * 0.15)
1304
+ turns_budget = int(max_tokens * 0.40)
1305
+ knowledge_budget = int(max_tokens * 0.35)
1306
+ topics_budget = int(max_tokens * 0.10)
1307
+
1308
+ hub.define_slot("decisions", max_tokens=decisions_budget, pinned=False, priority=8)
1309
+ hub.define_slot("turns", max_tokens=turns_budget, pinned=False, priority=7)
1310
+ hub.define_slot("knowledge", max_tokens=knowledge_budget, pinned=False, priority=6)
1311
+ hub.define_slot("topics", max_tokens=topics_budget, pinned=False, priority=5)
1312
+
1313
+ all_nodes = db.get_all_nodes()
1314
+ all_edges = db.get_all_edges()
1315
+
1316
+ # Collect and sort Turn nodes
1317
+ turns = [n for n in all_nodes if n.label == "Turn"]
1318
+ turns.sort(key=lambda n: n.properties.get("turn_index", 0), reverse=True)
1319
+ recent = turns[:recent_turns]
1320
+
1321
+ # Add turns in chronological order
1322
+ for turn_node in reversed(recent):
1323
+ props = turn_node.properties
1324
+ role_str = props.get("role", "user")
1325
+ role_map = {
1326
+ "user": ContextRole.USER,
1327
+ "assistant": ContextRole.ASSISTANT,
1328
+ "system": ContextRole.SYSTEM,
1329
+ "tool": ContextRole.TOOL,
1330
+ }
1331
+ ctx_role = role_map.get(role_str, ContextRole.INTERACTION)
1332
+ hub.add_text(
1333
+ props.get("content", ""),
1334
+ role=ctx_role,
1335
+ label=f"Turn ({role_str})",
1336
+ source=f"turn:{turn_node.id}",
1337
+ )
1338
+ hub.assign_to_slot(len(hub.items()) - 1, "turns")
1339
+
1340
+ # Follow REFERENCES edges from recent turns to knowledge nodes
1341
+ recent_ids = {n.id for n in recent}
1342
+ ref_ids = set()
1343
+ for edge in all_edges:
1344
+ if edge.label == "REFERENCES" and edge.source in recent_ids:
1345
+ ref_ids.add(edge.target)
1346
+
1347
+ for node in all_nodes:
1348
+ if node.id in ref_ids:
1349
+ props = node.properties
1350
+ if node.label == "Entity":
1351
+ text = f"[Entity] {props.get('entity_name', '')} ({props.get('entity_type', '')})"
1352
+ if props.get("description"):
1353
+ text += f": {props['description']}"
1354
+ elif node.label == "Document":
1355
+ text = f"[Document] {props.get('title', props.get('source', ''))}"
1356
+ elif node.label == "TextChunk":
1357
+ text = f"[Chunk] {props.get('content', '')[:500]}"
1358
+ else:
1359
+ text = f"[{node.label}] {str(props)[:300]}"
1360
+ hub.add_text(text, role=ContextRole.RETRIEVED, label=f"Referenced {node.label}",
1361
+ source=f"knowledge:{node.id}")
1362
+ hub.assign_to_slot(len(hub.items()) - 1, "knowledge")
1363
+
1364
+ # Decisions and Actions
1365
+ da_nodes = [
1366
+ n for n in all_nodes
1367
+ if n.label in ("Decision", "Action") and n.properties.get("status", "") != "done"
1368
+ ]
1369
+ da_nodes.sort(
1370
+ key=lambda n: n.properties.get("decided_at", n.properties.get("created_at", "")),
1371
+ reverse=True,
1372
+ )
1373
+ for node in da_nodes[:10]:
1374
+ props = node.properties
1375
+ if node.label == "Decision":
1376
+ text = f"[Decision] {props.get('summary', '')}"
1377
+ else:
1378
+ text = f"[Action] {props.get('description', '')} (status: {props.get('status', 'pending')})"
1379
+ hub.add_text(text, role=ContextRole.DECISION, label=node.label,
1380
+ source=f"{node.label.lower()}:{node.id}")
1381
+ hub.assign_to_slot(len(hub.items()) - 1, "decisions")
1382
+
1383
+ # Topics summary
1384
+ topics = [n for n in all_nodes if n.label == "Topic"]
1385
+ topics.sort(key=lambda n: n.properties.get("mention_count", 0), reverse=True)
1386
+ if topics:
1387
+ lines = [
1388
+ f"- {t.properties.get('name', '?')} (mentions: {t.properties.get('mention_count', 1)})"
1389
+ for t in topics[:15]
1390
+ ]
1391
+ hub.add_text("## Active Topics\n" + "\n".join(lines),
1392
+ role=ContextRole.BACKGROUND, label="Topics", source="topics:summary")
1393
+ hub.assign_to_slot(len(hub.items()) - 1, "topics")
1394
+
1395
+ return hub
1396
+
1397
+ def __repr__(self) -> str:
1398
+ return (
1399
+ f"ContextHub(items={len(self._items)}, "
1400
+ f"~{self.estimate_tokens()} tokens / {self.max_tokens} limit)"
1401
+ )