contextsynapse 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextsynapse/__init__.py +107 -0
- contextsynapse/__main__.py +9 -0
- contextsynapse/a2a/__init__.py +28 -0
- contextsynapse/a2a/client.py +120 -0
- contextsynapse/a2a/discovery.py +128 -0
- contextsynapse/a2a/handlers.py +164 -0
- contextsynapse/a2a/models.py +246 -0
- contextsynapse/a2a/streaming.py +92 -0
- contextsynapse/a2a/task_manager.py +275 -0
- contextsynapse/adapters/__init__.py +98 -0
- contextsynapse/adapters/_base.py +298 -0
- contextsynapse/adapters/autogen/__init__.py +4 -0
- contextsynapse/adapters/autogen/tools.py +34 -0
- contextsynapse/adapters/crewai/__init__.py +4 -0
- contextsynapse/adapters/crewai/tools.py +63 -0
- contextsynapse/adapters/langchain/__init__.py +19 -0
- contextsynapse/adapters/langchain/message_history.py +17 -0
- contextsynapse/adapters/langchain/retriever.py +158 -0
- contextsynapse/adapters/langchain/tools.py +43 -0
- contextsynapse/adapters/langgraph/__init__.py +21 -0
- contextsynapse/adapters/langgraph/checkpoint.py +223 -0
- contextsynapse/adapters/langgraph/context_tools.py +341 -0
- contextsynapse/adapters/langgraph/message_history.py +138 -0
- contextsynapse/adapters/langgraph/tools.py +43 -0
- contextsynapse/adapters/llamaindex/__init__.py +4 -0
- contextsynapse/adapters/llamaindex/tools.py +64 -0
- contextsynapse/adapters/openai/__init__.py +24 -0
- contextsynapse/adapters/openai/tools.py +206 -0
- contextsynapse/adapters/pydantic_ai/__init__.py +4 -0
- contextsynapse/adapters/pydantic_ai/tools.py +36 -0
- contextsynapse/adapters/swarm/__init__.py +4 -0
- contextsynapse/adapters/swarm/tools.py +44 -0
- contextsynapse/agents/__init__.py +6 -0
- contextsynapse/agents/auto_capture.py +80 -0
- contextsynapse/agents/dispatcher.py +299 -0
- contextsynapse/agents/notifications.py +112 -0
- contextsynapse/aiql/__init__.py +71 -0
- contextsynapse/aiql/cache.py +341 -0
- contextsynapse/aiql/compiler/__init__.py +62 -0
- contextsynapse/aiql/compiler/optimizer.py +126 -0
- contextsynapse/aiql/compiler/planner.py +366 -0
- contextsynapse/aiql/compiler/statistics.py +235 -0
- contextsynapse/aiql/compiler/validator.py +425 -0
- contextsynapse/aiql/engine/__init__.py +11 -0
- contextsynapse/aiql/engine/chunk_storage.py +2584 -0
- contextsynapse/aiql/engine/executor.py +7737 -0
- contextsynapse/aiql/engine/extraction_integration.py +442 -0
- contextsynapse/aiql/engine/graph_builder.py +405 -0
- contextsynapse/aiql/engine/node_creation_from_normalized.py +691 -0
- contextsynapse/aiql/engine/performance_monitor.py +289 -0
- contextsynapse/aiql/engine/postprocessing.py +365 -0
- contextsynapse/aiql/engine/ray_stage_executors.py +918 -0
- contextsynapse/aiql/engine/redis_cache.py +106 -0
- contextsynapse/aiql/engine/streaming_json.py +366 -0
- contextsynapse/aiql/engine/streaming_parquet.py +568 -0
- contextsynapse/aiql/engine/streaming_ray_chunking.py +419 -0
- contextsynapse/aiql/grammar/__init__.py +14 -0
- contextsynapse/aiql/grammar/aiql_grammar.py +1822 -0
- contextsynapse/aiql/grammar/base.py +71 -0
- contextsynapse/aiql/parser/__init__.py +46 -0
- contextsynapse/aiql/parser/aiql_parser.py +11330 -0
- contextsynapse/aiql/parser/base_parser.py +47 -0
- contextsynapse/api/__init__.py +5 -0
- contextsynapse/api/a2a_router.py +233 -0
- contextsynapse/api/admin_router.py +140 -0
- contextsynapse/api/agent_worker_router.py +978 -0
- contextsynapse/api/algorithms_router.py +317 -0
- contextsynapse/api/api.py +4144 -0
- contextsynapse/api/audit.py +204 -0
- contextsynapse/api/auth.py +441 -0
- contextsynapse/api/auth_router.py +537 -0
- contextsynapse/api/billing.py +261 -0
- contextsynapse/api/billing_router.py +275 -0
- contextsynapse/api/boundary_router.py +452 -0
- contextsynapse/api/cognition_router.py +116 -0
- contextsynapse/api/connector_router.py +237 -0
- contextsynapse/api/context_router.py +1619 -0
- contextsynapse/api/dashboard_router.py +11367 -0
- contextsynapse/api/errors.py +183 -0
- contextsynapse/api/events.py +88 -0
- contextsynapse/api/experiment_bootstrap.py +321 -0
- contextsynapse/api/experiment_orchestrator.py +1078 -0
- contextsynapse/api/experiment_router.py +2089 -0
- contextsynapse/api/feature_gate.py +89 -0
- contextsynapse/api/federation_router.py +108 -0
- contextsynapse/api/integrations.py +567 -0
- contextsynapse/api/integrations_router.py +461 -0
- contextsynapse/api/intelligence_router.py +1028 -0
- contextsynapse/api/metering.py +155 -0
- contextsynapse/api/metrics.py +112 -0
- contextsynapse/api/models.py +38 -0
- contextsynapse/api/monitoring_router.py +174 -0
- contextsynapse/api/pipeline_router.py +1437 -0
- contextsynapse/api/playground_conn.py +116 -0
- contextsynapse/api/projects_router.py +131 -0
- contextsynapse/api/queries_router.py +282 -0
- contextsynapse/api/quota.py +95 -0
- contextsynapse/api/rate_limit.py +64 -0
- contextsynapse/api/search_router.py +261 -0
- contextsynapse/api/shield_router.py +447 -0
- contextsynapse/api/tenant_guard.py +103 -0
- contextsynapse/api/tenant_quotas.py +231 -0
- contextsynapse/api/tenants.py +305 -0
- contextsynapse/api/users.py +646 -0
- contextsynapse/api/v1.py +167 -0
- contextsynapse/api/vertical_builder_router.py +387 -0
- contextsynapse/benchmark/__init__.py +15 -0
- contextsynapse/benchmark/__main__.py +64 -0
- contextsynapse/benchmark/agents.py +338 -0
- contextsynapse/benchmark/live_ab.py +338 -0
- contextsynapse/benchmark/reporter.py +149 -0
- contextsynapse/benchmark/runner.py +264 -0
- contextsynapse/benchmark/savings.py +266 -0
- contextsynapse/benchmark/token_counter.py +212 -0
- contextsynapse/benchmark/workloads.py +289 -0
- contextsynapse/buffer/__init__.py +154 -0
- contextsynapse/buffer/base.py +257 -0
- contextsynapse/buffer/duckdb_buffer.py +246 -0
- contextsynapse/buffer/file_buffer.py +229 -0
- contextsynapse/buffer/lmdb_buffer.py +286 -0
- contextsynapse/buffer/local_buffer.py +154 -0
- contextsynapse/buffer/redis_buffer.py +211 -0
- contextsynapse/buffer/sqlite_buffer.py +266 -0
- contextsynapse/cache.py +2 -0
- contextsynapse/cli/__init__.py +40 -0
- contextsynapse/cli/cli.py +475 -0
- contextsynapse/cli/config_cli.py +177 -0
- contextsynapse/cli/scaffold.py +465 -0
- contextsynapse/cognition/__init__.py +31 -0
- contextsynapse/cognition/derivation.py +55 -0
- contextsynapse/cognition/events.py +135 -0
- contextsynapse/cognition/feedback.py +52 -0
- contextsynapse/cognition/hallucination.py +90 -0
- contextsynapse/cognition/invalidation.py +70 -0
- contextsynapse/cognition/read_tracker.py +50 -0
- contextsynapse/collaboration.py +2 -0
- contextsynapse/comparison.py +2 -0
- contextsynapse/config/__init__.py +10 -0
- contextsynapse/config/connectors/amfi_nav.yaml +18 -0
- contextsynapse/config/connectors/auto_sector.yaml +20 -0
- contextsynapse/config/connectors/banking_sector.yaml +20 -0
- contextsynapse/config/connectors/energy_sector.yaml +20 -0
- contextsynapse/config/connectors/fmcg_sector.yaml +20 -0
- contextsynapse/config/connectors/nsdl_fpi.yaml +9 -0
- contextsynapse/config/connectors/nse_price.yaml +61 -0
- contextsynapse/config/connectors/pharma_sector.yaml +20 -0
- contextsynapse/config/connectors/rbi_dbie.yaml +12 -0
- contextsynapse/config/context_presets/ai_research.json +25 -0
- contextsynapse/config/context_presets/chatgpt_conversations.json +19 -0
- contextsynapse/config/context_presets/claude_conversations.json +19 -0
- contextsynapse/config/context_presets/legal_compliance.json +21 -0
- contextsynapse/config/context_presets/meeting_notes.json +21 -0
- contextsynapse/config/context_presets/slack_engineering.json +21 -0
- contextsynapse/config/context_presets/tech_climate.json +24 -0
- contextsynapse/config/context_presets/toi_politics.json +25 -0
- contextsynapse/config/database_config.py +453 -0
- contextsynapse/config/deployment.py +221 -0
- contextsynapse/config/gpt_action_schema.json +164 -0
- contextsynapse/config/indicators/india_macro.yaml +213 -0
- contextsynapse/config/logging_config.py +88 -0
- contextsynapse/config/mode_config.py +223 -0
- contextsynapse/config/schemas/api_spec.yaml +48 -0
- contextsynapse/config/schemas/architecture_doc.yaml +51 -0
- contextsynapse/config/schemas/chat_history.yaml +90 -0
- contextsynapse/config/schemas/chatgpt_conversation.yaml +101 -0
- contextsynapse/config/schemas/claude_conversation.yaml +148 -0
- contextsynapse/config/schemas/contract.yaml +48 -0
- contextsynapse/config/schemas/conversation.yaml +167 -0
- contextsynapse/config/schemas/database.yaml +74 -0
- contextsynapse/config/schemas/decision.yaml +72 -0
- contextsynapse/config/schemas/ecommerce.yaml +56 -0
- contextsynapse/config/schemas/education.yaml +42 -0
- contextsynapse/config/schemas/healthcare.yaml +133 -0
- contextsynapse/config/schemas/hr_document.yaml +48 -0
- contextsynapse/config/schemas/invoice.yaml +40 -0
- contextsynapse/config/schemas/knowledge_base.yaml +75 -0
- contextsynapse/config/schemas/legal.yaml +42 -0
- contextsynapse/config/schemas/marketing_content.yaml +48 -0
- contextsynapse/config/schemas/meeting_notes.yaml +51 -0
- contextsynapse/config/schemas/news_article.yaml +61 -0
- contextsynapse/config/schemas/real_estate.yaml +49 -0
- contextsynapse/config/schemas/requirements_doc.yaml +48 -0
- contextsynapse/config/schemas/research_paper.yaml +56 -0
- contextsynapse/config/schemas/resume.yaml +49 -0
- contextsynapse/config/schemas/retail.yaml +42 -0
- contextsynapse/config/schemas/rule_meta.yaml +173 -0
- contextsynapse/config/schemas/rules.yaml +72 -0
- contextsynapse/config/schemas/screener_meta.yaml +236 -0
- contextsynapse/config/schemas/sdlc.yaml +103 -0
- contextsynapse/config/schemas/session_graph.yaml +172 -0
- contextsynapse/config/schemas/support_ticket.yaml +48 -0
- contextsynapse/config/schemas/web.yaml +64 -0
- contextsynapse/config/sensors/earnings_calendar.yaml +52 -0
- contextsynapse/config/sensors/global_risk.yaml +53 -0
- contextsynapse/config/sensors/government_policy.yaml +54 -0
- contextsynapse/config/sensors/supply_chain.yaml +50 -0
- contextsynapse/config/sensors/templates/behavioral.yaml +56 -0
- contextsynapse/config/sensors/templates/equity_sentiment.yaml +48 -0
- contextsynapse/config/sensors/templates/global_macro.yaml +47 -0
- contextsynapse/config/sensors/templates/market_data.yaml +118 -0
- contextsynapse/config/sensors/templates/sector_technical.yaml +47 -0
- contextsynapse/config/sensors/weather.yaml +52 -0
- contextsynapse/config/storage_strategy.py +202 -0
- contextsynapse/config/templates/competitive_intel.yaml +39 -0
- contextsynapse/config/templates/geopolitical_risk.yaml +46 -0
- contextsynapse/config/templates/market_analysis.yaml +58 -0
- contextsynapse/connectors/__init__.py +15 -0
- contextsynapse/connectors/api_router.py +121 -0
- contextsynapse/connectors/base.py +357 -0
- contextsynapse/connectors/financial/audio.py +170 -0
- contextsynapse/connectors/financial/youtube.py +272 -0
- contextsynapse/connectors/github.py +164 -0
- contextsynapse/connectors/jira.py +120 -0
- contextsynapse/connectors/kafka_connector.py +124 -0
- contextsynapse/connectors/pipeline_connector.py +747 -0
- contextsynapse/connectors/registry.py +116 -0
- contextsynapse/connectors/scheduler.py +87 -0
- contextsynapse/connectors/universal.py +360 -0
- contextsynapse/context/__init__.py +96 -0
- contextsynapse/context/__main__.py +4 -0
- contextsynapse/context/acl.py +125 -0
- contextsynapse/context/agent_card.py +201 -0
- contextsynapse/context/agent_memory.py +814 -0
- contextsynapse/context/agents.py +472 -0
- contextsynapse/context/agents_redis.py +391 -0
- contextsynapse/context/assembled.py +204 -0
- contextsynapse/context/attribution.py +113 -0
- contextsynapse/context/blob.py +184 -0
- contextsynapse/context/boundaries.py +187 -0
- contextsynapse/context/boundary.py +507 -0
- contextsynapse/context/bundle.py +155 -0
- contextsynapse/context/cli.py +519 -0
- contextsynapse/context/collaboration.py +165 -0
- contextsynapse/context/compiler.py +604 -0
- contextsynapse/context/composite.py +536 -0
- contextsynapse/context/compression.py +147 -0
- contextsynapse/context/context_manager.py +888 -0
- contextsynapse/context/context_schema.py +90 -0
- contextsynapse/context/context_schema.yaml +105 -0
- contextsynapse/context/context_units.py +891 -0
- contextsynapse/context/conversation.py +488 -0
- contextsynapse/context/dedup.py +207 -0
- contextsynapse/context/document_processor.py +698 -0
- contextsynapse/context/embedding_hooks.py +338 -0
- contextsynapse/context/execution_schema.py +274 -0
- contextsynapse/context/fan_out.py +138 -0
- contextsynapse/context/frozen.py +80 -0
- contextsynapse/context/gravity.py +227 -0
- contextsynapse/context/hub.py +1401 -0
- contextsynapse/context/ingest.py +391 -0
- contextsynapse/context/injection.py +226 -0
- contextsynapse/context/interaction_graphifier.py +562 -0
- contextsynapse/context/layers.py +574 -0
- contextsynapse/context/projection.py +1195 -0
- contextsynapse/context/promotion.py +634 -0
- contextsynapse/context/proof.py +131 -0
- contextsynapse/context/propagation.py +577 -0
- contextsynapse/context/pubsub.py +135 -0
- contextsynapse/context/quality.py +1054 -0
- contextsynapse/context/quality_tracker.py +270 -0
- contextsynapse/context/scoping.py +370 -0
- contextsynapse/context/seed.py +992 -0
- contextsynapse/context/session.py +1142 -0
- contextsynapse/context/session_graph.py +593 -0
- contextsynapse/context/session_memory.py +309 -0
- contextsynapse/context/session_resolver.py +111 -0
- contextsynapse/context/skill_learning.py +237 -0
- contextsynapse/context/smart_budget.py +129 -0
- contextsynapse/context/source_policy.py +242 -0
- contextsynapse/context/store_factory.py +53 -0
- contextsynapse/context/sync.py +648 -0
- contextsynapse/context/token.py +236 -0
- contextsynapse/context/vector_integration.py +297 -0
- contextsynapse/context/webhooks.py +194 -0
- contextsynapse/context/working_memory.py +224 -0
- contextsynapse/core/__init__.py +18 -0
- contextsynapse/core/archiver.py +237 -0
- contextsynapse/core/change_stream.py +282 -0
- contextsynapse/core/checkpoint.py +310 -0
- contextsynapse/core/cloud_storage.py +387 -0
- contextsynapse/core/context_state.py +651 -0
- contextsynapse/core/cron.py +324 -0
- contextsynapse/core/datasource.py +643 -0
- contextsynapse/core/db.py +289 -0
- contextsynapse/core/federation.py +283 -0
- contextsynapse/core/graph_coordinator.py +201 -0
- contextsynapse/core/graph_intelligence.py +388 -0
- contextsynapse/core/graph_structures.py +106 -0
- contextsynapse/core/graph_sync.py +157 -0
- contextsynapse/core/hybrid_graph_storage.py +2998 -0
- contextsynapse/core/lifecycle.py +691 -0
- contextsynapse/core/multiworker.py +378 -0
- contextsynapse/core/pruning.py +312 -0
- contextsynapse/core/redis_registry.py +524 -0
- contextsynapse/core/registry.py +1043 -0
- contextsynapse/core/registry_factory.py +51 -0
- contextsynapse/core/registry_metadata.py +294 -0
- contextsynapse/core/replay.py +354 -0
- contextsynapse/core/replication.py +339 -0
- contextsynapse/core/time_travel.py +260 -0
- contextsynapse/core/write_behind.py +152 -0
- contextsynapse/core/write_context.py +39 -0
- contextsynapse/db/__init__.py +1 -0
- contextsynapse/db/postgres.py +267 -0
- contextsynapse/db/profile.py +236 -0
- contextsynapse/db/rules.py +328 -0
- contextsynapse/demo/__init__.py +0 -0
- contextsynapse/demo/report.py +147 -0
- contextsynapse/demo/runner.py +520 -0
- contextsynapse/demo/scenario.py +165 -0
- contextsynapse/demo/session_demo.py +280 -0
- contextsynapse/demo/session_report.py +143 -0
- contextsynapse/demo/universal_runner.py +257 -0
- contextsynapse/engine/__init__.py +15 -0
- contextsynapse/engine/core.py +555 -0
- contextsynapse/extraction/__init__.py +69 -0
- contextsynapse/extraction/cli.py +274 -0
- contextsynapse/extraction/config.py +107 -0
- contextsynapse/extraction/examples/example_usage.py +118 -0
- contextsynapse/extraction/examples/sample_manifest.json +53 -0
- contextsynapse/extraction/examples/sample_normalized.json +185 -0
- contextsynapse/extraction/extract_engine.py +440 -0
- contextsynapse/extraction/extractors/__init__.py +37 -0
- contextsynapse/extraction/extractors/chat_export_extractor.py +189 -0
- contextsynapse/extraction/extractors/csv_extractor.py +83 -0
- contextsynapse/extraction/extractors/docx_extractor.py +129 -0
- contextsynapse/extraction/extractors/excel_extractor.py +152 -0
- contextsynapse/extraction/extractors/html_extractor.py +146 -0
- contextsynapse/extraction/extractors/pdf_extractor.py +635 -0
- contextsynapse/extraction/extractors/pdf_extractor_layoutparser.py +408 -0
- contextsynapse/extraction/extractors/register_extractors.py +63 -0
- contextsynapse/extraction/extractors/text_extractor.py +65 -0
- contextsynapse/extraction/extractors/txt_extractor.py +120 -0
- contextsynapse/extraction/extractors/website_extractor.py +246 -0
- contextsynapse/extraction/fact_extractor.py +236 -0
- contextsynapse/extraction/file_manager.py +353 -0
- contextsynapse/extraction/hierarchy.py +239 -0
- contextsynapse/extraction/id_generator.py +450 -0
- contextsynapse/extraction/layout_detector.py +409 -0
- contextsynapse/extraction/llm_entity_extractor.py +301 -0
- contextsynapse/extraction/normalized_store.py +480 -0
- contextsynapse/extraction/normalizer.py +594 -0
- contextsynapse/extraction/parquet_writer.py +598 -0
- contextsynapse/extraction/post_processors/__init__.py +40 -0
- contextsynapse/extraction/post_processors/column_reorganizer.py +507 -0
- contextsynapse/extraction/ray_normalization.py +233 -0
- contextsynapse/extraction/ray_runner.py +207 -0
- contextsynapse/extraction/redis_schema_registry.py +222 -0
- contextsynapse/extraction/registry.py +204 -0
- contextsynapse/extraction/schema_loader.py +663 -0
- contextsynapse/extraction/schema_registry.py +230 -0
- contextsynapse/extraction/semantic_normalizer.py +245 -0
- contextsynapse/extraction/version_manager.py +314 -0
- contextsynapse/gateway/__init__.py +24 -0
- contextsynapse/gateway/action_emitter.py +212 -0
- contextsynapse/gateway/cost_tracker.py +508 -0
- contextsynapse/gateway/gateway.py +377 -0
- contextsynapse/gateway/policy.py +303 -0
- contextsynapse/governance/__init__.py +13 -0
- contextsynapse/governance/layer.py +513 -0
- contextsynapse/ingestion/__init__.py +15 -0
- contextsynapse/ingestion/amplifier.py +288 -0
- contextsynapse/ingestion/async_ingest.py +463 -0
- contextsynapse/ingestion/chunker.py +303 -0
- contextsynapse/ingestion/chunking.py +58 -0
- contextsynapse/ingestion/cleaner.py +190 -0
- contextsynapse/ingestion/cleanse.py +252 -0
- contextsynapse/ingestion/connectors/__init__.py +38 -0
- contextsynapse/ingestion/connectors/base.py +300 -0
- contextsynapse/ingestion/connectors/chatgpt_memory.py +247 -0
- contextsynapse/ingestion/connectors/claude_memory.py +274 -0
- contextsynapse/ingestion/connectors/sap_ingestor.py +208 -0
- contextsynapse/ingestion/content_detector.py +80 -0
- contextsynapse/ingestion/content_monitor.py +502 -0
- contextsynapse/ingestion/dedup.py +106 -0
- contextsynapse/ingestion/feed.py +640 -0
- contextsynapse/ingestion/filters.py +461 -0
- contextsynapse/ingestion/graph_builder.py +705 -0
- contextsynapse/ingestion/input_classifier.py +307 -0
- contextsynapse/ingestion/job_manager.py +571 -0
- contextsynapse/ingestion/llm_extractor.py +610 -0
- contextsynapse/ingestion/orchestrator.py +116 -0
- contextsynapse/ingestion/parsers/__init__.py +19 -0
- contextsynapse/ingestion/parsers/graph_parser_registry.py +103 -0
- contextsynapse/ingestion/parsers/graphml_parser.py +141 -0
- contextsynapse/ingestion/parsers/jsonld_parser.py +171 -0
- contextsynapse/ingestion/parsers/rdf_parser.py +150 -0
- contextsynapse/ingestion/pipeline_context.py +228 -0
- contextsynapse/ingestion/pipeline_store.py +1195 -0
- contextsynapse/ingestion/queue.py +133 -0
- contextsynapse/ingestion/reprocess.py +179 -0
- contextsynapse/ingestion/scenario_router.py +224 -0
- contextsynapse/ingestion/schema_extractor.py +759 -0
- contextsynapse/ingestion/schema_validator.py +274 -0
- contextsynapse/ingestion/sdlc_ingest.py +87 -0
- contextsynapse/ingestion/smart_ingest.py +2451 -0
- contextsynapse/ingestion/source_connector.py +171 -0
- contextsynapse/ingestion/stage_executor.py +2095 -0
- contextsynapse/ingestion/stages/__init__.py +9 -0
- contextsynapse/ingestion/stages/tabular_to_graph.py +289 -0
- contextsynapse/ingestion/strategies.py +386 -0
- contextsynapse/ingestion/universal/__init__.py +4 -0
- contextsynapse/ingestion/universal/_operator_registry.py +59 -0
- contextsynapse/ingestion/universal/ingest_content.py +32 -0
- contextsynapse/ingestion/universal/operators/__init__.py +1 -0
- contextsynapse/ingestion/universal/operators/base.py +19 -0
- contextsynapse/ingestion/universal/operators/build_edges.py +75 -0
- contextsynapse/ingestion/universal/operators/cluster_topics.py +397 -0
- contextsynapse/ingestion/universal/operators/deduplicate.py +45 -0
- contextsynapse/ingestion/universal/operators/detect_signals.py +90 -0
- contextsynapse/ingestion/universal/operators/embed.py +80 -0
- contextsynapse/ingestion/universal/operators/extract_entities.py +253 -0
- contextsynapse/ingestion/universal/operators/extract_preferences.py +117 -0
- contextsynapse/ingestion/universal/operators/filter_content.py +69 -0
- contextsynapse/ingestion/universal/operators/index_bm25.py +52 -0
- contextsynapse/ingestion/universal/operators/infer_domains.py +69 -0
- contextsynapse/ingestion/universal/operators/link_cross_reference.py +210 -0
- contextsynapse/ingestion/universal/operators/parse_records.py +66 -0
- contextsynapse/ingestion/universal/operators/resolve_entities.py +261 -0
- contextsynapse/ingestion/universal/operators/scanners/__init__.py +16 -0
- contextsynapse/ingestion/universal/operators/scanners/edges.py +173 -0
- contextsynapse/ingestion/universal/operators/scanners/git.py +187 -0
- contextsynapse/ingestion/universal/operators/scanners/github.py +135 -0
- contextsynapse/ingestion/universal/operators/scanners/github_api.py +509 -0
- contextsynapse/ingestion/universal/operators/scanners/helpers.py +67 -0
- contextsynapse/ingestion/universal/operators/scanners/jira.py +65 -0
- contextsynapse/ingestion/universal/operators/scanners/llm_enrichment.py +82 -0
- contextsynapse/ingestion/universal/operators/scanners/repo.py +149 -0
- contextsynapse/ingestion/universal/operators/sdlc_scan.py +213 -0
- contextsynapse/ingestion/universal/operators/spec_scanner.py +149 -0
- contextsynapse/ingestion/universal/operators/store_documents.py +72 -0
- contextsynapse/ingestion/universal/operators/synthesize_cu.py +180 -0
- contextsynapse/ingestion/universal/operators/validate_gate.py +65 -0
- contextsynapse/ingestion/universal/parsers/__init__.py +5 -0
- contextsynapse/ingestion/universal/parsers/record_parser.py +203 -0
- contextsynapse/ingestion/universal/parsers/turn_parser.py +329 -0
- contextsynapse/ingestion/universal/shared_signals.py +161 -0
- contextsynapse/ingestion/universal/stage_executor.py +219 -0
- contextsynapse/ingestion/web_crawler.py +265 -0
- contextsynapse/intelligence/__init__.py +53 -0
- contextsynapse/intelligence/alerts.py +198 -0
- contextsynapse/intelligence/collector.py +655 -0
- contextsynapse/intelligence/comparison.py +198 -0
- contextsynapse/intelligence/config.py +56 -0
- contextsynapse/intelligence/conflict_detector.py +182 -0
- contextsynapse/intelligence/context_gaps.py +192 -0
- contextsynapse/intelligence/context_radar.py +135 -0
- contextsynapse/intelligence/correlation_engine.py +416 -0
- contextsynapse/intelligence/correlation_template.py +120 -0
- contextsynapse/intelligence/dedup.py +296 -0
- contextsynapse/intelligence/event_bus.py +91 -0
- contextsynapse/intelligence/feedback_loop.py +101 -0
- contextsynapse/intelligence/freshness.py +307 -0
- contextsynapse/intelligence/fundamental_feed.py +551 -0
- contextsynapse/intelligence/geo_data.py +170 -0
- contextsynapse/intelligence/geo_reference.py +112 -0
- contextsynapse/intelligence/graph_fusion.py +465 -0
- contextsynapse/intelligence/heuristics.py +176 -0
- contextsynapse/intelligence/impact_tracker.py +336 -0
- contextsynapse/intelligence/macro_indicators.py +319 -0
- contextsynapse/intelligence/ocr.py +104 -0
- contextsynapse/intelligence/persistence.py +267 -0
- contextsynapse/intelligence/pipeline_health.py +130 -0
- contextsynapse/intelligence/price_feed.py +221 -0
- contextsynapse/intelligence/rate_limiter.py +130 -0
- contextsynapse/intelligence/reactive.py +411 -0
- contextsynapse/intelligence/runtime_context.py +390 -0
- contextsynapse/intelligence/sentiment_decay.py +426 -0
- contextsynapse/intelligence/session.py +277 -0
- contextsynapse/intelligence/signal_hierarchy.py +438 -0
- contextsynapse/intelligence/signals.py +185 -0
- contextsynapse/intelligence/source_watcher.py +132 -0
- contextsynapse/intelligence/table_enricher.py +256 -0
- contextsynapse/intelligence/tagger.py +319 -0
- contextsynapse/intelligence/timeseries.py +209 -0
- contextsynapse/intelligence/tracker.py +318 -0
- contextsynapse/intelligence/watchdog.py +392 -0
- contextsynapse/intelligence/watchdog_manager.py +263 -0
- contextsynapse/intelligence/webhook_receiver.py +165 -0
- contextsynapse/intelligence/weight_learner.py +234 -0
- contextsynapse/llm/__init__.py +8 -0
- contextsynapse/llm/client.py +348 -0
- contextsynapse/llm/rate_limiter.py +166 -0
- contextsynapse/logging_config.py +2 -0
- contextsynapse/marketplace/__init__.py +7 -0
- contextsynapse/marketplace/registry.py +173 -0
- contextsynapse/mcp/__init__.py +11 -0
- contextsynapse/mcp/__main__.py +4 -0
- contextsynapse/mcp/auth_middleware.py +212 -0
- contextsynapse/mcp/connection_pool.py +128 -0
- contextsynapse/mcp/server.py +986 -0
- contextsynapse/mcp/session_context.py +67 -0
- contextsynapse/memory/__init__.py +34 -0
- contextsynapse/memory/engine.py +475 -0
- contextsynapse/memory/temporal.py +327 -0
- contextsynapse/metadata/__init__.py +12 -0
- contextsynapse/metadata/metadata_db.py +272 -0
- contextsynapse/metadata/metadata_query_tool.py +300 -0
- contextsynapse/metadata/metadata_tracker.py +425 -0
- contextsynapse/models/__init__.py +27 -0
- contextsynapse/models/embedding_service.py +360 -0
- contextsynapse/models/model_registry.py +228 -0
- contextsynapse/multiworker.py +2 -0
- contextsynapse/pipelines/__init__.py +1 -0
- contextsynapse/pipelines/executor.py +851 -0
- contextsynapse/pipelines/filters.py +189 -0
- contextsynapse/pipelines/models.py +176 -0
- contextsynapse/pipelines/scheduler.py +268 -0
- contextsynapse/playground/__init__.py +1 -0
- contextsynapse/playground/templates.py +85 -0
- contextsynapse/plugins/__init__.py +43 -0
- contextsynapse/plugins/api.py +66 -0
- contextsynapse/plugins/base.py +350 -0
- contextsynapse/plugins/domain_interface.py +508 -0
- contextsynapse/plugins/domain_loader.py +241 -0
- contextsynapse/plugins/loader.py +148 -0
- contextsynapse/plugins/registry.py +265 -0
- contextsynapse/project/__init__.py +11 -0
- contextsynapse/project/code_context.py +807 -0
- contextsynapse/project/connectors/__init__.py +4 -0
- contextsynapse/project/connectors/github_connector.py +113 -0
- contextsynapse/project/connectors/jira_connector.py +160 -0
- contextsynapse/project/graph.py +1023 -0
- contextsynapse/project/project_context.py +545 -0
- contextsynapse/project/scanner_registry.py +61 -0
- contextsynapse/project/schema_manager.py +322 -0
- contextsynapse/project/schemas/knowledge.yaml +59 -0
- contextsynapse/project/schemas/sdlc.yaml +369 -0
- contextsynapse/project/sdlc_schema.py +172 -0
- contextsynapse/project/spec_parser.py +85 -0
- contextsynapse/project/task_lock.py +181 -0
- contextsynapse/project/task_stream.py +203 -0
- contextsynapse/project/team_tools.py +537 -0
- contextsynapse/py.typed +0 -0
- contextsynapse/realtime/__init__.py +6 -0
- contextsynapse/realtime/collaboration.py +147 -0
- contextsynapse/rules/__init__.py +10 -0
- contextsynapse/rules/engine.py +116 -0
- contextsynapse/rules/evaluators/__init__.py +6 -0
- contextsynapse/rules/evaluators/aggregate.py +205 -0
- contextsynapse/rules/evaluators/custom.py +170 -0
- contextsynapse/rules/evaluators/pre_action.py +125 -0
- contextsynapse/rules/evaluators/temporal.py +129 -0
- contextsynapse/rules/loader.py +52 -0
- contextsynapse/rules/models.py +124 -0
- contextsynapse/rules/universal.py +393 -0
- contextsynapse/scheduler.py +229 -0
- contextsynapse/schema/__init__.py +42 -0
- contextsynapse/schema/compiler.py +252 -0
- contextsynapse/schema/composer.py +158 -0
- contextsynapse/schema/dedup.py +73 -0
- contextsynapse/schema/derivation.py +172 -0
- contextsynapse/schema/sdl.py +326 -0
- contextsynapse/schema/validation_gate.py +124 -0
- contextsynapse/sdk/__init__.py +63 -0
- contextsynapse/sdk/agent.py +214 -0
- contextsynapse/sdk/client.py +269 -0
- contextsynapse/sdk/exceptions.py +23 -0
- contextsynapse/sdk/models.py +78 -0
- contextsynapse/sdk/session.py +408 -0
- contextsynapse/sdk/tracked.py +218 -0
- contextsynapse/sdk/worker.py +266 -0
- contextsynapse/sdk/workspace_client.py +98 -0
- contextsynapse/sdk/ws.py +41 -0
- contextsynapse/search/__init__.py +20 -0
- contextsynapse/search/embedding_cache.py +140 -0
- contextsynapse/search/enhanced_search.py +152 -0
- contextsynapse/search/fulltext.py +130 -0
- contextsynapse/search/graph_search.py +634 -0
- contextsynapse/search/lmdb_index.py +1218 -0
- contextsynapse/search/nl_to_aiql.py +199 -0
- contextsynapse/search/rag.py +1346 -0
- contextsynapse/search/rag_cache.py +167 -0
- contextsynapse/search/reasoning_chain.py +231 -0
- contextsynapse/search/redis_search.py +207 -0
- contextsynapse/search/retrieval_quality.py +314 -0
- contextsynapse/search/semantic_query.py +455 -0
- contextsynapse/search/text_resolver.py +159 -0
- contextsynapse/search/whoosh_search.py +373 -0
- contextsynapse/security/__init__.py +17 -0
- contextsynapse/security/audit.py +297 -0
- contextsynapse/security/audit_logger.py +97 -0
- contextsynapse/security/audit_trail.py +270 -0
- contextsynapse/security/auth_unified.py +284 -0
- contextsynapse/security/auto_tagger.py +230 -0
- contextsynapse/security/data_security.py +218 -0
- contextsynapse/security/encryption.py +207 -0
- contextsynapse/security/identity.py +447 -0
- contextsynapse/security/jwt_identity.py +59 -0
- contextsynapse/security/middleware.py +336 -0
- contextsynapse/security/pii.py +233 -0
- contextsynapse/security/rbac.py +96 -0
- contextsynapse/security/rls.py +205 -0
- contextsynapse/security/sanitize.py +233 -0
- contextsynapse/security/scoped_encryption.py +131 -0
- contextsynapse/security/tenant.py +401 -0
- contextsynapse/shield/__init__.py +21 -0
- contextsynapse/shield/anomaly.py +56 -0
- contextsynapse/shield/permissions.py +99 -0
- contextsynapse/shield/profile.py +103 -0
- contextsynapse/shield/shield.py +108 -0
- contextsynapse/shield/trust_engine.py +69 -0
- contextsynapse/skills/__init__.py +14 -0
- contextsynapse/skills/engine.py +233 -0
- contextsynapse/skills/loader.py +59 -0
- contextsynapse/skills/models.py +139 -0
- contextsynapse/storage/__init__.py +7 -0
- contextsynapse/storage/cache.py +208 -0
- contextsynapse/storage/columnar_store_v2.py +178 -0
- contextsynapse/storage/csr_graph_storage.py +587 -0
- contextsynapse/storage/csr_store.py +388 -0
- contextsynapse/storage/document_store.py +491 -0
- contextsynapse/storage/hnsw_index.py +294 -0
- contextsynapse/storage/hybrid_store_v2.py +278 -0
- contextsynapse/storage/lmdb_graph_storage.py +366 -0
- contextsynapse/storage/migration.py +279 -0
- contextsynapse/storage/namespace_store.py +346 -0
- contextsynapse/storage/pure_graph_storage.py +185 -0
- contextsynapse/storage/redis_graph_adapter.py +186 -0
- contextsynapse/storage/redis_graph_storage.py +375 -0
- contextsynapse/storage/router/__init__.py +29 -0
- contextsynapse/storage/router/duckdb_store.py +398 -0
- contextsynapse/storage/router/factory.py +83 -0
- contextsynapse/storage/router/interface.py +125 -0
- contextsynapse/storage/router/postgres_store.py +339 -0
- contextsynapse/storage/router/resolver.py +192 -0
- contextsynapse/storage/router/router.py +300 -0
- contextsynapse/storage/single_file_storage.py +391 -0
- contextsynapse/storage/storage_manager.py +214 -0
- contextsynapse/storage/storage_query_optimizer.py +314 -0
- contextsynapse/storage/wal.py +359 -0
- contextsynapse/temporal/__init__.py +0 -0
- contextsynapse/temporal/storage.py +106 -0
- contextsynapse/tests/__init__.py +11 -0
- contextsynapse/tests/regression/__init__.py +1 -0
- contextsynapse/tests/regression/add_metadata.py +395 -0
- contextsynapse/tests/regression/add_skip_flags.py +85 -0
- contextsynapse/tests/regression/cases/test_basic_create_node.yaml +20 -0
- contextsynapse/tests/regression/cases/test_basic_select.yaml +21 -0
- contextsynapse/tests/regression/cases/test_blockchain_audit_trail.yaml +37 -0
- contextsynapse/tests/regression/cases/test_blockchain_audit_trail_filtered.yaml +39 -0
- contextsynapse/tests/regression/cases/test_blockchain_blocks_range.yaml +39 -0
- contextsynapse/tests/regression/cases/test_blockchain_get_block.yaml +35 -0
- contextsynapse/tests/regression/cases/test_blockchain_latest_block.yaml +35 -0
- contextsynapse/tests/regression/cases/test_blockchain_length.yaml +37 -0
- contextsynapse/tests/regression/cases/test_blockchain_merkle_root.yaml +37 -0
- contextsynapse/tests/regression/cases/test_blockchain_verify.yaml +35 -0
- contextsynapse/tests/regression/cases/test_blockchain_verify_block.yaml +35 -0
- contextsynapse/tests/regression/cases/test_create_edge.yaml +28 -0
- contextsynapse/tests/regression/cases/test_delete_edge_basic.yaml +29 -0
- contextsynapse/tests/regression/cases/test_delete_node_basic.yaml +29 -0
- contextsynapse/tests/regression/cases/test_evaluation_mrr.yaml +40 -0
- contextsynapse/tests/regression/cases/test_evaluation_ndcg.yaml +40 -0
- contextsynapse/tests/regression/cases/test_evaluation_precision.yaml +40 -0
- contextsynapse/tests/regression/cases/test_evaluation_recall.yaml +40 -0
- contextsynapse/tests/regression/cases/test_group_by_basic.yaml +56 -0
- contextsynapse/tests/regression/cases/test_match_node_basic.yaml +29 -0
- contextsynapse/tests/regression/cases/test_match_node_with_order_limit.yaml +31 -0
- contextsynapse/tests/regression/cases/test_namespace_switch.yaml +18 -0
- contextsynapse/tests/regression/cases/test_pipeline_cascaded.yaml +61 -0
- contextsynapse/tests/regression/cases/test_pipeline_fixed_size_chunking.yaml +48 -0
- contextsynapse/tests/regression/cases/test_pipeline_gpt4_chunking.yaml +54 -0
- contextsynapse/tests/regression/cases/test_pipeline_hybrid_indexing.yaml +57 -0
- contextsynapse/tests/regression/cases/test_pipeline_large_embedding.yaml +52 -0
- contextsynapse/tests/regression/cases/test_pipeline_paragraph_chunking.yaml +49 -0
- contextsynapse/tests/regression/cases/test_pipeline_recursive_chunking.yaml +47 -0
- contextsynapse/tests/regression/cases/test_pipeline_run_pipeline.yaml +56 -0
- contextsynapse/tests/regression/cases/test_pipeline_section_based_chunking.yaml +53 -0
- contextsynapse/tests/regression/cases/test_pipeline_semantic_chunking.yaml +51 -0
- contextsynapse/tests/regression/cases/test_pipeline_with_entity_extraction.yaml +57 -0
- contextsynapse/tests/regression/cases/test_pipeline_with_relationships.yaml +62 -0
- contextsynapse/tests/regression/cases/test_rag_evaluation.yaml +38 -0
- contextsynapse/tests/regression/cases/test_rag_graph_bfs_with_eval.yaml +48 -0
- contextsynapse/tests/regression/cases/test_rag_graph_centrality.yaml +40 -0
- contextsynapse/tests/regression/cases/test_rag_graph_centrality_eval.yaml +35 -0
- contextsynapse/tests/regression/cases/test_rag_graph_community.yaml +40 -0
- contextsynapse/tests/regression/cases/test_rag_graph_community_eval.yaml +35 -0
- contextsynapse/tests/regression/cases/test_rag_graph_dfs_with_eval.yaml +35 -0
- contextsynapse/tests/regression/cases/test_rag_graph_multi_hop_eval.yaml +35 -0
- contextsynapse/tests/regression/cases/test_rag_graph_path.yaml +40 -0
- contextsynapse/tests/regression/cases/test_rag_graph_search_basic.yaml +38 -0
- contextsynapse/tests/regression/cases/test_rag_graph_search_bfs.yaml +52 -0
- contextsynapse/tests/regression/cases/test_rag_graph_search_dfs.yaml +41 -0
- contextsynapse/tests/regression/cases/test_rag_graph_search_multi_hop.yaml +43 -0
- contextsynapse/tests/regression/cases/test_rag_graph_search_shortest_path.yaml +41 -0
- contextsynapse/tests/regression/cases/test_rag_graph_search_weighted.yaml +43 -0
- contextsynapse/tests/regression/cases/test_rag_graph_shortest_path_eval.yaml +35 -0
- contextsynapse/tests/regression/cases/test_rag_graph_traversal.yaml +30 -0
- contextsynapse/tests/regression/cases/test_rag_graph_weighted_eval.yaml +35 -0
- contextsynapse/tests/regression/cases/test_rag_hybrid_search.yaml +33 -0
- contextsynapse/tests/regression/cases/test_rag_hybrid_with_prompt_embedding.yaml +44 -0
- contextsynapse/tests/regression/cases/test_rag_keyword_search.yaml +32 -0
- contextsynapse/tests/regression/cases/test_rag_multi_hop_graph.yaml +32 -0
- contextsynapse/tests/regression/cases/test_rag_path_based.yaml +34 -0
- contextsynapse/tests/regression/cases/test_rag_profile_based.yaml +32 -0
- contextsynapse/tests/regression/cases/test_rag_vector_search.yaml +32 -0
- contextsynapse/tests/regression/cases/test_rag_weighted_graph.yaml +34 -0
- contextsynapse/tests/regression/cases/test_rag_weighted_hybrid.yaml +31 -0
- contextsynapse/tests/regression/cases/test_semantic_boundary_chunking.yaml +57 -0
- contextsynapse/tests/regression/cases/test_semantic_boundary_detection.yaml +33 -0
- contextsynapse/tests/regression/cases/test_semantic_hash_function.yaml +30 -0
- contextsynapse/tests/regression/cases/test_semantic_hash_with_model.yaml +31 -0
- contextsynapse/tests/regression/cases/test_tcs_ingestion_fixed_size.yaml +44 -0
- contextsynapse/tests/regression/cases/test_tcs_ingestion_recursive.yaml +46 -0
- contextsynapse/tests/regression/cases/test_tcs_ingestion_section.yaml +46 -0
- contextsynapse/tests/regression/cases/test_tcs_ingestion_semantic.yaml +44 -0
- contextsynapse/tests/regression/cases/test_time_travel_as_of.yaml +35 -0
- contextsynapse/tests/regression/cases/test_time_travel_at_commit.yaml +33 -0
- contextsynapse/tests/regression/cases/test_time_travel_at_timestamp.yaml +33 -0
- contextsynapse/tests/regression/cases/test_time_travel_for_system_time_all.yaml +37 -0
- contextsynapse/tests/regression/cases/test_time_travel_versions_between.yaml +39 -0
- contextsynapse/tests/regression/cases/test_traverse_basic.yaml +31 -0
- contextsynapse/tests/regression/cases/test_traverse_max_depth.yaml +34 -0
- contextsynapse/tests/regression/cases/test_update_edge_basic.yaml +29 -0
- contextsynapse/tests/regression/evaluation_metrics.py +227 -0
- contextsynapse/tests/regression/regression_runner.py +1081 -0
- contextsynapse/tests/regression/setup_regression_data.py +212 -0
- contextsynapse/tests/regression/test_basic_only.py +84 -0
- contextsynapse/tests/regression/test_namespace_creation.py +29 -0
- contextsynapse/tests/regression/test_single_fix.py +61 -0
- contextsynapse/tests/regression/test_single_query.py +20 -0
- contextsynapse/tests/regression/validate_test_syntax.py +179 -0
- contextsynapse/tests/test_advanced_features.py +347 -0
- contextsynapse/tests/test_aiql_summary.py +283 -0
- contextsynapse/tests/test_all_aiql_queries.py +948 -0
- contextsynapse/tests/test_all_create_queries.py +213 -0
- contextsynapse/tests/test_api_vs_native.py +280 -0
- contextsynapse/tests/test_chunking_dedup.py +199 -0
- contextsynapse/tests/test_connectors.py +171 -0
- contextsynapse/tests/test_edges_and_match.py +202 -0
- contextsynapse/tests/test_extraction_pipeline_integration.py +581 -0
- contextsynapse/tests/test_name_property.py +224 -0
- contextsynapse/tests/test_node_edge_metadata.py +250 -0
- contextsynapse/tests/test_playground_conn.py +114 -0
- contextsynapse/tests/test_query_cache.py +213 -0
- contextsynapse/tests/test_source_policy_auth.py +348 -0
- contextsynapse/tests/test_temporal_and_domain.py +224 -0
- contextsynapse/tests/test_time_travel_e2e.py +571 -0
- contextsynapse/tests/test_traversal_and_aggregation.py +275 -0
- contextsynapse/tools/__init__.py +36 -0
- contextsynapse/tools/a2a_tools.py +106 -0
- contextsynapse/tools/cognition_tools.py +118 -0
- contextsynapse/tools/context.py +905 -0
- contextsynapse/tools/formatting.py +92 -0
- contextsynapse/tools/gateway_tools.py +186 -0
- contextsynapse/tools/graph.py +1700 -0
- contextsynapse/tools/intelligence.py +235 -0
- contextsynapse/tools/memory.py +118 -0
- contextsynapse/tools/registry.py +373 -0
- contextsynapse/tools/shield_tools.py +159 -0
- contextsynapse/tools/tasks.py +341 -0
- contextsynapse/tools/workspace_tools.py +124 -0
- contextsynapse/utils/__init__.py +15 -0
- contextsynapse/utils/config_loader.py +198 -0
- contextsynapse/utils/env_loader.py +156 -0
- contextsynapse/utils/file_utils.py +288 -0
- contextsynapse/utils/logging_optimizer.py +119 -0
- contextsynapse/vector/__init__.py +30 -0
- contextsynapse/vector/chroma_vector_store.py +152 -0
- contextsynapse/vector/custom_vector_store.py +537 -0
- contextsynapse/vector/faiss_vector_store.py +296 -0
- contextsynapse/vector/qdrant_vector_store.py +163 -0
- contextsynapse/vector/vector_db.py +953 -0
- contextsynapse/vector/vector_db_manager.py +197 -0
- contextsynapse/workspace/__init__.py +27 -0
- contextsynapse/workspace/base.py +94 -0
- contextsynapse/workspace/git.py +228 -0
- contextsynapse/workspace/github.py +78 -0
- contextsynapse/workspace/local.py +73 -0
- contextsynapse/workspace/tools.py +222 -0
- contextsynapse-1.0.0.dist-info/METADATA +847 -0
- contextsynapse-1.0.0.dist-info/RECORD +776 -0
- contextsynapse-1.0.0.dist-info/WHEEL +5 -0
- contextsynapse-1.0.0.dist-info/entry_points.txt +4 -0
- contextsynapse-1.0.0.dist-info/licenses/LICENSE +200 -0
- contextsynapse-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2998 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CSR-Only Graph Storage Implementation
|
|
3
|
+
High-performance CSR storage backend - NetworkX fallback disabled
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from typing import Dict, List, Any, Optional, Union, Set, Tuple
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
import uuid
|
|
9
|
+
import time
|
|
10
|
+
import logging
|
|
11
|
+
import json
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
|
|
14
|
+
from ..storage.csr_graph_storage import CSRGraphStorage, CSRGraphStorageAdapter, CSRNode, CSREdge
|
|
15
|
+
from .graph_structures import GraphNode, GraphEdge
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
# Optional imports (removed modules)
|
|
20
|
+
TEMPORAL_AVAILABLE = True
|
|
21
|
+
HASHING_AVAILABLE = False
|
|
22
|
+
|
|
23
|
+
# Try to import sortedcontainers for B-tree indexes
|
|
24
|
+
try:
|
|
25
|
+
from sortedcontainers import SortedDict
|
|
26
|
+
SORTED_CONTAINERS_AVAILABLE = True
|
|
27
|
+
except ImportError:
|
|
28
|
+
SORTED_CONTAINERS_AVAILABLE = False
|
|
29
|
+
# Fallback to dict
|
|
30
|
+
SortedDict = dict
|
|
31
|
+
logger.warning("sortedcontainers not available, using dict for indexes (slower)")
|
|
32
|
+
|
|
33
|
+
# RAG-critical node types that need immediate write-through (bypass buffer)
|
|
34
|
+
RAG_CRITICAL_NODE_TYPES = {
|
|
35
|
+
'Document', 'Chunk', 'TextChunk', 'Passage', 'Fact', 'Entity',
|
|
36
|
+
'Table', 'Image', 'Embedding', 'Index', 'KnowledgeGraph',
|
|
37
|
+
'ContextBoundary', 'CategoryBoundary',
|
|
38
|
+
# Session graph conversation layer — need immediate visibility for
|
|
39
|
+
# cross-layer REFERENCES edge linking and context building
|
|
40
|
+
'Turn', 'Decision', 'Question', 'Action', 'Topic', 'AgentPresence',
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
# Metadata node types (use fast buffer with 0.5s flush)
|
|
44
|
+
METADATA_NODE_TYPES = {
|
|
45
|
+
'Metadata', 'Config', 'Schema', 'IndexMetadata', 'Pipeline'
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
class VersionConflictError(Exception):
|
|
49
|
+
"""Raised when an optimistic-lock version check fails on node update."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, current_version: int, provided_version: int):
|
|
52
|
+
self.current_version = current_version
|
|
53
|
+
self.provided_version = provided_version
|
|
54
|
+
super().__init__(
|
|
55
|
+
f"Version conflict: stored version is {current_version}, "
|
|
56
|
+
f"but update provided version {provided_version}. "
|
|
57
|
+
f"Re-fetch the node and retry."
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class ContextSynapse:
|
|
62
|
+
"""ContextSynapse - CSR-only graph storage for high-performance operations (NetworkX disabled)."""
|
|
63
|
+
|
|
64
|
+
def __init__(self, name: str = "default", config: Optional[Dict] = None, storage_backend: str = 'csr'):
|
|
65
|
+
"""
|
|
66
|
+
Initialize Cognitive Database.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
name: Name of the graph database
|
|
70
|
+
config: Optional configuration dictionary
|
|
71
|
+
storage_backend: 'csr' only (NetworkX fallback disabled)
|
|
72
|
+
|
|
73
|
+
Raises:
|
|
74
|
+
ValueError: If storage_backend is not 'csr'
|
|
75
|
+
"""
|
|
76
|
+
self.name = name
|
|
77
|
+
self.config = config or {}
|
|
78
|
+
|
|
79
|
+
# CRITICAL: Force CSR-only mode - NetworkX fallback disabled
|
|
80
|
+
if storage_backend != 'csr':
|
|
81
|
+
logger.warning(f"NetworkX fallback disabled. Forcing CSR-only mode. Received: {storage_backend}")
|
|
82
|
+
storage_backend = 'csr'
|
|
83
|
+
|
|
84
|
+
self.storage_backend = 'csr' # Always CSR
|
|
85
|
+
self.use_csr = True # Always True - no fallback
|
|
86
|
+
|
|
87
|
+
# Enable single-file HDF5 storage by default (Oracle-type format)
|
|
88
|
+
if 'graph' not in self.config:
|
|
89
|
+
self.config['graph'] = {}
|
|
90
|
+
# NEO4J-LEVEL PERFORMANCE: Default to HDF5 single-file storage (faster than JSON)
|
|
91
|
+
if 'storage_strategy' not in self.config['graph']:
|
|
92
|
+
self.config['graph']['storage_strategy'] = 'single_file' # HDF5 single-file (faster)
|
|
93
|
+
if 'storage_format' not in self.config['graph']:
|
|
94
|
+
self.config['graph']['storage_format'] = 'single_file' # Backward compatibility
|
|
95
|
+
|
|
96
|
+
# CSR-only: No NetworkX storage
|
|
97
|
+
# Thread-safe write lock for concurrent ingestion
|
|
98
|
+
import threading
|
|
99
|
+
self._write_lock = threading.RLock()
|
|
100
|
+
|
|
101
|
+
# Maintain minimal compatibility structures for properties
|
|
102
|
+
self.node_properties = {} # Store node properties separately
|
|
103
|
+
self.edge_properties = {} # Store edge properties separately
|
|
104
|
+
self.edge_store = {} # Full edge metadata: edge_uuid -> {source, target, label, properties, weight}
|
|
105
|
+
self.unique_constraints = defaultdict(set)
|
|
106
|
+
# NOTE: alias_to_node removed - aliases are now query-scoped (stored in executor only)
|
|
107
|
+
self.global_strategy = 'DFS'
|
|
108
|
+
|
|
109
|
+
# Create node_index for compatibility
|
|
110
|
+
self.node_index = {}
|
|
111
|
+
|
|
112
|
+
# Create edge_index for compatibility
|
|
113
|
+
self.edge_index = {}
|
|
114
|
+
|
|
115
|
+
# NEW: Property indexes for fast filtering
|
|
116
|
+
if SORTED_CONTAINERS_AVAILABLE:
|
|
117
|
+
self._property_indexes: Dict[str, SortedDict] = {}
|
|
118
|
+
else:
|
|
119
|
+
self._property_indexes: Dict[str, Dict[Any, Set[str]]] = {}
|
|
120
|
+
|
|
121
|
+
# Performance metrics
|
|
122
|
+
self.operation_counts = {
|
|
123
|
+
'add_node': 0,
|
|
124
|
+
'add_edge': 0,
|
|
125
|
+
'get_neighbors': 0,
|
|
126
|
+
'traverse': 0
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
# Time travel / Temporal storage (always enabled)
|
|
130
|
+
self.temporal_enabled = True
|
|
131
|
+
self.temporal_storage = None
|
|
132
|
+
try:
|
|
133
|
+
from ..temporal.storage import TemporalStorage
|
|
134
|
+
base_path = self.config.get('base_path', 'contextcore_data')
|
|
135
|
+
self.temporal_storage = TemporalStorage(namespace=name, base_path=base_path)
|
|
136
|
+
self.temporal_storage.load_versions()
|
|
137
|
+
logger.debug(f"[TIME_TRAVEL] Temporal storage ready for namespace: {name}")
|
|
138
|
+
except Exception as e:
|
|
139
|
+
logger.warning(f"[TIME_TRAVEL] Failed to initialize temporal storage: {e}")
|
|
140
|
+
self.temporal_enabled = False
|
|
141
|
+
self.temporal_storage = None
|
|
142
|
+
|
|
143
|
+
# Semantic hashing (optional, enabled via config)
|
|
144
|
+
self.semantic_hashing_enabled = self.config.get('hashing', {}).get('enabled', False)
|
|
145
|
+
self.hasher = None
|
|
146
|
+
if self.semantic_hashing_enabled:
|
|
147
|
+
try:
|
|
148
|
+
from ..hashing.comprehensive_hashing import ComprehensiveHasher
|
|
149
|
+
self.hasher = ComprehensiveHasher()
|
|
150
|
+
logger.info(f"[SEMANTIC_HASH] Semantic hashing enabled for namespace: {name}")
|
|
151
|
+
except Exception as e:
|
|
152
|
+
logger.warning(f"[SEMANTIC_HASH] Failed to initialize semantic hasher: {e}")
|
|
153
|
+
self.semantic_hashing_enabled = False
|
|
154
|
+
|
|
155
|
+
# Initialize traceability layer (optional, for transparency and integrity)
|
|
156
|
+
# Check config file first, then local config
|
|
157
|
+
from ..config.database_config import DatabaseConfigManager
|
|
158
|
+
try:
|
|
159
|
+
config_manager = DatabaseConfigManager()
|
|
160
|
+
db_config = config_manager.config
|
|
161
|
+
traceability_enabled = db_config.traceability.enabled
|
|
162
|
+
except:
|
|
163
|
+
traceability_enabled = self.config.get('traceability', {}).get('enabled', False)
|
|
164
|
+
|
|
165
|
+
self.traceability_enabled = traceability_enabled
|
|
166
|
+
self.traceability_layer = None
|
|
167
|
+
if self.traceability_enabled:
|
|
168
|
+
try:
|
|
169
|
+
from ..traceability.traceability_layer import TraceabilityLayer
|
|
170
|
+
self.traceability_layer = TraceabilityLayer(name, base_path="contextcore_data")
|
|
171
|
+
logger.info(f"Traceability layer enabled for graph '{name}'")
|
|
172
|
+
except Exception as e:
|
|
173
|
+
logger.warning(f"Failed to initialize traceability layer: {e}")
|
|
174
|
+
|
|
175
|
+
# WAL (Write-Ahead Log) — lazy init on first write for speed
|
|
176
|
+
self.wal = None
|
|
177
|
+
self._wal_enabled = True
|
|
178
|
+
self._wal_name = name
|
|
179
|
+
try:
|
|
180
|
+
self._wal_enabled = self.config.get('wal', {}).get('enabled', True)
|
|
181
|
+
except Exception:
|
|
182
|
+
pass
|
|
183
|
+
|
|
184
|
+
# CRITICAL: Enable buffer manager by default with delayed block updates
|
|
185
|
+
# Buffer writes to memory first, then flushes to CSR storage after delay
|
|
186
|
+
buffer_config = self.config.get('buffer', {})
|
|
187
|
+
if 'enabled' not in buffer_config:
|
|
188
|
+
buffer_config['enabled'] = True # Enable by default
|
|
189
|
+
if 'config' not in buffer_config:
|
|
190
|
+
buffer_config['config'] = {}
|
|
191
|
+
buffer_config['config'].setdefault('max_size', 1000) # Max records before flush
|
|
192
|
+
buffer_config['config'].setdefault('flush_interval', 2.0) # Flush every 2 seconds (delayed block update)
|
|
193
|
+
buffer_config['config'].setdefault('batch_size', 100) # Records per batch
|
|
194
|
+
buffer_config['config'].setdefault('auto_flush', True) # Enable auto-flush
|
|
195
|
+
buffer_config['config'].setdefault('flush_threshold', 0.7) # Flush when buffer reaches 70% capacity (0.0-1.0)
|
|
196
|
+
buffer_config['config'].setdefault('threshold_flush_enabled', True) # Enable threshold-based flushing
|
|
197
|
+
|
|
198
|
+
self.buffer_enabled = buffer_config.get('enabled', True)
|
|
199
|
+
self.buffer_manager = None
|
|
200
|
+
# OPTIMIZATION: Pre-initialize buffer manager synchronously during __init__
|
|
201
|
+
# This avoids async initialization overhead during first node creation
|
|
202
|
+
if self.buffer_enabled:
|
|
203
|
+
try:
|
|
204
|
+
from ..buffer import create_buffer_manager
|
|
205
|
+
buffer_type = buffer_config.get('type', 'local')
|
|
206
|
+
buffer_params = buffer_config.get('config', {})
|
|
207
|
+
buffer_params['namespace'] = name # Add namespace for multi-tenant buffers
|
|
208
|
+
self.buffer_manager = create_buffer_manager(buffer_type, buffer_params)
|
|
209
|
+
# Store writer function reference for later use
|
|
210
|
+
self._buffer_writer_fn_ref = self._buffer_writer_fn
|
|
211
|
+
|
|
212
|
+
# Register cleanup on instance destruction
|
|
213
|
+
import atexit
|
|
214
|
+
atexit.register(self._cleanup_buffer_manager)
|
|
215
|
+
|
|
216
|
+
# OPTIMIZATION: Initialize buffer manager synchronously if possible
|
|
217
|
+
# This avoids async overhead during first node creation
|
|
218
|
+
try:
|
|
219
|
+
import asyncio
|
|
220
|
+
# Try to get existing event loop
|
|
221
|
+
try:
|
|
222
|
+
loop = asyncio.get_event_loop()
|
|
223
|
+
if loop.is_running():
|
|
224
|
+
# Loop is running, initialize async (will be done lazily)
|
|
225
|
+
self._buffer_initialized = False
|
|
226
|
+
else:
|
|
227
|
+
# Loop exists but not running, initialize synchronously
|
|
228
|
+
loop.run_until_complete(self.buffer_manager.initialize())
|
|
229
|
+
if buffer_params.get('auto_flush', True):
|
|
230
|
+
loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
|
|
231
|
+
self._buffer_initialized = True
|
|
232
|
+
logger.debug(f"Buffer manager pre-initialized for graph '{name}'")
|
|
233
|
+
except RuntimeError:
|
|
234
|
+
# No event loop, create one and initialize
|
|
235
|
+
loop = asyncio.new_event_loop()
|
|
236
|
+
asyncio.set_event_loop(loop)
|
|
237
|
+
loop.run_until_complete(self.buffer_manager.initialize())
|
|
238
|
+
if buffer_params.get('auto_flush', True):
|
|
239
|
+
loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
|
|
240
|
+
self._buffer_initialized = True
|
|
241
|
+
logger.debug(f"Buffer manager pre-initialized for graph '{name}' (new loop)")
|
|
242
|
+
except Exception as init_error:
|
|
243
|
+
# If sync initialization fails, fall back to lazy initialization
|
|
244
|
+
logger.debug(f"Buffer manager will be initialized lazily: {init_error}")
|
|
245
|
+
self._buffer_initialized = False
|
|
246
|
+
|
|
247
|
+
logger.debug(f"Buffer manager enabled for graph '{name}' (type: {buffer_type}, flush_interval: {buffer_params.get('flush_interval', 2.0)}s)")
|
|
248
|
+
except Exception as e:
|
|
249
|
+
logger.warning(f"Failed to initialize buffer manager: {e}")
|
|
250
|
+
self.buffer_enabled = False
|
|
251
|
+
self._buffer_initialized = False
|
|
252
|
+
else:
|
|
253
|
+
self._buffer_initialized = False
|
|
254
|
+
|
|
255
|
+
# Initialize graph storage backend: Redis (shared) or CSR (in-memory)
|
|
256
|
+
import os
|
|
257
|
+
graph_backend = os.environ.get("CONTEXTSYNAPSE_GRAPH_BACKEND") or os.environ.get("AICONTEXTDB_GRAPH_BACKEND", "csr")
|
|
258
|
+
self._using_redis_backend = False
|
|
259
|
+
|
|
260
|
+
if graph_backend == "redis" and os.environ.get("CONTEXTSYNAPSE_REDIS_URL") or os.environ.get("AICONTEXTDB_REDIS_URL"):
|
|
261
|
+
try:
|
|
262
|
+
from ..storage.redis_graph_adapter import RedisGraphAdapter
|
|
263
|
+
self.csr_adapter = RedisGraphAdapter(name)
|
|
264
|
+
self.csr_storage = self.csr_adapter # compatibility
|
|
265
|
+
self._using_redis_backend = True
|
|
266
|
+
# Disable buffer for Redis — Redis handles persistence directly
|
|
267
|
+
self.buffer_enabled = False
|
|
268
|
+
self.buffer_manager = None
|
|
269
|
+
self._buffer_initialized = True
|
|
270
|
+
logger.info("[REDIS] Graph '%s' using Redis backend (no in-memory loading)", name)
|
|
271
|
+
except Exception as e:
|
|
272
|
+
logger.warning("[REDIS] Redis graph backend failed, falling back to CSR: %s", e)
|
|
273
|
+
graph_backend = "csr"
|
|
274
|
+
|
|
275
|
+
# Check for LMDB backend
|
|
276
|
+
self._using_lmdb_backend = False
|
|
277
|
+
if not self._using_redis_backend:
|
|
278
|
+
storage_backend_env = os.environ.get("CONTEXTSYNAPSE_STORAGE_BACKEND") or os.environ.get("AICONTEXTDB_STORAGE_BACKEND", "csr")
|
|
279
|
+
if storage_backend_env == "lmdb":
|
|
280
|
+
try:
|
|
281
|
+
from ..storage.lmdb_graph_storage import LMDBGraphStorage, LMDB_AVAILABLE
|
|
282
|
+
if LMDB_AVAILABLE:
|
|
283
|
+
lmdb_path = os.environ.get("CONTEXTSYNAPSE_LMDB_PATH") or os.environ.get("AICONTEXTDB_LMDB_PATH", "contextcore_data/lmdb")
|
|
284
|
+
lmdb_dir = os.path.join(lmdb_path, self.name)
|
|
285
|
+
self.csr_storage = LMDBGraphStorage(lmdb_dir, namespace=self.name)
|
|
286
|
+
self.csr_adapter = self.csr_storage
|
|
287
|
+
self._using_lmdb_backend = True
|
|
288
|
+
logger.info("[STORAGE] Using LMDB backend: %s", lmdb_dir)
|
|
289
|
+
else:
|
|
290
|
+
logger.warning("[STORAGE] LMDB requested but lmdb/msgpack not installed, falling back to CSR")
|
|
291
|
+
except Exception as e:
|
|
292
|
+
logger.warning("[STORAGE] LMDB init failed (%s), falling back to CSR", e)
|
|
293
|
+
|
|
294
|
+
if not self._using_redis_backend and not self._using_lmdb_backend:
|
|
295
|
+
try:
|
|
296
|
+
self.csr_storage = CSRGraphStorage()
|
|
297
|
+
self.csr_adapter = CSRGraphStorageAdapter(self.csr_storage)
|
|
298
|
+
logger.debug("[CSR] Graph '%s' using in-memory CSR storage", name)
|
|
299
|
+
except Exception as e:
|
|
300
|
+
logger.error("[CSR] Failed to initialize CSR storage: %s", e)
|
|
301
|
+
raise RuntimeError(f"Graph storage initialization failed: {e}")
|
|
302
|
+
|
|
303
|
+
# Initialize Time Travel (always enabled)
|
|
304
|
+
self.temporal_enabled = True
|
|
305
|
+
self.temporal_storage = None
|
|
306
|
+
try:
|
|
307
|
+
from ..temporal.storage import TemporalStorage as _TemporalStorage
|
|
308
|
+
base_path = self.config.get('temporal', {}).get('base_path', self.config.get('base_path', 'contextcore_data'))
|
|
309
|
+
self.temporal_storage = _TemporalStorage(namespace=name, base_path=base_path)
|
|
310
|
+
self.temporal_storage.load_versions()
|
|
311
|
+
logger.debug(f"[TIME_TRAVEL] Temporal storage ready for namespace: {name}")
|
|
312
|
+
except Exception as e:
|
|
313
|
+
logger.warning(f"[TIME_TRAVEL] Failed to initialize temporal storage: {e}")
|
|
314
|
+
self.temporal_enabled = False
|
|
315
|
+
self.temporal_storage = None
|
|
316
|
+
|
|
317
|
+
# Initialize Semantic Hashing (if available and enabled)
|
|
318
|
+
self.hashing_enabled = self.config.get('hashing', {}).get('enabled', False) and HASHING_AVAILABLE
|
|
319
|
+
if self.hashing_enabled:
|
|
320
|
+
try:
|
|
321
|
+
self.hasher = get_global_hasher()
|
|
322
|
+
logger.info(f"[EMOJI] Semantic hashing enabled for namespace: {name}")
|
|
323
|
+
except Exception as e:
|
|
324
|
+
logger.warning(f"[EMOJI] Failed to initialize semantic hashing: {e}")
|
|
325
|
+
self.hashing_enabled = False
|
|
326
|
+
self.hasher = None
|
|
327
|
+
else:
|
|
328
|
+
self.hasher = None
|
|
329
|
+
|
|
330
|
+
def _clean_properties_for_indexing(self, properties: Dict[str, Any]) -> Dict[str, Any]:
|
|
331
|
+
"""Recursively clean properties to remove unhashable types that could cause indexing errors.
|
|
332
|
+
|
|
333
|
+
Converts nested dicts and lists to JSON strings to make them hashable.
|
|
334
|
+
"""
|
|
335
|
+
import copy
|
|
336
|
+
import json
|
|
337
|
+
cleaned = {}
|
|
338
|
+
for key, value in properties.items():
|
|
339
|
+
if isinstance(value, dict):
|
|
340
|
+
# Convert nested dict to JSON string
|
|
341
|
+
try:
|
|
342
|
+
cleaned[key] = json.dumps(value, sort_keys=True)
|
|
343
|
+
except (TypeError, ValueError):
|
|
344
|
+
# If can't serialize, skip it
|
|
345
|
+
continue
|
|
346
|
+
elif isinstance(value, list):
|
|
347
|
+
# Convert list to JSON string
|
|
348
|
+
try:
|
|
349
|
+
cleaned[key] = json.dumps(value, sort_keys=True)
|
|
350
|
+
except (TypeError, ValueError):
|
|
351
|
+
# If can't serialize, skip it
|
|
352
|
+
continue
|
|
353
|
+
else:
|
|
354
|
+
# Keep primitive types as-is
|
|
355
|
+
cleaned[key] = value
|
|
356
|
+
return cleaned
|
|
357
|
+
|
|
358
|
+
def _cleanup_buffer_manager(self):
|
|
359
|
+
"""Cleanup buffer manager on instance destruction."""
|
|
360
|
+
if self.buffer_manager:
|
|
361
|
+
try:
|
|
362
|
+
# Stop auto-flush if running
|
|
363
|
+
if hasattr(self.buffer_manager, 'stop_auto_flush'):
|
|
364
|
+
try:
|
|
365
|
+
loop = asyncio.get_event_loop()
|
|
366
|
+
if loop.is_running():
|
|
367
|
+
# Schedule cleanup
|
|
368
|
+
asyncio.create_task(self.buffer_manager.stop_auto_flush())
|
|
369
|
+
else:
|
|
370
|
+
# Run cleanup synchronously
|
|
371
|
+
loop.run_until_complete(self.buffer_manager.stop_auto_flush())
|
|
372
|
+
except (RuntimeError, AttributeError):
|
|
373
|
+
# No event loop - task will be cleaned up by Python
|
|
374
|
+
pass
|
|
375
|
+
|
|
376
|
+
# Close buffer manager
|
|
377
|
+
if hasattr(self.buffer_manager, 'close'):
|
|
378
|
+
try:
|
|
379
|
+
loop = asyncio.get_event_loop()
|
|
380
|
+
if loop.is_running():
|
|
381
|
+
asyncio.create_task(self.buffer_manager.close())
|
|
382
|
+
else:
|
|
383
|
+
loop.run_until_complete(self.buffer_manager.close())
|
|
384
|
+
except (RuntimeError, AttributeError):
|
|
385
|
+
pass
|
|
386
|
+
except Exception as e:
|
|
387
|
+
logger.debug(f"Error cleaning up buffer manager for {self.name}: {e}")
|
|
388
|
+
|
|
389
|
+
def load(self, file_path: str) -> bool:
|
|
390
|
+
"""
|
|
391
|
+
Load graph data from a file.
|
|
392
|
+
Tries to load CSR format first (for performance), falls back to JSON.
|
|
393
|
+
|
|
394
|
+
Args:
|
|
395
|
+
file_path: Path to the graph file
|
|
396
|
+
|
|
397
|
+
Returns:
|
|
398
|
+
True if successful, False otherwise
|
|
399
|
+
"""
|
|
400
|
+
try:
|
|
401
|
+
from pathlib import Path
|
|
402
|
+
from scipy.sparse import load_npz
|
|
403
|
+
|
|
404
|
+
file_path_obj = Path(file_path)
|
|
405
|
+
base_dir = file_path_obj.parent
|
|
406
|
+
base_name = file_path_obj.stem
|
|
407
|
+
|
|
408
|
+
# Check storage strategy and load accordingly
|
|
409
|
+
graph_config = self.config.get('graph', {})
|
|
410
|
+
storage_strategy = graph_config.get('storage_strategy') or graph_config.get('storage_format', 'single_file')
|
|
411
|
+
|
|
412
|
+
# Try single-file HDF5 first if strategy is single_file
|
|
413
|
+
if storage_strategy == 'single_file':
|
|
414
|
+
h5_path = file_path_obj.with_suffix('.h5')
|
|
415
|
+
if h5_path.exists():
|
|
416
|
+
return self._load_single_file(str(h5_path))
|
|
417
|
+
# Try multi-file optimized if strategy is multi_file_optimized
|
|
418
|
+
elif storage_strategy == 'multi_file_optimized':
|
|
419
|
+
if self._load_multi_file_optimized(str(file_path_obj)):
|
|
420
|
+
return True
|
|
421
|
+
# Auto-detect: try single-file first, then multi-file
|
|
422
|
+
else:
|
|
423
|
+
h5_path = file_path_obj.with_suffix('.h5')
|
|
424
|
+
if h5_path.exists():
|
|
425
|
+
return self._load_single_file(str(h5_path))
|
|
426
|
+
|
|
427
|
+
# Try to load CSR format first (faster for large graphs)
|
|
428
|
+
csr_file = base_dir / "csr" / f"{base_name}_csr.npz"
|
|
429
|
+
csr_metadata_file = base_dir / "csr" / f"{base_name}_csr_metadata.json"
|
|
430
|
+
|
|
431
|
+
if csr_file.exists() and csr_metadata_file.exists():
|
|
432
|
+
try:
|
|
433
|
+
logger.info(f"Loading graph from CSR format: {csr_file}")
|
|
434
|
+
|
|
435
|
+
# Load CSR matrix
|
|
436
|
+
csr_matrix_data = load_npz(str(csr_file))
|
|
437
|
+
|
|
438
|
+
# Load CSR metadata
|
|
439
|
+
with open(csr_metadata_file, 'r') as f:
|
|
440
|
+
csr_metadata = json.load(f)
|
|
441
|
+
|
|
442
|
+
node_list = csr_metadata.get('node_list', [])
|
|
443
|
+
node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
|
|
444
|
+
idx_to_node = {idx: node_id for node_id, idx in node_to_idx.items()}
|
|
445
|
+
edge_labels = csr_metadata.get('edge_labels', [])
|
|
446
|
+
edge_properties_list = csr_metadata.get('edge_properties', [])
|
|
447
|
+
|
|
448
|
+
# Load nodes from metadata (we'll need to get properties from JSON)
|
|
449
|
+
# First, load JSON to get node properties
|
|
450
|
+
with open(file_path, 'r') as f:
|
|
451
|
+
json_data = json.load(f)
|
|
452
|
+
|
|
453
|
+
# CRITICAL: Ensure CSR adapter is initialized
|
|
454
|
+
if self.use_csr and not self.csr_adapter:
|
|
455
|
+
from ..storage.csr_graph_storage import CSRGraphStorageAdapter
|
|
456
|
+
if self.csr_storage:
|
|
457
|
+
self.csr_adapter = CSRGraphStorageAdapter(self.csr_storage)
|
|
458
|
+
logger.info("Initialized CSR adapter during CSR format load")
|
|
459
|
+
|
|
460
|
+
# Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
|
|
461
|
+
|
|
462
|
+
# Create node properties map
|
|
463
|
+
node_props_map = {}
|
|
464
|
+
if 'nodes' in json_data:
|
|
465
|
+
for node_data in json_data['nodes']:
|
|
466
|
+
node_id = node_data.get('id')
|
|
467
|
+
if node_id:
|
|
468
|
+
node_props_map[node_id] = node_data.get('properties', {})
|
|
469
|
+
|
|
470
|
+
# Add nodes
|
|
471
|
+
for node_id in node_list:
|
|
472
|
+
label = 'Node'
|
|
473
|
+
properties = node_props_map.get(node_id, {})
|
|
474
|
+
# Get label from JSON data if available
|
|
475
|
+
if 'nodes' in json_data:
|
|
476
|
+
for node_data in json_data['nodes']:
|
|
477
|
+
if node_data.get('id') == node_id:
|
|
478
|
+
label = node_data.get('label', 'Node')
|
|
479
|
+
break
|
|
480
|
+
|
|
481
|
+
node = GraphNode(id=node_id, label=label, properties=properties)
|
|
482
|
+
properties_cleaned = self._clean_properties_for_indexing(properties)
|
|
483
|
+
node.properties, original_props = properties_cleaned, node.properties
|
|
484
|
+
self.add_node(node)
|
|
485
|
+
self.node_properties[node_id] = original_props
|
|
486
|
+
if node_id in self.node_index:
|
|
487
|
+
self.node_index[node_id].properties = original_props
|
|
488
|
+
|
|
489
|
+
# Add edges from CSR matrix
|
|
490
|
+
csr_matrix_data = csr_matrix_data.tocoo() # Convert to COO for iteration
|
|
491
|
+
for i, (row_idx, col_idx) in enumerate(zip(csr_matrix_data.row, csr_matrix_data.col)):
|
|
492
|
+
if i < len(edge_labels):
|
|
493
|
+
src = idx_to_node.get(row_idx)
|
|
494
|
+
dst = idx_to_node.get(col_idx)
|
|
495
|
+
edge_label = edge_labels[i] if i < len(edge_labels) else 'Edge'
|
|
496
|
+
edge_props = edge_properties_list[i] if i < len(edge_properties_list) else {}
|
|
497
|
+
|
|
498
|
+
if src and dst:
|
|
499
|
+
self.add_edge(GraphEdge(
|
|
500
|
+
id=f"{src}_{dst}_{i}",
|
|
501
|
+
source=src,
|
|
502
|
+
target=dst,
|
|
503
|
+
label=edge_label,
|
|
504
|
+
properties=edge_props
|
|
505
|
+
))
|
|
506
|
+
|
|
507
|
+
logger.info(f"[EMOJI] Loaded graph from CSR format: {len(node_list)} nodes, {len(edge_labels)} edges")
|
|
508
|
+
return True
|
|
509
|
+
|
|
510
|
+
except Exception as e:
|
|
511
|
+
logger.warning(f"Failed to load CSR format: {e}, falling back to JSON")
|
|
512
|
+
|
|
513
|
+
# Fallback to JSON format
|
|
514
|
+
logger.info(f"Loading graph from JSON format: {file_path}")
|
|
515
|
+
with open(file_path, 'r') as f:
|
|
516
|
+
data = json.load(f)
|
|
517
|
+
|
|
518
|
+
# Try to load node properties from Parquet (faster than JSON for large graphs)
|
|
519
|
+
parquet_file = base_dir / "node_properties" / f"{base_name}_nodes.parquet"
|
|
520
|
+
if parquet_file.exists():
|
|
521
|
+
try:
|
|
522
|
+
import pandas as pd
|
|
523
|
+
nodes_df = pd.read_parquet(parquet_file, engine='pyarrow')
|
|
524
|
+
logger.info(f"Loading node properties from Parquet: {parquet_file}")
|
|
525
|
+
|
|
526
|
+
# Update node properties from Parquet (if available)
|
|
527
|
+
parquet_nodes = {}
|
|
528
|
+
for _, row in nodes_df.iterrows():
|
|
529
|
+
node_id = row['node_id']
|
|
530
|
+
props = {}
|
|
531
|
+
for col in nodes_df.columns:
|
|
532
|
+
if col.startswith('prop_'):
|
|
533
|
+
prop_key = col.replace('prop_', '')
|
|
534
|
+
value = row[col]
|
|
535
|
+
# Try to parse JSON strings back to objects
|
|
536
|
+
if pd.notna(value) and isinstance(value, str) and (value.startswith('{') or value.startswith('[')):
|
|
537
|
+
try:
|
|
538
|
+
props[prop_key] = json.loads(value)
|
|
539
|
+
except:
|
|
540
|
+
props[prop_key] = value
|
|
541
|
+
elif pd.notna(value):
|
|
542
|
+
props[prop_key] = value
|
|
543
|
+
if props:
|
|
544
|
+
parquet_nodes[node_id] = props
|
|
545
|
+
|
|
546
|
+
# Merge Parquet properties into JSON data if they exist
|
|
547
|
+
if parquet_nodes:
|
|
548
|
+
logger.info(f"Loaded {len(parquet_nodes)} node properties from Parquet")
|
|
549
|
+
for node in data.get('nodes', []):
|
|
550
|
+
node_id = node.get('id')
|
|
551
|
+
if node_id in parquet_nodes:
|
|
552
|
+
# Merge Parquet properties (may be more up-to-date)
|
|
553
|
+
if 'properties' not in node:
|
|
554
|
+
node['properties'] = {}
|
|
555
|
+
node['properties'].update(parquet_nodes[node_id])
|
|
556
|
+
except ImportError:
|
|
557
|
+
logger.debug("PyArrow/pandas not available, using JSON node properties only")
|
|
558
|
+
except Exception as e:
|
|
559
|
+
logger.debug(f"Failed to load node properties from Parquet: {e}, using JSON")
|
|
560
|
+
|
|
561
|
+
# CRITICAL: Ensure CSR adapter is initialized
|
|
562
|
+
# CSR adapter is created in __init__, but ensure it exists
|
|
563
|
+
if self.use_csr and not self.csr_adapter:
|
|
564
|
+
from ..storage.csr_graph_storage import CSRGraphStorageAdapter
|
|
565
|
+
if self.csr_storage:
|
|
566
|
+
self.csr_adapter = CSRGraphStorageAdapter(self.csr_storage)
|
|
567
|
+
logger.info("Initialized CSR adapter during graph load")
|
|
568
|
+
|
|
569
|
+
# Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
|
|
570
|
+
|
|
571
|
+
# Load nodes
|
|
572
|
+
if 'nodes' in data:
|
|
573
|
+
for node_data in data['nodes']:
|
|
574
|
+
try:
|
|
575
|
+
node_id = node_data['id']
|
|
576
|
+
label = node_data.get('label', 'Node')
|
|
577
|
+
properties = node_data.get('properties', {})
|
|
578
|
+
# Ensure properties is a dict (not nested dicts that could cause unhashable errors)
|
|
579
|
+
if isinstance(properties, dict):
|
|
580
|
+
# Deep copy to avoid reference issues
|
|
581
|
+
import copy
|
|
582
|
+
properties = copy.deepcopy(properties)
|
|
583
|
+
# Clean properties for indexing (convert nested dicts/lists to JSON strings)
|
|
584
|
+
properties_cleaned = self._clean_properties_for_indexing(properties)
|
|
585
|
+
# Keep original properties for node, but use cleaned for indexing
|
|
586
|
+
node = GraphNode(id=node_id, label=label, properties=properties)
|
|
587
|
+
# Temporarily replace properties with cleaned version for add_node (to avoid indexing errors)
|
|
588
|
+
# Then restore original properties after indexing
|
|
589
|
+
node.properties, original_props = properties_cleaned, node.properties
|
|
590
|
+
self.add_node(node)
|
|
591
|
+
# Restore original properties after indexing
|
|
592
|
+
self.node_properties[node_id] = original_props
|
|
593
|
+
if node_id in self.node_index:
|
|
594
|
+
self.node_index[node_id].properties = original_props
|
|
595
|
+
else:
|
|
596
|
+
properties = {}
|
|
597
|
+
self.add_node(GraphNode(id=node_id, label=label, properties=properties))
|
|
598
|
+
except Exception as e:
|
|
599
|
+
logger.warning(f"[EMOJI][EMOJI] Skipping node {node_data.get('id', 'unknown')} due to error: {e}")
|
|
600
|
+
continue
|
|
601
|
+
|
|
602
|
+
# Load edges
|
|
603
|
+
if 'edges' in data:
|
|
604
|
+
for edge_data in data['edges']:
|
|
605
|
+
# Handle both 'src'/'dst' and 'source'/'target' formats
|
|
606
|
+
src = edge_data.get('src', edge_data.get('source', ''))
|
|
607
|
+
dest = edge_data.get('dst', edge_data.get('dest', edge_data.get('target', '')))
|
|
608
|
+
label = edge_data.get('label', edge_data.get('type', 'Edge'))
|
|
609
|
+
properties = edge_data.get('properties', {})
|
|
610
|
+
weight = edge_data.get('weight', 1.0)
|
|
611
|
+
|
|
612
|
+
if src and dest: # Only add edge if both source and target exist
|
|
613
|
+
self.add_edge(GraphEdge(
|
|
614
|
+
id=f"{src}_{dest}",
|
|
615
|
+
source=src,
|
|
616
|
+
target=dest,
|
|
617
|
+
label=label,
|
|
618
|
+
properties=properties
|
|
619
|
+
))
|
|
620
|
+
|
|
621
|
+
return True
|
|
622
|
+
except Exception as e:
|
|
623
|
+
logger.error(f"Error loading graph from {file_path}: {e}")
|
|
624
|
+
return False
|
|
625
|
+
|
|
626
|
+
def _load_single_file(self, h5_path: str) -> bool:
|
|
627
|
+
"""
|
|
628
|
+
Load graph from single-file HDF5 format.
|
|
629
|
+
|
|
630
|
+
Args:
|
|
631
|
+
h5_path: Path to HDF5 file
|
|
632
|
+
|
|
633
|
+
Returns:
|
|
634
|
+
True if successful
|
|
635
|
+
"""
|
|
636
|
+
try:
|
|
637
|
+
from ..storage.single_file_storage import SingleFileStorage
|
|
638
|
+
|
|
639
|
+
with SingleFileStorage(h5_path, mode='r') as storage:
|
|
640
|
+
data = storage.load_graph()
|
|
641
|
+
if not data:
|
|
642
|
+
return False
|
|
643
|
+
|
|
644
|
+
# Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
|
|
645
|
+
|
|
646
|
+
# Load nodes (always write_through — bypass async buffer during load)
|
|
647
|
+
for node_data in data.get('nodes', []):
|
|
648
|
+
node_id = node_data.get('id')
|
|
649
|
+
label = node_data.get('label', 'Node')
|
|
650
|
+
properties = node_data.get('properties', {})
|
|
651
|
+
|
|
652
|
+
node = GraphNode(id=node_id, label=label, properties=properties)
|
|
653
|
+
properties_cleaned = self._clean_properties_for_indexing(properties)
|
|
654
|
+
node.properties, original_props = properties_cleaned, node.properties
|
|
655
|
+
self.add_node(node, write_through=True)
|
|
656
|
+
self.node_properties[node_id] = original_props
|
|
657
|
+
if node_id in self.node_index:
|
|
658
|
+
self.node_index[node_id].properties = original_props
|
|
659
|
+
|
|
660
|
+
# Load edges
|
|
661
|
+
for edge_data in data.get('edges', []):
|
|
662
|
+
src = edge_data.get('src', edge_data.get('source', ''))
|
|
663
|
+
dst = edge_data.get('dst', edge_data.get('target', ''))
|
|
664
|
+
label = edge_data.get('label', 'Edge')
|
|
665
|
+
properties = edge_data.get('properties', {})
|
|
666
|
+
|
|
667
|
+
if src and dst:
|
|
668
|
+
self.add_edge(GraphEdge(
|
|
669
|
+
id=f"{src}_{dst}",
|
|
670
|
+
source=src,
|
|
671
|
+
target=dst,
|
|
672
|
+
label=label,
|
|
673
|
+
properties=properties
|
|
674
|
+
))
|
|
675
|
+
|
|
676
|
+
# Try to load CSR matrix
|
|
677
|
+
csr_matrix_data, csr_metadata = storage.load_csr_matrix()
|
|
678
|
+
if csr_matrix_data is not None:
|
|
679
|
+
logger.info("Loaded CSR matrix from HDF5")
|
|
680
|
+
# CSR matrix can be used for fast operations
|
|
681
|
+
# Note: Full CSR integration would require CSR adapter setup
|
|
682
|
+
|
|
683
|
+
logger.info(f"Loaded graph from single-file HDF5: {len(data.get('nodes', []))} nodes, {len(data.get('edges', []))} edges")
|
|
684
|
+
return True
|
|
685
|
+
|
|
686
|
+
except ImportError:
|
|
687
|
+
logger.error("h5py not available. Install with: pip install h5py")
|
|
688
|
+
return False
|
|
689
|
+
except Exception as e:
|
|
690
|
+
logger.error(f"Failed to load from single-file format: {e}")
|
|
691
|
+
return False
|
|
692
|
+
|
|
693
|
+
def flush(self):
|
|
694
|
+
"""Flush any buffered writes to CSR storage for immediate visibility."""
|
|
695
|
+
if self.buffer_enabled and self.buffer_manager:
|
|
696
|
+
try:
|
|
697
|
+
import asyncio
|
|
698
|
+
loop = asyncio.new_event_loop()
|
|
699
|
+
loop.run_until_complete(self.buffer_manager.flush())
|
|
700
|
+
loop.close()
|
|
701
|
+
except Exception:
|
|
702
|
+
pass
|
|
703
|
+
|
|
704
|
+
def save(self, file_path: str) -> bool:
|
|
705
|
+
"""
|
|
706
|
+
Save graph data to a file.
|
|
707
|
+
Saves both JSON format (for compatibility) and CSR format (for performance).
|
|
708
|
+
Optionally saves to single-file HDF5 format if configured.
|
|
709
|
+
|
|
710
|
+
This method is used by BOTH direct graph operations and API operations.
|
|
711
|
+
It ensures the same code path is used regardless of how the graph was created/modified.
|
|
712
|
+
|
|
713
|
+
Args:
|
|
714
|
+
file_path: Path to save the graph file
|
|
715
|
+
|
|
716
|
+
Returns:
|
|
717
|
+
True if successful, False otherwise
|
|
718
|
+
"""
|
|
719
|
+
try:
|
|
720
|
+
from pathlib import Path
|
|
721
|
+
import numpy as np
|
|
722
|
+
from scipy.sparse import save_npz
|
|
723
|
+
|
|
724
|
+
file_path_obj = Path(file_path)
|
|
725
|
+
base_dir = file_path_obj.parent
|
|
726
|
+
base_name = file_path_obj.stem
|
|
727
|
+
|
|
728
|
+
# Check storage strategy (prefer storage_strategy over storage_format for backward compatibility)
|
|
729
|
+
graph_config = self.config.get('graph', {})
|
|
730
|
+
storage_strategy = graph_config.get('storage_strategy') or graph_config.get('storage_format', 'single_file')
|
|
731
|
+
|
|
732
|
+
if storage_strategy == 'single_file':
|
|
733
|
+
return self._save_single_file(file_path)
|
|
734
|
+
elif storage_strategy == 'multi_file_optimized':
|
|
735
|
+
return self._save_multi_file_optimized(file_path)
|
|
736
|
+
# Fallback to legacy multi-file format
|
|
737
|
+
|
|
738
|
+
# 1. Save JSON format (for compatibility and human readability)
|
|
739
|
+
# Note: Aliases are NOT saved (they are query-scoped only)
|
|
740
|
+
|
|
741
|
+
# CSR-only: Get node/edge counts from CSR storage
|
|
742
|
+
node_count = self.csr_storage.get_node_count() if self.csr_storage else 0
|
|
743
|
+
edge_count = self.csr_storage.get_edge_count() if self.csr_storage else 0
|
|
744
|
+
|
|
745
|
+
data = {
|
|
746
|
+
'nodes': [],
|
|
747
|
+
'edges': [],
|
|
748
|
+
# 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
|
|
749
|
+
'metadata': {
|
|
750
|
+
'name': self.name,
|
|
751
|
+
'node_count': node_count,
|
|
752
|
+
'edge_count': edge_count,
|
|
753
|
+
'storage_backend': self.storage_backend,
|
|
754
|
+
'saved_at': time.time(),
|
|
755
|
+
'has_csr': self.csr_storage is not None and node_count > 0
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
# Save nodes from CSR storage
|
|
760
|
+
nodes_saved = 0
|
|
761
|
+
if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
|
|
762
|
+
nodes_dict = self.csr_storage.nodes if hasattr(self.csr_storage, 'nodes') else {}
|
|
763
|
+
logger.debug(f"[SAVE] CSR storage has {len(nodes_dict)} nodes in nodes dict")
|
|
764
|
+
|
|
765
|
+
for node_id, csr_node in nodes_dict.items():
|
|
766
|
+
# Handle both CSRNode objects and GraphNode objects
|
|
767
|
+
if hasattr(csr_node, 'node_type'):
|
|
768
|
+
node_type = csr_node.node_type
|
|
769
|
+
elif hasattr(csr_node, 'label'):
|
|
770
|
+
node_type = csr_node.label
|
|
771
|
+
else:
|
|
772
|
+
node_type = 'Node'
|
|
773
|
+
|
|
774
|
+
# Get properties from node_properties dict (stored separately)
|
|
775
|
+
node_props = self.node_properties.get(node_id, {})
|
|
776
|
+
if not node_props and hasattr(csr_node, 'properties'):
|
|
777
|
+
node_props = csr_node.properties
|
|
778
|
+
|
|
779
|
+
node_data = {
|
|
780
|
+
'id': node_id,
|
|
781
|
+
'label': node_type,
|
|
782
|
+
'properties': node_props
|
|
783
|
+
}
|
|
784
|
+
data['nodes'].append(node_data)
|
|
785
|
+
nodes_saved += 1
|
|
786
|
+
|
|
787
|
+
logger.info(f"[SAVE] Saved {nodes_saved} nodes from CSR storage")
|
|
788
|
+
|
|
789
|
+
# FALLBACK: If CSR storage has no nodes but node_index does, use node_index
|
|
790
|
+
if not data['nodes'] and hasattr(self, 'node_index') and self.node_index:
|
|
791
|
+
logger.warning(f"[SAVE] CSR storage has no nodes, using node_index fallback ({len(self.node_index)} nodes)")
|
|
792
|
+
for node_id, node in self.node_index.items():
|
|
793
|
+
node_data = {
|
|
794
|
+
'id': node_id,
|
|
795
|
+
'label': node.label if hasattr(node, 'label') else 'Node',
|
|
796
|
+
'properties': self.node_properties.get(node_id, node.properties if hasattr(node, 'properties') else {})
|
|
797
|
+
}
|
|
798
|
+
data['nodes'].append(node_data)
|
|
799
|
+
nodes_saved += 1
|
|
800
|
+
logger.info(f"[SAVE] Saved {nodes_saved} nodes from node_index fallback")
|
|
801
|
+
|
|
802
|
+
# Update node count in metadata
|
|
803
|
+
data['metadata']['node_count'] = len(data['nodes'])
|
|
804
|
+
|
|
805
|
+
# Save edges from CSR storage
|
|
806
|
+
saved_edge_pairs = set() # Track (source, target, label) to avoid duplicates
|
|
807
|
+
if self.csr_storage and hasattr(self.csr_storage, 'edge_data'):
|
|
808
|
+
for edge in self.csr_storage.edge_data:
|
|
809
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
810
|
+
edge_info = {
|
|
811
|
+
'src': edge.source_id,
|
|
812
|
+
'dst': edge.target_id,
|
|
813
|
+
'label': edge.edge_type,
|
|
814
|
+
'properties': self.edge_properties.get(edge_id, edge.properties),
|
|
815
|
+
'weight': edge.weight
|
|
816
|
+
}
|
|
817
|
+
data['edges'].append(edge_info)
|
|
818
|
+
saved_edge_pairs.add((edge.source_id, edge.target_id, edge.edge_type))
|
|
819
|
+
|
|
820
|
+
# Fallback: save edges from edge_store that weren't in CSR
|
|
821
|
+
# This catches edges where CSR add_edge silently failed (e.g., node not in CSR)
|
|
822
|
+
for edge_id, edge_meta in self.edge_store.items():
|
|
823
|
+
pair = (edge_meta['source'], edge_meta['target'], edge_meta.get('label', 'EDGE'))
|
|
824
|
+
if pair in saved_edge_pairs:
|
|
825
|
+
continue
|
|
826
|
+
edge_info = {
|
|
827
|
+
'src': edge_meta['source'],
|
|
828
|
+
'dst': edge_meta['target'],
|
|
829
|
+
'label': edge_meta.get('label', 'EDGE'),
|
|
830
|
+
'properties': edge_meta.get('properties', {}),
|
|
831
|
+
'weight': edge_meta.get('weight', 1.0)
|
|
832
|
+
}
|
|
833
|
+
data['edges'].append(edge_info)
|
|
834
|
+
saved_edge_pairs.add(pair)
|
|
835
|
+
|
|
836
|
+
# Write JSON file (for compatibility)
|
|
837
|
+
with open(file_path, 'w') as f:
|
|
838
|
+
json.dump(data, f, indent=2)
|
|
839
|
+
|
|
840
|
+
logger.info(f"Graph '{self.name}' saved to {file_path} (JSON format)")
|
|
841
|
+
|
|
842
|
+
# 3. Save node properties to Parquet (for fast property queries)
|
|
843
|
+
try:
|
|
844
|
+
import pandas as pd
|
|
845
|
+
parquet_dir = base_dir / "node_properties"
|
|
846
|
+
parquet_dir.mkdir(parents=True, exist_ok=True)
|
|
847
|
+
|
|
848
|
+
# Convert node properties to DataFrame
|
|
849
|
+
if data['nodes']:
|
|
850
|
+
nodes_df_data = []
|
|
851
|
+
for node in data['nodes']:
|
|
852
|
+
row = {
|
|
853
|
+
'node_id': node['id'],
|
|
854
|
+
'label': node['label']
|
|
855
|
+
}
|
|
856
|
+
# Flatten properties
|
|
857
|
+
props = node.get('properties', {})
|
|
858
|
+
for key, value in props.items():
|
|
859
|
+
# Handle nested properties by converting to string
|
|
860
|
+
if isinstance(value, (dict, list)):
|
|
861
|
+
row[f'prop_{key}'] = json.dumps(value)
|
|
862
|
+
else:
|
|
863
|
+
row[f'prop_{key}'] = value
|
|
864
|
+
nodes_df_data.append(row)
|
|
865
|
+
|
|
866
|
+
if nodes_df_data:
|
|
867
|
+
nodes_df = pd.DataFrame(nodes_df_data)
|
|
868
|
+
parquet_file = parquet_dir / f"{base_name}_nodes.parquet"
|
|
869
|
+
nodes_df.to_parquet(parquet_file, index=False, engine='pyarrow', compression='snappy')
|
|
870
|
+
logger.info(f"Node properties saved to Parquet: {parquet_file} ({len(nodes_df)} nodes)")
|
|
871
|
+
except ImportError:
|
|
872
|
+
logger.warning("PyArrow/pandas not available, skipping Parquet node properties storage")
|
|
873
|
+
except Exception as e:
|
|
874
|
+
logger.warning(f"Failed to save node properties to Parquet: {e}")
|
|
875
|
+
|
|
876
|
+
# 2. Save CSR format (CRITICAL: Always save CSR when enabled, not just when _should_use_csr())
|
|
877
|
+
# CSR is the primary storage format for performance
|
|
878
|
+
if self.csr_storage and self.use_csr:
|
|
879
|
+
try:
|
|
880
|
+
csr_dir = base_dir / "csr"
|
|
881
|
+
csr_dir.mkdir(parents=True, exist_ok=True)
|
|
882
|
+
|
|
883
|
+
# Save CSR matrix and metadata
|
|
884
|
+
csr_file = csr_dir / f"{base_name}_csr.npz"
|
|
885
|
+
csr_metadata_file = csr_dir / f"{base_name}_csr_metadata.json"
|
|
886
|
+
|
|
887
|
+
# Build CSR matrix from CSR storage edges
|
|
888
|
+
if not self.csr_storage or not hasattr(self.csr_storage, 'nodes'):
|
|
889
|
+
raise RuntimeError("CSR storage not available")
|
|
890
|
+
|
|
891
|
+
node_list = list(self.csr_storage.nodes.keys())
|
|
892
|
+
node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
|
|
893
|
+
num_nodes = len(node_list)
|
|
894
|
+
|
|
895
|
+
# Build edge lists for CSR
|
|
896
|
+
row_indices = []
|
|
897
|
+
col_indices = []
|
|
898
|
+
edge_labels = []
|
|
899
|
+
edge_properties_list = []
|
|
900
|
+
|
|
901
|
+
if hasattr(self.csr_storage, 'edge_data'):
|
|
902
|
+
for edge in self.csr_storage.edge_data:
|
|
903
|
+
if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
|
|
904
|
+
row_indices.append(node_to_idx[edge.source_id])
|
|
905
|
+
col_indices.append(node_to_idx[edge.target_id])
|
|
906
|
+
edge_labels.append(edge.edge_type)
|
|
907
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
908
|
+
edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
|
|
909
|
+
|
|
910
|
+
# CRITICAL: Always save CSR format, even if no edges (for node-only graphs)
|
|
911
|
+
# Create CSR matrix (use zeros if no edges, but still create the matrix)
|
|
912
|
+
from scipy.sparse import csr_matrix
|
|
913
|
+
if row_indices:
|
|
914
|
+
csr_matrix_data = csr_matrix(
|
|
915
|
+
(np.ones(len(row_indices)), (row_indices, col_indices)),
|
|
916
|
+
shape=(num_nodes, num_nodes)
|
|
917
|
+
)
|
|
918
|
+
else:
|
|
919
|
+
# Empty graph - create empty CSR matrix
|
|
920
|
+
csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
|
|
921
|
+
|
|
922
|
+
# Save CSR matrix
|
|
923
|
+
save_npz(str(csr_file), csr_matrix_data)
|
|
924
|
+
|
|
925
|
+
# Save CSR metadata (without aliases - they are query-scoped)
|
|
926
|
+
csr_metadata = {
|
|
927
|
+
'node_list': node_list,
|
|
928
|
+
'node_to_idx': node_to_idx,
|
|
929
|
+
'edge_labels': edge_labels,
|
|
930
|
+
'edge_properties': edge_properties_list,
|
|
931
|
+
'row_indices': row_indices,
|
|
932
|
+
'col_indices': col_indices,
|
|
933
|
+
'num_nodes': num_nodes,
|
|
934
|
+
'num_edges': len(row_indices),
|
|
935
|
+
# 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
|
|
936
|
+
'saved_at': time.time()
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
with open(csr_metadata_file, 'w') as f:
|
|
940
|
+
json.dump(csr_metadata, f, indent=2, default=str)
|
|
941
|
+
|
|
942
|
+
logger.info(f"CSR format saved to {csr_file} ({num_nodes} nodes, {len(row_indices)} edges)")
|
|
943
|
+
except Exception as e:
|
|
944
|
+
logger.error(f"[EMOJI] Failed to save CSR format: {e}")
|
|
945
|
+
import traceback
|
|
946
|
+
logger.debug(f"CSR save traceback: {traceback.format_exc()}")
|
|
947
|
+
# Don't fail completely - JSON is still saved as fallback
|
|
948
|
+
|
|
949
|
+
return True
|
|
950
|
+
except Exception as e:
|
|
951
|
+
logger.error(f"Failed to save graph '{self.name}': {e}")
|
|
952
|
+
return False
|
|
953
|
+
|
|
954
|
+
def _save_single_file(self, file_path: str) -> bool:
|
|
955
|
+
"""
|
|
956
|
+
Save graph to single-file HDF5 format (like Oracle's .dbf).
|
|
957
|
+
|
|
958
|
+
Args:
|
|
959
|
+
file_path: Path to save (will use .h5 extension)
|
|
960
|
+
|
|
961
|
+
Returns:
|
|
962
|
+
True if successful
|
|
963
|
+
"""
|
|
964
|
+
try:
|
|
965
|
+
from pathlib import Path
|
|
966
|
+
import numpy as np
|
|
967
|
+
from scipy.sparse import csr_matrix
|
|
968
|
+
from ..storage.single_file_storage import SingleFileStorage
|
|
969
|
+
|
|
970
|
+
# Use .h5 extension for single-file format
|
|
971
|
+
h5_path = Path(file_path).with_suffix('.h5')
|
|
972
|
+
|
|
973
|
+
# Prepare graph data (without aliases - they are query-scoped)
|
|
974
|
+
# CSR-only: Get node/edge counts from CSR storage
|
|
975
|
+
node_count = self.csr_storage.get_node_count() if self.csr_storage else 0
|
|
976
|
+
edge_count = self.csr_storage.get_edge_count() if self.csr_storage else 0
|
|
977
|
+
|
|
978
|
+
data = {
|
|
979
|
+
'nodes': [],
|
|
980
|
+
'edges': [],
|
|
981
|
+
# 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
|
|
982
|
+
'metadata': {
|
|
983
|
+
'name': self.name,
|
|
984
|
+
'node_count': node_count,
|
|
985
|
+
'edge_count': edge_count,
|
|
986
|
+
'storage_backend': self.storage_backend,
|
|
987
|
+
'saved_at': time.time(),
|
|
988
|
+
'has_csr': self.csr_storage is not None and node_count > 0
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
# Collect nodes from CSR storage
|
|
993
|
+
if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
|
|
994
|
+
for node_id, csr_node in self.csr_storage.nodes.items():
|
|
995
|
+
node_data = {
|
|
996
|
+
'id': node_id,
|
|
997
|
+
'label': csr_node.node_type,
|
|
998
|
+
'properties': self.node_properties.get(node_id, {})
|
|
999
|
+
}
|
|
1000
|
+
data['nodes'].append(node_data)
|
|
1001
|
+
|
|
1002
|
+
# Collect edges from CSR storage
|
|
1003
|
+
saved_edge_pairs = set()
|
|
1004
|
+
if self.csr_storage and hasattr(self.csr_storage, 'edge_data'):
|
|
1005
|
+
for edge in self.csr_storage.edge_data:
|
|
1006
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
1007
|
+
edge_info = {
|
|
1008
|
+
'src': edge.source_id,
|
|
1009
|
+
'dst': edge.target_id,
|
|
1010
|
+
'label': edge.edge_type,
|
|
1011
|
+
'properties': self.edge_properties.get(edge_id, edge.properties),
|
|
1012
|
+
'weight': edge.weight
|
|
1013
|
+
}
|
|
1014
|
+
data['edges'].append(edge_info)
|
|
1015
|
+
saved_edge_pairs.add((edge.source_id, edge.target_id, edge.edge_type))
|
|
1016
|
+
|
|
1017
|
+
# Fallback: save edges from edge_store not in CSR
|
|
1018
|
+
for edge_id, edge_meta in self.edge_store.items():
|
|
1019
|
+
pair = (edge_meta['source'], edge_meta['target'], edge_meta.get('label', 'EDGE'))
|
|
1020
|
+
if pair in saved_edge_pairs:
|
|
1021
|
+
continue
|
|
1022
|
+
edge_info = {
|
|
1023
|
+
'src': edge_meta['source'],
|
|
1024
|
+
'dst': edge_meta['target'],
|
|
1025
|
+
'label': edge_meta.get('label', 'EDGE'),
|
|
1026
|
+
'properties': edge_meta.get('properties', {}),
|
|
1027
|
+
'weight': edge_meta.get('weight', 1.0)
|
|
1028
|
+
}
|
|
1029
|
+
data['edges'].append(edge_info)
|
|
1030
|
+
saved_edge_pairs.add(pair)
|
|
1031
|
+
|
|
1032
|
+
# Save to HDF5
|
|
1033
|
+
# NEO4J-LEVEL PERFORMANCE: Use 'a' mode and delete/recreate groups to avoid file locking issues
|
|
1034
|
+
# This allows concurrent saves without "file already open" errors
|
|
1035
|
+
storage = None
|
|
1036
|
+
try:
|
|
1037
|
+
with SingleFileStorage(str(h5_path), mode='a') as storage:
|
|
1038
|
+
storage.save_graph(data)
|
|
1039
|
+
|
|
1040
|
+
# Save CSR matrix if available (CRITICAL: Inside try block, not except!)
|
|
1041
|
+
if self.csr_storage and self.use_csr:
|
|
1042
|
+
try:
|
|
1043
|
+
# Build CSR matrix from CSR storage
|
|
1044
|
+
if not hasattr(self.csr_storage, 'nodes'):
|
|
1045
|
+
raise RuntimeError("CSR storage nodes not available")
|
|
1046
|
+
|
|
1047
|
+
node_list = list(self.csr_storage.nodes.keys())
|
|
1048
|
+
node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
|
|
1049
|
+
num_nodes = len(node_list)
|
|
1050
|
+
|
|
1051
|
+
row_indices = []
|
|
1052
|
+
col_indices = []
|
|
1053
|
+
edge_labels = []
|
|
1054
|
+
edge_properties_list = []
|
|
1055
|
+
|
|
1056
|
+
if hasattr(self.csr_storage, 'edge_data'):
|
|
1057
|
+
for edge in self.csr_storage.edge_data:
|
|
1058
|
+
if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
|
|
1059
|
+
row_indices.append(node_to_idx[edge.source_id])
|
|
1060
|
+
col_indices.append(node_to_idx[edge.target_id])
|
|
1061
|
+
edge_labels.append(edge.edge_type)
|
|
1062
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
1063
|
+
edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
|
|
1064
|
+
|
|
1065
|
+
if row_indices:
|
|
1066
|
+
csr_matrix_data = csr_matrix(
|
|
1067
|
+
(np.ones(len(row_indices)), (row_indices, col_indices)),
|
|
1068
|
+
shape=(num_nodes, num_nodes)
|
|
1069
|
+
)
|
|
1070
|
+
else:
|
|
1071
|
+
csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
|
|
1072
|
+
|
|
1073
|
+
csr_metadata = {
|
|
1074
|
+
'node_list': node_list,
|
|
1075
|
+
'node_to_idx': node_to_idx,
|
|
1076
|
+
'edge_labels': edge_labels,
|
|
1077
|
+
'edge_properties': edge_properties_list,
|
|
1078
|
+
'num_nodes': num_nodes,
|
|
1079
|
+
'num_edges': len(row_indices)
|
|
1080
|
+
# 'alias_to_node': {} # REMOVED: Aliases are query-scoped, not persisted
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
storage.save_csr_matrix(csr_matrix_data, csr_metadata)
|
|
1084
|
+
logger.info(f"CSR matrix saved to HDF5: {num_nodes} nodes, {len(row_indices)} edges")
|
|
1085
|
+
except Exception as csr_error:
|
|
1086
|
+
logger.warning(f"Failed to save CSR matrix to HDF5: {csr_error}")
|
|
1087
|
+
except Exception as e:
|
|
1088
|
+
# If append mode fails (file locked), try write mode with retry
|
|
1089
|
+
logger.warning(f"Append mode failed, trying write mode: {e}")
|
|
1090
|
+
max_retries = 3
|
|
1091
|
+
for retry in range(max_retries):
|
|
1092
|
+
try:
|
|
1093
|
+
with SingleFileStorage(str(h5_path), mode='w') as storage:
|
|
1094
|
+
storage.save_graph(data)
|
|
1095
|
+
|
|
1096
|
+
# Save CSR matrix if available (also in retry path)
|
|
1097
|
+
if self.csr_storage and self.use_csr:
|
|
1098
|
+
try:
|
|
1099
|
+
# Build CSR matrix from CSR storage
|
|
1100
|
+
if not hasattr(self.csr_storage, 'nodes'):
|
|
1101
|
+
raise RuntimeError("CSR storage nodes not available")
|
|
1102
|
+
|
|
1103
|
+
node_list = list(self.csr_storage.nodes.keys())
|
|
1104
|
+
node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
|
|
1105
|
+
num_nodes = len(node_list)
|
|
1106
|
+
|
|
1107
|
+
row_indices = []
|
|
1108
|
+
col_indices = []
|
|
1109
|
+
edge_labels = []
|
|
1110
|
+
edge_properties_list = []
|
|
1111
|
+
|
|
1112
|
+
if hasattr(self.csr_storage, 'edge_data'):
|
|
1113
|
+
for edge in self.csr_storage.edge_data:
|
|
1114
|
+
if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
|
|
1115
|
+
row_indices.append(node_to_idx[edge.source_id])
|
|
1116
|
+
col_indices.append(node_to_idx[edge.target_id])
|
|
1117
|
+
edge_labels.append(edge.edge_type)
|
|
1118
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
1119
|
+
edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
|
|
1120
|
+
|
|
1121
|
+
if row_indices:
|
|
1122
|
+
csr_matrix_data = csr_matrix(
|
|
1123
|
+
(np.ones(len(row_indices)), (row_indices, col_indices)),
|
|
1124
|
+
shape=(num_nodes, num_nodes)
|
|
1125
|
+
)
|
|
1126
|
+
else:
|
|
1127
|
+
csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
|
|
1128
|
+
|
|
1129
|
+
csr_metadata = {
|
|
1130
|
+
'node_list': node_list,
|
|
1131
|
+
'node_to_idx': node_to_idx,
|
|
1132
|
+
'edge_labels': edge_labels,
|
|
1133
|
+
'edge_properties': edge_properties_list,
|
|
1134
|
+
'num_nodes': num_nodes,
|
|
1135
|
+
'num_edges': len(row_indices)
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
storage.save_csr_matrix(csr_matrix_data, csr_metadata)
|
|
1139
|
+
logger.info(f"CSR matrix saved to HDF5 (retry): {num_nodes} nodes, {len(row_indices)} edges")
|
|
1140
|
+
except Exception as csr_error:
|
|
1141
|
+
logger.warning(f"Failed to save CSR matrix to HDF5 (retry): {csr_error}")
|
|
1142
|
+
break
|
|
1143
|
+
except Exception as retry_error:
|
|
1144
|
+
if retry < max_retries - 1:
|
|
1145
|
+
time.sleep(0.1) # Wait 100ms before retry
|
|
1146
|
+
else:
|
|
1147
|
+
raise retry_error
|
|
1148
|
+
|
|
1149
|
+
logger.info(f"Graph '{self.name}' saved to single-file HDF5: {h5_path}")
|
|
1150
|
+
return True
|
|
1151
|
+
|
|
1152
|
+
except ImportError:
|
|
1153
|
+
logger.error("h5py not available. Install with: pip install h5py")
|
|
1154
|
+
return False
|
|
1155
|
+
except Exception as e:
|
|
1156
|
+
logger.error(f"Failed to save to single-file format: {e}")
|
|
1157
|
+
return False
|
|
1158
|
+
|
|
1159
|
+
def _save_multi_file_optimized(self, file_path: str) -> bool:
|
|
1160
|
+
"""
|
|
1161
|
+
Save graph using multi-file optimized format (different formats for different data types).
|
|
1162
|
+
|
|
1163
|
+
Uses:
|
|
1164
|
+
- JSON for graph structure (human-readable)
|
|
1165
|
+
- Parquet for node/edge properties (columnar, fast queries)
|
|
1166
|
+
- NPZ for CSR matrices (optimized for sparse graphs)
|
|
1167
|
+
- NPY for vectors (fast NumPy arrays)
|
|
1168
|
+
- HDF5 for images (efficient binary)
|
|
1169
|
+
- Parquet for tables (columnar analytics)
|
|
1170
|
+
|
|
1171
|
+
Args:
|
|
1172
|
+
file_path: Base path for saving (will create multiple files)
|
|
1173
|
+
|
|
1174
|
+
Returns:
|
|
1175
|
+
True if successful
|
|
1176
|
+
"""
|
|
1177
|
+
try:
|
|
1178
|
+
from pathlib import Path
|
|
1179
|
+
import numpy as np
|
|
1180
|
+
from scipy.sparse import save_npz
|
|
1181
|
+
|
|
1182
|
+
file_path_obj = Path(file_path)
|
|
1183
|
+
base_dir = file_path_obj.parent
|
|
1184
|
+
base_name = file_path_obj.stem
|
|
1185
|
+
|
|
1186
|
+
# Get multi-file optimized config
|
|
1187
|
+
graph_config = self.config.get('graph', {})
|
|
1188
|
+
multi_config = graph_config.get('multi_file_optimized', {})
|
|
1189
|
+
graph_format = multi_config.get('graph_format', 'csr') # Default: CSR for graph structure
|
|
1190
|
+
properties_format = multi_config.get('properties_format', 'parquet')
|
|
1191
|
+
csr_format = multi_config.get('csr_format', 'npz')
|
|
1192
|
+
|
|
1193
|
+
# 1. Save graph structure (CSR by default for multi-file optimized)
|
|
1194
|
+
# Note: Aliases are NOT saved (they are query-scoped only)
|
|
1195
|
+
|
|
1196
|
+
node_count = self.csr_storage.get_node_count() if self.csr_storage else 0
|
|
1197
|
+
edge_count = self.csr_storage.get_edge_count() if self.csr_storage else 0
|
|
1198
|
+
|
|
1199
|
+
graph_data = {
|
|
1200
|
+
'nodes': [],
|
|
1201
|
+
'edges': [],
|
|
1202
|
+
# 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
|
|
1203
|
+
'metadata': {
|
|
1204
|
+
'name': self.name,
|
|
1205
|
+
'node_count': node_count,
|
|
1206
|
+
'edge_count': edge_count,
|
|
1207
|
+
'storage_backend': self.storage_backend,
|
|
1208
|
+
'saved_at': time.time()
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
# Collect nodes from CSR storage
|
|
1213
|
+
if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
|
|
1214
|
+
for node_id, csr_node in self.csr_storage.nodes.items():
|
|
1215
|
+
node_data = {
|
|
1216
|
+
'id': node_id,
|
|
1217
|
+
'label': csr_node.node_type,
|
|
1218
|
+
'properties': self.node_properties.get(node_id, {})
|
|
1219
|
+
}
|
|
1220
|
+
graph_data['nodes'].append(node_data)
|
|
1221
|
+
|
|
1222
|
+
# Collect edges from CSR storage
|
|
1223
|
+
if self.csr_storage and hasattr(self.csr_storage, 'edge_data'):
|
|
1224
|
+
for edge in self.csr_storage.edge_data:
|
|
1225
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
1226
|
+
edge_info = {
|
|
1227
|
+
'src': edge.source_id,
|
|
1228
|
+
'dst': edge.target_id,
|
|
1229
|
+
'label': edge.edge_type,
|
|
1230
|
+
'properties': self.edge_properties.get(edge_id, edge.properties),
|
|
1231
|
+
'weight': edge.weight
|
|
1232
|
+
}
|
|
1233
|
+
graph_data['edges'].append(edge_info)
|
|
1234
|
+
|
|
1235
|
+
# Save graph structure (CSR format for graph, JSON as fallback/metadata)
|
|
1236
|
+
if graph_format == 'csr':
|
|
1237
|
+
# Graph structure is saved as CSR (see step 3 below)
|
|
1238
|
+
# Also save minimal JSON for metadata
|
|
1239
|
+
graph_file = file_path_obj.with_suffix('.json')
|
|
1240
|
+
minimal_graph_data = {
|
|
1241
|
+
# 'alias_to_node': {}, # REMOVED: Aliases are query-scoped, not persisted
|
|
1242
|
+
'metadata': graph_data['metadata']
|
|
1243
|
+
}
|
|
1244
|
+
with open(graph_file, 'w') as f:
|
|
1245
|
+
json.dump(minimal_graph_data, f, indent=2)
|
|
1246
|
+
logger.debug(f"Graph metadata saved to JSON: {graph_file}")
|
|
1247
|
+
elif graph_format == 'json':
|
|
1248
|
+
# Full JSON format (fallback)
|
|
1249
|
+
graph_file = file_path_obj.with_suffix('.json')
|
|
1250
|
+
with open(graph_file, 'w') as f:
|
|
1251
|
+
json.dump(graph_data, f, indent=2)
|
|
1252
|
+
logger.debug(f"Graph structure saved to JSON: {graph_file}")
|
|
1253
|
+
|
|
1254
|
+
# 2. Save node properties to Parquet (optimized for property queries)
|
|
1255
|
+
if properties_format == 'parquet':
|
|
1256
|
+
try:
|
|
1257
|
+
import pandas as pd
|
|
1258
|
+
parquet_dir = base_dir / "node_properties"
|
|
1259
|
+
parquet_dir.mkdir(parents=True, exist_ok=True)
|
|
1260
|
+
|
|
1261
|
+
if graph_data['nodes']:
|
|
1262
|
+
nodes_df_data = []
|
|
1263
|
+
for node in graph_data['nodes']:
|
|
1264
|
+
row = {'node_id': node['id'], 'label': node['label']}
|
|
1265
|
+
props = node.get('properties', {})
|
|
1266
|
+
for key, value in props.items():
|
|
1267
|
+
if isinstance(value, (dict, list)):
|
|
1268
|
+
row[f'prop_{key}'] = json.dumps(value)
|
|
1269
|
+
else:
|
|
1270
|
+
row[f'prop_{key}'] = value
|
|
1271
|
+
nodes_df_data.append(row)
|
|
1272
|
+
|
|
1273
|
+
if nodes_df_data:
|
|
1274
|
+
nodes_df = pd.DataFrame(nodes_df_data)
|
|
1275
|
+
parquet_file = parquet_dir / f"{base_name}_nodes.parquet"
|
|
1276
|
+
compression = multi_config.get('properties_compression', 'snappy')
|
|
1277
|
+
nodes_df.to_parquet(parquet_file, index=False, engine='pyarrow', compression=compression)
|
|
1278
|
+
logger.debug(f"Node properties saved to Parquet: {parquet_file}")
|
|
1279
|
+
except ImportError:
|
|
1280
|
+
logger.warning("PyArrow/pandas not available, skipping Parquet storage")
|
|
1281
|
+
except Exception as e:
|
|
1282
|
+
logger.warning(f"Failed to save node properties to Parquet: {e}")
|
|
1283
|
+
|
|
1284
|
+
# 3. Save CSR matrix (NPZ format - optimized for sparse graphs)
|
|
1285
|
+
# This is the PRIMARY graph structure storage for multi-file optimized strategy
|
|
1286
|
+
if (graph_format == 'csr' or csr_format == 'npz') and self.csr_storage and self.use_csr:
|
|
1287
|
+
try:
|
|
1288
|
+
csr_dir = base_dir / "csr"
|
|
1289
|
+
csr_dir.mkdir(parents=True, exist_ok=True)
|
|
1290
|
+
|
|
1291
|
+
node_list = list(self.csr_storage.nodes.keys())
|
|
1292
|
+
node_to_idx = {node_id: idx for idx, node_id in enumerate(node_list)}
|
|
1293
|
+
num_nodes = len(node_list)
|
|
1294
|
+
|
|
1295
|
+
row_indices = []
|
|
1296
|
+
col_indices = []
|
|
1297
|
+
edge_labels = []
|
|
1298
|
+
edge_properties_list = []
|
|
1299
|
+
|
|
1300
|
+
if hasattr(self.csr_storage, 'edge_data'):
|
|
1301
|
+
for edge in self.csr_storage.edge_data:
|
|
1302
|
+
if edge.source_id in node_to_idx and edge.target_id in node_to_idx:
|
|
1303
|
+
row_indices.append(node_to_idx[edge.source_id])
|
|
1304
|
+
col_indices.append(node_to_idx[edge.target_id])
|
|
1305
|
+
edge_labels.append(edge.edge_type)
|
|
1306
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
1307
|
+
edge_properties_list.append(self.edge_properties.get(edge_id, edge.properties))
|
|
1308
|
+
|
|
1309
|
+
from scipy.sparse import csr_matrix
|
|
1310
|
+
if row_indices:
|
|
1311
|
+
csr_matrix_data = csr_matrix(
|
|
1312
|
+
(np.ones(len(row_indices)), (row_indices, col_indices)),
|
|
1313
|
+
shape=(num_nodes, num_nodes)
|
|
1314
|
+
)
|
|
1315
|
+
else:
|
|
1316
|
+
csr_matrix_data = csr_matrix((num_nodes, num_nodes), dtype=np.float64)
|
|
1317
|
+
|
|
1318
|
+
csr_file = csr_dir / f"{base_name}_csr.npz"
|
|
1319
|
+
save_npz(str(csr_file), csr_matrix_data)
|
|
1320
|
+
|
|
1321
|
+
csr_metadata = {
|
|
1322
|
+
'node_list': node_list,
|
|
1323
|
+
'node_to_idx': node_to_idx,
|
|
1324
|
+
'edge_labels': edge_labels,
|
|
1325
|
+
'edge_properties': edge_properties_list,
|
|
1326
|
+
'num_nodes': num_nodes,
|
|
1327
|
+
'num_edges': len(row_indices)
|
|
1328
|
+
# 'alias_to_node': {} # REMOVED: Aliases are query-scoped, not persisted
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
csr_metadata_file = csr_dir / f"{base_name}_csr_metadata.json"
|
|
1332
|
+
with open(csr_metadata_file, 'w') as f:
|
|
1333
|
+
json.dump(csr_metadata, f, indent=2, default=str)
|
|
1334
|
+
|
|
1335
|
+
logger.debug(f"CSR matrix saved to NPZ: {csr_file}")
|
|
1336
|
+
except Exception as e:
|
|
1337
|
+
logger.warning(f"Failed to save CSR format: {e}")
|
|
1338
|
+
|
|
1339
|
+
logger.info(f"Graph '{self.name}' saved using multi-file optimized format")
|
|
1340
|
+
return True
|
|
1341
|
+
|
|
1342
|
+
except Exception as e:
|
|
1343
|
+
logger.error(f"Failed to save multi-file optimized format: {e}")
|
|
1344
|
+
return False
|
|
1345
|
+
|
|
1346
|
+
def _load_multi_file_optimized(self, file_path: str) -> bool:
|
|
1347
|
+
"""
|
|
1348
|
+
Load graph from multi-file optimized format.
|
|
1349
|
+
|
|
1350
|
+
Args:
|
|
1351
|
+
file_path: Base path for loading
|
|
1352
|
+
|
|
1353
|
+
Returns:
|
|
1354
|
+
True if successful
|
|
1355
|
+
"""
|
|
1356
|
+
try:
|
|
1357
|
+
from pathlib import Path
|
|
1358
|
+
from scipy.sparse import load_npz
|
|
1359
|
+
|
|
1360
|
+
file_path_obj = Path(file_path)
|
|
1361
|
+
base_dir = file_path_obj.parent
|
|
1362
|
+
base_name = file_path_obj.stem
|
|
1363
|
+
|
|
1364
|
+
# Get config
|
|
1365
|
+
graph_config = self.config.get('graph', {})
|
|
1366
|
+
multi_config = graph_config.get('multi_file_optimized', {})
|
|
1367
|
+
graph_format = multi_config.get('graph_format', 'csr') # Default: CSR for graph structure
|
|
1368
|
+
csr_format = multi_config.get('csr_format', 'npz')
|
|
1369
|
+
|
|
1370
|
+
# 1. Try loading CSR first (primary graph structure format for multi-file optimized)
|
|
1371
|
+
csr_file = base_dir / "csr" / f"{base_name}_csr.npz"
|
|
1372
|
+
csr_metadata_file = base_dir / "csr" / f"{base_name}_csr_metadata.json"
|
|
1373
|
+
|
|
1374
|
+
if csr_file.exists() and csr_metadata_file.exists() and (graph_format == 'csr' or csr_format == 'npz'):
|
|
1375
|
+
try:
|
|
1376
|
+
csr_matrix_data = load_npz(str(csr_file))
|
|
1377
|
+
with open(csr_metadata_file, 'r') as f:
|
|
1378
|
+
csr_metadata = json.load(f)
|
|
1379
|
+
|
|
1380
|
+
node_list = csr_metadata.get('node_list', [])
|
|
1381
|
+
edge_labels = csr_metadata.get('edge_labels', [])
|
|
1382
|
+
edge_properties_list = csr_metadata.get('edge_properties', [])
|
|
1383
|
+
# Note: aliases are NOT loaded (they are query-scoped only)
|
|
1384
|
+
|
|
1385
|
+
# Restore nodes to CSR storage
|
|
1386
|
+
for node_id in node_list:
|
|
1387
|
+
self.csr_storage.add_node(node_id, 'Node', {})
|
|
1388
|
+
|
|
1389
|
+
# Restore edges
|
|
1390
|
+
row_indices, col_indices = csr_matrix_data.nonzero()
|
|
1391
|
+
for i, (row_idx, col_idx) in enumerate(zip(row_indices, col_indices)):
|
|
1392
|
+
source_id = node_list[row_idx]
|
|
1393
|
+
target_id = node_list[col_idx]
|
|
1394
|
+
edge_label = edge_labels[i] if i < len(edge_labels) else 'RELATED_TO'
|
|
1395
|
+
edge_props = edge_properties_list[i] if i < len(edge_properties_list) else {}
|
|
1396
|
+
self.csr_storage.add_edge(source_id, target_id, edge_label, edge_props)
|
|
1397
|
+
|
|
1398
|
+
# Note: alias_to_node removed - aliases are query-scoped only (stored in executor)
|
|
1399
|
+
|
|
1400
|
+
logger.debug(f"Loaded graph from multi-file optimized format (CSR)")
|
|
1401
|
+
return True
|
|
1402
|
+
except Exception as e:
|
|
1403
|
+
logger.warning(f"Failed to load CSR format: {e}")
|
|
1404
|
+
|
|
1405
|
+
# 2. Fallback to JSON graph structure
|
|
1406
|
+
graph_file = file_path_obj.with_suffix('.json')
|
|
1407
|
+
if graph_file.exists() and graph_format == 'json':
|
|
1408
|
+
with open(graph_file, 'r') as f:
|
|
1409
|
+
graph_data = json.load(f)
|
|
1410
|
+
|
|
1411
|
+
# Restore nodes
|
|
1412
|
+
for node in graph_data.get('nodes', []):
|
|
1413
|
+
node_id = node['id']
|
|
1414
|
+
node_type = node.get('label', 'Node')
|
|
1415
|
+
properties = node.get('properties', {})
|
|
1416
|
+
self.csr_storage.add_node(node_id, node_type, properties)
|
|
1417
|
+
self.node_properties[node_id] = properties
|
|
1418
|
+
|
|
1419
|
+
# Restore edges
|
|
1420
|
+
for edge in graph_data.get('edges', []):
|
|
1421
|
+
source_id = edge['src']
|
|
1422
|
+
target_id = edge['dst']
|
|
1423
|
+
edge_type = edge.get('label', 'RELATED_TO')
|
|
1424
|
+
edge_props = edge.get('properties', {})
|
|
1425
|
+
self.csr_storage.add_edge(source_id, target_id, edge_type, edge_props)
|
|
1426
|
+
edge_id = f"{source_id}_{target_id}"
|
|
1427
|
+
self.edge_properties[edge_id] = edge_props
|
|
1428
|
+
|
|
1429
|
+
# Note: aliases are NOT loaded (they are query-scoped only)
|
|
1430
|
+
# No need to initialize alias_to_node - aliases are query-scoped only
|
|
1431
|
+
|
|
1432
|
+
logger.debug(f"Loaded graph from multi-file optimized format (JSON)")
|
|
1433
|
+
return True
|
|
1434
|
+
|
|
1435
|
+
return False
|
|
1436
|
+
|
|
1437
|
+
except Exception as e:
|
|
1438
|
+
logger.error(f"Failed to load multi-file optimized format: {e}")
|
|
1439
|
+
return False
|
|
1440
|
+
|
|
1441
|
+
def _should_use_csr(self) -> bool:
|
|
1442
|
+
"""Determine if CSR storage should be used. Always True - NetworkX disabled."""
|
|
1443
|
+
return True # CSR-only mode - always use CSR
|
|
1444
|
+
|
|
1445
|
+
def add_node(self, node: GraphNode, alias: Optional[str] = None, user_id: Optional[str] = None, write_through: Optional[bool] = None, _skip_temporal: bool = False):
|
|
1446
|
+
"""
|
|
1447
|
+
Add a node to the graph with smart routing based on node type.
|
|
1448
|
+
|
|
1449
|
+
RAG-critical nodes (Document, Chunk, Entity) are written immediately (write-through)
|
|
1450
|
+
to ensure they're visible for RAG queries without waiting for buffer flush.
|
|
1451
|
+
|
|
1452
|
+
This method is used by BOTH direct graph operations and API operations.
|
|
1453
|
+
It ensures the same code path is used regardless of how the node was created.
|
|
1454
|
+
|
|
1455
|
+
Args:
|
|
1456
|
+
node: GraphNode to add
|
|
1457
|
+
alias: Optional alias for the node
|
|
1458
|
+
user_id: Optional user ID for traceability
|
|
1459
|
+
write_through: Force write-through (bypass buffer). If None, auto-detect based on node type.
|
|
1460
|
+
"""
|
|
1461
|
+
|
|
1462
|
+
# STRATEGY: Default to write-through for immediate visibility.
|
|
1463
|
+
# Buffer is an optimization for bulk ingestion pipelines that explicitly
|
|
1464
|
+
# set write_through=False. Interactive AIQL queries and API operations
|
|
1465
|
+
# need nodes visible immediately for queries, saves, and multi-worker sync.
|
|
1466
|
+
if write_through is None:
|
|
1467
|
+
write_through = True
|
|
1468
|
+
|
|
1469
|
+
|
|
1470
|
+
# If write-through is enabled, write directly (bypass buffer)
|
|
1471
|
+
if write_through:
|
|
1472
|
+
logger.debug(f"Write-through enabled for node type '{node.label}' (RAG-critical)")
|
|
1473
|
+
self._add_node_direct(node, alias, user_id, _skip_temporal=_skip_temporal)
|
|
1474
|
+
return True
|
|
1475
|
+
|
|
1476
|
+
# If buffer is enabled, add to buffer instead of directly to graph
|
|
1477
|
+
if self.buffer_enabled and self.buffer_manager:
|
|
1478
|
+
# OPTIMIZATION: Check if buffer was pre-initialized during __init__
|
|
1479
|
+
# If not, initialize lazily (should be rare now)
|
|
1480
|
+
if not getattr(self, '_buffer_initialized', False) and not self.buffer_manager._initialized:
|
|
1481
|
+
import asyncio
|
|
1482
|
+
try:
|
|
1483
|
+
loop = asyncio.get_event_loop()
|
|
1484
|
+
if loop.is_running():
|
|
1485
|
+
# If loop is running, schedule initialization
|
|
1486
|
+
asyncio.create_task(self._init_buffer_async())
|
|
1487
|
+
else:
|
|
1488
|
+
loop.run_until_complete(self.buffer_manager.initialize())
|
|
1489
|
+
if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
|
|
1490
|
+
loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
|
|
1491
|
+
self._buffer_initialized = True
|
|
1492
|
+
except RuntimeError:
|
|
1493
|
+
loop = asyncio.new_event_loop()
|
|
1494
|
+
asyncio.set_event_loop(loop)
|
|
1495
|
+
loop.run_until_complete(self.buffer_manager.initialize())
|
|
1496
|
+
if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
|
|
1497
|
+
loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
|
|
1498
|
+
self._buffer_initialized = True
|
|
1499
|
+
|
|
1500
|
+
# Add to buffer (will be flushed later)
|
|
1501
|
+
record = {
|
|
1502
|
+
'operation': 'CREATE_NODE',
|
|
1503
|
+
'node_id': node.id,
|
|
1504
|
+
'label': node.label,
|
|
1505
|
+
'properties': node.properties,
|
|
1506
|
+
'alias': alias,
|
|
1507
|
+
'user_id': user_id
|
|
1508
|
+
}
|
|
1509
|
+
import asyncio
|
|
1510
|
+
try:
|
|
1511
|
+
loop = asyncio.get_event_loop()
|
|
1512
|
+
if loop.is_running():
|
|
1513
|
+
# If loop is running, schedule add_record
|
|
1514
|
+
asyncio.create_task(self.buffer_manager.add_record(record))
|
|
1515
|
+
else:
|
|
1516
|
+
loop.run_until_complete(self.buffer_manager.add_record(record))
|
|
1517
|
+
except RuntimeError:
|
|
1518
|
+
loop = asyncio.new_event_loop()
|
|
1519
|
+
asyncio.set_event_loop(loop)
|
|
1520
|
+
loop.run_until_complete(self.buffer_manager.add_record(record))
|
|
1521
|
+
|
|
1522
|
+
# STRATEGY: Check if buffer reached threshold and trigger immediate flush
|
|
1523
|
+
if self.config.get('buffer', {}).get('config', {}).get('threshold_flush_enabled', True):
|
|
1524
|
+
try:
|
|
1525
|
+
pending_count = loop.run_until_complete(self.buffer_manager.get_pending_count()) if not loop.is_running() else 0
|
|
1526
|
+
max_size = self.config.get('buffer', {}).get('config', {}).get('max_size', 1000)
|
|
1527
|
+
flush_threshold = self.config.get('buffer', {}).get('config', {}).get('flush_threshold', 0.7)
|
|
1528
|
+
threshold = int(max_size * flush_threshold)
|
|
1529
|
+
|
|
1530
|
+
if pending_count >= threshold:
|
|
1531
|
+
logger.debug(f"Buffer threshold reached ({pending_count}/{max_size} = {pending_count/max_size*100:.1f}%) - triggering flush")
|
|
1532
|
+
# Trigger immediate flush (non-blocking)
|
|
1533
|
+
if loop.is_running():
|
|
1534
|
+
asyncio.create_task(self.buffer_manager.flush_to_db(writer_fn=self._buffer_writer_fn_ref))
|
|
1535
|
+
else:
|
|
1536
|
+
loop.run_until_complete(self.buffer_manager.flush_to_db(writer_fn=self._buffer_writer_fn_ref))
|
|
1537
|
+
except Exception as e:
|
|
1538
|
+
logger.debug(f"Error checking buffer threshold: {e}")
|
|
1539
|
+
|
|
1540
|
+
logger.debug(f"Added node {node.id} to buffer")
|
|
1541
|
+
return True # Return True to indicate success (node is buffered)
|
|
1542
|
+
|
|
1543
|
+
# Direct addition (buffer disabled or called from buffer writer)
|
|
1544
|
+
self._add_node_direct(node, alias, user_id, _skip_temporal=_skip_temporal)
|
|
1545
|
+
|
|
1546
|
+
async def _init_buffer_async(self):
|
|
1547
|
+
"""Initialize buffer asynchronously."""
|
|
1548
|
+
await self.buffer_manager.initialize()
|
|
1549
|
+
if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
|
|
1550
|
+
await self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref)
|
|
1551
|
+
|
|
1552
|
+
async def _buffer_writer_fn(self, batch: List[Dict[str, Any]]) -> None:
|
|
1553
|
+
"""
|
|
1554
|
+
Writer function for buffer manager.
|
|
1555
|
+
Writes buffered records to the graph (CSR storage in memory).
|
|
1556
|
+
|
|
1557
|
+
NOTE: This writes to CSR storage in memory. To persist to disk, call save() separately.
|
|
1558
|
+
|
|
1559
|
+
Args:
|
|
1560
|
+
batch: List of buffered records
|
|
1561
|
+
"""
|
|
1562
|
+
if not batch:
|
|
1563
|
+
logger.debug("Buffer writer called with empty batch")
|
|
1564
|
+
return
|
|
1565
|
+
|
|
1566
|
+
logger.info(f"Buffer writer: Processing {len(batch)} records for graph '{self.name}'")
|
|
1567
|
+
|
|
1568
|
+
nodes_written = 0
|
|
1569
|
+
edges_written = 0
|
|
1570
|
+
errors = []
|
|
1571
|
+
|
|
1572
|
+
for record in batch:
|
|
1573
|
+
try:
|
|
1574
|
+
operation = record.get('operation')
|
|
1575
|
+
if operation == 'CREATE_NODE':
|
|
1576
|
+
# Create node from buffer record
|
|
1577
|
+
node = GraphNode(
|
|
1578
|
+
id=record['node_id'],
|
|
1579
|
+
label=record['label'],
|
|
1580
|
+
properties=record.get('properties', {})
|
|
1581
|
+
)
|
|
1582
|
+
alias = record.get('alias')
|
|
1583
|
+
user_id = record.get('user_id')
|
|
1584
|
+
# Call direct addition (bypassing buffer to avoid recursion)
|
|
1585
|
+
self._add_node_direct(node, alias, user_id)
|
|
1586
|
+
nodes_written += 1
|
|
1587
|
+
logger.debug(f"Buffer writer: Wrote node {node.id[:8]}... (label: {node.label})")
|
|
1588
|
+
elif operation == 'CREATE_EDGE':
|
|
1589
|
+
# Create edge from buffer record
|
|
1590
|
+
edge = GraphEdge(
|
|
1591
|
+
id=record.get('edge_id', f"{record['source']}_{record['target']}"),
|
|
1592
|
+
source=record['source'],
|
|
1593
|
+
target=record['target'],
|
|
1594
|
+
label=record['label'],
|
|
1595
|
+
properties=record.get('properties', {})
|
|
1596
|
+
)
|
|
1597
|
+
user_id = record.get('user_id')
|
|
1598
|
+
# Call direct addition (bypassing buffer to avoid recursion)
|
|
1599
|
+
self._add_edge_direct(edge, user_id)
|
|
1600
|
+
edges_written += 1
|
|
1601
|
+
logger.debug(f"Buffer writer: Wrote edge {edge.id[:8]}... ({edge.source} -> {edge.target})")
|
|
1602
|
+
else:
|
|
1603
|
+
logger.warning(f"Buffer writer: Unknown operation '{operation}' in record")
|
|
1604
|
+
errors.append(f"Unknown operation: {operation}")
|
|
1605
|
+
except Exception as e:
|
|
1606
|
+
error_msg = f"Error writing record {record.get('node_id') or record.get('edge_id', 'unknown')}: {e}"
|
|
1607
|
+
logger.error(error_msg)
|
|
1608
|
+
errors.append(error_msg)
|
|
1609
|
+
# Continue processing other records
|
|
1610
|
+
|
|
1611
|
+
logger.info(f"Buffer writer: Completed - {nodes_written} nodes, {edges_written} edges written to CSR storage (graph: '{self.name}')")
|
|
1612
|
+
if errors:
|
|
1613
|
+
logger.warning(f"Buffer writer: {len(errors)} errors occurred during flush")
|
|
1614
|
+
|
|
1615
|
+
# CRITICAL: Nodes are now in CSR storage (memory), but NOT persisted to disk yet
|
|
1616
|
+
# Trigger background save to persist flushed records to disk (non-blocking)
|
|
1617
|
+
# Only save if we wrote something and enough time has passed since last save
|
|
1618
|
+
if (nodes_written > 0 or edges_written > 0):
|
|
1619
|
+
try:
|
|
1620
|
+
# Import registry to trigger save
|
|
1621
|
+
from ..core.registry import graph_registry
|
|
1622
|
+
|
|
1623
|
+
# Trigger background save (non-blocking)
|
|
1624
|
+
import asyncio
|
|
1625
|
+
try:
|
|
1626
|
+
loop = asyncio.get_event_loop()
|
|
1627
|
+
if loop.is_running():
|
|
1628
|
+
# Schedule background save
|
|
1629
|
+
asyncio.create_task(self._background_save_async())
|
|
1630
|
+
else:
|
|
1631
|
+
# Run in executor if no event loop
|
|
1632
|
+
loop.run_in_executor(None, self._save_to_disk)
|
|
1633
|
+
except RuntimeError:
|
|
1634
|
+
# No event loop, run synchronously in thread
|
|
1635
|
+
import threading
|
|
1636
|
+
threading.Thread(target=self._save_to_disk, daemon=True).start()
|
|
1637
|
+
|
|
1638
|
+
logger.debug(f"Buffer writer: Triggered background save for graph '{self.name}' ({nodes_written} nodes, {edges_written} edges)")
|
|
1639
|
+
except Exception as e:
|
|
1640
|
+
logger.warning(f"Buffer writer: Failed to trigger background save: {e}")
|
|
1641
|
+
# Don't fail the flush if save fails - nodes are still in memory
|
|
1642
|
+
|
|
1643
|
+
def _save_to_disk(self):
|
|
1644
|
+
"""Synchronous save to disk (runs in background thread)."""
|
|
1645
|
+
try:
|
|
1646
|
+
from ..core.registry import graph_registry
|
|
1647
|
+
# Re-register in registry if evicted (LRU cleanup can remove graphs
|
|
1648
|
+
# while their pipeline is still running in a background thread)
|
|
1649
|
+
if self.name not in graph_registry.graphs:
|
|
1650
|
+
graph_registry.graphs[self.name] = self
|
|
1651
|
+
success = graph_registry.save_graph(self.name)
|
|
1652
|
+
if success:
|
|
1653
|
+
logger.debug(f"Buffer writer: Successfully saved graph '{self.name}' to disk")
|
|
1654
|
+
else:
|
|
1655
|
+
logger.warning(f"Buffer writer: Failed to save graph '{self.name}' to disk")
|
|
1656
|
+
except Exception as e:
|
|
1657
|
+
logger.error(f"Buffer writer: Error saving graph '{self.name}' to disk: {e}")
|
|
1658
|
+
|
|
1659
|
+
async def _background_save_async(self):
|
|
1660
|
+
"""Async background save (runs in event loop)."""
|
|
1661
|
+
try:
|
|
1662
|
+
from ..core.registry import graph_registry
|
|
1663
|
+
import asyncio
|
|
1664
|
+
# Re-register in registry if evicted by LRU cleanup
|
|
1665
|
+
if self.name not in graph_registry.graphs:
|
|
1666
|
+
graph_registry.graphs[self.name] = self
|
|
1667
|
+
loop = asyncio.get_event_loop()
|
|
1668
|
+
# Run save in executor to avoid blocking
|
|
1669
|
+
success = await loop.run_in_executor(None, graph_registry.save_graph, self.name)
|
|
1670
|
+
if success:
|
|
1671
|
+
logger.debug(f"Buffer writer: Successfully saved graph '{self.name}' to disk (async)")
|
|
1672
|
+
else:
|
|
1673
|
+
logger.warning(f"Buffer writer: Failed to save graph '{self.name}' to disk (async)")
|
|
1674
|
+
except Exception as e:
|
|
1675
|
+
logger.error(f"Buffer writer: Error saving graph '{self.name}' to disk (async): {e}")
|
|
1676
|
+
|
|
1677
|
+
def _ensure_wal(self):
|
|
1678
|
+
"""Lazy-init WAL on first write."""
|
|
1679
|
+
if self.wal is None and self._wal_enabled:
|
|
1680
|
+
try:
|
|
1681
|
+
from ..storage.wal import AIContextDBWALSystem, WALConfig
|
|
1682
|
+
import re as _re
|
|
1683
|
+
safe = _re.sub(r'[<>:"/\\|?*]', '_', self._wal_name)
|
|
1684
|
+
self.wal = AIContextDBWALSystem(WALConfig(wal_dir=f"contextcore_data/wal/{safe}"))
|
|
1685
|
+
except Exception:
|
|
1686
|
+
self._wal_enabled = False
|
|
1687
|
+
|
|
1688
|
+
def _add_node_direct(self, node: GraphNode, alias: Optional[str] = None, user_id: Optional[str] = None, _skip_temporal: bool = False):
|
|
1689
|
+
"""Direct node addition (bypasses buffer). WAL-logged for durability. Thread-safe."""
|
|
1690
|
+
with self._write_lock:
|
|
1691
|
+
return self._add_node_direct_inner(node, alias, user_id, _skip_temporal=_skip_temporal)
|
|
1692
|
+
|
|
1693
|
+
def _add_node_direct_inner(self, node: GraphNode, alias: Optional[str] = None, user_id: Optional[str] = None, _skip_temporal: bool = False):
|
|
1694
|
+
# WAL: log before writing to memory (write-ahead)
|
|
1695
|
+
self._ensure_wal()
|
|
1696
|
+
if self.wal:
|
|
1697
|
+
try:
|
|
1698
|
+
from ..storage.wal import OperationType
|
|
1699
|
+
self.wal.log_operation(OperationType.CREATE_NODE, {
|
|
1700
|
+
'node_id': node.id, 'label': node.label,
|
|
1701
|
+
'properties': node.properties,
|
|
1702
|
+
})
|
|
1703
|
+
except Exception:
|
|
1704
|
+
pass # WAL failure should not block writes
|
|
1705
|
+
|
|
1706
|
+
# Record operation in traceability layer
|
|
1707
|
+
if self.traceability_layer:
|
|
1708
|
+
self.traceability_layer.record_operation(
|
|
1709
|
+
operation_type='CREATE',
|
|
1710
|
+
entity_type='node',
|
|
1711
|
+
entity_id=node.id,
|
|
1712
|
+
data={'label': node.label, 'properties': node.properties, 'alias': alias},
|
|
1713
|
+
user_id=user_id,
|
|
1714
|
+
namespace=self.name
|
|
1715
|
+
)
|
|
1716
|
+
|
|
1717
|
+
# Add domain metadata if not present (default to namespace as domain)
|
|
1718
|
+
if 'domain' not in node.properties:
|
|
1719
|
+
node.properties['domain'] = self.name
|
|
1720
|
+
|
|
1721
|
+
# Provenance: inject write metadata
|
|
1722
|
+
from .write_context import get_write_context
|
|
1723
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
1724
|
+
_wctx = get_write_context()
|
|
1725
|
+
if '_created_at' not in node.properties:
|
|
1726
|
+
node.properties['_created_at'] = _dt.now(_tz.utc).isoformat()
|
|
1727
|
+
if '_agent_id' not in node.properties:
|
|
1728
|
+
node.properties['_agent_id'] = user_id or (_wctx.agent_id if _wctx else "") or ""
|
|
1729
|
+
if '_origin' not in node.properties:
|
|
1730
|
+
node.properties['_origin'] = (_wctx.origin if _wctx else "unknown")
|
|
1731
|
+
if '_verified' not in node.properties:
|
|
1732
|
+
node.properties['_verified'] = (_wctx.verified if _wctx else False)
|
|
1733
|
+
if '_region' not in node.properties:
|
|
1734
|
+
node.properties['_region'] = (_wctx.region if _wctx else "local")
|
|
1735
|
+
|
|
1736
|
+
# Provenance defaults (A4)
|
|
1737
|
+
if 'source_type' not in node.properties:
|
|
1738
|
+
node.properties['source_type'] = 'unknown'
|
|
1739
|
+
if 'confidence' not in node.properties:
|
|
1740
|
+
node.properties['confidence'] = 1.0
|
|
1741
|
+
if 'created_by' not in node.properties:
|
|
1742
|
+
node.properties['created_by'] = node.properties.get('_agent_id', '')
|
|
1743
|
+
|
|
1744
|
+
# Temporal defaults (A2)
|
|
1745
|
+
if 'valid_from' not in node.properties:
|
|
1746
|
+
node.properties['valid_from'] = node.properties.get('_created_at', _dt.now(_tz.utc).isoformat())
|
|
1747
|
+
if 'valid_to' not in node.properties:
|
|
1748
|
+
node.properties['valid_to'] = None
|
|
1749
|
+
if 'version' not in node.properties:
|
|
1750
|
+
node.properties['version'] = 1
|
|
1751
|
+
if 'status' not in node.properties:
|
|
1752
|
+
node.properties['status'] = 'active'
|
|
1753
|
+
|
|
1754
|
+
# Semantic hashing: Calculate content hash for deduplication
|
|
1755
|
+
if self.hashing_enabled and self.hasher:
|
|
1756
|
+
try:
|
|
1757
|
+
hash_result = self.hasher.hash_node(node.id, node.label, node.properties)
|
|
1758
|
+
# Store hash in properties for later deduplication
|
|
1759
|
+
if 'content_hash' not in node.properties:
|
|
1760
|
+
node.properties['content_hash'] = hash_result.hash_value
|
|
1761
|
+
if 'hash_algorithm' not in node.properties:
|
|
1762
|
+
node.properties['hash_algorithm'] = hash_result.algorithm
|
|
1763
|
+
except Exception as e:
|
|
1764
|
+
logger.warning(f"[SEMANTIC_HASH] Failed to hash node {node.id}: {e}")
|
|
1765
|
+
|
|
1766
|
+
# Time travel: Create version for new node (skip when called internally)
|
|
1767
|
+
if self.temporal_enabled and self.temporal_storage and not _skip_temporal:
|
|
1768
|
+
try:
|
|
1769
|
+
from datetime import datetime
|
|
1770
|
+
self.temporal_storage.create_version(
|
|
1771
|
+
entity_id=node.id,
|
|
1772
|
+
entity_type='node',
|
|
1773
|
+
properties=node.properties.copy(),
|
|
1774
|
+
operation='CREATE',
|
|
1775
|
+
timestamp=datetime.now()
|
|
1776
|
+
)
|
|
1777
|
+
except Exception as e:
|
|
1778
|
+
logger.warning(f"[TIME_TRAVEL] Failed to create version for node {node.id}: {e}")
|
|
1779
|
+
|
|
1780
|
+
# Debug - log what we're adding
|
|
1781
|
+
logger.debug(f"add_node called: node_id={node.id[:8]}..., alias={alias} (query-scoped, not persisted), label={node.label}, graph='{self.name}'")
|
|
1782
|
+
|
|
1783
|
+
# CSR-only: Store properties separately (no NetworkX)
|
|
1784
|
+
self.node_properties[node.id] = node.properties.copy()
|
|
1785
|
+
|
|
1786
|
+
# Store in node_index for compatibility
|
|
1787
|
+
self.node_index[node.id] = node
|
|
1788
|
+
|
|
1789
|
+
# NOTE: Aliases are query-scoped only (stored in executor.node_aliases, not here)
|
|
1790
|
+
# We accept the alias parameter for compatibility but don't store it persistently
|
|
1791
|
+
|
|
1792
|
+
# NEW: Update property indexes (use cleaned properties to avoid unhashable errors)
|
|
1793
|
+
# Clean properties before indexing to handle nested dicts/lists
|
|
1794
|
+
properties_cleaned = self._clean_properties_for_indexing(node.properties)
|
|
1795
|
+
self._update_property_indexes(node.id, properties_cleaned)
|
|
1796
|
+
# Store original properties (with nested structures) separately
|
|
1797
|
+
self.node_properties[node.id] = node.properties
|
|
1798
|
+
|
|
1799
|
+
# CRITICAL: Always add to CSR (mandatory - NetworkX disabled)
|
|
1800
|
+
if not self.csr_adapter:
|
|
1801
|
+
raise RuntimeError("CSR adapter not initialized. NetworkX fallback disabled.")
|
|
1802
|
+
# Add to CSR (pass None for alias since we don't persist aliases)
|
|
1803
|
+
success = self.csr_adapter.add_node(
|
|
1804
|
+
node.id,
|
|
1805
|
+
node.label,
|
|
1806
|
+
node.properties,
|
|
1807
|
+
None # Don't persist alias - query-scoped only
|
|
1808
|
+
)
|
|
1809
|
+
if not success:
|
|
1810
|
+
logger.warning(f"Failed to add node {node.id} to CSR storage")
|
|
1811
|
+
|
|
1812
|
+
# Update ContextMeta counters (A1)
|
|
1813
|
+
if node.label != "ContextMeta" and success:
|
|
1814
|
+
try:
|
|
1815
|
+
meta_node = self.csr_adapter.get_node("_context_meta")
|
|
1816
|
+
if meta_node:
|
|
1817
|
+
meta_node.properties["node_count"] = meta_node.properties.get("node_count", 0) + 1
|
|
1818
|
+
meta_node.properties["updated_at"] = _dt.now(_tz.utc).isoformat()
|
|
1819
|
+
existing = set(meta_node.properties.get("schema_summary", "").split(", ")) - {""}
|
|
1820
|
+
existing.add(node.label)
|
|
1821
|
+
meta_node.properties["schema_summary"] = ", ".join(sorted(existing))
|
|
1822
|
+
self.csr_adapter.update_node_properties("_context_meta", meta_node.properties)
|
|
1823
|
+
except Exception:
|
|
1824
|
+
pass # ContextMeta update failure should never block writes
|
|
1825
|
+
else:
|
|
1826
|
+
logger.debug(f"Successfully added node {node.id} to CSR storage")
|
|
1827
|
+
|
|
1828
|
+
# Graph intelligence: auto-link, quality score, topic tag
|
|
1829
|
+
try:
|
|
1830
|
+
from .graph_intelligence import get_graph_intelligence
|
|
1831
|
+
gi = get_graph_intelligence(self)
|
|
1832
|
+
gi.on_node_added(node, namespace=self.name)
|
|
1833
|
+
except Exception as e:
|
|
1834
|
+
logger.debug(f"Graph intelligence hook failed: {e}")
|
|
1835
|
+
|
|
1836
|
+
self.operation_counts['add_node'] += 1
|
|
1837
|
+
return True # Return True to indicate success
|
|
1838
|
+
|
|
1839
|
+
def remove_node(self, node_id: str, user_id: Optional[str] = None) -> bool:
|
|
1840
|
+
"""Remove a node from the graph. Thread-safe."""
|
|
1841
|
+
with self._write_lock:
|
|
1842
|
+
return self._remove_node_inner(node_id, user_id)
|
|
1843
|
+
|
|
1844
|
+
def _remove_node_inner(self, node_id: str, user_id: Optional[str] = None) -> bool:
|
|
1845
|
+
try:
|
|
1846
|
+
# Record operation in traceability layer
|
|
1847
|
+
if self.traceability_layer:
|
|
1848
|
+
node_data = {}
|
|
1849
|
+
# Get node from CSR storage
|
|
1850
|
+
csr_node = self.csr_adapter.get_node(node_id) if self.csr_adapter else None
|
|
1851
|
+
if csr_node:
|
|
1852
|
+
node_data = {
|
|
1853
|
+
'label': csr_node.node_type,
|
|
1854
|
+
'properties': self.node_properties.get(node_id, {})
|
|
1855
|
+
}
|
|
1856
|
+
self.traceability_layer.record_operation(
|
|
1857
|
+
operation_type='DELETE',
|
|
1858
|
+
entity_type='node',
|
|
1859
|
+
entity_id=node_id,
|
|
1860
|
+
data=node_data,
|
|
1861
|
+
user_id=user_id,
|
|
1862
|
+
namespace=self.name
|
|
1863
|
+
)
|
|
1864
|
+
# Check if node exists in CSR storage
|
|
1865
|
+
if not self.csr_adapter or not self.csr_adapter.get_node(node_id):
|
|
1866
|
+
return False
|
|
1867
|
+
|
|
1868
|
+
# Remove from CSR storage (manual removal since CSR doesn't have remove_node method)
|
|
1869
|
+
if self.csr_storage and hasattr(self.csr_storage, 'nodes'):
|
|
1870
|
+
if node_id in self.csr_storage.nodes:
|
|
1871
|
+
# Remove node from CSR storage
|
|
1872
|
+
csr_node = self.csr_storage.nodes[node_id]
|
|
1873
|
+
node_index = csr_node.index
|
|
1874
|
+
del self.csr_storage.nodes[node_id]
|
|
1875
|
+
del self.csr_storage.node_id_to_index[node_id]
|
|
1876
|
+
if node_index in self.csr_storage.index_to_node_id:
|
|
1877
|
+
del self.csr_storage.index_to_node_id[node_index]
|
|
1878
|
+
|
|
1879
|
+
# Remove from node types
|
|
1880
|
+
node_type = csr_node.node_type
|
|
1881
|
+
if node_type in self.csr_storage.node_types:
|
|
1882
|
+
self.csr_storage.node_types[node_type].discard(node_id)
|
|
1883
|
+
if not self.csr_storage.node_types[node_type]:
|
|
1884
|
+
del self.csr_storage.node_types[node_type]
|
|
1885
|
+
|
|
1886
|
+
# Remove from property index
|
|
1887
|
+
for prop_name, prop_index in self.csr_storage.property_index.items():
|
|
1888
|
+
for prop_value, node_set in list(prop_index.items()):
|
|
1889
|
+
node_set.discard(node_id)
|
|
1890
|
+
if not node_set:
|
|
1891
|
+
del prop_index[prop_value]
|
|
1892
|
+
|
|
1893
|
+
# Note: Edges pointing to/from this node should also be removed
|
|
1894
|
+
# This is a simplified removal - full implementation would require edge cleanup
|
|
1895
|
+
if hasattr(self.csr_storage, 'edge_data'):
|
|
1896
|
+
# Remove edges where this node is source or target
|
|
1897
|
+
edges_to_remove = []
|
|
1898
|
+
for i, edge in enumerate(self.csr_storage.edge_data):
|
|
1899
|
+
if edge.source_id == node_id or edge.target_id == node_id:
|
|
1900
|
+
edges_to_remove.append(i)
|
|
1901
|
+
# Remove in reverse order to maintain indices
|
|
1902
|
+
for i in reversed(edges_to_remove):
|
|
1903
|
+
edge = self.csr_storage.edge_data.pop(i)
|
|
1904
|
+
# Update edge types
|
|
1905
|
+
if edge.edge_type in self.csr_storage.edge_types:
|
|
1906
|
+
self.csr_storage.edge_types[edge.edge_type].discard(i)
|
|
1907
|
+
if not self.csr_storage.edge_types[edge.edge_type]:
|
|
1908
|
+
del self.csr_storage.edge_types[edge.edge_type]
|
|
1909
|
+
# Note: CSR row_ptr and col_indices would need to be rebuilt after edge removal
|
|
1910
|
+
# This is a simplified implementation
|
|
1911
|
+
|
|
1912
|
+
# Remove from properties
|
|
1913
|
+
if node_id in self.node_properties:
|
|
1914
|
+
del self.node_properties[node_id]
|
|
1915
|
+
|
|
1916
|
+
# Remove from alias mapping
|
|
1917
|
+
# NOTE: Aliases are query-scoped only, so no need to remove from persistent storage
|
|
1918
|
+
# (alias_to_node is kept for backward compatibility but is not used)
|
|
1919
|
+
|
|
1920
|
+
# Remove from CSR if enabled
|
|
1921
|
+
if self.csr_storage and hasattr(self.csr_storage, 'remove_node'):
|
|
1922
|
+
self.csr_storage.remove_node(node_id)
|
|
1923
|
+
|
|
1924
|
+
logger.info(f"Node '{node_id}' removed successfully")
|
|
1925
|
+
return True
|
|
1926
|
+
|
|
1927
|
+
except Exception as e:
|
|
1928
|
+
logger.error(f"Failed to remove node '{node_id}': {e}")
|
|
1929
|
+
return False
|
|
1930
|
+
|
|
1931
|
+
def get_node_by_alias(self, alias: str) -> Optional[str]:
|
|
1932
|
+
"""
|
|
1933
|
+
Get node ID by alias.
|
|
1934
|
+
|
|
1935
|
+
NOTE: Aliases are now query-scoped only (not persisted).
|
|
1936
|
+
This method returns None for query-scoped aliases.
|
|
1937
|
+
Use executor.node_aliases during query execution instead.
|
|
1938
|
+
"""
|
|
1939
|
+
# Aliases are query-scoped only - not persisted in graph
|
|
1940
|
+
# This method is kept for backward compatibility but will return None
|
|
1941
|
+
return None
|
|
1942
|
+
|
|
1943
|
+
def add_edge(self, edge: GraphEdge, user_id: Optional[str] = None):
|
|
1944
|
+
"""Add an edge to the graph."""
|
|
1945
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
1946
|
+
# Add domain metadata if not present (default to namespace as domain)
|
|
1947
|
+
if 'domain' not in edge.properties:
|
|
1948
|
+
edge.properties['domain'] = self.name
|
|
1949
|
+
# Edge metadata defaults (A5)
|
|
1950
|
+
if 'weight' not in edge.properties:
|
|
1951
|
+
edge.properties['weight'] = 1.0
|
|
1952
|
+
if 'confidence' not in edge.properties:
|
|
1953
|
+
edge.properties['confidence'] = 1.0
|
|
1954
|
+
if 'created_at' not in edge.properties:
|
|
1955
|
+
edge.properties['created_at'] = _dt.now(_tz.utc).isoformat()
|
|
1956
|
+
|
|
1957
|
+
# If buffer is enabled, add to buffer instead of directly to graph
|
|
1958
|
+
if self.buffer_enabled and self.buffer_manager:
|
|
1959
|
+
# Initialize buffer if not already initialized
|
|
1960
|
+
if not self.buffer_manager._initialized:
|
|
1961
|
+
import asyncio
|
|
1962
|
+
try:
|
|
1963
|
+
loop = asyncio.get_event_loop()
|
|
1964
|
+
if loop.is_running():
|
|
1965
|
+
asyncio.create_task(self._init_buffer_async())
|
|
1966
|
+
else:
|
|
1967
|
+
loop.run_until_complete(self.buffer_manager.initialize())
|
|
1968
|
+
if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
|
|
1969
|
+
loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
|
|
1970
|
+
except RuntimeError:
|
|
1971
|
+
loop = asyncio.new_event_loop()
|
|
1972
|
+
asyncio.set_event_loop(loop)
|
|
1973
|
+
loop.run_until_complete(self.buffer_manager.initialize())
|
|
1974
|
+
if self.config.get('buffer', {}).get('config', {}).get('auto_flush', True):
|
|
1975
|
+
loop.run_until_complete(self.buffer_manager.start_auto_flush(writer_fn=self._buffer_writer_fn_ref))
|
|
1976
|
+
|
|
1977
|
+
# Add to buffer (will be flushed later)
|
|
1978
|
+
record = {
|
|
1979
|
+
'operation': 'CREATE_EDGE',
|
|
1980
|
+
'edge_id': edge.id,
|
|
1981
|
+
'source': edge.source,
|
|
1982
|
+
'target': edge.target,
|
|
1983
|
+
'label': edge.label,
|
|
1984
|
+
'properties': edge.properties,
|
|
1985
|
+
'user_id': user_id
|
|
1986
|
+
}
|
|
1987
|
+
import asyncio
|
|
1988
|
+
try:
|
|
1989
|
+
loop = asyncio.get_event_loop()
|
|
1990
|
+
if loop.is_running():
|
|
1991
|
+
asyncio.create_task(self.buffer_manager.add_record(record))
|
|
1992
|
+
else:
|
|
1993
|
+
loop.run_until_complete(self.buffer_manager.add_record(record))
|
|
1994
|
+
except RuntimeError:
|
|
1995
|
+
loop = asyncio.new_event_loop()
|
|
1996
|
+
asyncio.set_event_loop(loop)
|
|
1997
|
+
loop.run_until_complete(self.buffer_manager.add_record(record))
|
|
1998
|
+
logger.debug(f"Added edge {edge.id} to buffer")
|
|
1999
|
+
# WRITE-THROUGH: Also add directly to CSR so edges are immediately queryable
|
|
2000
|
+
# (buffer reads fail in async context since loop.is_running() returns True)
|
|
2001
|
+
self._add_edge_direct(edge, user_id)
|
|
2002
|
+
return True # Return True to indicate success
|
|
2003
|
+
|
|
2004
|
+
# Direct addition (buffer disabled or called from buffer writer)
|
|
2005
|
+
return self._add_edge_direct(edge, user_id)
|
|
2006
|
+
|
|
2007
|
+
def _add_edge_direct(self, edge: GraphEdge, user_id: Optional[str] = None):
|
|
2008
|
+
"""Direct edge addition (bypasses buffer). WAL-logged for durability. Thread-safe."""
|
|
2009
|
+
with self._write_lock:
|
|
2010
|
+
return self._add_edge_direct_inner(edge, user_id)
|
|
2011
|
+
|
|
2012
|
+
def _add_edge_direct_inner(self, edge: GraphEdge, user_id: Optional[str] = None):
|
|
2013
|
+
try:
|
|
2014
|
+
# WAL: log before writing to memory (write-ahead)
|
|
2015
|
+
self._ensure_wal()
|
|
2016
|
+
if self.wal:
|
|
2017
|
+
try:
|
|
2018
|
+
from ..storage.wal import OperationType
|
|
2019
|
+
self.wal.log_operation(OperationType.CREATE_EDGE, {
|
|
2020
|
+
'edge_id': edge.id, 'source': edge.source,
|
|
2021
|
+
'target': edge.target, 'label': edge.label,
|
|
2022
|
+
'properties': edge.properties,
|
|
2023
|
+
})
|
|
2024
|
+
except Exception:
|
|
2025
|
+
pass # WAL failure should not block writes
|
|
2026
|
+
|
|
2027
|
+
# Add domain metadata if not present (default to namespace as domain)
|
|
2028
|
+
if 'domain' not in edge.properties:
|
|
2029
|
+
edge.properties['domain'] = self.name
|
|
2030
|
+
|
|
2031
|
+
# Provenance: inject write metadata
|
|
2032
|
+
from .write_context import get_write_context
|
|
2033
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
2034
|
+
_wctx = get_write_context()
|
|
2035
|
+
if '_created_at' not in edge.properties:
|
|
2036
|
+
edge.properties['_created_at'] = _dt.now(_tz.utc).isoformat()
|
|
2037
|
+
if '_agent_id' not in edge.properties:
|
|
2038
|
+
edge.properties['_agent_id'] = user_id or (_wctx.agent_id if _wctx else "") or ""
|
|
2039
|
+
if '_origin' not in edge.properties:
|
|
2040
|
+
edge.properties['_origin'] = (_wctx.origin if _wctx else "unknown")
|
|
2041
|
+
if '_verified' not in edge.properties:
|
|
2042
|
+
edge.properties['_verified'] = (_wctx.verified if _wctx else False)
|
|
2043
|
+
if '_region' not in edge.properties:
|
|
2044
|
+
edge.properties['_region'] = (_wctx.region if _wctx else "local")
|
|
2045
|
+
|
|
2046
|
+
# Time travel: Create version for new edge
|
|
2047
|
+
if self.temporal_enabled and self.temporal_storage:
|
|
2048
|
+
try:
|
|
2049
|
+
from datetime import datetime
|
|
2050
|
+
self.temporal_storage.create_version(
|
|
2051
|
+
entity_id=edge.id,
|
|
2052
|
+
entity_type='edge',
|
|
2053
|
+
properties=edge.properties.copy(),
|
|
2054
|
+
operation='CREATE',
|
|
2055
|
+
timestamp=datetime.now()
|
|
2056
|
+
)
|
|
2057
|
+
except Exception as e:
|
|
2058
|
+
logger.warning(f"[TIME_TRAVEL] Failed to create version for edge {edge.id}: {e}")
|
|
2059
|
+
|
|
2060
|
+
# Semantic hashing: Calculate content hash for deduplication
|
|
2061
|
+
if self.hashing_enabled and self.hasher:
|
|
2062
|
+
try:
|
|
2063
|
+
hash_result = self.hasher.hash_edge(edge.source, edge.target, edge.label, edge.properties)
|
|
2064
|
+
if 'content_hash' not in edge.properties:
|
|
2065
|
+
edge.properties['content_hash'] = hash_result.hash_value
|
|
2066
|
+
if 'hash_algorithm' not in edge.properties:
|
|
2067
|
+
edge.properties['hash_algorithm'] = hash_result.algorithm
|
|
2068
|
+
except Exception as e:
|
|
2069
|
+
logger.warning(f"[SEMANTIC_HASH] Failed to hash edge {edge.id}: {e}")
|
|
2070
|
+
|
|
2071
|
+
# Record operation in traceability layer
|
|
2072
|
+
if self.traceability_layer:
|
|
2073
|
+
self.traceability_layer.record_operation(
|
|
2074
|
+
operation_type='CREATE',
|
|
2075
|
+
entity_type='edge',
|
|
2076
|
+
entity_id=edge.id,
|
|
2077
|
+
data={
|
|
2078
|
+
'source': edge.source,
|
|
2079
|
+
'target': edge.target,
|
|
2080
|
+
'label': edge.label,
|
|
2081
|
+
'properties': edge.properties
|
|
2082
|
+
},
|
|
2083
|
+
user_id=user_id,
|
|
2084
|
+
namespace=self.name
|
|
2085
|
+
)
|
|
2086
|
+
# CSR-only: Store edge properties separately (no NetworkX)
|
|
2087
|
+
# Include _source/_target so get_all_edges can match CSR entries back to this edge
|
|
2088
|
+
props_copy = edge.properties.copy()
|
|
2089
|
+
props_copy['_source'] = edge.source
|
|
2090
|
+
props_copy['_target'] = edge.target
|
|
2091
|
+
self.edge_properties[edge.id] = props_copy
|
|
2092
|
+
# Store full edge metadata for reliable save/load
|
|
2093
|
+
self.edge_store[edge.id] = {
|
|
2094
|
+
'source': edge.source,
|
|
2095
|
+
'target': edge.target,
|
|
2096
|
+
'label': edge.label,
|
|
2097
|
+
'properties': edge.properties.copy(),
|
|
2098
|
+
'weight': getattr(edge, 'weight', 1.0) if hasattr(edge, 'weight') else 1.0
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
# CRITICAL: Always add to CSR (mandatory - NetworkX disabled)
|
|
2102
|
+
if not self.csr_adapter:
|
|
2103
|
+
raise RuntimeError("CSR adapter not initialized. NetworkX fallback disabled.")
|
|
2104
|
+
self.csr_adapter.add_edge(
|
|
2105
|
+
edge.source,
|
|
2106
|
+
edge.target,
|
|
2107
|
+
edge.label,
|
|
2108
|
+
edge.properties
|
|
2109
|
+
)
|
|
2110
|
+
|
|
2111
|
+
self.operation_counts['add_edge'] += 1
|
|
2112
|
+
|
|
2113
|
+
# Update ContextMeta edge_count
|
|
2114
|
+
try:
|
|
2115
|
+
meta_node = self.csr_adapter.get_node("_context_meta")
|
|
2116
|
+
if meta_node:
|
|
2117
|
+
meta_node.properties["edge_count"] = meta_node.properties.get("edge_count", 0) + 1
|
|
2118
|
+
meta_node.properties["updated_at"] = _dt.now(_tz.utc).isoformat()
|
|
2119
|
+
self.csr_adapter.update_node_properties("_context_meta", meta_node.properties)
|
|
2120
|
+
except Exception:
|
|
2121
|
+
pass
|
|
2122
|
+
# Only log edge additions in debug mode to reduce verbosity
|
|
2123
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
2124
|
+
logger.debug(f"Edge '{edge.source}' -> '{edge.target}' added successfully")
|
|
2125
|
+
return True
|
|
2126
|
+
|
|
2127
|
+
except Exception as e:
|
|
2128
|
+
logger.error(f"Failed to add edge '{edge.source}' -> '{edge.target}': {e}")
|
|
2129
|
+
return False
|
|
2130
|
+
|
|
2131
|
+
def get_node(self, node_id: str) -> Optional[GraphNode]:
|
|
2132
|
+
"""Get a node by ID. Checks both persistent storage and buffer (like Neo4j's page cache)."""
|
|
2133
|
+
# CSR-only: Get node from CSR storage first
|
|
2134
|
+
if not self.csr_adapter:
|
|
2135
|
+
# If no CSR adapter, check buffer only
|
|
2136
|
+
if self.buffer_enabled and self.buffer_manager:
|
|
2137
|
+
return self._get_buffered_node_by_id(node_id)
|
|
2138
|
+
return None
|
|
2139
|
+
|
|
2140
|
+
csr_node = self.csr_adapter.get_node(node_id)
|
|
2141
|
+
if csr_node:
|
|
2142
|
+
properties = self.node_properties.get(node_id, {})
|
|
2143
|
+
# node_id is now the UUID - use it directly
|
|
2144
|
+
# Extract name and alias from properties if present
|
|
2145
|
+
node_uuid = node_id # node_id is the UUID
|
|
2146
|
+
node_name = properties.get('name')
|
|
2147
|
+
node_alias = properties.get('alias')
|
|
2148
|
+
return GraphNode(
|
|
2149
|
+
id=node_id,
|
|
2150
|
+
label=csr_node.node_type,
|
|
2151
|
+
properties=properties,
|
|
2152
|
+
uuid=node_uuid,
|
|
2153
|
+
name=node_name,
|
|
2154
|
+
alias=node_alias
|
|
2155
|
+
)
|
|
2156
|
+
|
|
2157
|
+
# OPTIMIZATION: Also check buffer memory for pending nodes (immediate consistency)
|
|
2158
|
+
# This ensures nodes created in buffer are immediately visible (like Neo4j's page cache)
|
|
2159
|
+
if self.buffer_enabled and self.buffer_manager:
|
|
2160
|
+
buffered_node = self._get_buffered_node_by_id(node_id)
|
|
2161
|
+
if buffered_node:
|
|
2162
|
+
return buffered_node
|
|
2163
|
+
|
|
2164
|
+
return None
|
|
2165
|
+
|
|
2166
|
+
def update_node(
|
|
2167
|
+
self,
|
|
2168
|
+
node_id: str,
|
|
2169
|
+
properties: dict,
|
|
2170
|
+
expected_version: Optional[int] = None,
|
|
2171
|
+
author: Optional[str] = None,
|
|
2172
|
+
reason: Optional[str] = None,
|
|
2173
|
+
) -> "GraphNode":
|
|
2174
|
+
"""Update a node's properties with optional optimistic locking.
|
|
2175
|
+
|
|
2176
|
+
Snapshots old content in TemporalStorage before overwriting.
|
|
2177
|
+
Propagates stale flag to directly downstream nodes.
|
|
2178
|
+
|
|
2179
|
+
Args:
|
|
2180
|
+
node_id: ID of the node to update.
|
|
2181
|
+
properties: Property dict to merge into the existing node.
|
|
2182
|
+
expected_version: If set, raises VersionConflictError on mismatch.
|
|
2183
|
+
author: Agent/user making the change (stored as _updated_by).
|
|
2184
|
+
reason: Why this change is being made (stored as _change_reason).
|
|
2185
|
+
|
|
2186
|
+
Raises:
|
|
2187
|
+
KeyError: If the node does not exist.
|
|
2188
|
+
VersionConflictError: If expected_version is provided and does not
|
|
2189
|
+
match the stored version.
|
|
2190
|
+
"""
|
|
2191
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
2192
|
+
|
|
2193
|
+
existing = self.get_node(node_id)
|
|
2194
|
+
if existing is None:
|
|
2195
|
+
raise KeyError(f"Node {node_id!r} not found")
|
|
2196
|
+
|
|
2197
|
+
stored_version = existing.properties.get("version", 1)
|
|
2198
|
+
if expected_version is not None and expected_version != stored_version:
|
|
2199
|
+
raise VersionConflictError(
|
|
2200
|
+
current_version=stored_version,
|
|
2201
|
+
provided_version=expected_version,
|
|
2202
|
+
)
|
|
2203
|
+
|
|
2204
|
+
# Snapshot old content before overwriting
|
|
2205
|
+
if self.temporal_enabled and self.temporal_storage:
|
|
2206
|
+
try:
|
|
2207
|
+
self.temporal_storage.create_version(
|
|
2208
|
+
entity_id=node_id,
|
|
2209
|
+
entity_type="node",
|
|
2210
|
+
properties=existing.properties.copy(),
|
|
2211
|
+
operation="UPDATE",
|
|
2212
|
+
timestamp=_dt.now(),
|
|
2213
|
+
author=author or "",
|
|
2214
|
+
reason=reason or "",
|
|
2215
|
+
)
|
|
2216
|
+
except Exception as e:
|
|
2217
|
+
logger.warning(f"[TIME_TRAVEL] Failed to snapshot node {node_id}: {e}")
|
|
2218
|
+
|
|
2219
|
+
merged = dict(existing.properties)
|
|
2220
|
+
merged.update(properties)
|
|
2221
|
+
merged["version"] = stored_version + 1
|
|
2222
|
+
merged["_updated_at"] = _dt.now(_tz.utc).isoformat()
|
|
2223
|
+
if author:
|
|
2224
|
+
merged["_updated_by"] = author
|
|
2225
|
+
if reason:
|
|
2226
|
+
merged["_change_reason"] = reason
|
|
2227
|
+
|
|
2228
|
+
updated = GraphNode(id=node_id, label=existing.label, properties=merged)
|
|
2229
|
+
self.add_node(updated, _skip_temporal=True)
|
|
2230
|
+
|
|
2231
|
+
# Snapshot NEW content too (v2 in temporal storage)
|
|
2232
|
+
if self.temporal_enabled and self.temporal_storage:
|
|
2233
|
+
try:
|
|
2234
|
+
self.temporal_storage.create_version(
|
|
2235
|
+
entity_id=node_id,
|
|
2236
|
+
entity_type="node",
|
|
2237
|
+
properties=updated.properties.copy(),
|
|
2238
|
+
operation="UPDATE",
|
|
2239
|
+
timestamp=_dt.now(),
|
|
2240
|
+
author=author or "",
|
|
2241
|
+
reason=reason or "",
|
|
2242
|
+
)
|
|
2243
|
+
except Exception as e:
|
|
2244
|
+
logger.warning(f"[TIME_TRAVEL] Failed to snapshot updated node {node_id}: {e}")
|
|
2245
|
+
|
|
2246
|
+
# Propagate stale to directly downstream nodes
|
|
2247
|
+
self._propagate_stale(node_id, merged["version"])
|
|
2248
|
+
|
|
2249
|
+
return updated
|
|
2250
|
+
|
|
2251
|
+
def _propagate_stale(self, source_node_id: str, new_version: int) -> None:
|
|
2252
|
+
"""Flag all directly-downstream nodes as stale (one hop only).
|
|
2253
|
+
|
|
2254
|
+
Called automatically by update_node(). Never recursive — only
|
|
2255
|
+
direct outgoing neighbors are flagged.
|
|
2256
|
+
"""
|
|
2257
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
2258
|
+
|
|
2259
|
+
source_node = self.get_node(source_node_id)
|
|
2260
|
+
if not source_node:
|
|
2261
|
+
return
|
|
2262
|
+
|
|
2263
|
+
source_name = source_node.properties.get("name", source_node_id[:8])
|
|
2264
|
+
stale_reason = f"{source_name} updated to v{new_version}"
|
|
2265
|
+
stale_at = _dt.now(_tz.utc).isoformat()
|
|
2266
|
+
|
|
2267
|
+
try:
|
|
2268
|
+
neighbors = self.get_neighbors(source_node_id, direction="OUTGOING")
|
|
2269
|
+
except Exception as e:
|
|
2270
|
+
logger.warning(f"[STALE] Failed to get neighbors of {source_node_id}: {e}")
|
|
2271
|
+
return
|
|
2272
|
+
|
|
2273
|
+
for neighbor_id, _ in neighbors:
|
|
2274
|
+
neighbor = self.get_node(neighbor_id)
|
|
2275
|
+
if not neighbor:
|
|
2276
|
+
continue
|
|
2277
|
+
props = dict(neighbor.properties)
|
|
2278
|
+
props["_stale"] = True
|
|
2279
|
+
props["_stale_reason"] = stale_reason
|
|
2280
|
+
props["_stale_since"] = stale_at
|
|
2281
|
+
# Write directly via add_node to avoid triggering another update_node
|
|
2282
|
+
# cascade — stale writes are not versioned changes
|
|
2283
|
+
self.add_node(GraphNode(id=neighbor_id, label=neighbor.label, properties=props), _skip_temporal=True)
|
|
2284
|
+
|
|
2285
|
+
def confirm_current(self, node_id: str, author: str = "") -> "GraphNode":
|
|
2286
|
+
"""Clear the stale flag on a node — agent confirms it is still valid.
|
|
2287
|
+
|
|
2288
|
+
Args:
|
|
2289
|
+
node_id: UUID of the node to confirm.
|
|
2290
|
+
author: Agent or user making the confirmation.
|
|
2291
|
+
|
|
2292
|
+
Raises:
|
|
2293
|
+
KeyError: If node_id does not exist.
|
|
2294
|
+
"""
|
|
2295
|
+
from datetime import datetime as _dt, timezone as _tz
|
|
2296
|
+
|
|
2297
|
+
node = self.get_node(node_id)
|
|
2298
|
+
if node is None:
|
|
2299
|
+
raise KeyError(f"Node {node_id!r} not found")
|
|
2300
|
+
|
|
2301
|
+
props = dict(node.properties)
|
|
2302
|
+
props["_stale"] = False
|
|
2303
|
+
props.pop("_stale_reason", None)
|
|
2304
|
+
props.pop("_stale_since", None)
|
|
2305
|
+
if author:
|
|
2306
|
+
props["_stale_confirmed_by"] = author
|
|
2307
|
+
props["_stale_confirmed_at"] = _dt.now(_tz.utc).isoformat()
|
|
2308
|
+
|
|
2309
|
+
updated = GraphNode(id=node_id, label=node.label, properties=props)
|
|
2310
|
+
self.add_node(updated, _skip_temporal=True)
|
|
2311
|
+
return updated
|
|
2312
|
+
|
|
2313
|
+
def get_neighbors(self, node_id: str, edge_label: Optional[str] = None, direction: str = "OUTGOING") -> List[Tuple[str, str]]:
|
|
2314
|
+
"""Get neighbors of a node."""
|
|
2315
|
+
self.operation_counts['get_neighbors'] += 1
|
|
2316
|
+
|
|
2317
|
+
# CSR-only: Use CSR adapter (NetworkX disabled)
|
|
2318
|
+
if not self.csr_adapter:
|
|
2319
|
+
return []
|
|
2320
|
+
|
|
2321
|
+
if direction == "OUTGOING":
|
|
2322
|
+
csr_neighbors = self.csr_adapter.get_neighbors(node_id, edge_label)
|
|
2323
|
+
# Convert CSR format to expected format
|
|
2324
|
+
return [(neighbor_id, edge.source_id + "_" + edge.target_id) for neighbor_id, edge in csr_neighbors]
|
|
2325
|
+
else: # INCOMING
|
|
2326
|
+
# For incoming edges, traverse all edges to find those pointing to this node
|
|
2327
|
+
neighbors = []
|
|
2328
|
+
# Get all edges from CSR storage
|
|
2329
|
+
if hasattr(self.csr_storage, 'edge_data'):
|
|
2330
|
+
for edge in self.csr_storage.edge_data:
|
|
2331
|
+
if edge.target_id == node_id:
|
|
2332
|
+
if edge_label is None or edge.edge_type == edge_label:
|
|
2333
|
+
edge_id = f"{edge.source_id}_{edge.target_id}"
|
|
2334
|
+
neighbors.append((edge.source_id, edge_id))
|
|
2335
|
+
return neighbors
|
|
2336
|
+
|
|
2337
|
+
def check_unique_constraint(self, constraint_name: str, value: Any) -> bool:
|
|
2338
|
+
"""Check if a unique constraint is satisfied."""
|
|
2339
|
+
return value not in self.unique_constraints[constraint_name]
|
|
2340
|
+
|
|
2341
|
+
def add_unique_constraint(self, constraint_name: str, value: Any):
|
|
2342
|
+
"""Add a value to a unique constraint."""
|
|
2343
|
+
self.unique_constraints[constraint_name].add(value)
|
|
2344
|
+
|
|
2345
|
+
def get_all_nodes(self, label: Optional[str] = None, *, limit: Optional[int] = None, offset: int = 0) -> List[GraphNode]:
|
|
2346
|
+
"""Get all nodes, optionally filtered by label. Includes buffered nodes.
|
|
2347
|
+
|
|
2348
|
+
Args:
|
|
2349
|
+
label: Optional label filter.
|
|
2350
|
+
limit: Max number of nodes to return (None = unlimited).
|
|
2351
|
+
offset: Number of matching nodes to skip before collecting results.
|
|
2352
|
+
"""
|
|
2353
|
+
seen_ids = set()
|
|
2354
|
+
nodes = []
|
|
2355
|
+
skipped = 0
|
|
2356
|
+
|
|
2357
|
+
def _should_stop():
|
|
2358
|
+
return limit is not None and len(nodes) >= limit
|
|
2359
|
+
|
|
2360
|
+
def _collect(node):
|
|
2361
|
+
nonlocal skipped
|
|
2362
|
+
if _should_stop():
|
|
2363
|
+
return
|
|
2364
|
+
if offset > 0 and skipped < offset:
|
|
2365
|
+
skipped += 1
|
|
2366
|
+
return
|
|
2367
|
+
nodes.append(node)
|
|
2368
|
+
|
|
2369
|
+
# Primary source: storage adapter
|
|
2370
|
+
if self.csr_adapter:
|
|
2371
|
+
# Redis backend: use label index when available (O(k) vs O(n))
|
|
2372
|
+
if self._using_redis_backend:
|
|
2373
|
+
adapter_nodes = self.csr_adapter.get_all_nodes(node_type=label) if label else self.csr_adapter.get_all_nodes()
|
|
2374
|
+
for gn in adapter_nodes:
|
|
2375
|
+
if gn.id not in seen_ids:
|
|
2376
|
+
seen_ids.add(gn.id)
|
|
2377
|
+
_collect(gn)
|
|
2378
|
+
return nodes
|
|
2379
|
+
|
|
2380
|
+
# CSR backend: iterate node IDs
|
|
2381
|
+
if label:
|
|
2382
|
+
node_ids = self.csr_adapter.get_nodes_by_type(label)
|
|
2383
|
+
else:
|
|
2384
|
+
node_ids = list(self.csr_storage.nodes.keys()) if hasattr(self.csr_storage, 'nodes') else []
|
|
2385
|
+
|
|
2386
|
+
for node_id in node_ids:
|
|
2387
|
+
if _should_stop():
|
|
2388
|
+
break
|
|
2389
|
+
node = self.get_node(node_id)
|
|
2390
|
+
if node and node.id not in seen_ids:
|
|
2391
|
+
seen_ids.add(node.id)
|
|
2392
|
+
_collect(node)
|
|
2393
|
+
|
|
2394
|
+
# Secondary source: node_index (catches nodes not yet in CSR)
|
|
2395
|
+
if not _should_stop() and hasattr(self, 'node_index'):
|
|
2396
|
+
for node_id, node in self.node_index.items():
|
|
2397
|
+
if _should_stop():
|
|
2398
|
+
break
|
|
2399
|
+
if node_id in seen_ids:
|
|
2400
|
+
continue
|
|
2401
|
+
if label is None or node.label == label:
|
|
2402
|
+
seen_ids.add(node_id)
|
|
2403
|
+
_collect(node)
|
|
2404
|
+
|
|
2405
|
+
# Tertiary source: write buffer (unflushed nodes)
|
|
2406
|
+
if not _should_stop():
|
|
2407
|
+
buffered_nodes = self._get_buffered_nodes(label)
|
|
2408
|
+
for buffered_node in buffered_nodes:
|
|
2409
|
+
if _should_stop():
|
|
2410
|
+
break
|
|
2411
|
+
if buffered_node.id not in seen_ids:
|
|
2412
|
+
seen_ids.add(buffered_node.id)
|
|
2413
|
+
_collect(buffered_node)
|
|
2414
|
+
|
|
2415
|
+
return nodes
|
|
2416
|
+
|
|
2417
|
+
def _get_buffered_nodes(self, label: Optional[str] = None) -> List[GraphNode]:
|
|
2418
|
+
"""Get nodes from buffer memory that haven't been flushed yet."""
|
|
2419
|
+
buffered_nodes = []
|
|
2420
|
+
|
|
2421
|
+
if not self.buffer_enabled or not self.buffer_manager:
|
|
2422
|
+
return buffered_nodes
|
|
2423
|
+
|
|
2424
|
+
try:
|
|
2425
|
+
# Get pending records from buffer
|
|
2426
|
+
import asyncio
|
|
2427
|
+
|
|
2428
|
+
# Try to get records synchronously
|
|
2429
|
+
try:
|
|
2430
|
+
loop = asyncio.get_event_loop()
|
|
2431
|
+
if loop.is_running():
|
|
2432
|
+
# Can't run sync in async context, return empty
|
|
2433
|
+
return buffered_nodes
|
|
2434
|
+
else:
|
|
2435
|
+
# Run async method synchronously
|
|
2436
|
+
records = loop.run_until_complete(self._get_buffer_records_async())
|
|
2437
|
+
except RuntimeError:
|
|
2438
|
+
# No event loop, create one
|
|
2439
|
+
loop = asyncio.new_event_loop()
|
|
2440
|
+
asyncio.set_event_loop(loop)
|
|
2441
|
+
records = loop.run_until_complete(self._get_buffer_records_async())
|
|
2442
|
+
loop.close()
|
|
2443
|
+
|
|
2444
|
+
# Convert buffer records to GraphNode objects
|
|
2445
|
+
for record in records:
|
|
2446
|
+
operation = record.get('operation', '')
|
|
2447
|
+
|
|
2448
|
+
if operation == 'CREATE_NODE':
|
|
2449
|
+
node_label = record.get('label', '')
|
|
2450
|
+
node_id = record.get('node_id', '')
|
|
2451
|
+
properties = record.get('properties', {})
|
|
2452
|
+
|
|
2453
|
+
# Filter by label if specified
|
|
2454
|
+
if label is None or node_label == label:
|
|
2455
|
+
node = GraphNode(
|
|
2456
|
+
id=node_id,
|
|
2457
|
+
label=node_label,
|
|
2458
|
+
properties=properties
|
|
2459
|
+
)
|
|
2460
|
+
buffered_nodes.append(node)
|
|
2461
|
+
|
|
2462
|
+
except Exception as e:
|
|
2463
|
+
logger.debug(f"Error getting buffered nodes: {e}")
|
|
2464
|
+
|
|
2465
|
+
return buffered_nodes
|
|
2466
|
+
|
|
2467
|
+
def _get_buffered_node_by_id(self, node_id: str) -> Optional[GraphNode]:
|
|
2468
|
+
"""Get a specific node from buffer by ID (for immediate consistency)."""
|
|
2469
|
+
if not self.buffer_enabled or not self.buffer_manager:
|
|
2470
|
+
return None
|
|
2471
|
+
|
|
2472
|
+
try:
|
|
2473
|
+
# Get pending records from buffer synchronously
|
|
2474
|
+
records = self._get_buffer_records_sync()
|
|
2475
|
+
|
|
2476
|
+
for record in records:
|
|
2477
|
+
if record.get('operation') == 'CREATE_NODE':
|
|
2478
|
+
if record.get('node_id') == node_id:
|
|
2479
|
+
return GraphNode(
|
|
2480
|
+
id=record.get('node_id'),
|
|
2481
|
+
label=record.get('label', ''),
|
|
2482
|
+
properties=record.get('properties', {})
|
|
2483
|
+
)
|
|
2484
|
+
except Exception as e:
|
|
2485
|
+
logger.debug(f"Error getting buffered node by ID: {e}")
|
|
2486
|
+
|
|
2487
|
+
return None
|
|
2488
|
+
|
|
2489
|
+
def _get_buffer_records_sync(self) -> List[Dict[str, Any]]:
|
|
2490
|
+
"""Get buffer records synchronously (helper for sync contexts)."""
|
|
2491
|
+
import asyncio
|
|
2492
|
+
|
|
2493
|
+
if not self.buffer_manager:
|
|
2494
|
+
return []
|
|
2495
|
+
|
|
2496
|
+
try:
|
|
2497
|
+
loop = asyncio.get_event_loop()
|
|
2498
|
+
if loop.is_running():
|
|
2499
|
+
# Can't run sync in async context, return empty (will be available after flush)
|
|
2500
|
+
return []
|
|
2501
|
+
else:
|
|
2502
|
+
return loop.run_until_complete(self._get_buffer_records_async())
|
|
2503
|
+
except RuntimeError:
|
|
2504
|
+
# No event loop, create one
|
|
2505
|
+
loop = asyncio.new_event_loop()
|
|
2506
|
+
asyncio.set_event_loop(loop)
|
|
2507
|
+
try:
|
|
2508
|
+
return loop.run_until_complete(self._get_buffer_records_async())
|
|
2509
|
+
finally:
|
|
2510
|
+
loop.close()
|
|
2511
|
+
|
|
2512
|
+
async def _get_buffer_records_async(self) -> List[Dict[str, Any]]:
|
|
2513
|
+
"""Get all pending records from buffer asynchronously."""
|
|
2514
|
+
if not self.buffer_manager:
|
|
2515
|
+
return []
|
|
2516
|
+
|
|
2517
|
+
try:
|
|
2518
|
+
# Check buffer type and get records accordingly
|
|
2519
|
+
if hasattr(self.buffer_manager, '_records'):
|
|
2520
|
+
# Local buffer - direct access
|
|
2521
|
+
async with self.buffer_manager._lock:
|
|
2522
|
+
return list(self.buffer_manager._records)
|
|
2523
|
+
elif hasattr(self.buffer_manager, 'get_pending_records'):
|
|
2524
|
+
# Buffer manager with get_pending_records method
|
|
2525
|
+
return await self.buffer_manager.get_pending_records()
|
|
2526
|
+
else:
|
|
2527
|
+
# For other buffer types, we'd need to implement get_pending_records
|
|
2528
|
+
# For now, return empty list
|
|
2529
|
+
return []
|
|
2530
|
+
except Exception as e:
|
|
2531
|
+
logger.debug(f"Error getting buffer records: {e}")
|
|
2532
|
+
return []
|
|
2533
|
+
|
|
2534
|
+
def get_all_edges(self, label: Optional[str] = None, *, limit: Optional[int] = None, offset: int = 0) -> List[GraphEdge]:
|
|
2535
|
+
"""Get all edges, optionally filtered by label. Includes buffered edges.
|
|
2536
|
+
|
|
2537
|
+
Args:
|
|
2538
|
+
label: Optional label filter.
|
|
2539
|
+
limit: Max number of edges to return (None = unlimited).
|
|
2540
|
+
offset: Number of matching edges to skip before collecting results.
|
|
2541
|
+
"""
|
|
2542
|
+
edges = []
|
|
2543
|
+
_skipped = 0
|
|
2544
|
+
|
|
2545
|
+
def _at_limit():
|
|
2546
|
+
return limit is not None and len(edges) >= limit
|
|
2547
|
+
# Redis backend: use adapter directly
|
|
2548
|
+
if self._using_redis_backend and self.csr_adapter:
|
|
2549
|
+
for edge in self.csr_adapter.get_all_edges():
|
|
2550
|
+
if _at_limit():
|
|
2551
|
+
break
|
|
2552
|
+
edge_label = getattr(edge, 'edge_type', getattr(edge, 'label', ''))
|
|
2553
|
+
if label and edge_label != label:
|
|
2554
|
+
continue
|
|
2555
|
+
if offset > 0 and _skipped < offset:
|
|
2556
|
+
_skipped += 1
|
|
2557
|
+
continue
|
|
2558
|
+
edges.append(GraphEdge(
|
|
2559
|
+
id=getattr(edge, 'id', ''),
|
|
2560
|
+
source=getattr(edge, 'source_id', getattr(edge, 'source', '')),
|
|
2561
|
+
target=getattr(edge, 'target_id', getattr(edge, 'target', '')),
|
|
2562
|
+
label=edge_label,
|
|
2563
|
+
properties=getattr(edge, 'properties', {}),
|
|
2564
|
+
))
|
|
2565
|
+
return edges
|
|
2566
|
+
|
|
2567
|
+
# CSR backend: Get edges from CSR storage
|
|
2568
|
+
if not self.csr_storage or not hasattr(self.csr_storage, 'edge_data'):
|
|
2569
|
+
pass # Continue to check buffer
|
|
2570
|
+
else:
|
|
2571
|
+
for edge in self.csr_storage.edge_data:
|
|
2572
|
+
if _at_limit():
|
|
2573
|
+
break
|
|
2574
|
+
edge_label = edge.edge_type
|
|
2575
|
+
if label is None or edge_label == label:
|
|
2576
|
+
import uuid as uuid_module
|
|
2577
|
+
|
|
2578
|
+
# Try to find edge in edge_properties by matching source/target
|
|
2579
|
+
# Since edge_properties now uses UUID as key, we need to search
|
|
2580
|
+
edge_id = None
|
|
2581
|
+
properties = edge.properties.copy() if edge.properties else {}
|
|
2582
|
+
|
|
2583
|
+
# Search edge_properties for matching source/target
|
|
2584
|
+
# This handles both new (UUID key) and old (source_target key) formats
|
|
2585
|
+
old_format_id = f"{edge.source_id}_{edge.target_id}"
|
|
2586
|
+
if old_format_id in self.edge_properties:
|
|
2587
|
+
# Old format - migrate to UUID
|
|
2588
|
+
edge_id = str(uuid_module.uuid4())
|
|
2589
|
+
properties = self.edge_properties[old_format_id].copy()
|
|
2590
|
+
# Migrate to new format
|
|
2591
|
+
self.edge_properties[edge_id] = properties
|
|
2592
|
+
self.edge_properties.pop(old_format_id, None)
|
|
2593
|
+
else:
|
|
2594
|
+
# Search for edge with matching source/target in edge_properties
|
|
2595
|
+
for stored_id, stored_props in self.edge_properties.items():
|
|
2596
|
+
# Check if properties indicate this is our edge
|
|
2597
|
+
if (stored_props.get('_source') == edge.source_id or
|
|
2598
|
+
stored_props.get('source') == edge.source_id):
|
|
2599
|
+
if (stored_props.get('_target') == edge.target_id or
|
|
2600
|
+
stored_props.get('target') == edge.target_id):
|
|
2601
|
+
edge_id = stored_id
|
|
2602
|
+
properties = stored_props.copy()
|
|
2603
|
+
break
|
|
2604
|
+
|
|
2605
|
+
# If not found, generate new UUID (new edge or migration)
|
|
2606
|
+
if edge_id is None:
|
|
2607
|
+
edge_id = str(uuid_module.uuid4())
|
|
2608
|
+
# Store properties with new UUID key
|
|
2609
|
+
self.edge_properties[edge_id] = properties
|
|
2610
|
+
|
|
2611
|
+
# Remove uuid from properties if present (to avoid duplication)
|
|
2612
|
+
properties.pop('uuid', None)
|
|
2613
|
+
|
|
2614
|
+
# edge_id is now the UUID
|
|
2615
|
+
edge_uuid = edge_id
|
|
2616
|
+
edge_name = properties.get('name')
|
|
2617
|
+
edge_alias = properties.get('alias')
|
|
2618
|
+
|
|
2619
|
+
graph_edge = GraphEdge(
|
|
2620
|
+
id=edge_id, # This is the UUID
|
|
2621
|
+
source=edge.source_id, # Source node UUID
|
|
2622
|
+
target=edge.target_id, # Target node UUID
|
|
2623
|
+
label=edge_label,
|
|
2624
|
+
properties=properties,
|
|
2625
|
+
uuid=edge_uuid,
|
|
2626
|
+
name=edge_name,
|
|
2627
|
+
alias=edge_alias
|
|
2628
|
+
)
|
|
2629
|
+
if offset > 0 and _skipped < offset:
|
|
2630
|
+
_skipped += 1
|
|
2631
|
+
continue
|
|
2632
|
+
edges.append(graph_edge)
|
|
2633
|
+
|
|
2634
|
+
# OPTIMIZATION: Also check buffer memory for pending edges
|
|
2635
|
+
if not _at_limit():
|
|
2636
|
+
buffered_edges = self._get_buffered_edges(label)
|
|
2637
|
+
if buffered_edges:
|
|
2638
|
+
existing_ids = {edge.id for edge in edges}
|
|
2639
|
+
# Also deduplicate by (source, target, label) to handle write-through duplicates
|
|
2640
|
+
existing_triples = {(edge.source, edge.target, edge.label) for edge in edges}
|
|
2641
|
+
for buffered_edge in buffered_edges:
|
|
2642
|
+
if _at_limit():
|
|
2643
|
+
break
|
|
2644
|
+
triple = (buffered_edge.source, buffered_edge.target, buffered_edge.label)
|
|
2645
|
+
if buffered_edge.id not in existing_ids and triple not in existing_triples:
|
|
2646
|
+
if offset > 0 and _skipped < offset:
|
|
2647
|
+
_skipped += 1
|
|
2648
|
+
continue
|
|
2649
|
+
edges.append(buffered_edge)
|
|
2650
|
+
|
|
2651
|
+
return edges
|
|
2652
|
+
|
|
2653
|
+
def get_statistics(self) -> Dict[str, Any]:
|
|
2654
|
+
"""Return graph statistics for query optimization.
|
|
2655
|
+
|
|
2656
|
+
Returns node/edge counts by type using the label index (O(1) per label
|
|
2657
|
+
on Redis, O(k) on CSR) — avoids full scan.
|
|
2658
|
+
"""
|
|
2659
|
+
node_counts: Dict[str, int] = {}
|
|
2660
|
+
edge_counts: Dict[str, int] = {}
|
|
2661
|
+
total_nodes = 0
|
|
2662
|
+
total_edges = 0
|
|
2663
|
+
|
|
2664
|
+
if self._using_redis_backend and self.csr_adapter:
|
|
2665
|
+
# Use Redis SETs for O(1) count per label
|
|
2666
|
+
r = getattr(self.csr_adapter, '_r', None)
|
|
2667
|
+
prefix = getattr(self.csr_adapter, '_prefix', '')
|
|
2668
|
+
if r:
|
|
2669
|
+
total_nodes = r.scard(f"{prefix}nodes")
|
|
2670
|
+
total_edges = r.scard(f"{prefix}edges")
|
|
2671
|
+
# Scan ntype:* keys for label counts
|
|
2672
|
+
for key in r.scan_iter(f"{prefix}ntype:*", count=100):
|
|
2673
|
+
label_name = key.split("ntype:")[-1] if "ntype:" in key else ""
|
|
2674
|
+
if label_name:
|
|
2675
|
+
node_counts[label_name] = r.scard(key)
|
|
2676
|
+
for key in r.scan_iter(f"{prefix}etype:*", count=100):
|
|
2677
|
+
etype = key.split("etype:")[-1] if "etype:" in key else ""
|
|
2678
|
+
if etype:
|
|
2679
|
+
edge_counts[etype] = r.scard(key)
|
|
2680
|
+
elif self.csr_adapter:
|
|
2681
|
+
# CSR: use in-memory node_types dict
|
|
2682
|
+
nt = getattr(self.csr_adapter, 'node_types', {})
|
|
2683
|
+
for lbl, ids in nt.items():
|
|
2684
|
+
node_counts[lbl] = len(ids)
|
|
2685
|
+
total_nodes = sum(node_counts.values())
|
|
2686
|
+
total_edges = len(self.get_all_edges())
|
|
2687
|
+
|
|
2688
|
+
return {
|
|
2689
|
+
"total_nodes": total_nodes,
|
|
2690
|
+
"total_edges": total_edges,
|
|
2691
|
+
"node_count_by_label": node_counts,
|
|
2692
|
+
"edge_count_by_type": edge_counts,
|
|
2693
|
+
"label_index_available": True,
|
|
2694
|
+
}
|
|
2695
|
+
|
|
2696
|
+
def _get_buffered_edges(self, label: Optional[str] = None) -> List[GraphEdge]:
|
|
2697
|
+
"""Get edges from buffer memory that haven't been flushed yet."""
|
|
2698
|
+
buffered_edges = []
|
|
2699
|
+
|
|
2700
|
+
if not self.buffer_enabled or not self.buffer_manager:
|
|
2701
|
+
return buffered_edges
|
|
2702
|
+
|
|
2703
|
+
try:
|
|
2704
|
+
import asyncio
|
|
2705
|
+
|
|
2706
|
+
# Try to get records synchronously
|
|
2707
|
+
try:
|
|
2708
|
+
loop = asyncio.get_event_loop()
|
|
2709
|
+
if loop.is_running():
|
|
2710
|
+
return buffered_edges # Can't run sync in async context
|
|
2711
|
+
else:
|
|
2712
|
+
records = loop.run_until_complete(self._get_buffer_records_async())
|
|
2713
|
+
except RuntimeError:
|
|
2714
|
+
loop = asyncio.new_event_loop()
|
|
2715
|
+
asyncio.set_event_loop(loop)
|
|
2716
|
+
records = loop.run_until_complete(self._get_buffer_records_async())
|
|
2717
|
+
loop.close()
|
|
2718
|
+
|
|
2719
|
+
# Convert buffer records to GraphEdge objects
|
|
2720
|
+
for record in records:
|
|
2721
|
+
operation = record.get('operation', '')
|
|
2722
|
+
|
|
2723
|
+
if operation == 'CREATE_EDGE':
|
|
2724
|
+
edge_label = record.get('label', '')
|
|
2725
|
+
edge_id = record.get('edge_id', f"{record.get('source')}_{record.get('target')}")
|
|
2726
|
+
source = record.get('source', '')
|
|
2727
|
+
target = record.get('target', '')
|
|
2728
|
+
properties = record.get('properties', {})
|
|
2729
|
+
|
|
2730
|
+
# Filter by label if specified
|
|
2731
|
+
if label is None or edge_label == label:
|
|
2732
|
+
edge = GraphEdge(
|
|
2733
|
+
id=edge_id,
|
|
2734
|
+
source=source,
|
|
2735
|
+
target=target,
|
|
2736
|
+
label=edge_label,
|
|
2737
|
+
properties=properties
|
|
2738
|
+
)
|
|
2739
|
+
buffered_edges.append(edge)
|
|
2740
|
+
|
|
2741
|
+
except Exception as e:
|
|
2742
|
+
logger.debug(f"Error getting buffered edges: {e}")
|
|
2743
|
+
|
|
2744
|
+
return buffered_edges
|
|
2745
|
+
|
|
2746
|
+
def get_node_types(self) -> Set[str]:
|
|
2747
|
+
"""Get all unique node types (labels) in the graph."""
|
|
2748
|
+
# CSR-only: Get node types from CSR storage
|
|
2749
|
+
if not self.csr_storage:
|
|
2750
|
+
return set()
|
|
2751
|
+
return self.csr_storage.get_node_types()
|
|
2752
|
+
|
|
2753
|
+
def get_edge_types(self) -> Set[str]:
|
|
2754
|
+
"""Get all unique edge types (labels) in the graph."""
|
|
2755
|
+
# CSR-only: Get edge types from CSR storage
|
|
2756
|
+
if not self.csr_storage:
|
|
2757
|
+
return set()
|
|
2758
|
+
return self.csr_storage.get_edge_types()
|
|
2759
|
+
|
|
2760
|
+
def traverse_graph(self, start_node_id: str, edge_label: Optional[str] = None,
|
|
2761
|
+
max_depth: int = 1, direction: str = "OUTGOING") -> List[str]:
|
|
2762
|
+
"""Traverse the graph from a starting node."""
|
|
2763
|
+
self.operation_counts['traverse'] += 1
|
|
2764
|
+
|
|
2765
|
+
# CSR-only: Use CSR adapter (NetworkX disabled)
|
|
2766
|
+
if not self.csr_adapter:
|
|
2767
|
+
return []
|
|
2768
|
+
return self.csr_adapter.traverse(start_node_id, edge_label, max_depth, direction)
|
|
2769
|
+
|
|
2770
|
+
def get_nodes_by_property(self, property_name: str, property_value: Any) -> List[str]:
|
|
2771
|
+
"""Get nodes by property value."""
|
|
2772
|
+
# CSR-only: Use CSR adapter (NetworkX disabled)
|
|
2773
|
+
if not self.csr_adapter:
|
|
2774
|
+
return []
|
|
2775
|
+
return self.csr_adapter.get_nodes_by_property(property_name, property_value)
|
|
2776
|
+
|
|
2777
|
+
def _update_property_indexes(self, node_id: str, properties: Dict[str, Any]):
|
|
2778
|
+
"""Update property indexes when a node is added."""
|
|
2779
|
+
for prop_name, prop_value in properties.items():
|
|
2780
|
+
# Skip complex types (dict, list) that can't be indexed
|
|
2781
|
+
if isinstance(prop_value, (dict, list)):
|
|
2782
|
+
continue
|
|
2783
|
+
|
|
2784
|
+
# Skip None values
|
|
2785
|
+
if prop_value is None:
|
|
2786
|
+
continue
|
|
2787
|
+
|
|
2788
|
+
# Test hashability to prevent "unhashable type" errors
|
|
2789
|
+
try:
|
|
2790
|
+
# Try to use value as dict key (tests hashability)
|
|
2791
|
+
_ = {prop_value: True}
|
|
2792
|
+
except TypeError:
|
|
2793
|
+
# Value is not hashable, skip indexing
|
|
2794
|
+
continue
|
|
2795
|
+
|
|
2796
|
+
if prop_name not in self._property_indexes:
|
|
2797
|
+
if SORTED_CONTAINERS_AVAILABLE:
|
|
2798
|
+
self._property_indexes[prop_name] = SortedDict()
|
|
2799
|
+
else:
|
|
2800
|
+
self._property_indexes[prop_name] = {}
|
|
2801
|
+
|
|
2802
|
+
if SORTED_CONTAINERS_AVAILABLE:
|
|
2803
|
+
# Use SortedDict — coerce key to str to avoid mixed-type comparison errors
|
|
2804
|
+
_idx_key = str(prop_value) if not isinstance(prop_value, str) else prop_value
|
|
2805
|
+
if _idx_key not in self._property_indexes[prop_name]:
|
|
2806
|
+
self._property_indexes[prop_name][_idx_key] = set()
|
|
2807
|
+
self._property_indexes[prop_name][_idx_key].add(node_id)
|
|
2808
|
+
else:
|
|
2809
|
+
# Use regular dict
|
|
2810
|
+
if prop_value not in self._property_indexes[prop_name]:
|
|
2811
|
+
self._property_indexes[prop_name][prop_value] = set()
|
|
2812
|
+
self._property_indexes[prop_name][prop_value].add(node_id)
|
|
2813
|
+
|
|
2814
|
+
def filter_by_property(self, prop_name: str, prop_value: Any) -> Set[str]:
|
|
2815
|
+
"""Fast property filtering using index."""
|
|
2816
|
+
if prop_name not in self._property_indexes:
|
|
2817
|
+
return set()
|
|
2818
|
+
|
|
2819
|
+
index = self._property_indexes[prop_name]
|
|
2820
|
+
return index.get(prop_value, set())
|
|
2821
|
+
|
|
2822
|
+
def filter_by_property_range(self, prop_name: str, op: str, value: Any) -> Set[str]:
|
|
2823
|
+
"""Filter by property range using index."""
|
|
2824
|
+
if prop_name not in self._property_indexes:
|
|
2825
|
+
return set()
|
|
2826
|
+
|
|
2827
|
+
index = self._property_indexes[prop_name]
|
|
2828
|
+
filtered = set()
|
|
2829
|
+
|
|
2830
|
+
if op == '=':
|
|
2831
|
+
return index.get(value, set())
|
|
2832
|
+
elif op == '>':
|
|
2833
|
+
# For now, use dict fallback for range queries
|
|
2834
|
+
for k, node_ids in index.items():
|
|
2835
|
+
if k > value:
|
|
2836
|
+
filtered.update(node_ids)
|
|
2837
|
+
elif op == '<':
|
|
2838
|
+
for k, node_ids in index.items():
|
|
2839
|
+
if k < value:
|
|
2840
|
+
filtered.update(node_ids)
|
|
2841
|
+
elif op == '>=':
|
|
2842
|
+
for k, node_ids in index.items():
|
|
2843
|
+
if k >= value:
|
|
2844
|
+
filtered.update(node_ids)
|
|
2845
|
+
elif op == '<=':
|
|
2846
|
+
for k, node_ids in index.items():
|
|
2847
|
+
if k <= value:
|
|
2848
|
+
filtered.update(node_ids)
|
|
2849
|
+
|
|
2850
|
+
return filtered
|
|
2851
|
+
|
|
2852
|
+
def get_statistics(self) -> Dict[str, Any]:
|
|
2853
|
+
"""Get comprehensive graph statistics."""
|
|
2854
|
+
# CSR-only: Get statistics from CSR storage
|
|
2855
|
+
if not self.csr_storage:
|
|
2856
|
+
return {
|
|
2857
|
+
'node_count': 0,
|
|
2858
|
+
'edge_count': 0,
|
|
2859
|
+
'node_types': 0,
|
|
2860
|
+
'edge_types': 0,
|
|
2861
|
+
'storage_backend': 'CSR',
|
|
2862
|
+
'operation_counts': self.operation_counts
|
|
2863
|
+
}
|
|
2864
|
+
|
|
2865
|
+
csr_stats = self.csr_storage.get_statistics()
|
|
2866
|
+
csr_stats['storage_backend'] = 'CSR'
|
|
2867
|
+
csr_stats['operation_counts'] = self.operation_counts
|
|
2868
|
+
return csr_stats
|
|
2869
|
+
|
|
2870
|
+
def optimize(self):
|
|
2871
|
+
"""Optimize the graph storage."""
|
|
2872
|
+
# CSR-only: Optimize CSR storage
|
|
2873
|
+
if self.csr_storage:
|
|
2874
|
+
self.csr_storage.optimize()
|
|
2875
|
+
logger.info("CSR storage optimized")
|
|
2876
|
+
|
|
2877
|
+
def migrate_to_csr(self):
|
|
2878
|
+
"""Migrate existing data to CSR storage. No-op in CSR-only mode."""
|
|
2879
|
+
logger.info("CSR-only mode: No migration needed (NetworkX disabled)")
|
|
2880
|
+
return True
|
|
2881
|
+
|
|
2882
|
+
def clear(self):
|
|
2883
|
+
"""Clear all data."""
|
|
2884
|
+
# CSR-only: Clear CSR storage and properties
|
|
2885
|
+
self.node_properties.clear()
|
|
2886
|
+
self.edge_properties.clear()
|
|
2887
|
+
self.unique_constraints.clear()
|
|
2888
|
+
# NOTE: alias_to_node removed - aliases are query-scoped only
|
|
2889
|
+
|
|
2890
|
+
if self.csr_storage:
|
|
2891
|
+
self.csr_storage.clear()
|
|
2892
|
+
|
|
2893
|
+
# NOTE: alias_mapping in csr_adapter is not used (aliases are query-scoped only)
|
|
2894
|
+
|
|
2895
|
+
self.operation_counts = {k: 0 for k in self.operation_counts}
|
|
2896
|
+
logger.info("Graph storage cleared (CSR-only mode)")
|
|
2897
|
+
|
|
2898
|
+
# Performance comparison utility
|
|
2899
|
+
def compare_storage_performance():
|
|
2900
|
+
"""Compare performance between NetworkX and CSR storage."""
|
|
2901
|
+
import time
|
|
2902
|
+
import random
|
|
2903
|
+
|
|
2904
|
+
print("Storage Performance Comparison")
|
|
2905
|
+
print("=" * 50)
|
|
2906
|
+
|
|
2907
|
+
# Test parameters
|
|
2908
|
+
num_nodes = 1000
|
|
2909
|
+
num_edges = 2000
|
|
2910
|
+
|
|
2911
|
+
# Generate test data
|
|
2912
|
+
nodes = []
|
|
2913
|
+
edges = []
|
|
2914
|
+
|
|
2915
|
+
for i in range(num_nodes):
|
|
2916
|
+
nodes.append(GraphNode(
|
|
2917
|
+
id=f"node_{i}",
|
|
2918
|
+
label="Person",
|
|
2919
|
+
properties={"id": i, "name": f"Person_{i}", "age": random.randint(18, 80)}
|
|
2920
|
+
))
|
|
2921
|
+
|
|
2922
|
+
for i in range(num_edges):
|
|
2923
|
+
source = f"node_{random.randint(0, num_nodes-1)}"
|
|
2924
|
+
target = f"node_{random.randint(0, num_nodes-1)}"
|
|
2925
|
+
edges.append(GraphEdge(
|
|
2926
|
+
id=f"edge_{i}",
|
|
2927
|
+
source=source,
|
|
2928
|
+
target=target,
|
|
2929
|
+
label="KNOWS",
|
|
2930
|
+
properties={"weight": random.random()}
|
|
2931
|
+
))
|
|
2932
|
+
|
|
2933
|
+
# Test NetworkX-only storage
|
|
2934
|
+
print("\nTesting NetworkX-only storage:")
|
|
2935
|
+
nx_storage = HybridGraphStorage('networkx')
|
|
2936
|
+
|
|
2937
|
+
start_time = time.time()
|
|
2938
|
+
for node in nodes:
|
|
2939
|
+
nx_storage.add_node(node)
|
|
2940
|
+
nx_node_time = time.time() - start_time
|
|
2941
|
+
|
|
2942
|
+
start_time = time.time()
|
|
2943
|
+
for edge in edges:
|
|
2944
|
+
nx_storage.add_edge(edge)
|
|
2945
|
+
nx_edge_time = time.time() - start_time
|
|
2946
|
+
|
|
2947
|
+
start_time = time.time()
|
|
2948
|
+
for i in range(100):
|
|
2949
|
+
start_node = f"node_{random.randint(0, num_nodes-1)}"
|
|
2950
|
+
nx_storage.traverse_graph(start_node, max_depth=2)
|
|
2951
|
+
nx_traverse_time = time.time() - start_time
|
|
2952
|
+
|
|
2953
|
+
# Test CSR storage
|
|
2954
|
+
print("\nTesting CSR storage:")
|
|
2955
|
+
csr_storage = HybridGraphStorage('csr')
|
|
2956
|
+
|
|
2957
|
+
start_time = time.time()
|
|
2958
|
+
for node in nodes:
|
|
2959
|
+
csr_storage.add_node(node)
|
|
2960
|
+
csr_node_time = time.time() - start_time
|
|
2961
|
+
|
|
2962
|
+
start_time = time.time()
|
|
2963
|
+
for edge in edges:
|
|
2964
|
+
csr_storage.add_edge(edge)
|
|
2965
|
+
csr_edge_time = time.time() - start_time
|
|
2966
|
+
|
|
2967
|
+
start_time = time.time()
|
|
2968
|
+
for i in range(100):
|
|
2969
|
+
start_node = f"node_{random.randint(0, num_nodes-1)}"
|
|
2970
|
+
csr_storage.traverse_graph(start_node, max_depth=2)
|
|
2971
|
+
csr_traverse_time = time.time() - start_time
|
|
2972
|
+
|
|
2973
|
+
# Results
|
|
2974
|
+
print(f"\nResults ({num_nodes} nodes, {num_edges} edges):")
|
|
2975
|
+
print(f"Add Nodes:")
|
|
2976
|
+
print(f" NetworkX: {nx_node_time:.4f}s")
|
|
2977
|
+
print(f" CSR: {csr_node_time:.4f}s")
|
|
2978
|
+
print(f" Speedup: {nx_node_time/csr_node_time:.2f}x")
|
|
2979
|
+
|
|
2980
|
+
print(f"\nAdd Edges:")
|
|
2981
|
+
print(f" NetworkX: {nx_edge_time:.4f}s")
|
|
2982
|
+
print(f" CSR: {csr_edge_time:.4f}s")
|
|
2983
|
+
print(f" Speedup: {nx_edge_time/csr_edge_time:.2f}x")
|
|
2984
|
+
|
|
2985
|
+
print(f"\nTraverse (100 iterations):")
|
|
2986
|
+
print(f" NetworkX: {nx_traverse_time:.4f}s")
|
|
2987
|
+
print(f" CSR: {csr_traverse_time:.4f}s")
|
|
2988
|
+
print(f" Speedup: {nx_traverse_time/csr_traverse_time:.2f}x")
|
|
2989
|
+
|
|
2990
|
+
print(f"\nStorage Statistics:")
|
|
2991
|
+
print("NetworkX:", nx_storage.get_statistics())
|
|
2992
|
+
print("CSR:", csr_storage.get_statistics())
|
|
2993
|
+
|
|
2994
|
+
# Backward compatibility alias
|
|
2995
|
+
AIContextDB = ContextSynapse
|
|
2996
|
+
|
|
2997
|
+
if __name__ == "__main__":
|
|
2998
|
+
compare_storage_performance()
|