cognee 1.2.2.dev0__py3-none-any.whl → 1.2.2.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cognee/api/v1/improve/improve.py +26 -0
- cognee/api/v1/recall/recall.py +2 -1
- cognee/api/v1/search/search.py +2 -1
- cognee/base_config.py +3 -0
- cognee/context_global_variables.py +16 -1
- cognee/infrastructure/databases/graph/graph_db_interface.py +16 -0
- cognee/infrastructure/databases/graph/ladybug/adapter.py +83 -0
- cognee/infrastructure/databases/vector/lancedb/LanceDBAdapter.py +56 -0
- cognee/infrastructure/databases/vector/lancedb/LanceDBDatasetDatabaseHandler.py +2 -1
- cognee/infrastructure/databases/vector/vector_db_interface.py +15 -0
- cognee/modules/chunking/models/DocumentChunk.py +4 -0
- cognee/modules/engine/models/Entity.py +5 -0
- cognee/modules/graph/cognee_graph/CogneeGraph.py +4 -3
- cognee/modules/migration/import_source.py +10 -0
- cognee/modules/migration/loader.py +8 -6
- cognee/modules/migration/snapshot.py +42 -8
- cognee/modules/retrieval/graph_completion_context_extension_retriever.py +2 -1
- cognee/modules/retrieval/graph_completion_cot_retriever.py +2 -1
- cognee/modules/retrieval/graph_completion_decomposition_retriever.py +2 -1
- cognee/modules/retrieval/graph_completion_retriever.py +2 -1
- cognee/modules/retrieval/graph_summary_completion_retriever.py +2 -1
- cognee/modules/retrieval/hybrid/chunks.py +13 -1
- cognee/modules/retrieval/hybrid/ranking.py +12 -0
- cognee/modules/retrieval/hybrid_retriever.py +75 -1
- cognee/modules/retrieval/temporal_retriever.py +2 -1
- cognee/modules/retrieval/utils/brute_force_triplet_search.py +3 -2
- cognee/modules/search/methods/get_search_type_retriever_instance.py +5 -1
- cognee/modules/search/methods/search.py +12 -13
- cognee/modules/session_distillation/distill.py +3 -1
- cognee/modules/truth_subspace/__init__.py +0 -0
- cognee/modules/truth_subspace/align.py +89 -0
- cognee/modules/truth_subspace/build.py +299 -0
- cognee/modules/truth_subspace/centroids.py +198 -0
- cognee/modules/truth_subspace/constants.py +7 -0
- cognee/modules/truth_subspace/models.py +12 -0
- cognee/tests/test_cogx_roundtrip.py +174 -0
- cognee/tests/unit/infrastructure/databases/graph/test_ladybug_truth_state.py +39 -0
- cognee/tests/unit/infrastructure/databases/vector/test_lancedb_raw_vectors.py +79 -0
- cognee/tests/unit/migration/test_import_source.py +43 -1
- cognee/tests/unit/migration/test_provenance_ctx.py +75 -0
- cognee/tests/unit/migration/test_snapshot.py +53 -0
- cognee/tests/unit/modules/retrieval/hybrid/test_truth_epoch_ranking.py +44 -0
- cognee/tests/unit/modules/retrieval/test_hybrid_truth_context.py +60 -0
- cognee/tests/unit/modules/session_distillation/test_distillation_helpers.py +23 -1
- cognee/tests/unit/test_context_global_variables.py +21 -0
- cognee/tests/unit/truth_subspace/__init__.py +0 -0
- cognee/tests/unit/truth_subspace/test_align.py +112 -0
- cognee/tests/unit/truth_subspace/test_build_centroid_slots.py +95 -0
- cognee/tests/unit/truth_subspace/test_centroids.py +125 -0
- {cognee-1.2.2.dev0.dist-info → cognee-1.2.2.dev2.dist-info}/METADATA +100 -20
- {cognee-1.2.2.dev0.dist-info → cognee-1.2.2.dev2.dist-info}/RECORD +55 -38
- {cognee-1.2.2.dev0.dist-info → cognee-1.2.2.dev2.dist-info}/WHEEL +0 -0
- {cognee-1.2.2.dev0.dist-info → cognee-1.2.2.dev2.dist-info}/entry_points.txt +0 -0
- {cognee-1.2.2.dev0.dist-info → cognee-1.2.2.dev2.dist-info}/licenses/LICENSE +0 -0
- {cognee-1.2.2.dev0.dist-info → cognee-1.2.2.dev2.dist-info}/licenses/NOTICE.md +0 -0
cognee/api/v1/improve/improve.py
CHANGED
|
@@ -40,6 +40,7 @@ async def improve(
|
|
|
40
40
|
node_name: Optional[List[str]] = None,
|
|
41
41
|
session_ids: Optional[List[str]] = None,
|
|
42
42
|
build_global_context_index: bool = False,
|
|
43
|
+
build_truth_subspace: bool = False,
|
|
43
44
|
**kwargs: Unpack[ImproveKwargs],
|
|
44
45
|
):
|
|
45
46
|
"""Enrich an existing knowledge graph with additional context and rules.
|
|
@@ -86,6 +87,10 @@ async def improve(
|
|
|
86
87
|
context index after default enrichment. Skipped in background
|
|
87
88
|
mode because ordered background pipeline chaining is not
|
|
88
89
|
supported yet.
|
|
90
|
+
build_truth_subspace: Opt-in flag (default ``False``) for building the
|
|
91
|
+
truth subspace from distilled session learnings after distillation
|
|
92
|
+
and before enrichment. Only runs when ``session_ids`` is provided.
|
|
93
|
+
Off by default = no behaviour change.
|
|
89
94
|
**kwargs: Additional options -- see ``ImproveKwargs``.
|
|
90
95
|
|
|
91
96
|
Returns:
|
|
@@ -113,6 +118,7 @@ async def improve(
|
|
|
113
118
|
"session_ids": ",".join(session_ids) if session_ids else "",
|
|
114
119
|
"run_in_background": run_in_background,
|
|
115
120
|
"build_global_context_index": build_global_context_index,
|
|
121
|
+
"build_truth_subspace": build_truth_subspace,
|
|
116
122
|
"cognee_version": cognee_version,
|
|
117
123
|
},
|
|
118
124
|
)
|
|
@@ -199,6 +205,26 @@ async def improve(
|
|
|
199
205
|
if distilled:
|
|
200
206
|
stages_run.append("distill_sessions")
|
|
201
207
|
|
|
208
|
+
# Stage 2d: build the truth subspace from distilled session
|
|
209
|
+
# learnings (opt-in, default OFF). Runs after distillation so
|
|
210
|
+
# freshly accepted lessons are available as anchors, and before
|
|
211
|
+
# enrichment. Non-fatal — never blocks the rest of improve().
|
|
212
|
+
if build_truth_subspace:
|
|
213
|
+
try:
|
|
214
|
+
from cognee.modules.truth_subspace.build import (
|
|
215
|
+
build_truth_subspace as _build_truth_subspace,
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
result_ts = await _build_truth_subspace(
|
|
219
|
+
dataset=dataset,
|
|
220
|
+
session_ids=session_ids,
|
|
221
|
+
user=user,
|
|
222
|
+
)
|
|
223
|
+
logger.info("improve: truth subspace built -> %s", result_ts)
|
|
224
|
+
stages_run.append("build_truth_subspace")
|
|
225
|
+
except Exception as e:
|
|
226
|
+
logger.warning("improve: truth subspace build failed (non-fatal): %s", e)
|
|
227
|
+
|
|
202
228
|
# Stage 3: default enrichment (triplet embeddings)
|
|
203
229
|
from cognee.modules.memify import memify
|
|
204
230
|
|
cognee/api/v1/recall/recall.py
CHANGED
|
@@ -5,6 +5,7 @@ from uuid import UUID
|
|
|
5
5
|
from pydantic import BaseModel, Field
|
|
6
6
|
from typing_extensions import TypedDict
|
|
7
7
|
|
|
8
|
+
from cognee.base_config import get_base_config
|
|
8
9
|
from cognee.context_global_variables import set_session_user_context_variable
|
|
9
10
|
from cognee.exceptions import CogneeValidationError
|
|
10
11
|
from cognee.infrastructure.databases.vector.embeddings.config import EmbeddingConfig
|
|
@@ -375,7 +376,7 @@ async def recall(
|
|
|
375
376
|
context_profile: str = "qa",
|
|
376
377
|
wide_search_top_k: int | None = 100,
|
|
377
378
|
triplet_distance_penalty: float | None = 6.5,
|
|
378
|
-
feedback_influence: float =
|
|
379
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
379
380
|
verbose: bool = False,
|
|
380
381
|
retriever_specific_config: dict | None = None,
|
|
381
382
|
neighborhood_depth: int | None = None,
|
cognee/api/v1/search/search.py
CHANGED
|
@@ -8,6 +8,7 @@ from cognee.infrastructure.databases.vector.embeddings.config import EmbeddingCo
|
|
|
8
8
|
from cognee.infrastructure.llm.config import LLMConfig
|
|
9
9
|
from cognee.modules.search.types import SearchResult, SearchType
|
|
10
10
|
from cognee.modules.users.methods import get_default_user
|
|
11
|
+
from cognee.base_config import get_base_config
|
|
11
12
|
from cognee.modules.search.methods import search as search_function
|
|
12
13
|
from cognee.modules.data.methods import get_authorized_existing_datasets
|
|
13
14
|
from cognee.modules.data.exceptions import DatasetNotFoundError
|
|
@@ -43,7 +44,7 @@ async def search(
|
|
|
43
44
|
session_id: Optional[str] = None,
|
|
44
45
|
wide_search_top_k: Optional[int] = 100,
|
|
45
46
|
triplet_distance_penalty: Optional[float] = 6.5,
|
|
46
|
-
feedback_influence: float =
|
|
47
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
47
48
|
verbose: bool = False,
|
|
48
49
|
retriever_specific_config: Optional[dict] = None,
|
|
49
50
|
neighborhood_depth: Optional[int] = None,
|
cognee/base_config.py
CHANGED
|
@@ -14,6 +14,9 @@ class BaseConfig(BaseSettings):
|
|
|
14
14
|
cache_root_directory: str = get_absolute_path(".cognee_cache")
|
|
15
15
|
logs_root_directory: str = os.getenv("COGNEE_LOGS_DIR", str(Path.home() / ".cognee" / "logs"))
|
|
16
16
|
monitoring_tool: object = Observer.NONE
|
|
17
|
+
# Default blend weight for the learned feedback signal during graph search.
|
|
18
|
+
# Opt-in by default to preserve existing retrieval behavior.
|
|
19
|
+
default_feedback_influence: float = float(os.getenv("DEFAULT_FEEDBACK_INFLUENCE", "0.0"))
|
|
17
20
|
|
|
18
21
|
@pydantic.model_validator(mode="after")
|
|
19
22
|
def validate_paths(self):
|
|
@@ -27,6 +27,7 @@ from cognee.infrastructure.databases.utils.resolve_dataset_database_connection_i
|
|
|
27
27
|
# for different async tasks, threads and processes
|
|
28
28
|
vector_db_config = ContextVar("vector_db_config", default=None)
|
|
29
29
|
graph_db_config = ContextVar("graph_db_config", default=None)
|
|
30
|
+
current_dataset_id = ContextVar("current_dataset_id", default=None)
|
|
30
31
|
# Note: same mechanism for LLM and embedding configs so that the LiteLLM client
|
|
31
32
|
# and the embedding engine can use per-context (e.g. per-request) configs.
|
|
32
33
|
llm_config = ContextVar("llm_config", default=None)
|
|
@@ -111,7 +112,14 @@ class DatabaseContextManager:
|
|
|
111
112
|
Note: Single-use object, should not be reused across multiple calls.
|
|
112
113
|
"""
|
|
113
114
|
|
|
114
|
-
__slots__ = (
|
|
115
|
+
__slots__ = (
|
|
116
|
+
"_dataset",
|
|
117
|
+
"_user_id",
|
|
118
|
+
"_llm_config",
|
|
119
|
+
"_embedding_config",
|
|
120
|
+
"_applied",
|
|
121
|
+
"_dataset_token",
|
|
122
|
+
)
|
|
115
123
|
|
|
116
124
|
def __init__(
|
|
117
125
|
self,
|
|
@@ -125,10 +133,13 @@ class DatabaseContextManager:
|
|
|
125
133
|
self._llm_config = llm_config
|
|
126
134
|
self._embedding_config = embedding_config
|
|
127
135
|
self._applied = False
|
|
136
|
+
self._dataset_token = None
|
|
128
137
|
|
|
129
138
|
async def apply_database_context_variables(
|
|
130
139
|
self, dataset: Union[str, UUID], user_id: UUID
|
|
131
140
|
) -> None:
|
|
141
|
+
self._dataset_token = current_dataset_id.set(str(dataset) if dataset is not None else None)
|
|
142
|
+
|
|
132
143
|
# LLM and embedding configs are an explicit, caller-provided override and
|
|
133
144
|
# are intentionally applied regardless of backend access control: callers
|
|
134
145
|
# may want per-context LLM/embedding configs even in single-tenant mode.
|
|
@@ -255,6 +266,10 @@ class DatabaseContextManager:
|
|
|
255
266
|
return self
|
|
256
267
|
|
|
257
268
|
async def __aexit__(self, exc_type, exc, tb) -> None:
|
|
269
|
+
if self._dataset_token is not None:
|
|
270
|
+
current_dataset_id.reset(self._dataset_token)
|
|
271
|
+
self._dataset_token = None
|
|
272
|
+
|
|
258
273
|
if not backend_access_control_enabled():
|
|
259
274
|
return None
|
|
260
275
|
|
|
@@ -354,6 +354,22 @@ class GraphDBInterface(ABC):
|
|
|
354
354
|
"""
|
|
355
355
|
raise NotImplementedError("set_node_feedback_weights is not implemented for this adapter")
|
|
356
356
|
|
|
357
|
+
async def get_node_truth_state(self, node_ids: List[str]) -> Dict[str, Dict[str, Any]]:
|
|
358
|
+
"""
|
|
359
|
+
Retrieve node truth alignment state for multiple node ids.
|
|
360
|
+
Returns only found node ids.
|
|
361
|
+
"""
|
|
362
|
+
raise NotImplementedError("get_node_truth_state is not implemented for this adapter")
|
|
363
|
+
|
|
364
|
+
async def set_node_truth_state(
|
|
365
|
+
self, node_truth_state: Dict[str, Dict[str, Any]]
|
|
366
|
+
) -> Dict[str, bool]:
|
|
367
|
+
"""
|
|
368
|
+
Persist node truth alignment state for multiple node ids.
|
|
369
|
+
Returns per-id update success.
|
|
370
|
+
"""
|
|
371
|
+
raise NotImplementedError("set_node_truth_state is not implemented for this adapter")
|
|
372
|
+
|
|
357
373
|
async def get_edge_feedback_weights(self, edge_object_ids: List[str]) -> Dict[str, float]:
|
|
358
374
|
"""
|
|
359
375
|
Retrieve edge feedback weights for multiple edge_object_ids.
|
|
@@ -1556,6 +1556,48 @@ class LadybugAdapter(GraphDBInterface):
|
|
|
1556
1556
|
rows_dicts = self._rows_to_dicts(result, ["node_id"])
|
|
1557
1557
|
return {str(r["node_id"]) for r in rows_dicts if r.get("node_id") is not None}
|
|
1558
1558
|
|
|
1559
|
+
def _build_node_truth_state_updates(
|
|
1560
|
+
self,
|
|
1561
|
+
nodes: List[Dict[str, Any]],
|
|
1562
|
+
node_truth_state: Dict[str, Dict[str, Any]],
|
|
1563
|
+
) -> List[Dict[str, Any]]:
|
|
1564
|
+
"""Build UNWIND items for node truth state updates."""
|
|
1565
|
+
updates = []
|
|
1566
|
+
for node in nodes:
|
|
1567
|
+
node_id = node.get("id")
|
|
1568
|
+
if not isinstance(node_id, str) or node_id not in node_truth_state:
|
|
1569
|
+
continue
|
|
1570
|
+
state = node_truth_state[node_id]
|
|
1571
|
+
properties = {
|
|
1572
|
+
k: v
|
|
1573
|
+
for k, v in node.items()
|
|
1574
|
+
if k not in {"id", "name", "type", "created_at", "updated_at"}
|
|
1575
|
+
}
|
|
1576
|
+
properties["truth_alignment"] = list(state.get("truth_alignment") or [])
|
|
1577
|
+
if state.get("truth_epoch") is not None:
|
|
1578
|
+
properties["truth_epoch"] = int(state["truth_epoch"])
|
|
1579
|
+
updates.append(
|
|
1580
|
+
{"node_id": node_id, "properties": json.dumps(properties, cls=JSONEncoder)}
|
|
1581
|
+
)
|
|
1582
|
+
return updates
|
|
1583
|
+
|
|
1584
|
+
async def _execute_node_truth_state_updates(self, updates: List[Dict[str, Any]]) -> Set[str]:
|
|
1585
|
+
"""Run node truth state UNWIND/SET; return set of updated node_ids."""
|
|
1586
|
+
if not updates:
|
|
1587
|
+
return set()
|
|
1588
|
+
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
|
1589
|
+
query = """
|
|
1590
|
+
UNWIND $items AS item
|
|
1591
|
+
MATCH (n:Node)
|
|
1592
|
+
WHERE n.id = item.node_id
|
|
1593
|
+
SET n.properties = item.properties,
|
|
1594
|
+
n.updated_at = timestamp($updated_at)
|
|
1595
|
+
RETURN n.id AS node_id
|
|
1596
|
+
"""
|
|
1597
|
+
result = await self.query(query, {"items": updates, "updated_at": now})
|
|
1598
|
+
rows_dicts = self._rows_to_dicts(result, ["node_id"])
|
|
1599
|
+
return {str(r["node_id"]) for r in rows_dicts if r.get("node_id") is not None}
|
|
1600
|
+
|
|
1559
1601
|
def _build_edge_feedback_updates(
|
|
1560
1602
|
self,
|
|
1561
1603
|
edge_rows: List[Dict[str, Any]],
|
|
@@ -1642,6 +1684,47 @@ class LadybugAdapter(GraphDBInterface):
|
|
|
1642
1684
|
updated_ids = await self._execute_node_feedback_updates(updates)
|
|
1643
1685
|
return {nid: (nid in updated_ids) for nid in node_ids}
|
|
1644
1686
|
|
|
1687
|
+
async def get_node_truth_state(self, node_ids: List[str]) -> Dict[str, Dict[str, Any]]:
|
|
1688
|
+
if not node_ids:
|
|
1689
|
+
return {}
|
|
1690
|
+
valid_node_ids = [node_id for node_id in node_ids if isinstance(node_id, str) and node_id]
|
|
1691
|
+
if not valid_node_ids:
|
|
1692
|
+
return {}
|
|
1693
|
+
nodes = await self.get_nodes(valid_node_ids)
|
|
1694
|
+
result: Dict[str, Dict[str, Any]] = {}
|
|
1695
|
+
for node in nodes:
|
|
1696
|
+
node_id = node.get("id")
|
|
1697
|
+
if not isinstance(node_id, str):
|
|
1698
|
+
continue
|
|
1699
|
+
value = node.get("truth_alignment", [])
|
|
1700
|
+
epoch = node.get("truth_epoch")
|
|
1701
|
+
if isinstance(value, (list, tuple)):
|
|
1702
|
+
alignment = list(value)
|
|
1703
|
+
else:
|
|
1704
|
+
alignment = []
|
|
1705
|
+
try:
|
|
1706
|
+
truth_epoch = int(epoch) if epoch is not None else None
|
|
1707
|
+
except (TypeError, ValueError):
|
|
1708
|
+
truth_epoch = None
|
|
1709
|
+
result[node_id] = {"truth_alignment": alignment, "truth_epoch": truth_epoch}
|
|
1710
|
+
return result
|
|
1711
|
+
|
|
1712
|
+
async def set_node_truth_state(
|
|
1713
|
+
self, node_truth_state: Dict[str, Dict[str, Any]]
|
|
1714
|
+
) -> Dict[str, bool]:
|
|
1715
|
+
if not node_truth_state:
|
|
1716
|
+
return {}
|
|
1717
|
+
node_ids = list(node_truth_state.keys())
|
|
1718
|
+
valid_node_ids = [nid for nid in node_ids if isinstance(nid, str) and nid]
|
|
1719
|
+
if not valid_node_ids:
|
|
1720
|
+
return {nid: False for nid in node_ids}
|
|
1721
|
+
nodes = await self.get_nodes(valid_node_ids)
|
|
1722
|
+
updates = self._build_node_truth_state_updates(nodes, node_truth_state)
|
|
1723
|
+
if not updates:
|
|
1724
|
+
return {nid: False for nid in node_ids}
|
|
1725
|
+
updated_ids = await self._execute_node_truth_state_updates(updates)
|
|
1726
|
+
return {nid: (nid in updated_ids) for nid in node_ids}
|
|
1727
|
+
|
|
1645
1728
|
async def get_edge_feedback_weights(self, edge_object_ids: List[str]) -> Dict[str, float]:
|
|
1646
1729
|
if not edge_object_ids:
|
|
1647
1730
|
return {}
|
|
@@ -546,6 +546,62 @@ class LanceDBAdapter(VectorDBInterface):
|
|
|
546
546
|
collection_name, collection, payload_schema, lance_data_points
|
|
547
547
|
)
|
|
548
548
|
|
|
549
|
+
async def upsert_raw_vectors(
|
|
550
|
+
self,
|
|
551
|
+
collection_name: str,
|
|
552
|
+
points: list[dict],
|
|
553
|
+
payload_schema: Optional[type[BaseModel]] = None,
|
|
554
|
+
) -> None:
|
|
555
|
+
"""Upsert caller-provided vectors without invoking the embedding engine."""
|
|
556
|
+
if not points:
|
|
557
|
+
return
|
|
558
|
+
if payload_schema is None:
|
|
559
|
+
raise ValueError("payload_schema is required for LanceDB raw vector upserts")
|
|
560
|
+
|
|
561
|
+
vector_size = self.embedding_engine.get_vector_size()
|
|
562
|
+
LanceDataPoint = self._make_lance_datapoint_cls(payload_schema, vector_size)
|
|
563
|
+
|
|
564
|
+
if not await self.has_collection(collection_name):
|
|
565
|
+
async with self.VECTOR_DB_LOCK:
|
|
566
|
+
if not await self.has_collection(collection_name):
|
|
567
|
+
connection = await self.get_connection()
|
|
568
|
+
await connection.create_table(
|
|
569
|
+
name=collection_name,
|
|
570
|
+
schema=self._schema_for_create_table(LanceDataPoint),
|
|
571
|
+
exist_ok=True,
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
collection = await self.get_collection(collection_name)
|
|
575
|
+
|
|
576
|
+
raw_points = []
|
|
577
|
+
for point in points:
|
|
578
|
+
point_id = point.get("id")
|
|
579
|
+
vector = point.get("vector")
|
|
580
|
+
payload_value = point.get("payload")
|
|
581
|
+
if point_id is None:
|
|
582
|
+
raise ValueError("Raw vector point is missing id")
|
|
583
|
+
if not isinstance(vector, list):
|
|
584
|
+
raise ValueError("Raw vector point vector must be a list")
|
|
585
|
+
if len(vector) != vector_size:
|
|
586
|
+
raise ValueError(
|
|
587
|
+
f"Raw vector size {len(vector)} does not match expected size {vector_size}"
|
|
588
|
+
)
|
|
589
|
+
raw_points.append(
|
|
590
|
+
LanceDataPoint(
|
|
591
|
+
id=str(point_id),
|
|
592
|
+
vector=vector,
|
|
593
|
+
payload=payload_schema.model_validate(payload_value).model_dump(),
|
|
594
|
+
)
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
async with self.VECTOR_DB_LOCK:
|
|
598
|
+
await (
|
|
599
|
+
collection.merge_insert("id")
|
|
600
|
+
.when_matched_update_all()
|
|
601
|
+
.when_not_matched_insert_all()
|
|
602
|
+
.execute(self._records_for_write(raw_points))
|
|
603
|
+
)
|
|
604
|
+
|
|
549
605
|
async def _migrate_collection_schema(
|
|
550
606
|
self,
|
|
551
607
|
collection_name: str,
|
|
@@ -8,6 +8,7 @@ from cognee.modules.users.models import DatasetDatabase
|
|
|
8
8
|
from cognee.base_config import get_base_config
|
|
9
9
|
from cognee.infrastructure.databases.vector import get_vectordb_config
|
|
10
10
|
from cognee.infrastructure.databases.dataset_database_handler import DatasetDatabaseHandlerInterface
|
|
11
|
+
from cognee.infrastructure.files.storage.get_file_storage import get_file_storage
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
class LanceDBDatasetDatabaseHandler(DatasetDatabaseHandlerInterface):
|
|
@@ -29,7 +30,7 @@ class LanceDBDatasetDatabaseHandler(DatasetDatabaseHandlerInterface):
|
|
|
29
30
|
databases_directory_path = os.path.join(
|
|
30
31
|
base_config.system_root_directory, "databases", str(user.id)
|
|
31
32
|
)
|
|
32
|
-
|
|
33
|
+
await get_file_storage(databases_directory_path).ensure_directory_exists()
|
|
33
34
|
|
|
34
35
|
vector_db_name = f"{dataset_id}.lance.db"
|
|
35
36
|
|
|
@@ -63,6 +63,21 @@ class VectorDBInterface(Protocol):
|
|
|
63
63
|
"""
|
|
64
64
|
raise NotImplementedError
|
|
65
65
|
|
|
66
|
+
async def upsert_raw_vectors(
|
|
67
|
+
self,
|
|
68
|
+
collection_name: str,
|
|
69
|
+
points: list[dict],
|
|
70
|
+
payload_schema: Optional[Any] = None,
|
|
71
|
+
) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Upsert already-computed vectors into the specified collection.
|
|
74
|
+
|
|
75
|
+
This is for small system-owned vector state where re-embedding from text would be
|
|
76
|
+
incorrect. Adapters that do not support raw vector writes should raise
|
|
77
|
+
NotImplementedError.
|
|
78
|
+
"""
|
|
79
|
+
raise NotImplementedError("upsert_raw_vectors is not implemented for this adapter")
|
|
80
|
+
|
|
66
81
|
@abstractmethod
|
|
67
82
|
async def retrieve(self, collection_name: str, data_point_ids: list[str]):
|
|
68
83
|
"""
|
|
@@ -38,4 +38,8 @@ class DocumentChunk(DataPoint):
|
|
|
38
38
|
importance_weight: Optional[float] = 0.5
|
|
39
39
|
document_id: Optional[str] = None
|
|
40
40
|
document_name: Optional[str] = None
|
|
41
|
+
# Optional truth-alignment fields; never embedded (kept out of index_fields)
|
|
42
|
+
# and not part of id/dedup.
|
|
43
|
+
truth_alignment: Optional[list[float]] = None
|
|
44
|
+
truth_epoch: Optional[int] = None
|
|
41
45
|
metadata: dict = {"index_fields": ["text"]}
|
|
@@ -9,6 +9,11 @@ class Entity(DataPoint):
|
|
|
9
9
|
is_a: Optional[EntityType] = None
|
|
10
10
|
description: str
|
|
11
11
|
relations: List[tuple] = []
|
|
12
|
+
# Optional truth-alignment fields; never embedded (kept out of index_fields)
|
|
13
|
+
# and not part of id/dedup (kept out of identity_fields).
|
|
14
|
+
truth_alignment: Optional[list[float]] = None
|
|
15
|
+
truth_subspace_signature: Optional[str] = None
|
|
16
|
+
truth_epoch: Optional[int] = None
|
|
12
17
|
# identity_fields makes the id deterministic and namespaced by class
|
|
13
18
|
# (``Entity:<name>``) when constructed without an explicit id — the same
|
|
14
19
|
# value ``Entity.id_for(name)`` produces. Prevents the random-uuid4 footgun.
|
|
@@ -10,6 +10,7 @@ from cognee.modules.graph.exceptions import (
|
|
|
10
10
|
from cognee.infrastructure.databases.graph.graph_db_interface import GraphDBInterface
|
|
11
11
|
from cognee.modules.graph.cognee_graph.CogneeGraphElements import Node, Edge
|
|
12
12
|
from cognee.modules.graph.cognee_graph.CogneeAbstractGraph import CogneeAbstractGraph
|
|
13
|
+
from cognee.base_config import get_base_config
|
|
13
14
|
import heapq
|
|
14
15
|
|
|
15
16
|
logger = get_logger("CogneeGraph")
|
|
@@ -36,7 +37,7 @@ class CogneeGraph(CogneeAbstractGraph):
|
|
|
36
37
|
self.edges_by_distance_key = {}
|
|
37
38
|
self.directed = directed
|
|
38
39
|
self.triplet_distance_penalty = 6.5
|
|
39
|
-
self.feedback_influence =
|
|
40
|
+
self.feedback_influence = get_base_config().default_feedback_influence
|
|
40
41
|
|
|
41
42
|
def add_node(self, node: Node) -> None:
|
|
42
43
|
if node.id in self.nodes:
|
|
@@ -236,7 +237,7 @@ class CogneeGraph(CogneeAbstractGraph):
|
|
|
236
237
|
node_name_filter_operator: str = "OR",
|
|
237
238
|
relevant_ids_to_filter: Optional[List[str]] = None,
|
|
238
239
|
triplet_distance_penalty: float = 6.5,
|
|
239
|
-
feedback_influence: float =
|
|
240
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
240
241
|
) -> None:
|
|
241
242
|
if node_dimension < 1 or edge_dimension < 1:
|
|
242
243
|
raise InvalidDimensionsError()
|
|
@@ -322,7 +323,7 @@ class CogneeGraph(CogneeAbstractGraph):
|
|
|
322
323
|
node_dimension: int = 1,
|
|
323
324
|
edge_dimension: int = 1,
|
|
324
325
|
triplet_distance_penalty: float = 6.5,
|
|
325
|
-
feedback_influence: float =
|
|
326
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
326
327
|
) -> None:
|
|
327
328
|
"""
|
|
328
329
|
Project a neighborhood subgraph from the database around seed nodes.
|
|
@@ -77,6 +77,16 @@ async def import_memory_source(
|
|
|
77
77
|
external_id, node ids from entity names) make re-running an interrupted
|
|
78
78
|
or repeated import safe.
|
|
79
79
|
"""
|
|
80
|
+
from cognee.modules.migrations.startup import run_migrations_and_block
|
|
81
|
+
|
|
82
|
+
# Imports are writes, so they take the same migration gate as
|
|
83
|
+
# remember()/cognify() (the remember() MemorySource dispatch happens
|
|
84
|
+
# before its own gate). This also records the data-migration revision —
|
|
85
|
+
# stamping a fresh store at head — BEFORE the imported rows arrive;
|
|
86
|
+
# without it the populated store has no recorded revision and the first
|
|
87
|
+
# migration-aware startup replays the entire data chain over it.
|
|
88
|
+
await run_migrations_and_block(dataset_name, user)
|
|
89
|
+
|
|
80
90
|
node_set = node_set or [f"import:{source.source_system}"]
|
|
81
91
|
|
|
82
92
|
if source.mode == "preserve" and getattr(source, "replayable", False):
|
|
@@ -11,6 +11,7 @@ Two targets, selected by the source's import mode:
|
|
|
11
11
|
the source (``valid_at``/``invalid_at``) is preserved as edge properties.
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
+
import dataclasses
|
|
14
15
|
from dataclasses import dataclass, field
|
|
15
16
|
from datetime import datetime
|
|
16
17
|
from types import SimpleNamespace
|
|
@@ -427,6 +428,12 @@ def _provenance_ctx(ctx):
|
|
|
427
428
|
add_data_points stamps ledger provenance from ``ctx.data_item.id``, which
|
|
428
429
|
exists on Data ORM records but not on raw ingestion DataItems. Substitute
|
|
429
430
|
the DataItem's deterministic ``data_id`` so provenance still lands.
|
|
431
|
+
|
|
432
|
+
Only ``data_item`` is replaced; every other PipelineContext field is
|
|
433
|
+
carried over via ``dataclasses.replace`` so consumers that read ctx
|
|
434
|
+
attributes (e.g. ``add_data_points`` reads ``ctx.pipeline_run_id``) never
|
|
435
|
+
see a field-stripped context — a hand-copied field list here silently
|
|
436
|
+
drops any parameter later added to PipelineContext.
|
|
430
437
|
"""
|
|
431
438
|
if ctx is None:
|
|
432
439
|
return None
|
|
@@ -434,12 +441,7 @@ def _provenance_ctx(ctx):
|
|
|
434
441
|
if data_item is None or hasattr(data_item, "id"):
|
|
435
442
|
return ctx
|
|
436
443
|
data_id = getattr(data_item, "data_id", None)
|
|
437
|
-
return SimpleNamespace(
|
|
438
|
-
user=getattr(ctx, "user", None),
|
|
439
|
-
dataset=getattr(ctx, "dataset", None),
|
|
440
|
-
data_item=SimpleNamespace(id=data_id) if data_id else None,
|
|
441
|
-
extras=getattr(ctx, "extras", None),
|
|
442
|
-
)
|
|
444
|
+
return dataclasses.replace(ctx, data_item=SimpleNamespace(id=data_id) if data_id else None)
|
|
443
445
|
|
|
444
446
|
|
|
445
447
|
# Edge batches can run larger than node batches: edges are small tuples and
|
|
@@ -19,7 +19,7 @@ so no property is lost.
|
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
import json
|
|
22
|
-
from typing import Any, Dict, List, Optional, Type, Union
|
|
22
|
+
from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
|
|
23
23
|
|
|
24
24
|
from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, create_model, field_validator
|
|
25
25
|
|
|
@@ -59,13 +59,41 @@ class _DynamicDataPoint(DataPoint):
|
|
|
59
59
|
model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
|
|
60
60
|
|
|
61
61
|
|
|
62
|
-
_dynamic_models: Dict[str, Type[DataPoint]] = {}
|
|
62
|
+
_dynamic_models: Dict[Tuple[str, Tuple[str, ...], Tuple[str, ...]], Type[DataPoint]] = {}
|
|
63
63
|
|
|
64
64
|
|
|
65
|
-
def _dynamic_model(
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
def _dynamic_model(
|
|
66
|
+
type_name: str,
|
|
67
|
+
property_names: Optional[Iterable[str]] = None,
|
|
68
|
+
index_fields: Optional[List[str]] = None,
|
|
69
|
+
) -> Type[DataPoint]:
|
|
70
|
+
"""A DataPoint subclass named ``type_name`` that DECLARES the record's
|
|
71
|
+
properties as fields and bakes ``index_fields`` into the CLASS-level
|
|
72
|
+
metadata default.
|
|
73
|
+
|
|
74
|
+
Instance-level state is not enough on either count: downstream model
|
|
75
|
+
machinery only sees declared fields. ``get_graph_from_model`` iterates
|
|
76
|
+
declared fields when collecting node properties (``extra="allow"`` values
|
|
77
|
+
are silently dropped — imported DocumentChunk text never reached storage),
|
|
78
|
+
and rebuilds nodes through ``copy_model``, which re-applies class field
|
|
79
|
+
defaults (a plain DataPoint metadata default of ``index_fields: []``
|
|
80
|
+
dropped the embeddable-field markers, so nothing was vector-indexed).
|
|
81
|
+
"""
|
|
82
|
+
extra_fields = tuple(
|
|
83
|
+
sorted(
|
|
84
|
+
name
|
|
85
|
+
for name in (property_names or ())
|
|
86
|
+
if name != "metadata" and name not in DataPoint.model_fields
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
key = (type_name, extra_fields, tuple(index_fields or ()))
|
|
90
|
+
if key not in _dynamic_models:
|
|
91
|
+
field_definitions: Dict[str, Any] = {name: (Any, None) for name in extra_fields}
|
|
92
|
+
field_definitions["metadata"] = (dict, {"index_fields": list(index_fields or [])})
|
|
93
|
+
_dynamic_models[key] = create_model(
|
|
94
|
+
type_name, __base__=_DynamicDataPoint, **field_definitions
|
|
95
|
+
)
|
|
96
|
+
return _dynamic_models[key]
|
|
69
97
|
|
|
70
98
|
|
|
71
99
|
def _clean_properties(properties: Dict[str, Any]) -> Dict[str, Any]:
|
|
@@ -101,14 +129,20 @@ def rehydrate_node(
|
|
|
101
129
|
except Exception: # noqa: BLE001 — fall back to a dynamic model
|
|
102
130
|
pass
|
|
103
131
|
|
|
132
|
+
# Carry the record's properties and embeddable fields onto the dynamic
|
|
133
|
+
# CLASS so they survive downstream field iteration and copy_model
|
|
134
|
+
# rebuilds (see _dynamic_model).
|
|
135
|
+
metadata = props.get("metadata")
|
|
136
|
+
index_fields = metadata.get("index_fields") if isinstance(metadata, dict) else None
|
|
137
|
+
|
|
104
138
|
try:
|
|
105
139
|
# Created lazily: only when the registered class is absent or fails.
|
|
106
|
-
return _dynamic_model(type_name)(**props)
|
|
140
|
+
return _dynamic_model(type_name, props, index_fields)(**props)
|
|
107
141
|
except Exception: # noqa: BLE001 — final fallback below
|
|
108
142
|
pass
|
|
109
143
|
|
|
110
144
|
base_safe = {key: value for key, value in props.items() if key in _BASE_FIELDS}
|
|
111
|
-
return _dynamic_model(type_name)(**base_safe)
|
|
145
|
+
return _dynamic_model(type_name, base_safe, index_fields)(**base_safe)
|
|
112
146
|
|
|
113
147
|
|
|
114
148
|
class GraphEdge(BaseModel):
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
from typing import Optional, List, Type, Union
|
|
3
3
|
|
|
4
|
+
from cognee.base_config import get_base_config
|
|
4
5
|
from cognee.modules.graph.cognee_graph.CogneeGraphElements import Edge
|
|
5
6
|
from cognee.modules.retrieval.utils.query_state import QueryState
|
|
6
7
|
from cognee.modules.retrieval.utils.validate_queries import validate_retriever_input
|
|
@@ -28,7 +29,7 @@ class GraphCompletionContextExtensionRetriever(GraphCompletionRetriever):
|
|
|
28
29
|
node_name_filter_operator: str = "OR",
|
|
29
30
|
wide_search_top_k: Optional[int] = 100,
|
|
30
31
|
triplet_distance_penalty: Optional[float] = 6.5,
|
|
31
|
-
feedback_influence: float =
|
|
32
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
32
33
|
context_extension_rounds: int = 4,
|
|
33
34
|
session_id: Optional[str] = None,
|
|
34
35
|
response_model: Type = str,
|
|
@@ -4,6 +4,7 @@ from typing import Optional, List, Type, Any, Union
|
|
|
4
4
|
|
|
5
5
|
from pydantic import BaseModel
|
|
6
6
|
|
|
7
|
+
from cognee.base_config import get_base_config
|
|
7
8
|
from cognee.modules.graph.cognee_graph.CogneeGraphElements import Edge
|
|
8
9
|
from cognee.modules.retrieval.utils.query_state import QueryState
|
|
9
10
|
from cognee.modules.retrieval.utils.validate_queries import validate_retriever_input
|
|
@@ -64,7 +65,7 @@ class GraphCompletionCotRetriever(GraphCompletionRetriever):
|
|
|
64
65
|
node_name_filter_operator: str = "OR",
|
|
65
66
|
wide_search_top_k: Optional[int] = 100,
|
|
66
67
|
triplet_distance_penalty: Optional[float] = 6.5,
|
|
67
|
-
feedback_influence: float =
|
|
68
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
68
69
|
max_iter: int = 4,
|
|
69
70
|
session_id: Optional[str] = None,
|
|
70
71
|
response_model: Type = str,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
from typing import Any, List, Optional, Type
|
|
3
3
|
|
|
4
|
+
from cognee.base_config import get_base_config
|
|
4
5
|
from cognee.infrastructure.databases.unified import get_unified_engine
|
|
5
6
|
from cognee.infrastructure.llm.LLMGateway import LLMGateway
|
|
6
7
|
from cognee.infrastructure.llm.prompts import read_query_prompt
|
|
@@ -40,7 +41,7 @@ class GraphCompletionDecompositionRetriever(GraphCompletionRetriever):
|
|
|
40
41
|
node_name_filter_operator: str = "OR",
|
|
41
42
|
wide_search_top_k: Optional[int] = 100,
|
|
42
43
|
triplet_distance_penalty: Optional[float] = 6.5,
|
|
43
|
-
feedback_influence: float =
|
|
44
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
44
45
|
session_id: Optional[str] = None,
|
|
45
46
|
response_model: Type = str,
|
|
46
47
|
neighborhood_depth: Optional[int] = None,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
from typing import Any, Dict, List, Optional, Type, Union
|
|
3
3
|
|
|
4
|
+
from cognee.base_config import get_base_config
|
|
4
5
|
from cognee.infrastructure.engine import DataPoint
|
|
5
6
|
from cognee.modules.graph.cognee_graph.CogneeGraphElements import Edge
|
|
6
7
|
from cognee.modules.retrieval.utils.validate_queries import validate_retriever_input
|
|
@@ -51,7 +52,7 @@ class GraphCompletionRetriever(BaseRetriever):
|
|
|
51
52
|
node_name_filter_operator: str = "OR",
|
|
52
53
|
wide_search_top_k: Optional[int] = 100,
|
|
53
54
|
triplet_distance_penalty: Optional[float] = 6.5,
|
|
54
|
-
feedback_influence: float =
|
|
55
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
55
56
|
session_id: Optional[str] = None,
|
|
56
57
|
response_model: Type = str,
|
|
57
58
|
neighborhood_depth: Optional[int] = None,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from typing import Optional, Type, List
|
|
2
2
|
|
|
3
|
+
from cognee.base_config import get_base_config
|
|
3
4
|
from cognee.modules.retrieval.graph_completion_retriever import GraphCompletionRetriever
|
|
4
5
|
from cognee.modules.retrieval.utils.completion import summarize_text
|
|
5
6
|
|
|
@@ -28,7 +29,7 @@ class GraphSummaryCompletionRetriever(GraphCompletionRetriever):
|
|
|
28
29
|
node_name_filter_operator: str = "OR",
|
|
29
30
|
wide_search_top_k: Optional[int] = 100,
|
|
30
31
|
triplet_distance_penalty: Optional[float] = 6.5,
|
|
31
|
-
feedback_influence: float =
|
|
32
|
+
feedback_influence: float = get_base_config().default_feedback_influence,
|
|
32
33
|
session_id: Optional[str] = None,
|
|
33
34
|
include_references: bool = False,
|
|
34
35
|
):
|