cognee 0.2.1.dev7__py3-none-any.whl → 0.2.2.dev1__py3-none-any.whl

cognee/modules/graph/utils/get_graph_from_model.py

@@ -4,43 +4,50 @@ from cognee.infrastructure.engine import DataPoint, Edge
 from cognee.modules.storage.utils import copy_model
 
 
-def _extract_field_info(field_value: Any) -> Tuple[str, Any, Optional[Edge]]:
-    """Extract field type, actual value, and edge metadata from a field value."""
-
-    # Handle tuple[Edge, DataPoint]
-    if (
-        isinstance(field_value, tuple)
-        and len(field_value) == 2
-        and isinstance(field_value[0], Edge)
-        and isinstance(field_value[1], DataPoint)
-    ):
-        return "single_datapoint_with_edge", field_value[1], field_value[0]
-
-    # Handle tuple[Edge, list[DataPoint]]
+def _extract_field_data(field_value: Any) -> List[Tuple[Optional[Edge], List[DataPoint]]]:
+    """Extract edge metadata and datapoints from a field value."""
+    # Handle single DataPoint
+    if isinstance(field_value, DataPoint):
+        return [(None, [field_value])]
+
+    # Handle list - could contain DataPoints, edge tuples, or mixed
+    if isinstance(field_value, list) and len(field_value) > 0:
+        result = []
+        for item in field_value:
+            # Handle tuple[Edge, DataPoint or list[DataPoint]]
+            if isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], Edge):
+                edge, data_value = item
+                if isinstance(data_value, DataPoint):
+                    result.append((edge, [data_value]))
+                elif (
+                    isinstance(data_value, list)
+                    and len(data_value) > 0
+                    and isinstance(data_value[0], DataPoint)
+                ):
+                    result.append((edge, data_value))
+            # Handle single DataPoint in list
+            elif isinstance(item, DataPoint):
+                result.append((None, [item]))
+        return result
+
+    # Handle tuple[Edge, DataPoint or list[DataPoint]]
     if (
         isinstance(field_value, tuple)
         and len(field_value) == 2
         and isinstance(field_value[0], Edge)
-        and isinstance(field_value[1], list)
-        and len(field_value[1]) > 0
-        and isinstance(field_value[1][0], DataPoint)
     ):
-        return "list_datapoint_with_edge", field_value[1], field_value[0]
-
-    # Handle single DataPoint
-    if isinstance(field_value, DataPoint):
-        return "single_datapoint", field_value, None
-
-    # Handle list of DataPoints
-    if (
-        isinstance(field_value, list)
-        and len(field_value) > 0
-        and isinstance(field_value[0], DataPoint)
-    ):
-        return "list_datapoint", field_value, None
+        edge_metadata, data_value = field_value
+        if isinstance(data_value, DataPoint):
+            return [(edge_metadata, [data_value])]
+        elif (
+            isinstance(data_value, list)
+            and len(data_value) > 0
+            and isinstance(data_value[0], DataPoint)
+        ):
+            return [(edge_metadata, data_value)]
 
-    # Regular property
-    return "property", field_value, None
+    # Regular property or empty list
+    return []
 
 
 def _create_edge_properties(
@@ -80,30 +87,49 @@ def _get_relationship_key(field_name: str, edge_metadata: Optional[Edge]) -> str
 
 def _generate_property_key(data_point_id: str, relationship_key: str, target_id: str) -> str:
     """Generate a unique property key for visited_properties tracking."""
-    return f"{data_point_id}{relationship_key}{target_id}"
+    return f"{data_point_id}_{relationship_key}_{target_id}"
 
 
 def _process_datapoint_field(
     data_point_id: str,
     field_name: str,
-    datapoints: List[DataPoint],
-    edge_metadata: Optional[Edge],
+    edge_datapoint_pairs: List[Tuple[Optional[Edge], List[DataPoint]]],
     visited_properties: Dict[str, bool],
     properties_to_visit: set,
     excluded_properties: set,
 ) -> None:
-    """Process a field containing DataPoint(s), handling both single and list cases."""
+    """Process a field containing DataPoints, always working with lists."""
     excluded_properties.add(field_name)
-    relationship_key = _get_relationship_key(field_name, edge_metadata)
 
-    for index, datapoint in enumerate(datapoints):
-        property_key = _generate_property_key(data_point_id, relationship_key, str(datapoint.id))
-        if property_key in visited_properties:
+    for edge_metadata, datapoints in edge_datapoint_pairs:
+        relationship_key = _get_relationship_key(field_name, edge_metadata)
+
+        for datapoint in datapoints:
+            property_key = _generate_property_key(
+                data_point_id, relationship_key, str(datapoint.id)
+            )
+            if property_key in visited_properties:
+                continue
+
+            # Always use field_name since we're working with lists
+            properties_to_visit.add(field_name)
+
+
+def _targets_generator(
+    data_point: DataPoint,
+    properties_to_visit: set,
+) -> Tuple[DataPoint, str, Optional[Edge]]:
+    """Generator that yields (target_datapoint, field_name, edge_metadata) tuples."""
+    for field_name in properties_to_visit:
+        field_value = getattr(data_point, field_name)
+        edge_datapoint_pairs = _extract_field_data(field_value)
+
+        if not edge_datapoint_pairs:
             continue
 
-        # For single datapoint, use field_name; for list, use field_name.index
-        field_identifier = field_name if len(datapoints) == 1 else f"{field_name}.{index}"
-        properties_to_visit.add(field_identifier)
+        for edge_metadata, datapoints in edge_datapoint_pairs:
+            for target_datapoint in datapoints:
+                yield target_datapoint, field_name, edge_metadata
 
 
 async def get_graph_from_model(
@@ -143,26 +169,17 @@ async def get_graph_from_model(
         if field_name == "metadata":
             continue
 
-        field_type, actual_value, edge_metadata = _extract_field_info(field_value)
+        edge_datapoint_pairs = _extract_field_data(field_value)
 
-        if field_type == "property":
+        if not edge_datapoint_pairs:
+            # Regular property
             data_point_properties[field_name] = field_value
-        elif field_type in ["single_datapoint", "single_datapoint_with_edge"]:
-            _process_datapoint_field(
-                data_point_id,
-                field_name,
-                [actual_value],
-                edge_metadata,
-                visited_properties,
-                properties_to_visit,
-                excluded_properties,
-            )
-        elif field_type in ["list_datapoint", "list_datapoint_with_edge"]:
+        else:
+            # DataPoint relationship
             _process_datapoint_field(
                 data_point_id,
                 field_name,
-                actual_value,
-                edge_metadata,
+                edge_datapoint_pairs,
                 visited_properties,
                 properties_to_visit,
                 excluded_properties,
@@ -176,41 +193,15 @@ async def get_graph_from_model(
         nodes.append(SimpleDataPointModel(**data_point_properties))
         added_nodes[data_point_id] = True
 
-    # Process all relationships
-    for field_name_with_index in properties_to_visit:
-        # Parse field name and index
-        if "." in field_name_with_index:
-            field_name, index_str = field_name_with_index.split(".")
-            index = int(index_str)
-        else:
-            field_name, index = field_name_with_index, None
-
-        # Get field value and extract edge metadata
-        field_value = getattr(data_point, field_name)
-        edge_metadata = None
-
-        if (
-            isinstance(field_value, tuple)
-            and len(field_value) == 2
-            and isinstance(field_value[0], Edge)
-        ):
-            edge_metadata, field_value = field_value
-
-        # Get specific datapoint - handle both single and list cases
-        if index is not None:
-            # List case: extract specific item by index
-            target_datapoint = field_value[index]
-        elif isinstance(field_value, list):
-            # Single datapoint case that was wrapped in a list
-            target_datapoint = field_value[0]
-        else:
-            # True single datapoint case
-            target_datapoint = field_value
+    # Process all relationships using generator
+    for target_datapoint, field_name, edge_metadata in _targets_generator(
+        data_point, properties_to_visit
+    ):
+        relationship_name = _get_relationship_key(field_name, edge_metadata)
 
         # Create edge if not already added
-        edge_key = f"{data_point_id}{target_datapoint.id}{field_name}"
+        edge_key = f"{data_point_id}_{target_datapoint.id}_{field_name}"
         if edge_key not in added_edges:
-            relationship_name = _get_relationship_key(field_name, edge_metadata)
             edge_properties = _create_edge_properties(
                 data_point.id, target_datapoint.id, relationship_name, edge_metadata
             )
@@ -218,23 +209,24 @@ async def get_graph_from_model(
             added_edges[edge_key] = True
 
         # Mark property as visited - CRITICAL for preventing infinite loops
-        relationship_key = _get_relationship_key(field_name, edge_metadata)
         property_key = _generate_property_key(
-            data_point_id, relationship_key, str(target_datapoint.id)
+            data_point_id, relationship_name, str(target_datapoint.id)
         )
         visited_properties[property_key] = True
 
         # Recursively process target node if not already processed
-        if str(target_datapoint.id) not in added_nodes:
-            child_nodes, child_edges = await get_graph_from_model(
-                target_datapoint,
-                include_root=True,
-                added_nodes=added_nodes,
-                added_edges=added_edges,
-                visited_properties=visited_properties,
-            )
-            nodes.extend(child_nodes)
-            edges.extend(child_edges)
+        if str(target_datapoint.id) in added_nodes:
+            continue
+
+        child_nodes, child_edges = await get_graph_from_model(
+            target_datapoint,
+            include_root=True,
+            added_nodes=added_nodes,
+            added_edges=added_edges,
+            visited_properties=visited_properties,
+        )
+        nodes.extend(child_nodes)
+        edges.extend(child_edges)
 
     return nodes, edges
 

cognee/modules/ingestion/data_types/TextData.py

@@ -1,5 +1,6 @@
 from typing import BinaryIO
 from contextlib import asynccontextmanager
+import hashlib
 from cognee.infrastructure.data.utils.extract_keywords import extract_keywords
 from .IngestionData import IngestionData
 
@@ -16,9 +17,9 @@ class TextData(IngestionData):
         self.data = data
 
     def get_identifier(self):
-        keywords = extract_keywords(self.data)
+        metadata = self.get_metadata()
 
-        return "text/plain" + "_" + "|".join(keywords)
+        return metadata["content_hash"]
 
     def get_metadata(self):
         self.ensure_metadata()
@@ -29,6 +30,11 @@ class TextData(IngestionData):
         if self.metadata is None:
             self.metadata = {}
 
+        data_contents = self.data.encode("utf-8")
+        hash_contents = hashlib.md5(data_contents).hexdigest()
+        self.metadata["name"] = "text_" + hash_contents + ".txt"
+        self.metadata["content_hash"] = hash_contents
+
     @asynccontextmanager
     async def get_data(self):
         yield self.data

cognee/modules/ingestion/save_data_to_file.py

@@ -1,7 +1,7 @@
-import hashlib
 from typing import BinaryIO, Union
 from cognee.infrastructure.files.storage import get_file_storage, get_storage_config
 from .classify import classify
+import hashlib
 
 
 async def save_data_to_file(data: Union[str, BinaryIO], filename: str = None):

cognee/modules/pipelines/exceptions/__init__.py

@@ -0,0 +1 @@
+from .exceptions import PipelineRunFailedError

cognee/modules/pipelines/exceptions/exceptions.py

@@ -0,0 +1,12 @@
+from cognee.exceptions import CogneeApiError
+from fastapi import status
+
+
+class PipelineRunFailedError(CogneeApiError):
+    def __init__(
+        self,
+        message: str = "Pipeline run failed.",
+        name: str = "PipelineRunFailedError",
+        status_code: int = status.HTTP_422_UNPROCESSABLE_ENTITY,
+    ):
+        super().__init__(message, name, status_code)

cognee/modules/pipelines/models/DataItemStatus.py

@@ -0,0 +1,5 @@
+import enum
+
+
+class DataItemStatus(str, enum.Enum):
+    DATA_ITEM_PROCESSING_COMPLETED = "DATA_ITEM_PROCESSING_COMPLETED"

cognee/modules/pipelines/models/PipelineRunInfo.py

@@ -9,6 +9,7 @@ class PipelineRunInfo(BaseModel):
     dataset_id: UUID
     dataset_name: str
     payload: Optional[Any] = None
+    data_ingestion_info: Optional[list] = None
 
     model_config = {
         "arbitrary_types_allowed": True,
@@ -30,6 +31,11 @@ class PipelineRunCompleted(PipelineRunInfo):
     pass
 
 
+class PipelineRunAlreadyCompleted(PipelineRunInfo):
+    status: str = "PipelineRunAlreadyCompleted"
+    pass
+
+
 class PipelineRunErrored(PipelineRunInfo):
     status: str = "PipelineRunErrored"
     pass

cognee/modules/pipelines/models/__init__.py

@@ -6,3 +6,4 @@ from .PipelineRunInfo import (
     PipelineRunCompleted,
     PipelineRunErrored,
 )
+from .DataItemStatus import DataItemStatus

cognee/modules/pipelines/operations/pipeline.py

@@ -52,6 +52,7 @@ async def cognee_pipeline(
     pipeline_name: str = "custom_pipeline",
     vector_db_config: dict = None,
     graph_db_config: dict = None,
+    incremental_loading: bool = False,
 ):
     # Note: These context variables allow different value assignment for databases in Cognee
     #       per async task, thread, process and etc.
@@ -69,7 +70,10 @@ async def cognee_pipeline(
         cognee_pipeline.first_run = True
 
     if cognee_pipeline.first_run:
-        from cognee.infrastructure.llm.utils import test_llm_connection, test_embedding_connection
+        from cognee.infrastructure.llm.utils import (
+            test_llm_connection,
+            test_embedding_connection,
+        )
 
         # Test LLM and Embedding configuration once before running Cognee
         await test_llm_connection()
@@ -106,6 +110,7 @@ async def cognee_pipeline(
             data=data,
             pipeline_name=pipeline_name,
             context={"dataset": dataset},
+            incremental_loading=incremental_loading,
         ):
             yield run_info
 
@@ -117,6 +122,7 @@ async def run_pipeline(
     data=None,
     pipeline_name: str = "custom_pipeline",
     context: dict = None,
+    incremental_loading=False,
 ):
     check_dataset_name(dataset.name)
 
@@ -184,7 +190,9 @@ async def run_pipeline(
         if not isinstance(task, Task):
             raise ValueError(f"Task {task} is not an instance of Task")
 
-    pipeline_run = run_tasks(tasks, dataset_id, data, user, pipeline_name, context)
+    pipeline_run = run_tasks(
+        tasks, dataset_id, data, user, pipeline_name, context, incremental_loading
+    )
 
     async for pipeline_run_info in pipeline_run:
         yield pipeline_run_info