cognee 0.5.5.dev0__tar.gz → 0.5.5.dev1__tar.gz

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/.dockerignore

@@ -8,7 +8,7 @@ assets
 notebooks
 cognee-frontend
 
-.dlt
+examples/python/.dlt
 .venv
 .github
 

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/.env.template

@@ -254,6 +254,14 @@ COGNEE_CLOUD_AUTH_TOKEN="your-api-key"
 # URL where the frontend is served, defaults to http://localhost:3000
 UI_APP_URL=http://localhost:3000
 
+################################################################################
+#  🔄 DLT Ingestion Settings
+################################################################################
+
+# Maximum number of rows to read per table from DLT sources.
+# Set to 0 to read all rows (no limit). Default: 50
+#DLT_MAX_ROWS_PER_TABLE=50
+
 ################################################################################
 #  🛠️ DEV Settings
 ################################################################################

cognee-0.5.5.dev1/.gitattributes

@@ -0,0 +1,5 @@
+# ignore jupyter notebooks in the language bar on github
+notebooks/** linguist-vendored
+
+# Ensure shell scripts always use LF line endings (fixes Docker on Windows)
+*.sh text eol=lf

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/Dockerfile

@@ -53,7 +53,8 @@ COPY --from=uv /app /app
 # COPY --from=uv /app/.venv /app/.venv
 # COPY --from=uv /root/.local /root/.local
 
-RUN chmod +x /app/entrypoint.sh
+# Strip Windows carriage returns (fixes "no such file" on Windows Docker)
+RUN sed -i 's/\r$//' /app/entrypoint.sh && chmod +x /app/entrypoint.sh
 
 # Place executables in the environment at the front of the path
 ENV PATH="/app/.venv/bin:$PATH"

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognee
-Version: 0.5.5.dev0
+Version: 0.5.5.dev1
 Summary: Cognee - is a library for enriching LLM context with a semantic layer for better understanding and reasoning.
 Project-URL: Homepage, https://www.cognee.ai
 Project-URL: Repository, https://github.com/topoteretes/cognee

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/api/v1/add/add.py

@@ -1,5 +1,6 @@
 from uuid import UUID
 from typing import Union, BinaryIO, List, Optional, Any
+
 from cognee.modules.users.models import User
 from cognee.modules.pipelines import Task, run_pipeline
 from cognee.modules.pipelines.layers.resolve_authorized_user_dataset import (
@@ -11,13 +12,22 @@ from cognee.modules.pipelines.layers.reset_dataset_pipeline_run_status import (
 from cognee.modules.engine.operations.setup import setup
 from cognee.tasks.ingestion import ingest_data, resolve_data_directories
 from cognee.tasks.ingestion.data_item import DataItem
+from cognee.tasks.ingestion.resolve_dlt_sources import resolve_dlt_sources
 from cognee.shared.logging_utils import get_logger
 
 logger = get_logger()
 
 
 async def add(
-    data: Union[BinaryIO, list[BinaryIO], str, list[str], DataItem, list[DataItem]],
+    data: Union[
+        BinaryIO,
+        list[BinaryIO],
+        str,
+        list[str],
+        DataItem,
+        list[DataItem],
+        Any,  # DltResource, SourceFactory, or other dlt types
+    ],
     dataset_name: str = "main_dataset",
     user: User = None,
     node_set: Optional[List[str]] = None,
@@ -27,6 +37,7 @@ async def add(
     preferred_loaders: Optional[List[Union[str, dict[str, dict[str, Any]]]]] = None,
     incremental_loading: bool = True,
     data_per_batch: Optional[int] = 20,
+    **kwargs,
 ):
     """
     Add data to Cognee for knowledge graph processing.
@@ -192,6 +203,15 @@ async def add(
         dataset_name=dataset_name, dataset_id=dataset_id, user=user
     )
 
+    # Expand DLT resources (and auto-detected CSV/connection strings) into
+    # standard DataItems before the pipeline sees them.
+    data = await resolve_dlt_sources(
+        data,
+        dataset_name=dataset_name,
+        user=user,
+        **kwargs,
+    )
+
     await reset_dataset_pipeline_run_status(
         authorized_dataset.id, user, pipeline_names=["add_pipeline", "cognify_pipeline"]
     )

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/api/v1/add/routers/get_add_router.py

@@ -2,8 +2,9 @@ from uuid import UUID
 
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse
-from fastapi import Form, File, UploadFile, Depends
-from typing import List, Optional, Union, Literal
+from fastapi import Form, File, UploadFile as UF, Depends
+from typing import List, Optional, Union, Literal, Annotated
+from pydantic import WithJsonSchema
 
 from cognee.modules.users.models import User
 from cognee.modules.users.methods import get_authenticated_user
@@ -15,6 +16,10 @@ from cognee import __version__ as cognee_version
 
 logger = get_logger()
 
+# NOTE: Needed because of: https://github.com/fastapi/fastapi/discussions/14975
+#       Once issue is resolved on Swagger side it can be removed.
+UploadFile = Annotated[UF, WithJsonSchema({"type": "string", "format": "binary"})]
+
 
 def get_add_router() -> APIRouter:
     router = APIRouter()

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/api/v1/cognify/cognify.py

@@ -1,6 +1,6 @@
 import asyncio
 from pydantic import BaseModel
-from typing import Union, Optional
+from typing import Union, Optional, List
 from uuid import UUID
 
 from cognee.modules.cognify.config import get_cognify_config
@@ -26,6 +26,7 @@ from cognee.tasks.documents import (
 from cognee.tasks.graph import extract_graph_from_data
 from cognee.tasks.storage import add_data_points
 from cognee.tasks.summarization import summarize_text
+from cognee.tasks.ingestion.extract_dlt_fk_edges import extract_dlt_fk_edges
 from cognee.modules.pipelines.layers.pipeline_execution_mode import get_pipeline_executor
 from cognee.tasks.temporal_graph.extract_events_and_entities import extract_events_and_timestamps
 from cognee.tasks.temporal_graph.extract_knowledge_graph_from_events import (
@@ -324,6 +325,7 @@ async def get_default_tasks(  # TODO: Find out a better way to do this (Boris's
             embed_triplets=embed_triplets,
             task_config={"batch_size": chunks_per_batch},
         ),
+        Task(extract_dlt_fk_edges),
     ]
 
     return default_tasks

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/api/v1/update/routers/get_update_router.py

@@ -1,10 +1,11 @@
 from fastapi.responses import JSONResponse
-from fastapi import File, UploadFile, Depends, Form
-from typing import Optional
+from fastapi import File, UploadFile as UF, Depends, Form
+from typing import Optional, Annotated
 from fastapi import APIRouter
 from fastapi.encoders import jsonable_encoder
 from typing import List
 from uuid import UUID
+from pydantic import WithJsonSchema
 from cognee.shared.logging_utils import get_logger
 from cognee.modules.users.models import User
 from cognee.modules.users.methods import get_authenticated_user
@@ -14,6 +15,10 @@ from cognee.modules.pipelines.models.PipelineRunInfo import (
     PipelineRunErrored,
 )
 
+# NOTE: Needed because of: https://github.com/fastapi/fastapi/discussions/14975
+#       Once issue is resolved on Swagger side it can be removed.
+UploadFile = Annotated[UF, WithJsonSchema({"type": "string", "format": "binary"})]
+
 logger = get_logger()
 
 

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/infrastructure/databases/relational/sqlalchemy/SqlAlchemyAdapter.py

@@ -408,7 +408,10 @@ class SQLAlchemyAdapter:
                 # Load table information from schema into MetaData
                 await connection.run_sync(metadata.reflect, schema=schema_name)
                 # Define the full table name
-                full_table_name = f"{schema_name}.{table_name}"
+                if schema_name is None:
+                    full_table_name = table_name
+                else:
+                    full_table_name = f"{schema_name}.{table_name}"
                 # Check if table is in list of tables for the given schema
                 if full_table_name in metadata.tables:
                     return metadata.tables[full_table_name]

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/infrastructure/databases/vector/embeddings/EmbeddingEngine.py

@@ -22,7 +22,7 @@ class EmbeddingEngine(Protocol):
             - list[list[float]]: A list of lists, where each sublist contains the encoded
               representation of the corresponding text input.
         """
-        raise NotImplementedError()
+        raise NotImplementedError("Subclasses must implement embed_text()")
 
     def get_vector_size(self) -> int:
         """
@@ -33,7 +33,7 @@ class EmbeddingEngine(Protocol):
 
             - int: An integer representing the number of dimensions in the embedding vector.
         """
-        raise NotImplementedError()
+        raise NotImplementedError("Subclasses must implement get_vector_size()")
 
     def get_batch_size(self) -> int:
         """
@@ -42,4 +42,4 @@ class EmbeddingEngine(Protocol):
         Returns:
 
         """
-        raise NotImplementedError()
+        raise NotImplementedError("Subclasses must implement get_batch_size()")

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/infrastructure/llm/utils.py

@@ -64,8 +64,13 @@ def get_model_max_completion_tokens(model_name: str):
     max_completion_tokens = None
 
     if model_name in litellm.model_cost:
-        max_completion_tokens = litellm.model_cost[model_name]["max_tokens"]
-        logger.debug(f"Max input tokens for {model_name}: {max_completion_tokens}")
+        if "max_tokens" in litellm.model_cost[model_name]:
+            max_completion_tokens = litellm.model_cost[model_name]["max_tokens"]
+            logger.debug(f"Max input tokens for {model_name}: {max_completion_tokens}")
+        else:
+            logger.debug(
+                f"Model max_tokens not found in LiteLLM's model_cost for model {model_name}."
+            )
     else:
         logger.debug("Model not found in LiteLLM's model_cost.")
 

cognee-0.5.5.dev1/cognee/memify_pipelines/memify_default_tasks.py

@@ -0,0 +1,21 @@
+from cognee.modules.pipelines.tasks.task import Task
+from cognee.tasks.memify.get_triplet_datapoints import get_triplet_datapoints
+from cognee.tasks.memify.extract_user_sessions import extract_user_sessions
+from cognee.tasks.memify.cognify_session import cognify_session
+from cognee.tasks.storage.index_data_points import index_data_points
+
+
+def get_default_memify_extraction_tasks():
+    return [Task(get_triplet_datapoints, triplets_batch_size=100)]
+
+
+def get_default_memify_enrichment_tasks():
+    return [Task(index_data_points, task_config={"batch_size": 100})]
+
+
+def get_session_memify_tasks():
+    """Return (extraction_tasks, enrichment_tasks) for session cognification."""
+    return (
+        [Task(extract_user_sessions)],
+        [Task(cognify_session)],
+    )

cognee-0.5.5.dev1/cognee/modules/data/processing/document_types/DltRowDocument.py

@@ -0,0 +1,36 @@
+from cognee.modules.chunking.Chunker import Chunker
+from cognee.modules.engine.utils.generate_node_id import generate_node_id
+from .Document import Document
+
+
+class DltRowDocument(Document):
+    """Document type for DLT-ingested relational rows.
+
+    Skips chunking entirely — yields a single DocumentChunk containing
+    the enriched schema context text. The graph for DLT data is built
+    deterministically by extract_dlt_fk_edges, so no LLM extraction
+    is needed.
+    """
+
+    type: str = "dlt_row"
+    mime_type: str = "application/x-dlt-row"
+
+    async def read(self, chunker_cls: Chunker, max_chunk_size: int):
+        from cognee.modules.chunking.models.DocumentChunk import DocumentChunk
+        from cognee.infrastructure.files.utils.open_data_file import open_data_file
+
+        async with open_data_file(self.raw_data_location, mode="r", encoding="utf-8") as file:
+            text = file.read()
+
+        if not text or not text.strip():
+            return
+
+        yield DocumentChunk(
+            id=generate_node_id(f"{self.id}_chunk_0"),
+            text=text.strip(),
+            chunk_size=len(text.strip().split()),
+            chunk_index=0,
+            cut_type="dlt_row",
+            is_part_of=self,
+            contains=[],
+        )

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/modules/data/processing/document_types/__init__.py

@@ -5,3 +5,4 @@ from .ImageDocument import ImageDocument
 from .AudioDocument import AudioDocument
 from .UnstructuredDocument import UnstructuredDocument
 from .CsvDocument import CsvDocument
+from .DltRowDocument import DltRowDocument

cognee-0.5.5.dev1/cognee/modules/ingestion/data_types/IngestionData.py

@@ -0,0 +1,14 @@
+from typing import Protocol, BinaryIO, Union
+
+
+class IngestionData(Protocol):
+    data: Union[str, BinaryIO] = None
+
+    def get_data(self):
+        raise NotImplementedError("Subclasses must implement get_data()")
+
+    def get_identifier(self):
+        raise NotImplementedError("Subclasses must implement get_identifier()")
+
+    def get_metadata(self):
+        raise NotImplementedError("Subclasses must implement get_metadata()")

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/modules/memify/memify.py

@@ -86,7 +86,7 @@ async def memify(
         ```
     """
 
-    # Use default coding rules tasks if no tasks were provided
+    # Use default triplet embedding tasks if no tasks were provided
     if not extraction_tasks:
         extraction_tasks = get_default_memify_extraction_tasks()
     if not enrichment_tasks:

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/modules/observability/__init__.py

@@ -61,7 +61,8 @@ def new_span(name: str):
     """Context manager that creates an OTEL span if tracing is enabled, or yields None."""
     if is_tracing_enabled():
         tracer = get_tracer()
-        with tracer.start_as_current_span(name) as span:
-            yield span
-    else:
-        yield _NullSpan()
+        if tracer is not None:
+            with tracer.start_as_current_span(name) as span:
+                yield span
+                return
+    yield _NullSpan()

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/modules/pipelines/operations/run_tasks_data_item.py

@@ -64,12 +64,19 @@ async def run_tasks_data_item_incremental(
     # If incremental_loading of data is set to True don't process documents already processed by pipeline
     # If data is being added to Cognee for the first time calculate the id of the data
     if not isinstance(data_item, Data):
-        file_path = await save_data_item_to_storage(data_item)
-        # Ingest data and add metadata
-        async with open_data_file(file_path) as file:
-            classified_data = ingestion.classify(file)
-            # data_id is the hash of file contents + owner id to avoid duplicate data
-            data_id = await ingestion.identify(classified_data, user)
+        # If the DataItem carries a stable data_id (e.g. from DLT), prefer it
+        # over the content-hash-based ID so lookups stay consistent.
+        from cognee.tasks.ingestion.data_item import DataItem as DataItemType
+
+        if isinstance(data_item, DataItemType) and data_item.data_id is not None:
+            data_id = data_item.data_id
+        else:
+            file_path = await save_data_item_to_storage(data_item)
+            # Ingest data and add metadata
+            async with open_data_file(file_path) as file:
+                classified_data = ingestion.classify(file)
+                # data_id is the hash of file contents + owner id to avoid duplicate data
+                data_id = await ingestion.identify(classified_data, user)
     else:
         # If data was already processed by Cognee get data id
         data_id = data_item.id

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/modules/storage/utils/__init__.py

@@ -20,7 +20,13 @@ class JSONEncoder(json.JSONEncoder):
         return json.JSONEncoder.default(self, obj)
 
 
-def copy_model(model: DataPoint, include_fields: dict = {}, exclude_fields: list = []):
+def copy_model(
+    model: DataPoint, include_fields: dict | None = None, exclude_fields: list | None = None
+):
+    if include_fields is None:
+        include_fields = {}
+    if exclude_fields is None:
+        exclude_fields = []
     fields = {
         name: (field.annotation, field.default if field.default is not None else PydanticUndefined)
         for name, field in model.model_fields.items()

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/shared/utils.py

@@ -94,7 +94,7 @@ async def _send_telemetry_request(payload: dict) -> None:
         logger.debug("Telemetry request failed: %s", e)
 
 
-def send_telemetry(event_name: str, user_id: Union[str, UUID], additional_properties: dict = {}):
+def send_telemetry(event_name: str, user_id: Union[str, UUID], additional_properties: dict = None):
     if additional_properties is None:
         additional_properties = {}
     if os.getenv("TELEMETRY_DISABLED"):

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/tasks/documents/classify_documents.py

@@ -9,10 +9,12 @@ from cognee.modules.data.processing.document_types import (
     TextDocument,
     UnstructuredDocument,
     CsvDocument,
+    DltRowDocument,
 )
 from cognee.modules.engine.models.node_set import NodeSet
 from cognee.modules.engine.utils.generate_node_id import generate_node_id
 from cognee.tasks.documents.exceptions import WrongDataDocumentInputError
+from cognee.tasks.ingestion.dlt_utils import is_dlt_sourced
 
 EXTENSION_TO_DOCUMENT_CLASS = {
     "pdf": PdfDocument,  # Text documents
@@ -122,7 +124,12 @@ async def classify_documents(data_documents: list[Data]) -> list[Document]:
 
     documents = []
     for data_item in data_documents:
-        document = EXTENSION_TO_DOCUMENT_CLASS[data_item.extension](
+        if is_dlt_sourced(data_item):
+            doc_class = DltRowDocument
+        else:
+            doc_class = EXTENSION_TO_DOCUMENT_CLASS[data_item.extension]
+
+        document = doc_class(
             id=data_item.id,
             title=f"{data_item.name}.{data_item.extension}",
             raw_data_location=data_item.raw_data_location,

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/tasks/graph/extract_graph_from_data.py

@@ -160,9 +160,21 @@ async def extract_graph_from_data(
     if not isinstance(graph_model, type) or not issubclass(graph_model, BaseModel):
         raise InvalidGraphModelError(graph_model)
 
+    # Skip LLM extraction for DLT row chunks — their graph is built
+    # deterministically by extract_dlt_fk_edges from schema metadata.
+    from cognee.modules.data.processing.document_types import DltRowDocument
+
+    dlt_chunks = [
+        c for c in data_chunks if isinstance(getattr(c, "is_part_of", None), DltRowDocument)
+    ]
+    non_dlt_chunks = [c for c in data_chunks if c not in dlt_chunks]
+
+    if not non_dlt_chunks:
+        return data_chunks
+
     calculate_chunk_graphs = kwargs.get("calculate_chunk_graphs")
     if callable(calculate_chunk_graphs):
-        extracted = calculate_chunk_graphs(data_chunks, graph_model, custom_prompt, **kwargs)
+        extracted = calculate_chunk_graphs(non_dlt_chunks, graph_model, custom_prompt, **kwargs)
         chunk_graphs = await extracted if inspect.isawaitable(extracted) else extracted
     else:
         chunk_graphs = await asyncio.gather(
@@ -170,7 +182,7 @@ async def extract_graph_from_data(
                 extract_content_graph(
                     chunk.text, graph_model, custom_prompt=custom_prompt, **kwargs
                 )
-                for chunk in data_chunks
+                for chunk in non_dlt_chunks
             ]
         )
 
@@ -207,8 +219,8 @@ async def extract_graph_from_data(
     pipeline_name = context.get("pipeline_name") if isinstance(context, dict) else None
     task_name = "extract_graph_from_data"
 
-    return await integrate_chunk_graphs(
-        data_chunks,
+    integrated = await integrate_chunk_graphs(
+        non_dlt_chunks,
         chunk_graphs,
         graph_model,
         ontology_resolver,
@@ -217,3 +229,5 @@ async def extract_graph_from_data(
         task_name=task_name,
         **kwargs,
     )
+
+    return integrated + dlt_chunks

{cognee-0.5.5.dev0 → cognee-0.5.5.dev1}/cognee/tasks/ingestion/__init__.py

@@ -11,3 +11,4 @@ from .save_data_item_to_storage import save_data_item_to_storage
 from .ingest_data import ingest_data
 from .resolve_data_directories import resolve_data_directories
 from .migrate_relational_database import migrate_relational_database
+from .resolve_dlt_sources import resolve_dlt_sources

cognee-0.5.5.dev1/cognee/tasks/ingestion/config.py

@@ -0,0 +1,18 @@
+from functools import lru_cache
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class IngestionConfig(BaseSettings):
+    dlt_max_rows_per_table: int = 50
+
+    model_config = SettingsConfigDict(env_file=".env", extra="allow")
+
+    def to_dict(self) -> dict:
+        return {
+            "dlt_max_rows_per_table": self.dlt_max_rows_per_table,
+        }
+
+
+@lru_cache
+def get_ingestion_config():
+    return IngestionConfig()

cognee-0.5.5.dev1/cognee/tasks/ingestion/create_dlt_source.py

@@ -0,0 +1,87 @@
+import os
+import re
+from typing import Optional
+
+DB_CONNECTION_PATTERNS = [
+    "postgresql://",
+    "postgres://",
+    "mysql://",
+    "mysql+pymysql://",
+    "sqlite:///",
+    "mssql://",
+    "oracle://",
+]
+
+
+def is_connection_string(data: str) -> bool:
+    return any(data.startswith(p) for p in DB_CONNECTION_PATTERNS)
+
+
+def is_csv_path(data: str) -> bool:
+    return data.lower().endswith(".csv") and not data.startswith(("http://", "https://"))
+
+
+def create_dlt_source_from_connection_string(
+    connection_string: str,
+    query: Optional[str] = None,
+):
+    """Auto-generate a dlt source from a database connection string with optional SQL query filtering."""
+    from dlt.sources.sql_database import sql_database
+    import sqlalchemy
+
+    # SQLite paths must be absolute for SQLAlchemy to find the file.
+    # sqlite:/// = relative, sqlite://// = absolute
+    if connection_string.startswith("sqlite:///") and not connection_string.startswith(
+        "sqlite:////"
+    ):
+        relative_path = connection_string[len("sqlite:///") :]
+        connection_string = "sqlite:///" + os.path.abspath(relative_path)
+
+    if query:
+        table_name, where_clause = _parse_sql_query(query)
+
+        def query_adapter_callback(q, table):
+            if table.name == table_name:
+                return q.where(sqlalchemy.text(where_clause))
+            return q
+
+        source = sql_database(
+            credentials=connection_string,
+            table_names=[table_name],
+            query_adapter_callback=query_adapter_callback,
+        )
+    else:
+        source = sql_database(credentials=connection_string)
+
+    return source
+
+
+def create_dlt_source_from_csv(csv_path: str):
+    """Auto-generate a dlt resource from a CSV file path."""
+    from dlt.sources.filesystem import filesystem, read_csv
+
+    parent_dir = os.path.dirname(os.path.abspath(csv_path))
+    filename = os.path.basename(csv_path)
+
+    return (
+        filesystem(
+            bucket_url=f"file://{parent_dir}",
+            file_glob=filename,
+        )
+        | read_csv()
+    )
+
+
+def _parse_sql_query(query: str) -> tuple:
+    """Extract table name and WHERE clause from a SELECT query.
+    Returns (table_name, where_clause) or raises ValueError."""
+    match = re.match(
+        r"SELECT\s+.+?\s+FROM\s+(\w+)(?:\s+WHERE\s+(.+))?",
+        query.strip(),
+        re.IGNORECASE | re.DOTALL,
+    )
+    if not match:
+        raise ValueError(f"Cannot parse SQL query: {query}")
+    table_name = match.group(1)
+    where_clause = match.group(2) or "1=1"
+    return table_name, where_clause

cognee-0.5.5.dev1/cognee/tasks/ingestion/data_item.py

@@ -0,0 +1,11 @@
+from dataclasses import dataclass, field
+from typing import Any, Optional
+from uuid import UUID
+
+
+@dataclass
+class DataItem:
+    data: Any
+    label: Optional[str] = None
+    external_metadata: Optional[dict] = field(default=None)
+    data_id: Optional[UUID] = None

cognee-0.5.5.dev1/cognee/tasks/ingestion/dlt_row_data.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class DltRowData:
+    """Carries per-row information from ingest_dlt_source back to ingest_data."""
+
+    table_name: str
+    primary_key_column: str
+    primary_key_value: str
+    row_data: dict
+    content_hash: str
+    schema_info: object  # list[dict] for SQLite, dict for Postgres
+    schema_hash: str  # Hash of schema structure for evolution detection
+    foreign_keys: list
+    dlt_db_name: str
+    dataset_name: str

cognee-0.5.5.dev1/cognee/tasks/ingestion/dlt_utils.py

@@ -0,0 +1,38 @@
+"""Shared utilities for DLT ingestion."""
+
+import json
+
+
+def is_dlt_sourced(metadata) -> bool:
+    """Check whether external_metadata indicates a DLT source.
+
+    Accepts a dict, a JSON string, or an object with an ``external_metadata``
+    attribute.  Returns True when ``source == "dlt"``.
+    """
+    ext = getattr(metadata, "external_metadata", metadata)
+    if isinstance(ext, dict):
+        return ext.get("source") == "dlt"
+    if isinstance(ext, str):
+        try:
+            return json.loads(ext).get("source") == "dlt"
+        except (json.JSONDecodeError, TypeError):
+            pass
+    return False
+
+
+def parse_external_metadata(obj) -> dict | None:
+    """Parse external_metadata from an object (may be JSON string or dict).
+
+    Returns the parsed dict or None if parsing fails / not present.
+    """
+    raw = getattr(obj, "external_metadata", None)
+    if not raw:
+        return None
+    if isinstance(raw, dict):
+        return raw
+    if isinstance(raw, str):
+        try:
+            return json.loads(raw)
+        except (json.JSONDecodeError, TypeError):
+            return None
+    return None