cognee 0.2.2.dev1__tar.gz → 0.2.3.dev0__tar.gz

{cognee-0.2.2.dev1 → cognee-0.2.3.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognee
-Version: 0.2.2.dev1
+Version: 0.2.3.dev0
 Summary: Cognee - is a library for enriching LLM context with a semantic layer for better understanding and reasoning.
 Project-URL: Homepage, https://www.cognee.ai
 Project-URL: Repository, https://github.com/topoteretes/cognee
@@ -230,8 +230,8 @@ Your contributions are at the core of making this a true open source project. An
 
 ## 📦 Installation
 
-You can install Cognee using either **pip**, **poetry**, **uv** or any other python package manager.
-Cognee supports Python 3.8 to 3.12
+You can install Cognee using either **uv**, **pip**, **poetry** or any other python package manager.
+Cognee supports Python 3.10 to 3.13
 
 ### With pip
 
@@ -241,7 +241,7 @@ pip install cognee
 
 ## Local Cognee installation
 
-You can install the local Cognee repo using **pip**, **poetry** and **uv**.
+You can install the local Cognee repo using **uv**, **pip** and **poetry**.
 For local pip installation please make sure your pip version is above version 21.3.
 
 ### with UV with all optional dependencies

{cognee-0.2.2.dev1 → cognee-0.2.3.dev0}/README.md

@@ -91,8 +91,8 @@ Your contributions are at the core of making this a true open source project. An
 
 ## 📦 Installation
 
-You can install Cognee using either **pip**, **poetry**, **uv** or any other python package manager.
-Cognee supports Python 3.8 to 3.12
+You can install Cognee using either **uv**, **pip**, **poetry** or any other python package manager.
+Cognee supports Python 3.10 to 3.13
 
 ### With pip
 
@@ -102,7 +102,7 @@ pip install cognee
 
 ## Local Cognee installation
 
-You can install the local Cognee repo using **pip**, **poetry** and **uv**.
+You can install the local Cognee repo using **uv**, **pip** and **poetry**.
 For local pip installation please make sure your pip version is above version 21.3.
 
 ### with UV with all optional dependencies

{cognee-0.2.2.dev1 → cognee-0.2.3.dev0}/cognee/infrastructure/loaders/LoaderInterface.py

@@ -58,7 +58,7 @@ class LoaderInterface(ABC):
         pass
 
     @abstractmethod
-    async def load(self, file_path: str, file_stream: Optional[Any] = None, **kwargs):
+    async def load(self, file_path: str, **kwargs):
         """
         Load and process the file, returning standardized result.
 

cognee-0.2.3.dev0/distributed/Dockerfile

@@ -0,0 +1,34 @@
+FROM python:3.11-slim
+
+# Set environment variables
+ENV PIP_NO_CACHE_DIR=true
+ENV PATH="${PATH}:/root/.poetry/bin"
+ENV PYTHONPATH=/app
+ENV RUN_MODE=modal
+ENV SKIP_MIGRATIONS=true
+ENV COGNEE_DISTRIBUTED=true
+
+# System dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libpq-dev \
+    git \
+    curl \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY pyproject.toml poetry.lock README.md /app/
+
+RUN pip install poetry
+
+RUN poetry config virtualenvs.create false
+
+RUN poetry install --extras neo4j --extras postgres --extras aws --extras distributed --no-root
+
+COPY cognee/ /app/cognee
+COPY distributed/ /app/distributed
+RUN chmod +x /app/distributed/entrypoint.sh
+
+ENTRYPOINT ["/app/distributed/entrypoint.sh"]

cognee-0.2.3.dev0/distributed/app.py

@@ -0,0 +1,4 @@
+from modal import App
+
+
+app = App("cognee_modal_distributed")

cognee-0.2.3.dev0/distributed/entrypoint.py

@@ -0,0 +1,71 @@
+import os
+import asyncio
+
+import cognee
+from cognee.api.v1.prune import prune
+from cognee.shared.logging_utils import get_logger
+from cognee.modules.engine.operations.setup import setup
+
+from distributed.app import app
+from distributed.queues import add_nodes_and_edges_queue, add_data_points_queue
+from distributed.workers.graph_saving_worker import graph_saving_worker
+from distributed.workers.data_point_saving_worker import data_point_saving_worker
+
+logger = get_logger()
+
+
+os.environ["COGNEE_DISTRIBUTED"] = "True"
+
+
+@app.local_entrypoint()
+async def main():
+    # Clear queues
+    await add_nodes_and_edges_queue.clear.aio()
+    await add_data_points_queue.clear.aio()
+
+    number_of_graph_saving_workers = 1  # Total number of graph_saving_worker to spawn
+    number_of_data_point_saving_workers = 5  # Total number of graph_saving_worker to spawn
+
+    results = []
+    consumer_futures = []
+
+    # await prune.prune_data()  # We don't want to delete files on s3
+    # Delete DBs and saved files from metastore
+    await prune.prune_system(metadata=True)
+
+    await setup()
+
+    # Start graph_saving_worker functions
+    for _ in range(number_of_graph_saving_workers):
+        worker_future = graph_saving_worker.spawn()
+        consumer_futures.append(worker_future)
+
+    # Start data_point_saving_worker functions
+    for _ in range(number_of_data_point_saving_workers):
+        worker_future = data_point_saving_worker.spawn()
+        consumer_futures.append(worker_future)
+
+    s3_bucket_path = os.getenv("S3_BUCKET_PATH")
+    s3_data_path = "s3://" + s3_bucket_path
+
+    await cognee.add(s3_data_path, dataset_name="s3-files")
+
+    await cognee.cognify(datasets=["s3-files"])
+
+    # Push empty tuple into the queue to signal the end of data.
+    await add_nodes_and_edges_queue.put.aio(())
+    await add_data_points_queue.put.aio(())
+
+    for consumer_future in consumer_futures:
+        try:
+            print("Finished but waiting for saving workers to finish.")
+            consumer_final = consumer_future.get()
+            print(f"All workers are done: {consumer_final}")
+        except Exception as e:
+            logger.error(e)
+
+    print(results)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

cognee-0.2.3.dev0/distributed/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "$@"
+
+exec "$@" # Runs the command passed to the entrypoint script.

cognee-0.2.3.dev0/distributed/modal_image.py

@@ -0,0 +1,11 @@
+import pathlib
+from os import path
+from modal import Image
+from logging import getLogger
+
+logger = getLogger("modal_image_creation")
+
+image = Image.from_dockerfile(
+    path=pathlib.Path(path.join(path.dirname(__file__), "Dockerfile")).resolve(),
+    force_build=False,
+).add_local_python_source("cognee", "entrypoint")