cognee 0.2.2.dev1__py3-none-any.whl → 0.2.3.dev0__py3-none-any.whl

cognee/infrastructure/loaders/LoaderInterface.py

@@ -58,7 +58,7 @@ class LoaderInterface(ABC):
         pass
 
     @abstractmethod
-    async def load(self, file_path: str, file_stream: Optional[Any] = None, **kwargs):
+    async def load(self, file_path: str, **kwargs):
         """
         Load and process the file, returning standardized result.
 

{cognee-0.2.2.dev1.dist-info → cognee-0.2.3.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognee
-Version: 0.2.2.dev1
+Version: 0.2.3.dev0
 Summary: Cognee - is a library for enriching LLM context with a semantic layer for better understanding and reasoning.
 Project-URL: Homepage, https://www.cognee.ai
 Project-URL: Repository, https://github.com/topoteretes/cognee
@@ -230,8 +230,8 @@ Your contributions are at the core of making this a true open source project. An
 
 ## 📦 Installation
 
-You can install Cognee using either **pip**, **poetry**, **uv** or any other python package manager.
-Cognee supports Python 3.8 to 3.12
+You can install Cognee using either **uv**, **pip**, **poetry** or any other python package manager.
+Cognee supports Python 3.10 to 3.13
 
 ### With pip
 
@@ -241,7 +241,7 @@ pip install cognee
 
 ## Local Cognee installation
 
-You can install the local Cognee repo using **pip**, **poetry** and **uv**.
+You can install the local Cognee repo using **uv**, **pip** and **poetry**.
 For local pip installation please make sure your pip version is above version 21.3.
 
 ### with UV with all optional dependencies

{cognee-0.2.2.dev1.dist-info → cognee-0.2.3.dev0.dist-info}/RECORD

@@ -314,7 +314,7 @@ cognee/infrastructure/llm/tokenizer/Mistral/adapter.py,sha256=ZaAuCl9ka-ANzykzzH
 cognee/infrastructure/llm/tokenizer/TikToken/__init__.py,sha256=m1P0VJKiWAKiftLRLLDfEVSuufIxy-T24yMjVm4FsgY,39
 cognee/infrastructure/llm/tokenizer/TikToken/adapter.py,sha256=IDFWGQe5Z8PmhDu0bRoQLreYqicUtzGCEp-_9F4_amE,3581
 cognee/infrastructure/loaders/LoaderEngine.py,sha256=brNSioTh6Ya6yMWAnVhqX5G93iixSbyM16cUvbt5z5k,5064
-cognee/infrastructure/loaders/LoaderInterface.py,sha256=PKkmQRQ9RHyVyL8mwg1BCe5pcRl8JQYgayQLUJimlIM,1967
+cognee/infrastructure/loaders/LoaderInterface.py,sha256=sJjcHAiEkzraBfKcgj9NoLi3WZnQqnf1H2qXewJwoxw,1932
 cognee/infrastructure/loaders/__init__.py,sha256=onz7sA9r6p5Vq28IsD12eOqF0JixUvB3WDBO4t8hY0g,643
 cognee/infrastructure/loaders/create_loader_engine.py,sha256=mgo1RlLJ4E3exdEE03CDsrpdeMoucwrOql-1hZ8Ko00,964
 cognee/infrastructure/loaders/get_loader_engine.py,sha256=cPJefAHFAWU1HXQoWqCpwh8vRtoz1poPSw6wKyECg8k,517
@@ -746,8 +746,25 @@ cognee/tests/unit/processing/chunks/chunk_by_sentence_test.py,sha256=j0zcTrGBe_s
 cognee/tests/unit/processing/chunks/chunk_by_word_test.py,sha256=tkZJgZMAIwyqqjc9aEeor4r4_1CD7PlyXQCCTiWw8U4,1177
 cognee/tests/unit/processing/chunks/test_input.py,sha256=T1TFdZWBuV7Mwm6kD-It2sMCRTozGjgP8YcMNgT7j_E,9463
 cognee/tests/unit/processing/utils/utils_test.py,sha256=VQ03g2r2thojM7dLnY7egtXXCpJuxcEcSxPtTZa4b0U,1503
-cognee-0.2.2.dev1.dist-info/METADATA,sha256=V7LPZ7zEEZGxPB45SaRiaYdICRmjFSEMiTMhjrK8vf0,14674
-cognee-0.2.2.dev1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognee-0.2.2.dev1.dist-info/licenses/LICENSE,sha256=pHHjSQj1DD8SDppW88MMs04TPk7eAanL1c5xj8NY7NQ,11344
-cognee-0.2.2.dev1.dist-info/licenses/NOTICE.md,sha256=6L3saP3kSpcingOxDh-SGjMS8GY79Rlh2dBNLaO0o5c,339
-cognee-0.2.2.dev1.dist-info/RECORD,,
+distributed/Dockerfile,sha256=co4eAKKe8wOWmvdrbvBP_S7aYQSho6SdDFgnYrLarX4,756
+distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+distributed/app.py,sha256=ZbnFimZvsgDuXWA6F-5o1KQJcJUGbJkuqS9MZbtdBSA,62
+distributed/entrypoint.py,sha256=tLF8eEfW5oG6obyzBGhJsqy9SXm6xfr5HYU45sMtQEw,2221
+distributed/entrypoint.sh,sha256=6j9Lu_6sxws7E-XGOdtfMWmdm8D3awjtGblB0JsL1I0,94
+distributed/modal_image.py,sha256=TSS0mF8-bjTiMHzyQu2xgq3AX74MRnJKyDgFroHvwiQ,320
+distributed/poetry.lock,sha256=KcXMRPszF6A1SKWSqp1Kwg0CFuq4yJ7nn3TUWJ8LNrA,952845
+distributed/pyproject.toml,sha256=cAU_AZEkP3G2hCtqKFUMVkb2yXMrBn5ZAURlGUoYZjU,4990
+distributed/queues.py,sha256=FF2nyfu1rsDTNKEEC2IAn_A0sULudt4JFxdffSyRPb0,212
+distributed/test.py,sha256=NftfFht_2TP898KwdyJNE6FRYeaDGt4l_74KPVVsGJc,552
+distributed/utils.py,sha256=rW2s7xkMPbh6IANFRVaXPI1pJf9SKRufUw4SfIFIFME,586
+distributed/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+distributed/tasks/queued_add_data_points.py,sha256=thOrQldx4NlEILPoftYtd4jKtYuio3AtugDUdzLPBB4,568
+distributed/tasks/queued_add_edges.py,sha256=kz1DHE05y-kNHORQJjYWHUi6Q1QWUp_v3Dl13ymEZLM,452
+distributed/tasks/queued_add_nodes.py,sha256=aqK4Ij--ADwUWknxYpiwbYrpa6CcvFfqHWbUZW4Kh3A,452
+distributed/workers/data_point_saving_worker.py,sha256=jFmA0-P_0Ru2IUDrSug0wML-5goAKrGtlBm5BA5Ryw4,3229
+distributed/workers/graph_saving_worker.py,sha256=oUYl99CdhlrPAIsUOHbHnS3d4XhGoV0_OIbCO8wYzRg,3648
+cognee-0.2.3.dev0.dist-info/METADATA,sha256=w7q58QNsWxZWW1m8dlmAcISxg6Y4a_SJq0ix7b4fsKo,14675
+cognee-0.2.3.dev0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognee-0.2.3.dev0.dist-info/licenses/LICENSE,sha256=pHHjSQj1DD8SDppW88MMs04TPk7eAanL1c5xj8NY7NQ,11344
+cognee-0.2.3.dev0.dist-info/licenses/NOTICE.md,sha256=6L3saP3kSpcingOxDh-SGjMS8GY79Rlh2dBNLaO0o5c,339
+cognee-0.2.3.dev0.dist-info/RECORD,,

distributed/Dockerfile

@@ -0,0 +1,34 @@
+FROM python:3.11-slim
+
+# Set environment variables
+ENV PIP_NO_CACHE_DIR=true
+ENV PATH="${PATH}:/root/.poetry/bin"
+ENV PYTHONPATH=/app
+ENV RUN_MODE=modal
+ENV SKIP_MIGRATIONS=true
+ENV COGNEE_DISTRIBUTED=true
+
+# System dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libpq-dev \
+    git \
+    curl \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY pyproject.toml poetry.lock README.md /app/
+
+RUN pip install poetry
+
+RUN poetry config virtualenvs.create false
+
+RUN poetry install --extras neo4j --extras postgres --extras aws --extras distributed --no-root
+
+COPY cognee/ /app/cognee
+COPY distributed/ /app/distributed
+RUN chmod +x /app/distributed/entrypoint.sh
+
+ENTRYPOINT ["/app/distributed/entrypoint.sh"]

distributed/app.py

@@ -0,0 +1,4 @@
+from modal import App
+
+
+app = App("cognee_modal_distributed")

distributed/entrypoint.py

@@ -0,0 +1,71 @@
+import os
+import asyncio
+
+import cognee
+from cognee.api.v1.prune import prune
+from cognee.shared.logging_utils import get_logger
+from cognee.modules.engine.operations.setup import setup
+
+from distributed.app import app
+from distributed.queues import add_nodes_and_edges_queue, add_data_points_queue
+from distributed.workers.graph_saving_worker import graph_saving_worker
+from distributed.workers.data_point_saving_worker import data_point_saving_worker
+
+logger = get_logger()
+
+
+os.environ["COGNEE_DISTRIBUTED"] = "True"
+
+
+@app.local_entrypoint()
+async def main():
+    # Clear queues
+    await add_nodes_and_edges_queue.clear.aio()
+    await add_data_points_queue.clear.aio()
+
+    number_of_graph_saving_workers = 1  # Total number of graph_saving_worker to spawn
+    number_of_data_point_saving_workers = 5  # Total number of graph_saving_worker to spawn
+
+    results = []
+    consumer_futures = []
+
+    # await prune.prune_data()  # We don't want to delete files on s3
+    # Delete DBs and saved files from metastore
+    await prune.prune_system(metadata=True)
+
+    await setup()
+
+    # Start graph_saving_worker functions
+    for _ in range(number_of_graph_saving_workers):
+        worker_future = graph_saving_worker.spawn()
+        consumer_futures.append(worker_future)
+
+    # Start data_point_saving_worker functions
+    for _ in range(number_of_data_point_saving_workers):
+        worker_future = data_point_saving_worker.spawn()
+        consumer_futures.append(worker_future)
+
+    s3_bucket_path = os.getenv("S3_BUCKET_PATH")
+    s3_data_path = "s3://" + s3_bucket_path
+
+    await cognee.add(s3_data_path, dataset_name="s3-files")
+
+    await cognee.cognify(datasets=["s3-files"])
+
+    # Push empty tuple into the queue to signal the end of data.
+    await add_nodes_and_edges_queue.put.aio(())
+    await add_data_points_queue.put.aio(())
+
+    for consumer_future in consumer_futures:
+        try:
+            print("Finished but waiting for saving workers to finish.")
+            consumer_final = consumer_future.get()
+            print(f"All workers are done: {consumer_final}")
+        except Exception as e:
+            logger.error(e)
+
+    print(results)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

distributed/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "$@"
+
+exec "$@" # Runs the command passed to the entrypoint script.

distributed/modal_image.py

@@ -0,0 +1,11 @@
+import pathlib
+from os import path
+from modal import Image
+from logging import getLogger
+
+logger = getLogger("modal_image_creation")
+
+image = Image.from_dockerfile(
+    path=pathlib.Path(path.join(path.dirname(__file__), "Dockerfile")).resolve(),
+    force_build=False,
+).add_local_python_source("cognee", "entrypoint")