cognee 0.5.1.dev0__py3-none-any.whl → 0.5.2.dev0__py3-none-any.whl

cognee/infrastructure/files/utils/get_data_file_path.py

@@ -1,44 +1,50 @@
 import os
-from urllib.parse import urlparse
+import posixpath
+from urllib.parse import urlparse, unquote
 
 
-def get_data_file_path(file_path: str):
-    # Check if this is a file URI BEFORE normalizing (which corrupts URIs)
-    if file_path.startswith("file://"):
-        # Remove first occurrence of file:// prefix
-        pure_file_path = file_path.replace("file://", "", 1)
-        # Normalize the file URI for Windows - replace backslashes with forward slashes
-        normalized_file_uri = os.path.normpath(pure_file_path)
+def get_data_file_path(file_path: str) -> str:
+    """Normalize file paths from various URI schemes to filesystem paths.
 
-        # Convert path to proper file system path
+    Handles file://, s3://, and regular filesystem paths. Decodes
+    percent-encoded characters and preserves UNC network paths.
+    """
+    parsed = urlparse(file_path)
+
+    if parsed.scheme == "file":
+        # file:///path/to/file -> /path/to/file
+        fs_path = unquote(parsed.path)
+
+        if os.name == "nt" and parsed.netloc:
+            # Handle UNC paths (file://server/share/...)
+            fs_path = f"//{parsed.netloc}{fs_path}"
+
+        # Normalize the file URI for Windows - handle drive letters correctly
         if os.name == "nt":  # Windows
-            # Handle Windows drive letters correctly
-            fs_path = normalized_file_uri
+            # Handle Windows drive letters correctly: /C:/path -> C:/path
             if (
                 (fs_path.startswith("/") or fs_path.startswith("\\"))
-                and len(fs_path) > 1
+                and len(fs_path) > 2
                 and fs_path[2] == ":"
+                and fs_path[1].isalpha()
             ):
                 fs_path = fs_path[1:]
-        else:
-            # Unix - like systems
-            fs_path = normalized_file_uri
 
-        # Now split the actual filesystem path
-        actual_fs_path = os.path.normpath(fs_path)
-        return actual_fs_path
+        return os.path.normpath(fs_path)
 
-    elif file_path.startswith("s3://"):
+    elif parsed.scheme == "s3":
         # Handle S3 URLs without normalization (which corrupts them)
-        parsed_url = urlparse(file_path)
+        if not parsed.path or parsed.path == "/":
+            return f"s3://{parsed.netloc}{parsed.path}"
 
-        normalized_url = (
-            f"s3://{parsed_url.netloc}{os.sep}{os.path.normpath(parsed_url.path).lstrip(os.sep)}"
-        )
+        normalized_path = posixpath.normpath(parsed.path).lstrip("/")
 
-        return normalized_url
+        return f"s3://{parsed.netloc}/{normalized_path}"
 
-    else:
+    elif parsed.scheme == "":
         # Regular file path - normalize separators
-        normalized_path = os.path.normpath(file_path)
-        return normalized_path
+        return os.path.normpath(file_path)
+
+    else:
+        # Other schemes (http, etc.) - return as is or handle as needed
+        return file_path

cognee/infrastructure/llm/prompts/extract_query_time.txt

@@ -10,4 +10,4 @@ Extraction rules:
 5. Current-time references ("now", "current", "today"): If the query explicitly refers to the present, set both starts_at and ends_at to now (the ingestion timestamp).
 6. "Who is" and "Who was" questions: These imply a general identity or biographical inquiry without a specific temporal scope. Set both starts_at and ends_at to None.
 7. Ordering rule: Always ensure the earlier date is assigned to starts_at and the later date to ends_at.
-8. No temporal information: If no valid or inferable time reference is found, set both starts_at and ends_at to None.
+8. No temporal information: If no valid or inferable time reference is found, set both starts_at and ends_at to None.

cognee/infrastructure/llm/prompts/generate_event_entity_prompt.txt

@@ -22,4 +22,4 @@ The `attributes` should be a list of dictionaries, each containing:
 - Relationships should be technical with one or at most two words. If two words, use underscore camelcase style
 - Relationships could imply general meaning like: subject, object, participant, recipient, agent, instrument, tool, source, cause, effect, purpose, manner, resource, etc.
 - You can combine two words to form a relationship name: subject_role, previous_owner, etc.
-- Focus on how the entity specifically relates to the event
+- Focus on how the entity specifically relates to the event

cognee/infrastructure/llm/prompts/generate_event_graph_prompt.txt

@@ -27,4 +27,4 @@ class Event(BaseModel):
     time_from: Optional[Timestamp] = None
     time_to: Optional[Timestamp] = None
     location: Optional[str] = None
-```
+```

cognee/infrastructure/llm/prompts/generate_graph_prompt.txt

@@ -19,8 +19,8 @@ The aim is to achieve simplicity and clarity in the knowledge graph.
   - **Naming Convention**: Use snake_case for relationship names, e.g., `acted_in`.
 # 3. Coreference Resolution
   - **Maintain Entity Consistency**: When extracting entities, it's vital to ensure consistency.
-  If an entity, such as "John Doe", is mentioned multiple times in the text but is referred to by different names or pronouns (e.g., "Joe", "he"),
-  always use the most complete identifier for that entity throughout the knowledge graph. In this example, use "John Doe" as the Persons ID.
+  If an entity, is mentioned multiple times in the text but is referred to by different names or pronouns,
+  always use the most complete identifier for that entity throughout the knowledge graph.
 Remember, the knowledge graph should be coherent and easily understandable, so maintaining consistency in entity references is crucial.
 # 4. Strict Compliance
 Adhere to the rules strictly. Non-compliance will result in termination

cognee/infrastructure/llm/prompts/generate_graph_prompt_guided.txt

@@ -22,7 +22,7 @@ You are an advanced algorithm designed to extract structured information to buil
 3. **Coreference Resolution**:
    - Maintain one consistent node ID for each real-world entity.
    - Resolve aliases, acronyms, and pronouns to the most complete form.
-   - *Example*: Always use "John Doe" even if later referred to as "Doe" or "he".
+   - *Example*: Always use full identifier even if later referred to as in a similar but slightly different way
 
 **Property & Data Guidelines**:
 

cognee/infrastructure/llm/prompts/generate_graph_prompt_oneshot.txt

@@ -42,10 +42,10 @@ You are an advanced algorithm designed to extract structured information from un
 - **Rule**: Resolve all aliases, acronyms, and pronouns to one canonical identifier.
 
 > **One-Shot Example**:
-> **Input**: "John Doe is an author. Later, Doe published a book. He is well-known."
+> **Input**: "X is an author. Later, Doe published a book. He is well-known."
 > **Output Node**:
 > ```
-> John Doe (Person)
+> X (Person)
 > ```
 
 ---

cognee/infrastructure/llm/prompts/generate_graph_prompt_simple.txt

@@ -15,7 +15,7 @@ You are an advanced algorithm that extracts structured data into a knowledge gra
    - Properties are key-value pairs; do not use escaped quotes.
 
 3. **Coreference Resolution**
-   - Use a single, complete identifier for each entity (e.g., always "John Doe" not "Joe" or "he").
+   - Use a single, complete identifier for each entity
 
 4. **Relationship Labels**:
    - Use descriptive, lowercase, snake_case names for edges.

cognee/infrastructure/llm/prompts/generate_graph_prompt_strict.txt

@@ -26,7 +26,7 @@ Use **basic atomic types** for node labels. Always prefer general types over spe
   - Good: "Alan Turing", "Google Inc.", "World War II"
   - Bad: "Entity_001", "1234", "he", "they"
 - Never use numeric or autogenerated IDs.
-- Prioritize **most complete form** of entity names for consistency (e.g., always use "John Doe" instead of "John" or "he").
+- Prioritize **most complete form** of entity names for consistency
 
 2. Dates, Numbers, and Properties
 ---------------------------------

cognee/infrastructure/llm/prompts/search_type_selector_prompt.txt

@@ -2,12 +2,12 @@ You are an expert query analyzer for a **GraphRAG system**. Your primary goal is
 
 Here are the available `SearchType` tools and their specific functions:
 
-- **`SUMMARIES`**: The `SUMMARIES` search type retrieves summarized information from the knowledge graph. 
+- **`SUMMARIES`**: The `SUMMARIES` search type retrieves summarized information from the knowledge graph.
 
-  **Best for:** 
+  **Best for:**
 
-  - Getting concise overviews of topics 
-  - Summarizing large amounts of information 
+  - Getting concise overviews of topics
+  - Summarizing large amounts of information
   - Quick understanding of complex subjects
 
   **Best for:**
@@ -16,7 +16,7 @@ Here are the available `SearchType` tools and their specific functions:
   - Understanding relationships between concepts
   - Exploring the structure of your knowledge graph
 
-* **`CHUNKS`**: The `CHUNKS` search type retrieves specific facts and information chunks from the knowledge graph. 
+* **`CHUNKS`**: The `CHUNKS` search type retrieves specific facts and information chunks from the knowledge graph.
 
   **Best for:**
 
@@ -122,4 +122,4 @@ Response: `NATURAL_LANGUAGE`
 
 
 
-Your response MUST be a single word, consisting of only the chosen `SearchType` name. Do not provide any explanation.
+Your response MUST be a single word, consisting of only the chosen `SearchType` name. Do not provide any explanation.

cognee/infrastructure/llm/prompts/test.txt

@@ -1 +1 @@
-Respond with: test
+Respond with: test

cognee/infrastructure/llm/prompts/translate_content.txt

@@ -0,0 +1,19 @@
+You are an expert translator with deep knowledge of languages, cultures, and linguistics.
+
+Your task is to:
+1. Detect the source language of the provided text if not specified
+2. Translate the text accurately to the target language
+3. Preserve the original meaning, tone, and intent
+4. Maintain proper grammar and natural phrasing in the target language
+
+Guidelines:
+- Preserve technical terms, proper nouns, and specialized vocabulary appropriately
+- Maintain formatting such as paragraphs, lists, and emphasis where applicable
+- If the text contains code, URLs, or other non-translatable content, preserve them as-is
+- Handle idioms and cultural references thoughtfully, adapting when necessary
+- Ensure the translation reads naturally to a native speaker of the target language
+
+Provide the translation in a structured format with:
+- The translated text
+- The detected source language (ISO 639-1 code like "en", "es", "fr", "de", etc.)
+- Any notes about the translation (optional, for ambiguous terms or cultural adaptations)

cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/get_llm_client.py

@@ -34,6 +34,7 @@ class LLMProvider(Enum):
     GEMINI = "gemini"
     MISTRAL = "mistral"
     BEDROCK = "bedrock"
+    LLAMA_CPP = "llama_cpp"
 
 
 def get_llm_client(raise_api_key_error: bool = True):
@@ -187,5 +188,28 @@ def get_llm_client(raise_api_key_error: bool = True):
             instructor_mode=llm_config.llm_instructor_mode.lower(),
         )
 
+    elif provider == LLMProvider.LLAMA_CPP:
+        from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.llama_cpp.adapter import (
+            LlamaCppAPIAdapter,
+        )
+
+        # Get optional local mode parameters (will be None if not set)
+        # TODO: refactor llm_config to include these parameters, currently they cannot be defined and defaults are used
+        model_path = getattr(llm_config, "llama_cpp_model_path", None)
+        n_ctx = getattr(llm_config, "llama_cpp_n_ctx", 2048)
+        n_gpu_layers = getattr(llm_config, "llama_cpp_n_gpu_layers", 0)
+        chat_format = getattr(llm_config, "llama_cpp_chat_format", "chatml")
+
+        return LlamaCppAPIAdapter(
+            model=llm_config.llm_model,
+            max_completion_tokens=max_completion_tokens,
+            instructor_mode=llm_config.llm_instructor_mode.lower(),
+            endpoint=llm_config.llm_endpoint,
+            api_key=llm_config.llm_api_key,
+            model_path=model_path,
+            n_ctx=n_ctx,
+            n_gpu_layers=n_gpu_layers,
+            chat_format=chat_format,
+        )
     else:
         raise UnsupportedLLMProviderError(provider)

cognee/infrastructure/llm/structured_output_framework/litellm_instructor/llm/llama_cpp/adapter.py

@@ -0,0 +1,191 @@
+"""Adapter for Instructor-backed Structured Output Framework for Llama CPP"""
+
+import litellm
+import logging
+import instructor
+from typing import Type, Optional
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+
+from cognee.infrastructure.llm.structured_output_framework.litellm_instructor.llm.llm_interface import (
+    LLMInterface,
+)
+from cognee.shared.logging_utils import get_logger
+from cognee.shared.rate_limiting import llm_rate_limiter_context_manager
+
+from tenacity import (
+    retry,
+    stop_after_delay,
+    wait_exponential_jitter,
+    retry_if_not_exception_type,
+    before_sleep_log,
+)
+
+logger = get_logger()
+
+
+class LlamaCppAPIAdapter(LLMInterface):
+    """
+    Adapter for Llama CPP LLM provider with support for TWO modes:
+
+    1. SERVER MODE (OpenAI-compatible):
+       - Connects to llama-cpp-python server via HTTP (local or remote)
+       - Uses instructor.from_openai()
+       - Requires: endpoint, api_key, model
+
+    2. LOCAL MODE (In-process):
+       - Loads model directly using llama-cpp-python library
+       - Uses instructor.patch() on llama.Llama object
+       - Requires: model_path
+
+    Public methods:
+    - acreate_structured_output
+
+    Instance variables:
+    - name
+    - model (for server mode) or model_path (for local mode)
+    - mode_type: "server" or "local"
+    - max_completion_tokens
+    - aclient
+    """
+
+    name: str
+    model: Optional[str]
+    model_path: Optional[str]
+    mode_type: str  # "server" or "local"
+    default_instructor_mode = instructor.Mode.JSON
+
+    def __init__(
+        self,
+        name: str = "LlamaCpp",
+        max_completion_tokens: int = 2048,
+        instructor_mode: Optional[str] = None,
+        # Server mode parameters
+        endpoint: Optional[str] = None,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        # Local mode parameters
+        model_path: Optional[str] = None,
+        n_ctx: int = 2048,
+        n_gpu_layers: int = 0,
+        chat_format: str = "chatml",
+    ):
+        self.name = name
+        self.max_completion_tokens = max_completion_tokens
+        self.instructor_mode = instructor_mode if instructor_mode else self.default_instructor_mode
+
+        # Determine which mode to use
+        if model_path:
+            self._init_local_mode(model_path, n_ctx, n_gpu_layers, chat_format)
+        elif endpoint:
+            self._init_server_mode(endpoint, api_key, model)
+        else:
+            raise ValueError(
+                "Must provide either 'model_path' (for local mode) or 'endpoint' (for server mode)"
+            )
+
+    def _init_local_mode(self, model_path: str, n_ctx: int, n_gpu_layers: int, chat_format: str):
+        """Initialize local mode using llama-cpp-python library directly"""
+        try:
+            import llama_cpp
+        except ImportError:
+            raise ImportError(
+                "llama-cpp-python is not installed. Install with: pip install llama-cpp-python"
+            )
+
+        logger.info(f"Initializing LlamaCpp in LOCAL mode with model: {model_path}")
+
+        self.mode_type = "local"
+        self.model_path = model_path
+        self.model = None
+
+        # Initialize llama-cpp-python with the model
+        self.llama = llama_cpp.Llama(
+            model_path=model_path,
+            n_gpu_layers=n_gpu_layers,  # -1 for all GPU, 0 for CPU only
+            chat_format=chat_format,
+            n_ctx=n_ctx,
+            verbose=False,
+        )
+
+        self.aclient = instructor.patch(
+            create=self.llama.create_chat_completion_openai_v1,
+            mode=instructor.Mode(self.instructor_mode),
+        )
+
+    def _init_server_mode(self, endpoint: str, api_key: Optional[str], model: Optional[str]):
+        """Initialize server mode connecting to llama-cpp-python server"""
+        logger.info(f"Initializing LlamaCpp in SERVER mode with endpoint: {endpoint}")
+
+        self.mode_type = "server"
+        self.model = model
+        self.model_path = None
+        self.endpoint = endpoint
+        self.api_key = api_key
+
+        # Use instructor.from_openai() for server mode (OpenAI-compatible API)
+        self.aclient = instructor.from_openai(
+            AsyncOpenAI(base_url=self.endpoint, api_key=self.api_key),
+            mode=instructor.Mode(self.instructor_mode),
+        )
+
+    @retry(
+        stop=stop_after_delay(128),
+        wait=wait_exponential_jitter(8, 128),
+        retry=retry_if_not_exception_type(litellm.exceptions.NotFoundError),
+        before_sleep=before_sleep_log(logger, logging.DEBUG),
+        reraise=True,
+    )
+    async def acreate_structured_output(
+        self, text_input: str, system_prompt: str, response_model: Type[BaseModel], **kwargs
+    ) -> BaseModel:
+        """
+        Generate a structured output from the LLM using the provided text and system prompt.
+
+        Works in both local and server modes transparently.
+
+        Parameters:
+        -----------
+            - text_input (str): The input text provided by the user.
+            - system_prompt (str): The system prompt that guides the response generation.
+            - response_model (Type[BaseModel]): The model type that the response should conform to.
+
+        Returns:
+        --------
+            - BaseModel: A structured output that conforms to the specified response model.
+        """
+        async with llm_rate_limiter_context_manager():
+            # Prepare messages (system first, then user is more standard)
+            messages = [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": text_input},
+            ]
+
+            if self.mode_type == "server":
+                # Server mode: use async client with OpenAI-compatible API
+                response = await self.aclient.chat.completions.create(
+                    model=self.model,
+                    messages=messages,
+                    response_model=response_model,
+                    max_retries=2,
+                    max_completion_tokens=self.max_completion_tokens,
+                    **kwargs,
+                )
+
+            else:
+                import asyncio
+
+                # Local mode: instructor.patch() returns a SYNC callable
+                # Per docs: https://python.useinstructor.com/integrations/llama-cpp-python/
+                def _call_sync():
+                    return self.aclient(
+                        messages=messages,
+                        response_model=response_model,
+                        max_tokens=self.max_completion_tokens,
+                        **kwargs,
+                    )
+
+                # Run sync function in thread pool to avoid blocking
+                response = await asyncio.to_thread(_call_sync)
+
+        return response

cognee/modules/chunking/models/DocumentChunk.py

@@ -33,5 +33,4 @@ class DocumentChunk(DataPoint):
     cut_type: str
     is_part_of: Document
     contains: List[Union[Entity, Event, tuple[Edge, Entity]]] = None
-
     metadata: dict = {"index_fields": ["text"]}

cognee/modules/cognify/config.py

@@ -9,6 +9,7 @@ class CognifyConfig(BaseSettings):
     classification_model: object = DefaultContentPrediction
     summarization_model: object = SummarizedContent
     triplet_embedding: bool = False
+    chunks_per_batch: Optional[int] = None
     model_config = SettingsConfigDict(env_file=".env", extra="allow")
 
     def to_dict(self) -> dict:
@@ -16,6 +17,7 @@ class CognifyConfig(BaseSettings):
             "classification_model": self.classification_model,
             "summarization_model": self.summarization_model,
             "triplet_embedding": self.triplet_embedding,
+            "chunks_per_batch": self.chunks_per_batch,
         }
 
 

cognee/modules/data/models/Data.py

@@ -36,6 +36,7 @@ class Data(Base):
     data_size = Column(Integer, nullable=True)  # File size in bytes
     created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
     updated_at = Column(DateTime(timezone=True), onupdate=lambda: datetime.now(timezone.utc))
+    last_accessed = Column(DateTime(timezone=True), nullable=True)
 
     datasets = relationship(
         "Dataset",

cognee/modules/engine/models/Entity.py

@@ -7,5 +7,4 @@ class Entity(DataPoint):
     name: str
     is_a: Optional[EntityType] = None
     description: str
-
     metadata: dict = {"index_fields": ["name"]}

cognee/modules/engine/operations/setup.py

@@ -15,3 +15,9 @@ async def setup():
     """
     await create_relational_db_and_tables()
     await create_pgvector_db_and_tables()
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(setup())