cognee 0.2.4__py3-none-any.whl → 0.3.0.dev0__py3-none-any.whl

cognee/modules/sync/models/SyncOperation.py

@@ -0,0 +1,142 @@
+from uuid import uuid4
+from enum import Enum
+from typing import Optional, List
+from datetime import datetime, timezone
+from sqlalchemy import (
+    Column,
+    Text,
+    DateTime,
+    UUID as SQLAlchemy_UUID,
+    Integer,
+    Enum as SQLEnum,
+    JSON,
+)
+
+from cognee.infrastructure.databases.relational import Base
+
+
+class SyncStatus(str, Enum):
+    """Enumeration of possible sync operation statuses."""
+
+    STARTED = "started"
+    IN_PROGRESS = "in_progress"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+class SyncOperation(Base):
+    """
+    Database model for tracking sync operations.
+
+    This model stores information about background sync operations,
+    allowing users to monitor progress and query the status of their sync requests.
+    """
+
+    __tablename__ = "sync_operations"
+
+    # Primary identifiers
+    id = Column(SQLAlchemy_UUID, primary_key=True, default=uuid4, doc="Database primary key")
+    run_id = Column(Text, unique=True, index=True, doc="Public run ID returned to users")
+
+    # Status and progress tracking
+    status = Column(
+        SQLEnum(SyncStatus), default=SyncStatus.STARTED, doc="Current status of the sync operation"
+    )
+    progress_percentage = Column(Integer, default=0, doc="Progress percentage (0-100)")
+
+    # Operation metadata
+    dataset_ids = Column(JSON, doc="Array of dataset IDs being synced")
+    dataset_names = Column(JSON, doc="Array of dataset names being synced")
+    user_id = Column(SQLAlchemy_UUID, index=True, doc="ID of the user who initiated the sync")
+
+    # Timing information
+    created_at = Column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(timezone.utc),
+        doc="When the sync was initiated",
+    )
+    started_at = Column(DateTime(timezone=True), doc="When the actual sync processing began")
+    completed_at = Column(
+        DateTime(timezone=True), doc="When the sync finished (success or failure)"
+    )
+
+    # Operation details
+    total_records_to_sync = Column(Integer, doc="Total number of records to sync")
+    total_records_to_download = Column(Integer, doc="Total number of records to download")
+    total_records_to_upload = Column(Integer, doc="Total number of records to upload")
+
+    records_downloaded = Column(Integer, default=0, doc="Number of records successfully downloaded")
+    records_uploaded = Column(Integer, default=0, doc="Number of records successfully uploaded")
+    bytes_downloaded = Column(Integer, default=0, doc="Total bytes downloaded from cloud")
+    bytes_uploaded = Column(Integer, default=0, doc="Total bytes uploaded to cloud")
+
+    # Data lineage tracking per dataset
+    dataset_sync_hashes = Column(
+        JSON, doc="Mapping of dataset_id -> {uploaded: [hashes], downloaded: [hashes]}"
+    )
+
+    # Error handling
+    error_message = Column(Text, doc="Error message if sync failed")
+    retry_count = Column(Integer, default=0, doc="Number of retry attempts")
+
+    def get_duration_seconds(self) -> Optional[float]:
+        """Get the duration of the sync operation in seconds."""
+        if not self.created_at:
+            return None
+
+        end_time = self.completed_at or datetime.now(timezone.utc)
+        return (end_time - self.created_at).total_seconds()
+
+    def get_progress_info(self) -> dict:
+        """Get comprehensive progress information."""
+        total_records_processed = (self.records_downloaded or 0) + (self.records_uploaded or 0)
+        total_bytes_transferred = (self.bytes_downloaded or 0) + (self.bytes_uploaded or 0)
+
+        return {
+            "status": self.status.value,
+            "progress_percentage": self.progress_percentage,
+            "records_processed": f"{total_records_processed}/{self.total_records_to_sync or 'unknown'}",
+            "records_downloaded": self.records_downloaded or 0,
+            "records_uploaded": self.records_uploaded or 0,
+            "bytes_transferred": total_bytes_transferred,
+            "bytes_downloaded": self.bytes_downloaded or 0,
+            "bytes_uploaded": self.bytes_uploaded or 0,
+            "duration_seconds": self.get_duration_seconds(),
+            "error_message": self.error_message,
+            "dataset_sync_hashes": self.dataset_sync_hashes or {},
+        }
+
+    def _get_all_sync_hashes(self) -> List[str]:
+        """Get all content hashes for data created/modified during this sync operation."""
+        all_hashes = set()
+        dataset_hashes = self.dataset_sync_hashes or {}
+
+        for dataset_id, operations in dataset_hashes.items():
+            if isinstance(operations, dict):
+                all_hashes.update(operations.get("uploaded", []))
+                all_hashes.update(operations.get("downloaded", []))
+
+        return list(all_hashes)
+
+    def _get_dataset_sync_hashes(self, dataset_id: str) -> dict:
+        """Get uploaded/downloaded hashes for a specific dataset."""
+        dataset_hashes = self.dataset_sync_hashes or {}
+        return dataset_hashes.get(dataset_id, {"uploaded": [], "downloaded": []})
+
+    def was_data_synced(self, content_hash: str, dataset_id: str = None) -> bool:
+        """
+        Check if a specific piece of data was part of this sync operation.
+
+        Args:
+            content_hash: The content hash to check for
+            dataset_id: Optional - check only within this dataset
+        """
+        if dataset_id:
+            dataset_hashes = self.get_dataset_sync_hashes(dataset_id)
+            return content_hash in dataset_hashes.get(
+                "uploaded", []
+            ) or content_hash in dataset_hashes.get("downloaded", [])
+
+        all_hashes = self.get_all_sync_hashes()
+        return content_hash in all_hashes

cognee/modules/sync/models/__init__.py

@@ -0,0 +1,3 @@
+from .SyncOperation import SyncOperation, SyncStatus
+
+__all__ = ["SyncOperation", "SyncStatus"]

cognee/modules/users/__init__.py

@@ -1,2 +1 @@
 from .get_user_db import get_user_db
-from .get_user_db import get_async_session

cognee/modules/users/methods/__init__.py

@@ -4,4 +4,7 @@ from .delete_user import delete_user
 from .get_default_user import get_default_user
 from .get_user_by_email import get_user_by_email
 from .create_default_user import create_default_user
-from .get_authenticated_user import get_authenticated_user
+from .get_authenticated_user import (
+    get_authenticated_user,
+    REQUIRE_AUTHENTICATION,
+)

cognee/modules/users/methods/create_user.py

@@ -1,6 +1,10 @@
+from uuid import uuid4
 from fastapi_users.exceptions import UserAlreadyExists
-from cognee.modules.users.exceptions import TenantNotFoundError
+
 from cognee.infrastructure.databases.relational import get_relational_engine
+from cognee.modules.notebooks.methods import create_notebook
+from cognee.modules.notebooks.models.Notebook import NotebookCell
+from cognee.modules.users.exceptions import TenantNotFoundError
 from cognee.modules.users.get_user_manager import get_user_manager_context
 from cognee.modules.users.get_user_db import get_user_db_context
 from cognee.modules.users.models.User import UserCreate
@@ -56,6 +60,27 @@ async def create_user(
                     if auto_login:
                         await session.refresh(user)
 
+                    await create_notebook(
+                        user_id=user.id,
+                        notebook_name="Welcome to cognee 🧠",
+                        cells=[
+                            NotebookCell(
+                                id=uuid4(),
+                                name="Welcome",
+                                content="Cognee is your toolkit for turning text into a structured knowledge graph, optionally enhanced by ontologies, and then querying it with advanced retrieval techniques. This notebook will guide you through a simple example.",
+                                type="markdown",
+                            ),
+                            NotebookCell(
+                                id=uuid4(),
+                                name="Example",
+                                content="",
+                                type="code",
+                            ),
+                        ],
+                        deletable=False,
+                        session=session,
+                    )
+
                     return user
     except UserAlreadyExists as error:
         print(f"User {email} already exists")

cognee/modules/users/methods/get_authenticated_user.py

@@ -1,48 +1,42 @@
+import os
+from typing import Optional
+from fastapi import Depends, HTTPException
+from ..models import User
 from ..get_fastapi_users import get_fastapi_users
+from .get_default_user import get_default_user
+from cognee.shared.logging_utils import get_logger
 
 
-fastapi_users = get_fastapi_users()
-
-get_authenticated_user = fastapi_users.current_user(active=True)
-
-# from types import SimpleNamespace
-
-# from ..get_fastapi_users import get_fastapi_users
-# from fastapi import HTTPException, Security
-# from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-# import os
-# import jwt
-
-# from uuid import UUID
-
-# fastapi_users = get_fastapi_users()
+logger = get_logger("get_authenticated_user")
 
-# # Allows Swagger to understand authorization type and allow single sign on for the Swagger docs to test backend
-# bearer_scheme = HTTPBearer(scheme_name="BearerAuth", description="Paste **Bearer <JWT>**")
+# Check environment variable to determine authentication requirement
+REQUIRE_AUTHENTICATION = (
+    os.getenv("REQUIRE_AUTHENTICATION", "false").lower() == "true"
+    or os.getenv("ENABLE_BACKEND_ACCESS_CONTROL", "false").lower() == "true"
+)
 
+fastapi_users = get_fastapi_users()
 
-# async def get_authenticated_user(
-#     creds: HTTPAuthorizationCredentials = Security(bearer_scheme),
-# ) -> SimpleNamespace:
-#     """
-#     Extract and validate the JWT presented in the Authorization header.
-#     """
-#     if creds is None:  # header missing
-#         raise HTTPException(status_code=401, detail="Not authenticated")
-
-#     if creds.scheme.lower() != "bearer":  # shouldn't happen extra guard
-#         raise HTTPException(status_code=401, detail="Invalid authentication scheme")
-
-#     token = creds.credentials
-#     try:
-#         payload = jwt.decode(
-#             token, os.getenv("FASTAPI_USERS_JWT_SECRET", "super_secret"), algorithms=["HS256"]
-#         )
-
-#         auth_data = SimpleNamespace(id=UUID(payload["user_id"]))
-#         return auth_data
-
-#     except jwt.ExpiredSignatureError:
-#         raise HTTPException(status_code=401, detail="Token has expired")
-#     except jwt.InvalidTokenError:
-#         raise HTTPException(status_code=401, detail="Invalid token")
+_auth_dependency = fastapi_users.current_user(active=True, optional=not REQUIRE_AUTHENTICATION)
+
+
+async def get_authenticated_user(
+    user: Optional[User] = Depends(_auth_dependency),
+) -> User:
+    """
+    Get authenticated user with environment-controlled behavior:
+    - If REQUIRE_AUTHENTICATION=true: Enforces authentication (raises 401 if not authenticated)
+    - If REQUIRE_AUTHENTICATION=false: Falls back to default user if not authenticated
+
+    Always returns a User object for consistent typing.
+    """
+    if user is None:
+        # When authentication is optional and user is None, use default user
+        try:
+            user = await get_default_user()
+        except Exception as e:
+            # Convert any get_default_user failure into a proper HTTP 500 error
+            logger.error(f"Failed to create default user: {str(e)}")
+            raise HTTPException(status_code=500, detail=f"Failed to create default user: {str(e)}")
+
+    return user

cognee/modules/users/methods/get_default_user.py

@@ -29,7 +29,9 @@ async def get_default_user() -> SimpleNamespace:
 
             # We return a SimpleNamespace to have the same user type as our SaaS
             # SimpleNamespace is just a dictionary which can be accessed through attributes
-            auth_data = SimpleNamespace(id=user.id, tenant_id=user.tenant_id, roles=[])
+            auth_data = SimpleNamespace(
+                id=user.id, email=user.email, tenant_id=user.tenant_id, roles=[]
+            )
             return auth_data
     except Exception as error:
         if "principals" in str(error.args):

cognee/modules/users/permissions/methods/get_specific_user_permission_datasets.py

@@ -1,4 +1,5 @@
 from uuid import UUID
+from typing import Optional
 from cognee.modules.data.models.Dataset import Dataset
 from cognee.modules.users.permissions.methods.get_all_user_permission_datasets import (
     get_all_user_permission_datasets,
@@ -8,7 +9,7 @@ from cognee.modules.users.methods import get_user
 
 
 async def get_specific_user_permission_datasets(
-    user_id: UUID, permission_type: str, dataset_ids: list[UUID] = None
+    user_id: UUID, permission_type: str, dataset_ids: Optional[list[UUID]] = None
 ) -> list[Dataset]:
     """
         Return a list of datasets user has given permission for. If a list of datasets is provided,

cognee/root_dir.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Optional
 
 ROOT_DIR = Path(__file__).resolve().parent
 
@@ -6,3 +7,21 @@ ROOT_DIR = Path(__file__).resolve().parent
 def get_absolute_path(path_from_root: str) -> str:
     absolute_path = ROOT_DIR / path_from_root
     return str(absolute_path.resolve())
+
+
+def ensure_absolute_path(path: str) -> str:
+    """Ensures a path is absolute.
+
+    Args:
+        path: The path to validate.
+
+    Returns:
+        Absolute path as string
+    """
+    if path is None:
+        raise ValueError("Path cannot be None")
+    path_obj = Path(path).expanduser()
+    if path_obj.is_absolute():
+        return str(path_obj.resolve())
+
+    raise ValueError(f"Path must be absolute. Got relative path: {path}")

cognee/shared/logging_utils.py

@@ -268,7 +268,7 @@ def setup_logging(log_level=None, name=None):
     global _is_structlog_configured
 
     # Regular detailed logging for non-CLI usage
-    log_level = log_level if log_level else log_levels[os.getenv("LOG_LEVEL", "INFO")]
+    log_level = log_level if log_level else log_levels[os.getenv("LOG_LEVEL", "INFO").upper()]
 
     # Configure external library logging early to suppress verbose output
     configure_external_library_logging()

cognee/tasks/codingagents/coding_rule_associations.py

@@ -0,0 +1,127 @@
+from uuid import NAMESPACE_OID, uuid5
+
+from cognee.infrastructure.databases.graph import get_graph_engine
+from cognee.infrastructure.databases.vector import get_vector_engine
+
+from cognee.low_level import DataPoint
+from cognee.infrastructure.llm import LLMGateway
+from cognee.shared.logging_utils import get_logger
+from cognee.modules.engine.models import NodeSet
+from cognee.tasks.storage import add_data_points, index_graph_edges
+from typing import Optional, List, Any
+from pydantic import Field
+
+logger = get_logger("coding_rule_association")
+
+
+class Rule(DataPoint):
+    """A single developer rule extracted from text."""
+
+    text: str = Field(..., description="The coding rule associated with the conversation")
+    belongs_to_set: Optional[NodeSet] = None
+    metadata: dict = {"index_fields": ["rule"]}
+
+
+class RuleSet(DataPoint):
+    """Collection of parsed rules."""
+
+    rules: List[Rule] = Field(
+        ...,
+        description="List of developer rules extracted from the input text. Each rule represents a coding best practice or guideline.",
+    )
+
+
+async def get_existing_rules(rules_nodeset_name: str) -> List[str]:
+    graph_engine = await get_graph_engine()
+    nodes_data, _ = await graph_engine.get_nodeset_subgraph(
+        node_type=NodeSet, node_name=[rules_nodeset_name]
+    )
+
+    existing_rules = [
+        item[1]["text"]
+        for item in nodes_data
+        if isinstance(item, tuple)
+        and len(item) == 2
+        and isinstance(item[1], dict)
+        and "text" in item[1]
+    ]
+
+    return existing_rules
+
+
+async def get_origin_edges(data: str, rules: List[Rule]) -> list[Any]:
+    vector_engine = get_vector_engine()
+
+    origin_chunk = await vector_engine.search("DocumentChunk_text", data, limit=1)
+
+    try:
+        origin_id = origin_chunk[0].id
+    except (AttributeError, KeyError, TypeError, IndexError):
+        origin_id = None
+
+    relationships = []
+
+    if origin_id and isinstance(rules, (list, tuple)) and len(rules) > 0:
+        for rule in rules:
+            try:
+                rule_id = getattr(rule, "id", None)
+                if rule_id is not None:
+                    rel_name = "rule_associated_from"
+                    relationships.append(
+                        (
+                            rule_id,
+                            origin_id,
+                            rel_name,
+                            {
+                                "relationship_name": rel_name,
+                                "source_node_id": rule_id,
+                                "target_node_id": origin_id,
+                                "ontology_valid": False,
+                            },
+                        )
+                    )
+            except Exception as e:
+                logger.info(f"Warning: Skipping invalid rule due to error: {e}")
+    else:
+        logger.info("No valid origin_id or rules provided.")
+
+    return relationships
+
+
+async def add_rule_associations(
+    data: str,
+    rules_nodeset_name: str,
+    user_prompt_location: str = "coding_rule_association_agent_user.txt",
+    system_prompt_location: str = "coding_rule_association_agent_system.txt",
+):
+    if isinstance(data, list):
+        # If data is a list of strings join all strings in list
+        data = " ".join(data)
+
+    graph_engine = await get_graph_engine()
+    existing_rules = await get_existing_rules(rules_nodeset_name=rules_nodeset_name)
+    existing_rules = "\n".join(f"- {rule}" for rule in existing_rules)
+
+    user_context = {"chat": data, "rules": existing_rules}
+
+    user_prompt = LLMGateway.render_prompt(user_prompt_location, context=user_context)
+    system_prompt = LLMGateway.render_prompt(system_prompt_location, context={})
+
+    rule_list = await LLMGateway.acreate_structured_output(
+        text_input=user_prompt, system_prompt=system_prompt, response_model=RuleSet
+    )
+
+    rules_nodeset = NodeSet(
+        id=uuid5(NAMESPACE_OID, name=rules_nodeset_name), name=rules_nodeset_name
+    )
+    for rule in rule_list.rules:
+        rule.belongs_to_set = rules_nodeset
+
+    edges_to_save = await get_origin_edges(data=data, rules=rule_list.rules)
+
+    await add_data_points(data_points=rule_list.rules)
+
+    if len(edges_to_save) > 0:
+        await graph_engine.add_edges(edges_to_save)
+
+    await index_graph_edges()

cognee/tasks/ingestion/save_data_item_to_storage.py

@@ -1,11 +1,15 @@
 import os
+from pathlib import Path
 from urllib.parse import urlparse
 from typing import Union, BinaryIO, Any
 
 from cognee.modules.ingestion.exceptions import IngestionError
 from cognee.modules.ingestion import save_data_to_file
+from cognee.shared.logging_utils import get_logger
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+logger = get_logger()
+
 
 class SaveDataSettings(BaseSettings):
     accept_local_file_path: bool = True
@@ -30,6 +34,16 @@ async def save_data_item_to_storage(data_item: Union[BinaryIO, str, Any]) -> str
     if isinstance(data_item, str):
         parsed_url = urlparse(data_item)
 
+        try:
+            # In case data item is a string with a relative path transform data item to absolute path and check
+            # if the file exists
+            abs_path = (Path.cwd() / Path(data_item)).resolve()
+            abs_path.is_file()
+        except (OSError, ValueError):
+            # In case file path is too long it's most likely not a relative path
+            logger.debug(f"Data item was too long to be a possible file path: {abs_path}")
+            abs_path = Path("")
+
         # data is s3 file path
         if parsed_url.scheme == "s3":
             return data_item
@@ -56,6 +70,15 @@ async def save_data_item_to_storage(data_item: Union[BinaryIO, str, Any]) -> str
                 return file_path
             else:
                 raise IngestionError(message="Local files are not accepted.")
+        # Data is a relative file path
+        elif abs_path.is_file():
+            if settings.accept_local_file_path:
+                # Normalize path separators before creating file URL
+                normalized_path = os.path.normpath(abs_path)
+                # Use forward slashes in file URLs for consistency
+                url_path = normalized_path.replace(os.sep, "/")
+                file_path = "file://" + url_path
+                return file_path
 
         # data is text, save it to data storage and return the file path
         return await save_data_to_file(data_item)

cognee/tasks/memify/__init__.py

@@ -0,0 +1,2 @@
+from .extract_subgraph import extract_subgraph
+from .extract_subgraph_chunks import extract_subgraph_chunks

cognee/tasks/memify/extract_subgraph.py

@@ -0,0 +1,7 @@
+from cognee.modules.graph.cognee_graph.CogneeGraph import CogneeGraph
+
+
+async def extract_subgraph(subgraphs: list[CogneeGraph]):
+    for subgraph in subgraphs:
+        for edge in subgraph.edges:
+            yield edge

cognee/tasks/memify/extract_subgraph_chunks.py

@@ -0,0 +1,11 @@
+from cognee.modules.graph.cognee_graph.CogneeGraph import CogneeGraph
+
+
+async def extract_subgraph_chunks(subgraphs: list[CogneeGraph]):
+    """
+    Get all Document Chunks from subgraphs and forward to next task in pipeline
+    """
+    for subgraph in subgraphs:
+        for node in subgraph.nodes.values():
+            if node.attributes["type"] == "DocumentChunk":
+                yield node.attributes["text"]