basic-memory 0.0.0__py3-none-any.whl

basic_memory/file_utils.py

@@ -0,0 +1,248 @@
+"""Utilities for file operations."""
+import hashlib
+from pathlib import Path
+from typing import Dict, Any, Tuple
+
+import yaml
+from loguru import logger
+
+
+class FileError(Exception):
+    """Base exception for file operations."""
+    pass
+
+
+class FileWriteError(FileError):
+    """Raised when file operations fail."""
+    pass
+
+
+class ParseError(FileError):
+    """Raised when parsing file content fails."""
+    pass
+
+
+async def compute_checksum(content: str) -> str:
+    """
+    Compute SHA-256 checksum of content.
+    
+    Args:
+        content: Text content to hash
+        
+    Returns:
+        SHA-256 hex digest
+        
+    Raises:
+        FileError: If checksum computation fails
+    """
+    try:
+        return hashlib.sha256(content.encode()).hexdigest()
+    except Exception as e:
+        logger.error(f"Failed to compute checksum: {e}")
+        raise FileError(f"Failed to compute checksum: {e}")
+
+
+async def ensure_directory(path: Path) -> None:
+    """
+    Ensure directory exists, creating if necessary.
+    
+    Args:
+        path: Directory path to ensure
+        
+    Raises:
+        FileWriteError: If directory creation fails
+    """
+    try:
+        path.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        logger.error(f"Failed to create directory: {path}: {e}")
+        raise FileWriteError(f"Failed to create directory {path}: {e}")
+
+
+async def write_file_atomic(path: Path, content: str) -> None:
+    """
+    Write file with atomic operation using temporary file.
+    
+    Args:
+        path: Target file path
+        content: Content to write
+        
+    Raises:
+        FileWriteError: If write operation fails
+    """
+    temp_path = path.with_suffix(".tmp")
+    try:
+        temp_path.write_text(content)
+        
+        # TODO check for path.exists()
+        temp_path.replace(path)
+        logger.debug(f"wrote file: {path}")
+    except Exception as e:
+        temp_path.unlink(missing_ok=True)
+        logger.error(f"Failed to write file: {path}: {e}")
+        raise FileWriteError(f"Failed to write file {path}: {e}")
+
+
+def has_frontmatter(content: str) -> bool:
+    """
+    Check if content contains YAML frontmatter.
+
+    Args:
+        content: Content to check
+
+    Returns:
+        True if content has frontmatter delimiter (---), False otherwise
+    """
+    content = content.strip()
+    return content.startswith("---") and "---" in content[3:]
+
+
+def parse_frontmatter(content: str) -> Dict[str, Any]:
+    """
+    Parse YAML frontmatter from content.
+
+    Args:
+        content: Content with YAML frontmatter
+
+    Returns:
+        Dictionary of frontmatter values
+
+    Raises:
+        ParseError: If frontmatter is invalid or parsing fails
+    """
+    try:
+        if not has_frontmatter(content):
+            raise ParseError("Content has no frontmatter")
+
+        # Split on first two occurrences of ---
+        parts = content.split("---", 2)
+        if len(parts) < 3:
+            raise ParseError("Invalid frontmatter format")
+
+        # Parse YAML
+        try:
+            frontmatter = yaml.safe_load(parts[1])
+            # Handle empty frontmatter (None from yaml.safe_load)
+            if frontmatter is None:
+                return {}
+            if not isinstance(frontmatter, dict):
+                raise ParseError("Frontmatter must be a YAML dictionary")
+            return frontmatter
+
+        except yaml.YAMLError as e:
+            raise ParseError(f"Invalid YAML in frontmatter: {e}")
+
+    except Exception as e:
+        if not isinstance(e, ParseError):
+            logger.error(f"Failed to parse frontmatter: {e}")
+            raise ParseError(f"Failed to parse frontmatter: {e}")
+        raise
+
+
+def remove_frontmatter(content: str) -> str:
+    """
+    Remove YAML frontmatter from content.
+
+    Args:
+        content: Content with frontmatter
+
+    Returns:
+        Content with frontmatter removed
+
+    Raises:
+        ParseError: If frontmatter format is invalid
+    """
+    try:
+        if not has_frontmatter(content):
+            return content.strip()
+
+        # Split on first two occurrences of ---
+        parts = content.split("---", 2)
+        if len(parts) < 3:
+            raise ParseError("Invalid frontmatter format")
+
+        return parts[2].strip()
+
+    except Exception as e:
+        if not isinstance(e, ParseError):
+            logger.error(f"Failed to remove frontmatter: {e}")
+            raise ParseError(f"Failed to remove frontmatter: {e}")
+        raise
+
+
+def remove_frontmatter_lenient(content: str) -> str:
+    """
+    Remove frontmatter markers and anything between them without validation.
+    
+    This is a more permissive version of remove_frontmatter that doesn't
+    try to validate the YAML content. It simply removes everything between
+    the first two '---' markers if they exist.
+
+    Args:
+        content: Content that may contain frontmatter
+
+    Returns:
+        Content with any frontmatter markers and content removed
+    """
+    content = content.strip()
+    if not content.startswith("---"):
+        return content
+
+    # Find the second marker
+    rest = content[3:].strip()
+    if "---" not in rest:
+        return content
+
+    # Split on the second marker and take everything after
+    parts = rest.split("---", 1)
+    return parts[1].strip()
+
+
+async def add_frontmatter(content: str, frontmatter: Dict[str, Any]) -> str:
+    """
+    Add YAML frontmatter to content.
+    
+    Args:
+        content: Main content text
+        frontmatter: Key-value pairs for frontmatter
+        
+    Returns:
+        Content with YAML frontmatter prepended
+        
+    Raises:
+        ParseError: If YAML serialization fails
+    """
+    try:
+        yaml_fm = yaml.dump(frontmatter, sort_keys=False)
+        return f"---\n{yaml_fm}---\n\n{content.strip()}"
+    except yaml.YAMLError as e:
+        logger.error(f"Failed to add frontmatter: {e}")
+        raise ParseError(f"Failed to add frontmatter: {e}")
+
+
+async def parse_content_with_frontmatter(content: str) -> Tuple[Dict[str, Any], str]:
+    """
+    Parse both frontmatter and content.
+
+    Args:
+        content: Text content with optional frontmatter
+
+    Returns:
+        Tuple of (frontmatter dict, content without frontmatter)
+
+    Raises:
+        ParseError: If parsing fails
+    """
+    try:
+        if not has_frontmatter(content):
+            return {}, content.strip()
+
+        frontmatter = parse_frontmatter(content)
+        remaining = remove_frontmatter(content)
+        return frontmatter, remaining
+
+    except Exception as e:
+        if not isinstance(e, ParseError):
+            logger.error(f"Failed to parse content with frontmatter: {e}")
+            raise ParseError(f"Failed to parse content with frontmatter: {e}")
+        raise

basic_memory/markdown/__init__.py

@@ -0,0 +1,19 @@
+"""Base package for markdown parsing."""
+
+from basic_memory.file_utils import ParseError
+from basic_memory.markdown.entity_parser import EntityParser
+from basic_memory.markdown.schemas import (
+    EntityMarkdown,
+    EntityFrontmatter,
+    Observation,
+    Relation,
+)
+
+__all__ = [
+    "EntityMarkdown",
+    "EntityFrontmatter",
+    "EntityParser",
+    "Observation",
+    "Relation",
+    "ParseError",
+]

basic_memory/markdown/entity_parser.py

@@ -0,0 +1,137 @@
+"""Parser for markdown files into Entity objects.
+
+Uses markdown-it with plugins to parse structured data from markdown content.
+"""
+from dataclasses import dataclass, field
+from pathlib import Path
+from datetime import datetime
+from typing import Any, Optional
+import dateparser
+
+from markdown_it import MarkdownIt
+import frontmatter
+
+from basic_memory.markdown.plugins import observation_plugin, relation_plugin
+from basic_memory.markdown.schemas import (
+    EntityMarkdown,
+    EntityFrontmatter,
+    Observation,
+    Relation,
+)
+
+md = MarkdownIt().use(observation_plugin).use(relation_plugin)
+
+@dataclass
+class EntityContent:
+    content: str 
+    observations: list[Observation] = field(default_factory=list) 
+    relations: list[Relation] = field(default_factory=list) 
+
+
+def parse(content: str) -> EntityContent:
+    """Parse markdown content into EntityMarkdown."""
+
+    # Parse content for observations and relations using markdown-it
+    observations = []
+    relations = []
+
+    if content:
+        for token in md.parse(content):
+            # check for observations and relations
+            if token.meta:
+                if "observation" in token.meta:
+                    obs = token.meta["observation"]
+                    observation = Observation.model_validate(obs)
+                    observations.append(observation)
+                if "relations" in token.meta:
+                    rels = token.meta["relations"]
+                    relations.extend([Relation.model_validate(r) for r in rels])
+
+    return EntityContent(
+        content=content,
+        observations=observations,
+        relations=relations,
+    )
+
+def parse_tags(tags: Any) -> list[str]:
+    """Parse tags into list of strings."""
+    if isinstance(tags, str):
+        return [t.strip() for t in tags.split(",") if t.strip()]
+    if isinstance(tags, (list, tuple)):
+        return [str(t).strip() for t in tags if str(t).strip()]
+    return []
+
+class EntityParser:
+    """Parser for markdown files into Entity objects."""
+
+    def __init__(self, base_path: Path):
+        """Initialize parser with base path for relative permalink generation."""
+        self.base_path = base_path.resolve()
+
+
+    def relative_path(self, file_path: Path) -> str:
+        """Get file path relative to base_path.
+
+        Example:
+            base_path: /project/root
+            file_path: /project/root/design/models/data.md
+            returns: "design/models/data"
+        """
+        # Get relative path and remove .md extension
+        rel_path = file_path.resolve().relative_to(self.base_path)
+        if rel_path.suffix.lower() == ".md":
+            return str(rel_path.with_suffix(""))
+        return str(rel_path)
+
+    def parse_date(self, value: Any) -> Optional[datetime]:
+        """Parse date strings using dateparser for maximum flexibility.
+
+        Supports human friendly formats like:
+        - 2024-01-15
+        - Jan 15, 2024
+        - 2024-01-15 10:00 AM
+        - yesterday
+        - 2 days ago
+        """
+        if isinstance(value, datetime):
+            return value
+        if isinstance(value, str):
+            try:
+                parsed = dateparser.parse(value)
+                if parsed:
+                    return parsed
+            except Exception:
+                pass
+        return None
+
+    async def parse_file(self, file_path: Path) -> EntityMarkdown:
+        """Parse markdown file into EntityMarkdown."""
+        
+        absolute_path = self.base_path / file_path
+        # Parse frontmatter and content using python-frontmatter
+        post = frontmatter.load(str(absolute_path))
+        
+        # Extract file stat info
+        file_stats = absolute_path.stat()
+
+        metadata = post.metadata
+        metadata["title"] = post.metadata.get("title", file_path.name)
+        metadata["type"] = metadata.get("type", "note")
+        metadata["tags"] = parse_tags(post.metadata.get("tags", []))
+
+        # frontmatter
+        entity_frontmatter = EntityFrontmatter(
+            metadata=post.metadata,
+        )
+
+        entity_content = parse(post.content)
+
+        return EntityMarkdown(
+            frontmatter=entity_frontmatter,
+            content=post.content,
+            observations=entity_content.observations,
+            relations=entity_content.relations,
+            created=datetime.fromtimestamp(file_stats.st_ctime),
+            modified=datetime.fromtimestamp(file_stats.st_mtime),
+        )
+

basic_memory/markdown/markdown_processor.py

@@ -0,0 +1,153 @@
+"""Process markdown files with structured sections.
+
+This module follows a Read -> Modify -> Write pattern for all file operations:
+1. Read entire file and parse into EntityMarkdown schema
+2. Modify the schema (add relation, update content, etc)
+3. Write entire file atomically using temp file + swap
+
+No in-place updates are performed. Each write reconstructs the entire file from the schema.
+The file format has two distinct types of content:
+1. User content - Free form text that is preserved exactly as written
+2. Structured sections - Observations and Relations that are always formatted
+   in a standard way and can be overwritten since they're tracked in our schema
+"""
+
+from pathlib import Path
+from typing import Optional
+from collections import OrderedDict
+
+import frontmatter
+from frontmatter import Post
+from loguru import logger
+
+from basic_memory import file_utils
+from basic_memory.markdown.entity_parser import EntityParser
+from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation
+
+
+class DirtyFileError(Exception):
+    """Raised when attempting to write to a file that has been modified."""
+
+    pass
+
+
+class MarkdownProcessor:
+    """Process markdown files while preserving content and structure.
+
+    This class handles the file I/O aspects of our markdown processing. It:
+    1. Uses EntityParser for reading/parsing files into our schema
+    2. Handles writing files with proper frontmatter
+    3. Formats structured sections (observations/relations) consistently
+    4. Preserves user content exactly as written
+    5. Performs atomic writes using temp files
+
+    It does NOT:
+    1. Modify the schema directly (that's done by services)
+    2. Handle in-place updates (everything is read->modify->write)
+    3. Track schema changes (that's done by the database)
+    """
+
+    def __init__(self, entity_parser: EntityParser):
+        """Initialize processor with base path and parser."""
+        self.entity_parser = entity_parser
+
+    async def read_file(self, path: Path) -> EntityMarkdown:
+        """Read and parse file into EntityMarkdown schema.
+
+        This is step 1 of our read->modify->write pattern.
+        We use EntityParser to handle all the markdown parsing.
+        """
+        return await self.entity_parser.parse_file(path)
+
+    async def write_file(
+        self,
+        path: Path,
+        markdown: EntityMarkdown,
+        expected_checksum: Optional[str] = None,
+    ) -> str:
+        """Write EntityMarkdown schema back to file.
+
+        This is step 3 of our read->modify->write pattern.
+        The entire file is rewritten atomically on each update.
+
+        File Structure:
+        ---
+        frontmatter fields
+        ---
+        user content area (preserved exactly)
+
+        ## Observations (if any)
+        formatted observations
+
+        ## Relations (if any)
+        formatted relations
+
+        Args:
+            path: Where to write the file
+            markdown: Complete schema to write
+            expected_checksum: If provided, verify file hasn't changed
+
+        Returns:
+            Checksum of written file
+
+        Raises:
+            DirtyFileError: If file has been modified (when expected_checksum provided)
+        """
+        # Dirty check if needed
+        if expected_checksum is not None:
+            current_content = path.read_text()
+            current_checksum = await file_utils.compute_checksum(current_content)
+            if current_checksum != expected_checksum:
+                raise DirtyFileError(f"File {path} has been modified")
+
+        # Convert frontmatter to dict
+        frontmatter_dict = OrderedDict()
+        frontmatter_dict["title"] =  markdown.frontmatter.title
+        frontmatter_dict["type"] =  markdown.frontmatter.type
+        frontmatter_dict["permalink"] = markdown.frontmatter.permalink
+        
+        metadata = markdown.frontmatter.metadata or {}
+        for k,v in metadata.items():
+            frontmatter_dict[k] = v
+        
+        # Start with user content (or minimal title for new files)
+        content = markdown.content or f"# {markdown.frontmatter.title}\n"
+
+        # Add structured sections with proper spacing
+        content = content.rstrip()  # Remove trailing whitespace
+
+        # add a blank line if we have semantic content
+        if markdown.observations or markdown.relations:
+            content += "\n"
+            
+        if markdown.observations:
+            content += self.format_observations(markdown.observations)
+        if markdown.relations:
+            content += self.format_relations(markdown.relations)
+
+        # Create Post object for frontmatter
+        post = Post(content, **frontmatter_dict)
+        final_content = frontmatter.dumps(post, sort_keys=False)
+
+        logger.debug(f"writing file {path} with content:\n{final_content}")
+
+        # Write atomically and return checksum of updated file
+        path.parent.mkdir(parents=True, exist_ok=True)
+        await file_utils.write_file_atomic(path, final_content)
+        return await file_utils.compute_checksum(final_content)
+
+    def format_observations(self, observations: list[Observation]) -> str:
+        """Format observations section in standard way.
+
+        Format: - [category] content #tag1 #tag2 (context)
+        """
+        lines = [f"{obs}" for obs in observations]
+        return "\n".join(lines) + "\n"
+
+    def format_relations(self, relations: list[Relation]) -> str:
+        """Format relations section in standard way.
+
+        Format: - relation_type [[target]] (context)
+        """
+        lines = [f"{rel}" for rel in relations]
+        return "\n".join(lines) + "\n"