basic-memory 0.17.1__py3-none-any.whl

basic_memory/markdown/entity_parser.py

@@ -0,0 +1,279 @@
+"""Parser for markdown files into Entity objects.
+
+Uses markdown-it with plugins to parse structured data from markdown content.
+"""
+
+from dataclasses import dataclass, field
+from datetime import date, datetime
+from pathlib import Path
+from typing import Any, Optional
+
+import dateparser
+import frontmatter
+import yaml
+from loguru import logger
+from markdown_it import MarkdownIt
+
+from basic_memory.markdown.plugins import observation_plugin, relation_plugin
+from basic_memory.markdown.schemas import (
+    EntityFrontmatter,
+    EntityMarkdown,
+    Observation,
+    Relation,
+)
+from basic_memory.utils import parse_tags
+
+
+md = MarkdownIt().use(observation_plugin).use(relation_plugin)
+
+
+def normalize_frontmatter_value(value: Any) -> Any:
+    """Normalize frontmatter values to safe types for processing.
+
+    PyYAML automatically converts various string-like values into native Python types:
+    - Date strings ("2025-10-24") → datetime.date objects
+    - Numbers ("1.0") → int or float
+    - Booleans ("true") → bool
+    - Lists → list objects
+
+    This can cause AttributeError when code expects strings and calls string methods
+    like .strip() on these values (see GitHub issue #236).
+
+    This function normalizes all frontmatter values to safe types:
+    - Dates/datetimes → ISO format strings
+    - Numbers (int/float) → strings
+    - Booleans → strings ("True"/"False")
+    - Lists → preserved as lists, but items are recursively normalized
+    - Dicts → preserved as dicts, but values are recursively normalized
+    - Strings → kept as-is
+    - None → kept as None
+
+    Args:
+        value: The frontmatter value to normalize
+
+    Returns:
+        The normalized value safe for string operations
+
+    Example:
+        >>> normalize_frontmatter_value(datetime.date(2025, 10, 24))
+        '2025-10-24'
+        >>> normalize_frontmatter_value([datetime.date(2025, 10, 24), "tag", 123])
+        ['2025-10-24', 'tag', '123']
+        >>> normalize_frontmatter_value(True)
+        'True'
+    """
+    # Convert date/datetime objects to ISO format strings
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, date):
+        return value.isoformat()
+
+    # Convert boolean to string (must come before int check since bool is subclass of int)
+    if isinstance(value, bool):
+        return str(value)
+
+    # Convert numbers to strings
+    if isinstance(value, (int, float)):
+        return str(value)
+
+    # Recursively process lists (preserve as list, normalize items)
+    if isinstance(value, list):
+        return [normalize_frontmatter_value(item) for item in value]
+
+    # Recursively process dicts (preserve as dict, normalize values)
+    if isinstance(value, dict):
+        return {key: normalize_frontmatter_value(val) for key, val in value.items()}
+
+    # Keep strings and None as-is
+    return value
+
+
+def normalize_frontmatter_metadata(metadata: dict) -> dict:
+    """Normalize all values in frontmatter metadata dict.
+
+    Converts date/datetime objects to ISO format strings to prevent
+    AttributeError when code expects strings (GitHub issue #236).
+
+    Args:
+        metadata: The frontmatter metadata dictionary
+
+    Returns:
+        A new dictionary with all values normalized
+    """
+    return {key: normalize_frontmatter_value(value) for key, value in metadata.items()}
+
+
+@dataclass
+class EntityContent:
+    content: str
+    observations: list[Observation] = field(default_factory=list)
+    relations: list[Relation] = field(default_factory=list)
+
+
+def parse(content: str) -> EntityContent:
+    """Parse markdown content into EntityMarkdown."""
+
+    # Parse content for observations and relations using markdown-it
+    observations = []
+    relations = []
+
+    if content:
+        for token in md.parse(content):
+            # check for observations and relations
+            if token.meta:
+                if "observation" in token.meta:
+                    obs = token.meta["observation"]
+                    observation = Observation.model_validate(obs)
+                    observations.append(observation)
+                if "relations" in token.meta:
+                    rels = token.meta["relations"]
+                    relations.extend([Relation.model_validate(r) for r in rels])
+
+    return EntityContent(
+        content=content,
+        observations=observations,
+        relations=relations,
+    )
+
+
+# def parse_tags(tags: Any) -> list[str]:
+#     """Parse tags into list of strings."""
+#     if isinstance(tags, (list, tuple)):
+#         return [str(t).strip() for t in tags if str(t).strip()]
+#     return [t.strip() for t in tags.split(",") if t.strip()]
+
+
+class EntityParser:
+    """Parser for markdown files into Entity objects."""
+
+    def __init__(self, base_path: Path):
+        """Initialize parser with base path for relative permalink generation."""
+        self.base_path = base_path.resolve()
+
+    def parse_date(self, value: Any) -> Optional[datetime]:
+        """Parse date strings using dateparser for maximum flexibility.
+
+        Supports human friendly formats like:
+        - 2024-01-15
+        - Jan 15, 2024
+        - 2024-01-15 10:00 AM
+        - yesterday
+        - 2 days ago
+        """
+        if isinstance(value, datetime):
+            return value
+        if isinstance(value, str):
+            parsed = dateparser.parse(value)
+            if parsed:
+                return parsed
+        return None
+
+    async def parse_file(self, path: Path | str) -> EntityMarkdown:
+        """Parse markdown file into EntityMarkdown."""
+
+        # Check if the path is already absolute
+        if (
+            isinstance(path, Path)
+            and path.is_absolute()
+            or (isinstance(path, str) and Path(path).is_absolute())
+        ):
+            absolute_path = Path(path)
+        else:
+            absolute_path = self.get_file_path(path)
+
+        # Parse frontmatter and content using python-frontmatter
+        file_content = absolute_path.read_text(encoding="utf-8")
+        return await self.parse_file_content(absolute_path, file_content)
+
+    def get_file_path(self, path):
+        """Get absolute path for a file using the base path for the project."""
+        return self.base_path / path
+
+    async def parse_file_content(self, absolute_path, file_content):
+        """Parse markdown content from file stats.
+
+        Delegates to parse_markdown_content() for actual parsing logic.
+        Exists for backwards compatibility with code that passes file paths.
+        """
+        # Extract file stat info for timestamps
+        file_stats = absolute_path.stat()
+
+        # Delegate to parse_markdown_content with timestamps from file stats
+        return await self.parse_markdown_content(
+            file_path=absolute_path,
+            content=file_content,
+            mtime=file_stats.st_mtime,
+            ctime=file_stats.st_ctime,
+        )
+
+    async def parse_markdown_content(
+        self,
+        file_path: Path,
+        content: str,
+        mtime: Optional[float] = None,
+        ctime: Optional[float] = None,
+    ) -> EntityMarkdown:
+        """Parse markdown content without requiring file to exist on disk.
+
+        Useful for parsing content from S3 or other remote sources where the file
+        is not available locally.
+
+        Args:
+            file_path: Path for metadata (doesn't need to exist on disk)
+            content: Markdown content as string
+            mtime: Optional modification time (Unix timestamp)
+            ctime: Optional creation time (Unix timestamp)
+
+        Returns:
+            EntityMarkdown with parsed content
+        """
+        # Strip BOM before parsing (can be present in files from Windows or certain sources)
+        # See issue #452
+        from basic_memory.file_utils import strip_bom
+
+        content = strip_bom(content)
+
+        # Parse frontmatter with proper error handling for malformed YAML
+        try:
+            post = frontmatter.loads(content)
+        except yaml.YAMLError as e:
+            logger.warning(
+                f"Failed to parse YAML frontmatter in {file_path}: {e}. "
+                f"Treating file as plain markdown without frontmatter."
+            )
+            post = frontmatter.Post(content, metadata={})
+
+        # Normalize frontmatter values
+        metadata = normalize_frontmatter_metadata(post.metadata)
+
+        # Ensure required fields have defaults
+        title = metadata.get("title")
+        if not title or title == "None":
+            metadata["title"] = file_path.stem
+        else:
+            metadata["title"] = title
+
+        entity_type = metadata.get("type")
+        metadata["type"] = entity_type if entity_type is not None else "note"
+
+        tags = parse_tags(metadata.get("tags", []))  # pyright: ignore
+        if tags:
+            metadata["tags"] = tags
+
+        # Parse content for observations and relations
+        entity_frontmatter = EntityFrontmatter(metadata=metadata)
+        entity_content = parse(post.content)
+
+        # Use provided timestamps or current time as fallback
+        now = datetime.now().astimezone()
+        created = datetime.fromtimestamp(ctime).astimezone() if ctime else now
+        modified = datetime.fromtimestamp(mtime).astimezone() if mtime else now
+
+        return EntityMarkdown(
+            frontmatter=entity_frontmatter,
+            content=post.content,
+            observations=entity_content.observations,
+            relations=entity_content.relations,
+            created=created,
+            modified=modified,
+        )

basic_memory/markdown/markdown_processor.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+from collections import OrderedDict
+
+from frontmatter import Post
+from loguru import logger
+
+
+from basic_memory import file_utils
+from basic_memory.file_utils import dump_frontmatter
+from basic_memory.markdown.entity_parser import EntityParser
+from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation
+
+if TYPE_CHECKING:
+    from basic_memory.config import BasicMemoryConfig
+
+
+class DirtyFileError(Exception):
+    """Raised when attempting to write to a file that has been modified."""
+
+    pass
+
+
+class MarkdownProcessor:
+    """Process markdown files while preserving content and structure.
+
+    used only for import
+
+    This class handles the file I/O aspects of our markdown processing. It:
+    1. Uses EntityParser for reading/parsing files into our schema
+    2. Handles writing files with proper frontmatter
+    3. Formats structured sections (observations/relations) consistently
+    4. Preserves user content exactly as written
+    5. Performs atomic writes using temp files
+
+    It does NOT:
+    1. Modify the schema directly (that's done by services)
+    2. Handle in-place updates (everything is read->modify->write)
+    3. Track schema changes (that's done by the database)
+    """
+
+    def __init__(
+        self,
+        entity_parser: EntityParser,
+        app_config: Optional["BasicMemoryConfig"] = None,
+    ):
+        """Initialize processor with parser and optional config."""
+        self.entity_parser = entity_parser
+        self.app_config = app_config
+
+    async def read_file(self, path: Path) -> EntityMarkdown:
+        """Read and parse file into EntityMarkdown schema.
+
+        This is step 1 of our read->modify->write pattern.
+        We use EntityParser to handle all the markdown parsing.
+        """
+        return await self.entity_parser.parse_file(path)
+
+    async def write_file(
+        self,
+        path: Path,
+        markdown: EntityMarkdown,
+        expected_checksum: Optional[str] = None,
+    ) -> str:
+        """Write EntityMarkdown schema back to file.
+
+        This is step 3 of our read->modify->write pattern.
+        The entire file is rewritten atomically on each update.
+
+        File Structure:
+        ---
+        frontmatter fields
+        ---
+        user content area (preserved exactly)
+
+        ## Observations (if any)
+        formatted observations
+
+        ## Relations (if any)
+        formatted relations
+
+        Args:
+            path: Where to write the file
+            markdown: Complete schema to write
+            expected_checksum: If provided, verify file hasn't changed
+
+        Returns:
+            Checksum of written file
+
+        Raises:
+            DirtyFileError: If file has been modified (when expected_checksum provided)
+        """
+        # Dirty check if needed
+        if expected_checksum is not None:
+            current_content = path.read_text(encoding="utf-8")
+            current_checksum = await file_utils.compute_checksum(current_content)
+            if current_checksum != expected_checksum:
+                raise DirtyFileError(f"File {path} has been modified")
+
+        # Convert frontmatter to dict
+        frontmatter_dict = OrderedDict()
+        frontmatter_dict["title"] = markdown.frontmatter.title
+        frontmatter_dict["type"] = markdown.frontmatter.type
+        frontmatter_dict["permalink"] = markdown.frontmatter.permalink
+
+        metadata = markdown.frontmatter.metadata or {}
+        for k, v in metadata.items():
+            frontmatter_dict[k] = v
+
+        # Start with user content (or minimal title for new files)
+        content = markdown.content or f"# {markdown.frontmatter.title}\n"
+
+        # Add structured sections with proper spacing
+        content = content.rstrip()  # Remove trailing whitespace
+
+        # add a blank line if we have semantic content
+        if markdown.observations or markdown.relations:
+            content += "\n"
+
+        if markdown.observations:
+            content += self.format_observations(markdown.observations)
+        if markdown.relations:
+            content += self.format_relations(markdown.relations)
+
+        # Create Post object for frontmatter
+        post = Post(content, **frontmatter_dict)
+        final_content = dump_frontmatter(post)
+
+        logger.debug(f"writing file {path} with content:\n{final_content}")
+
+        # Write atomically and return checksum of updated file
+        path.parent.mkdir(parents=True, exist_ok=True)
+        await file_utils.write_file_atomic(path, final_content)
+
+        # Format file if configured (MarkdownProcessor always handles markdown files)
+        content_for_checksum = final_content
+        if self.app_config:
+            formatted_content = await file_utils.format_file(
+                path, self.app_config, is_markdown=True
+            )
+            if formatted_content is not None:
+                content_for_checksum = formatted_content
+
+        return await file_utils.compute_checksum(content_for_checksum)
+
+    def format_observations(self, observations: list[Observation]) -> str:
+        """Format observations section in standard way.
+
+        Format: - [category] content #tag1 #tag2 (context)
+        """
+        lines = [f"{obs}" for obs in observations]
+        return "\n".join(lines) + "\n"
+
+    def format_relations(self, relations: list[Relation]) -> str:
+        """Format relations section in standard way.
+
+        Format: - relation_type [[target]] (context)
+        """
+        lines = [f"{rel}" for rel in relations]
+        return "\n".join(lines) + "\n"

basic_memory/markdown/plugins.py

@@ -0,0 +1,242 @@
+"""Markdown-it plugins for Basic Memory markdown parsing."""
+
+from typing import List, Any, Dict
+from markdown_it import MarkdownIt
+from markdown_it.token import Token
+
+
+# Observation handling functions
+def is_observation(token: Token) -> bool:
+    """Check if token looks like our observation format."""
+    import re
+
+    if token.type != "inline":  # pragma: no cover
+        return False
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
+    if not content:  # pragma: no cover
+        return False
+    # if it's a markdown_task, return false
+    if content.startswith("[ ]") or content.startswith("[x]") or content.startswith("[-]"):
+        return False
+
+    # Exclude markdown links: [text](url)
+    if re.match(r"^\[.*?\]\(.*?\)$", content):
+        return False
+
+    # Exclude wiki links: [[text]]
+    if re.match(r"^\[\[.*?\]\]$", content):
+        return False
+
+    # Check for proper observation format: [category] content
+    match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
+    # Check for standalone hashtags (words starting with #)
+    # This excludes # in HTML attributes like color="#4285F4"
+    has_tags = any(part.startswith("#") for part in content.split())
+    return bool(match) or has_tags
+
+
+def parse_observation(token: Token) -> Dict[str, Any]:
+    """Extract observation parts from token."""
+    import re
+
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
+
+    # Parse [category] with regex
+    match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
+    category = None
+    if match:
+        category = match.group(1).strip()
+        content = match.group(2).strip()
+    else:
+        # Handle empty brackets [] followed by content
+        empty_match = re.match(r"^\[\]\s+(.+)", content)
+        if empty_match:
+            content = empty_match.group(1).strip()
+
+    # Parse (context)
+    context = None
+    if content.endswith(")"):
+        start = content.rfind("(")
+        if start != -1:
+            context = content[start + 1 : -1].strip()
+            content = content[:start].strip()
+
+    # Extract tags and keep original content
+    tags = []
+    parts = content.split()
+    for part in parts:
+        if part.startswith("#"):
+            if "#" in part[1:]:
+                subtags = [t for t in part.split("#") if t]
+                tags.extend(subtags)
+            else:
+                tags.append(part[1:])
+
+    return {
+        "category": category,
+        "content": content,
+        "tags": tags if tags else None,
+        "context": context,
+    }
+
+
+# Relation handling functions
+def is_explicit_relation(token: Token) -> bool:
+    """Check if token looks like our relation format."""
+    if token.type != "inline":  # pragma: no cover
+        return False
+
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
+    return "[[" in content and "]]" in content
+
+
+def parse_relation(token: Token) -> Dict[str, Any] | None:
+    """Extract relation parts from token."""
+    # Remove bullet point if present
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
+
+    # Extract [[target]]
+    target = None
+    rel_type = "relates_to"  # default
+    context = None
+
+    start = content.find("[[")
+    end = content.find("]]")
+
+    if start != -1 and end != -1:
+        # Get text before link as relation type
+        before = content[:start].strip()
+        if before:
+            rel_type = before
+
+        # Get target
+        target = content[start + 2 : end].strip()
+
+        # Look for context after
+        after = content[end + 2 :].strip()
+        if after.startswith("(") and after.endswith(")"):
+            context = after[1:-1].strip() or None
+
+    if not target:  # pragma: no cover
+        return None
+
+    return {"type": rel_type, "target": target, "context": context}
+
+
+def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
+    """Find wiki-style links in regular content."""
+    relations = []
+    start = 0
+
+    while True:
+        # Find next outer-most [[
+        start = content.find("[[", start)
+        if start == -1:  # pragma: no cover
+            break
+
+        # Find matching ]]
+        depth = 1
+        pos = start + 2
+        end = -1
+
+        while pos < len(content):
+            if content[pos : pos + 2] == "[[":
+                depth += 1
+                pos += 2
+            elif content[pos : pos + 2] == "]]":
+                depth -= 1
+                if depth == 0:
+                    end = pos
+                    break
+                pos += 2
+            else:
+                pos += 1
+
+        if end == -1:
+            # No matching ]] found
+            break
+
+        target = content[start + 2 : end].strip()
+        if target:
+            relations.append({"type": "links_to", "target": target, "context": None})
+
+        start = end + 2
+
+    return relations
+
+
+def observation_plugin(md: MarkdownIt) -> None:
+    """Plugin for parsing observation format:
+    - [category] Content text #tag1 #tag2 (context)
+    - Content text #tag1 (context)  # No category is also valid
+    """
+
+    def observation_rule(state: Any) -> None:
+        """Process observations in token stream."""
+        tokens = state.tokens
+
+        for idx in range(len(tokens)):
+            token = tokens[idx]
+
+            # Initialize meta for all tokens
+            token.meta = token.meta or {}
+
+            # Parse observations in list items
+            if token.type == "inline" and is_observation(token):
+                obs = parse_observation(token)
+                if obs["content"]:  # Only store if we have content
+                    token.meta["observation"] = obs
+
+    # Add the rule after inline processing
+    md.core.ruler.after("inline", "observations", observation_rule)
+
+
+def relation_plugin(md: MarkdownIt) -> None:
+    """Plugin for parsing relation formats:
+
+    Explicit relations:
+    - relation_type [[target]] (context)
+
+    Implicit relations (links in content):
+    Some text with [[target]] reference
+    """
+
+    def relation_rule(state: Any) -> None:
+        """Process relations in token stream."""
+        tokens = state.tokens
+        in_list_item = False
+
+        for idx in range(len(tokens)):
+            token = tokens[idx]
+
+            # Track list nesting
+            if token.type == "list_item_open":
+                in_list_item = True
+            elif token.type == "list_item_close":
+                in_list_item = False
+
+            # Initialize meta for all tokens
+            token.meta = token.meta or {}
+
+            # Only process inline tokens
+            if token.type == "inline":
+                # Check for explicit relations in list items
+                if in_list_item and is_explicit_relation(token):
+                    rel = parse_relation(token)
+                    if rel:
+                        token.meta["relations"] = [rel]
+
+                # Always check for inline links in any text
+                else:
+                    content = token.tag or token.content
+                    if "[[" in content:
+                        rels = parse_inline_relations(content)
+                        if rels:
+                            token.meta["relations"] = token.meta.get("relations", []) + rels
+
+    # Add the rule after inline processing
+    md.core.ruler.after("inline", "relations", relation_rule)