basic-memory 0.7.0__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/markdown/entity_parser.py

@@ -4,25 +4,105 @@ Uses markdown-it with plugins to parse structured data from markdown content.
 """
 
 from dataclasses import dataclass, field
+from datetime import date, datetime
 from pathlib import Path
-from datetime import datetime
 from typing import Any, Optional
-import dateparser
 
-from markdown_it import MarkdownIt
+import dateparser
 import frontmatter
+import yaml
+from loguru import logger
+from markdown_it import MarkdownIt
 
 from basic_memory.markdown.plugins import observation_plugin, relation_plugin
 from basic_memory.markdown.schemas import (
-    EntityMarkdown,
     EntityFrontmatter,
+    EntityMarkdown,
     Observation,
     Relation,
 )
+from basic_memory.utils import parse_tags
+
 
 md = MarkdownIt().use(observation_plugin).use(relation_plugin)
 
 
+def normalize_frontmatter_value(value: Any) -> Any:
+    """Normalize frontmatter values to safe types for processing.
+
+    PyYAML automatically converts various string-like values into native Python types:
+    - Date strings ("2025-10-24") → datetime.date objects
+    - Numbers ("1.0") → int or float
+    - Booleans ("true") → bool
+    - Lists → list objects
+
+    This can cause AttributeError when code expects strings and calls string methods
+    like .strip() on these values (see GitHub issue #236).
+
+    This function normalizes all frontmatter values to safe types:
+    - Dates/datetimes → ISO format strings
+    - Numbers (int/float) → strings
+    - Booleans → strings ("True"/"False")
+    - Lists → preserved as lists, but items are recursively normalized
+    - Dicts → preserved as dicts, but values are recursively normalized
+    - Strings → kept as-is
+    - None → kept as None
+
+    Args:
+        value: The frontmatter value to normalize
+
+    Returns:
+        The normalized value safe for string operations
+
+    Example:
+        >>> normalize_frontmatter_value(datetime.date(2025, 10, 24))
+        '2025-10-24'
+        >>> normalize_frontmatter_value([datetime.date(2025, 10, 24), "tag", 123])
+        ['2025-10-24', 'tag', '123']
+        >>> normalize_frontmatter_value(True)
+        'True'
+    """
+    # Convert date/datetime objects to ISO format strings
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, date):
+        return value.isoformat()
+
+    # Convert boolean to string (must come before int check since bool is subclass of int)
+    if isinstance(value, bool):
+        return str(value)
+
+    # Convert numbers to strings
+    if isinstance(value, (int, float)):
+        return str(value)
+
+    # Recursively process lists (preserve as list, normalize items)
+    if isinstance(value, list):
+        return [normalize_frontmatter_value(item) for item in value]
+
+    # Recursively process dicts (preserve as dict, normalize values)
+    if isinstance(value, dict):
+        return {key: normalize_frontmatter_value(val) for key, val in value.items()}
+
+    # Keep strings and None as-is
+    return value
+
+
+def normalize_frontmatter_metadata(metadata: dict) -> dict:
+    """Normalize all values in frontmatter metadata dict.
+
+    Converts date/datetime objects to ISO format strings to prevent
+    AttributeError when code expects strings (GitHub issue #236).
+
+    Args:
+        metadata: The frontmatter metadata dictionary
+
+    Returns:
+        A new dictionary with all values normalized
+    """
+    return {key: normalize_frontmatter_value(value) for key, value in metadata.items()}
+
+
 @dataclass
 class EntityContent:
     content: str
@@ -56,11 +136,11 @@ def parse(content: str) -> EntityContent:
     )
 
 
-def parse_tags(tags: Any) -> list[str]:
-    """Parse tags into list of strings."""
-    if isinstance(tags, (list, tuple)):
-        return [str(t).strip() for t in tags if str(t).strip()]
-    return [t.strip() for t in tags.split(",") if t.strip()]
+# def parse_tags(tags: Any) -> list[str]:
+#     """Parse tags into list of strings."""
+#     if isinstance(tags, (list, tuple)):
+#         return [str(t).strip() for t in tags if str(t).strip()]
+#     return [t.strip() for t in tags.split(",") if t.strip()]
 
 
 class EntityParser:
@@ -88,33 +168,112 @@ class EntityParser:
                 return parsed
         return None
 
-    async def parse_file(self, file_path: Path) -> EntityMarkdown:
+    async def parse_file(self, path: Path | str) -> EntityMarkdown:
         """Parse markdown file into EntityMarkdown."""
 
-        absolute_path = self.base_path / file_path
+        # Check if the path is already absolute
+        if (
+            isinstance(path, Path)
+            and path.is_absolute()
+            or (isinstance(path, str) and Path(path).is_absolute())
+        ):
+            absolute_path = Path(path)
+        else:
+            absolute_path = self.get_file_path(path)
+
         # Parse frontmatter and content using python-frontmatter
-        post = frontmatter.load(str(absolute_path))
+        file_content = absolute_path.read_text(encoding="utf-8")
+        return await self.parse_file_content(absolute_path, file_content)
 
-        # Extract file stat info
-        file_stats = absolute_path.stat()
+    def get_file_path(self, path):
+        """Get absolute path for a file using the base path for the project."""
+        return self.base_path / path
 
-        metadata = post.metadata
-        metadata["title"] = post.metadata.get("title", file_path.name)
-        metadata["type"] = post.metadata.get("type", "note")
-        metadata["tags"] = parse_tags(post.metadata.get("tags", []))
+    async def parse_file_content(self, absolute_path, file_content):
+        """Parse markdown content from file stats.
 
-        # frontmatter
-        entity_frontmatter = EntityFrontmatter(
-            metadata=post.metadata,
+        Delegates to parse_markdown_content() for actual parsing logic.
+        Exists for backwards compatibility with code that passes file paths.
+        """
+        # Extract file stat info for timestamps
+        file_stats = absolute_path.stat()
+
+        # Delegate to parse_markdown_content with timestamps from file stats
+        return await self.parse_markdown_content(
+            file_path=absolute_path,
+            content=file_content,
+            mtime=file_stats.st_mtime,
+            ctime=file_stats.st_ctime,
         )
 
+    async def parse_markdown_content(
+        self,
+        file_path: Path,
+        content: str,
+        mtime: Optional[float] = None,
+        ctime: Optional[float] = None,
+    ) -> EntityMarkdown:
+        """Parse markdown content without requiring file to exist on disk.
+
+        Useful for parsing content from S3 or other remote sources where the file
+        is not available locally.
+
+        Args:
+            file_path: Path for metadata (doesn't need to exist on disk)
+            content: Markdown content as string
+            mtime: Optional modification time (Unix timestamp)
+            ctime: Optional creation time (Unix timestamp)
+
+        Returns:
+            EntityMarkdown with parsed content
+        """
+        # Strip BOM before parsing (can be present in files from Windows or certain sources)
+        # See issue #452
+        from basic_memory.file_utils import strip_bom
+
+        content = strip_bom(content)
+
+        # Parse frontmatter with proper error handling for malformed YAML
+        try:
+            post = frontmatter.loads(content)
+        except yaml.YAMLError as e:
+            logger.warning(
+                f"Failed to parse YAML frontmatter in {file_path}: {e}. "
+                f"Treating file as plain markdown without frontmatter."
+            )
+            post = frontmatter.Post(content, metadata={})
+
+        # Normalize frontmatter values
+        metadata = normalize_frontmatter_metadata(post.metadata)
+
+        # Ensure required fields have defaults
+        title = metadata.get("title")
+        if not title or title == "None":
+            metadata["title"] = file_path.stem
+        else:
+            metadata["title"] = title
+
+        entity_type = metadata.get("type")
+        metadata["type"] = entity_type if entity_type is not None else "note"
+
+        tags = parse_tags(metadata.get("tags", []))  # pyright: ignore
+        if tags:
+            metadata["tags"] = tags
+
+        # Parse content for observations and relations
+        entity_frontmatter = EntityFrontmatter(metadata=metadata)
         entity_content = parse(post.content)
 
+        # Use provided timestamps or current time as fallback
+        now = datetime.now().astimezone()
+        created = datetime.fromtimestamp(ctime).astimezone() if ctime else now
+        modified = datetime.fromtimestamp(mtime).astimezone() if mtime else now
+
         return EntityMarkdown(
             frontmatter=entity_frontmatter,
             content=post.content,
             observations=entity_content.observations,
             relations=entity_content.relations,
-            created=datetime.fromtimestamp(file_stats.st_ctime),
-            modified=datetime.fromtimestamp(file_stats.st_mtime),
+            created=created,
+            modified=modified,
         )

basic_memory/markdown/markdown_processor.py

@@ -1,15 +1,19 @@
 from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 from collections import OrderedDict
 
-import frontmatter
 from frontmatter import Post
 from loguru import logger
 
+
 from basic_memory import file_utils
+from basic_memory.file_utils import dump_frontmatter
 from basic_memory.markdown.entity_parser import EntityParser
 from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation
 
+if TYPE_CHECKING:  # pragma: no cover
+    from basic_memory.config import BasicMemoryConfig
+
 
 class DirtyFileError(Exception):
     """Raised when attempting to write to a file that has been modified."""
@@ -35,9 +39,14 @@ class MarkdownProcessor:
     3. Track schema changes (that's done by the database)
     """
 
-    def __init__(self, entity_parser: EntityParser):
-        """Initialize processor with base path and parser."""
+    def __init__(
+        self,
+        entity_parser: EntityParser,
+        app_config: Optional["BasicMemoryConfig"] = None,
+    ):
+        """Initialize processor with parser and optional config."""
         self.entity_parser = entity_parser
+        self.app_config = app_config
 
     async def read_file(self, path: Path) -> EntityMarkdown:
         """Read and parse file into EntityMarkdown schema.
@@ -83,7 +92,7 @@ class MarkdownProcessor:
         """
         # Dirty check if needed
         if expected_checksum is not None:
-            current_content = path.read_text()
+            current_content = path.read_text(encoding="utf-8")
             current_checksum = await file_utils.compute_checksum(current_content)
             if current_checksum != expected_checksum:
                 raise DirtyFileError(f"File {path} has been modified")
@@ -115,14 +124,68 @@ class MarkdownProcessor:
 
         # Create Post object for frontmatter
         post = Post(content, **frontmatter_dict)
-        final_content = frontmatter.dumps(post, sort_keys=False)
+        final_content = dump_frontmatter(post)
 
         logger.debug(f"writing file {path} with content:\n{final_content}")
 
         # Write atomically and return checksum of updated file
         path.parent.mkdir(parents=True, exist_ok=True)
         await file_utils.write_file_atomic(path, final_content)
-        return await file_utils.compute_checksum(final_content)
+
+        # Format file if configured (MarkdownProcessor always handles markdown files)
+        content_for_checksum = final_content
+        if self.app_config:
+            formatted_content = await file_utils.format_file(  # pragma: no cover
+                path, self.app_config, is_markdown=True
+            )
+            if formatted_content is not None:  # pragma: no cover
+                content_for_checksum = formatted_content  # pragma: no cover
+
+        return await file_utils.compute_checksum(content_for_checksum)
+
+    def to_markdown_string(self, markdown: EntityMarkdown) -> str:
+        """Convert EntityMarkdown to markdown string with frontmatter.
+
+        This method handles serialization only - it does not write to files.
+        Use FileService.write_file() to persist the output.
+
+        This enables cloud environments to override file operations via
+        dependency injection while reusing the serialization logic.
+
+        Args:
+            markdown: EntityMarkdown schema to serialize
+
+        Returns:
+            Complete markdown string with frontmatter, content, and structured sections
+        """
+        # Convert frontmatter to dict
+        frontmatter_dict = OrderedDict()
+        frontmatter_dict["title"] = markdown.frontmatter.title
+        frontmatter_dict["type"] = markdown.frontmatter.type
+        frontmatter_dict["permalink"] = markdown.frontmatter.permalink
+
+        metadata = markdown.frontmatter.metadata or {}
+        for k, v in metadata.items():
+            frontmatter_dict[k] = v
+
+        # Start with user content (or minimal title for new files)
+        content = markdown.content or f"# {markdown.frontmatter.title}\n"
+
+        # Add structured sections with proper spacing
+        content = content.rstrip()  # Remove trailing whitespace
+
+        # Add a blank line if we have semantic content
+        if markdown.observations or markdown.relations:
+            content += "\n"
+
+        if markdown.observations:
+            content += self.format_observations(markdown.observations)
+        if markdown.relations:
+            content += self.format_relations(markdown.relations)
+
+        # Create Post object for frontmatter
+        post = Post(content, **frontmatter_dict)
+        return dump_frontmatter(post)
 
     def format_observations(self, observations: list[Observation]) -> str:
         """Format observations section in standard way.

basic_memory/markdown/plugins.py

@@ -8,34 +8,52 @@ from markdown_it.token import Token
 # Observation handling functions
 def is_observation(token: Token) -> bool:
     """Check if token looks like our observation format."""
+    import re
+
     if token.type != "inline":  # pragma: no cover
         return False
-
-    content = token.content.strip()
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
     if not content:  # pragma: no cover
         return False
-
     # if it's a markdown_task, return false
     if content.startswith("[ ]") or content.startswith("[x]") or content.startswith("[-]"):
         return False
 
-    has_category = content.startswith("[") and "]" in content
-    has_tags = "#" in content
-    return has_category or has_tags
+    # Exclude markdown links: [text](url)
+    if re.match(r"^\[.*?\]\(.*?\)$", content):
+        return False
+
+    # Exclude wiki links: [[text]]
+    if re.match(r"^\[\[.*?\]\]$", content):
+        return False
+
+    # Check for proper observation format: [category] content
+    match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
+    # Check for standalone hashtags (words starting with #)
+    # This excludes # in HTML attributes like color="#4285F4"
+    has_tags = any(part.startswith("#") for part in content.split())
+    return bool(match) or has_tags
 
 
 def parse_observation(token: Token) -> Dict[str, Any]:
     """Extract observation parts from token."""
-    # Strip bullet point if present
-    content = token.content.strip()
+    import re
+
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
 
-    # Parse [category]
+    # Parse [category] with regex
+    match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
     category = None
-    if content.startswith("["):
-        end = content.find("]")
-        if end != -1:
-            category = content[1:end].strip() or None  # Convert empty to None
-            content = content[end + 1 :].strip()
+    if match:
+        category = match.group(1).strip()
+        content = match.group(2).strip()
+    else:
+        # Handle empty brackets [] followed by content
+        empty_match = re.match(r"^\[\]\s+(.+)", content)
+        if empty_match:
+            content = empty_match.group(1).strip()
 
     # Parse (context)
     context = None
@@ -50,9 +68,7 @@ def parse_observation(token: Token) -> Dict[str, Any]:
     parts = content.split()
     for part in parts:
         if part.startswith("#"):
-            # Handle multiple #tags stuck together
             if "#" in part[1:]:
-                # Split on # but keep non-empty tags
                 subtags = [t for t in part.split("#") if t]
                 tags.extend(subtags)
             else:
@@ -72,14 +88,16 @@ def is_explicit_relation(token: Token) -> bool:
     if token.type != "inline":  # pragma: no cover
         return False
 
-    content = token.content.strip()
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
     return "[[" in content and "]]" in content
 
 
 def parse_relation(token: Token) -> Dict[str, Any] | None:
     """Extract relation parts from token."""
     # Remove bullet point if present
-    content = token.content.strip()
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
 
     # Extract [[target]]
     target = None
@@ -144,7 +162,7 @@ def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
 
         target = content[start + 2 : end].strip()
         if target:
-            relations.append({"type": "links to", "target": target, "context": None})
+            relations.append({"type": "links_to", "target": target, "context": None})
 
         start = end + 2
 
@@ -213,10 +231,12 @@ def relation_plugin(md: MarkdownIt) -> None:
                         token.meta["relations"] = [rel]
 
                 # Always check for inline links in any text
-                elif "[[" in token.content:
-                    rels = parse_inline_relations(token.content)
-                    if rels:
-                        token.meta["relations"] = token.meta.get("relations", []) + rels
+                else:
+                    content = token.tag or token.content
+                    if "[[" in content:
+                        rels = parse_inline_relations(content)
+                        if rels:
+                            token.meta["relations"] = token.meta.get("relations", []) + rels
 
     # Add the rule after inline processing
     md.core.ruler.after("inline", "relations", relation_rule)

basic_memory/markdown/schemas.py

@@ -42,7 +42,7 @@ class EntityFrontmatter(BaseModel):
 
     @property
     def tags(self) -> List[str]:
-        return self.metadata.get("tags") if self.metadata else []  # pyright: ignore
+        return self.metadata.get("tags") if self.metadata else None  # pyright: ignore
 
     @property
     def title(self) -> str:

basic_memory/markdown/utils.py

@@ -1,17 +1,22 @@
 """Utilities for converting between markdown and entity models."""
 
 from pathlib import Path
-from typing import Optional, Any
+from typing import Any, Optional
+
 
 from frontmatter import Post
 
+from basic_memory.file_utils import has_frontmatter, remove_frontmatter, parse_frontmatter
 from basic_memory.markdown import EntityMarkdown
-from basic_memory.models import Entity, Observation as ObservationModel
-from basic_memory.utils import generate_permalink
+from basic_memory.models import Entity
+from basic_memory.models import Observation as ObservationModel
 
 
 def entity_model_from_markdown(
-    file_path: Path, markdown: EntityMarkdown, entity: Optional[Entity] = None
+    file_path: Path,
+    markdown: EntityMarkdown,
+    entity: Optional[Entity] = None,
+    project_id: Optional[int] = None,
 ) -> Entity:
     """
     Convert markdown entity to model. Does not include relations.
@@ -20,6 +25,7 @@ def entity_model_from_markdown(
         file_path: Path to the markdown file
         markdown: Parsed markdown entity
         entity: Optional existing entity to update
+        project_id: Project ID for new observations (uses entity.project_id if not provided)
 
     Returns:
         Entity model populated from markdown
@@ -31,17 +37,16 @@ def entity_model_from_markdown(
     if not markdown.created or not markdown.modified:  # pragma: no cover
         raise ValueError("Both created and modified dates are required in markdown")
 
-    # Generate permalink if not provided
-    permalink = markdown.frontmatter.permalink or generate_permalink(file_path)
-
     # Create or update entity
     model = entity or Entity()
 
     # Update basic fields
     model.title = markdown.frontmatter.title
     model.entity_type = markdown.frontmatter.type
-    model.permalink = permalink
-    model.file_path = str(file_path)
+    # Only update permalink if it exists in frontmatter, otherwise preserve existing
+    if markdown.frontmatter.permalink is not None:
+        model.permalink = markdown.frontmatter.permalink
+    model.file_path = file_path.as_posix()
     model.content_type = "text/markdown"
     model.created_at = markdown.created
     model.updated_at = markdown.modified
@@ -50,9 +55,13 @@ def entity_model_from_markdown(
     metadata = markdown.frontmatter.metadata or {}
     model.entity_metadata = {k: str(v) for k, v in metadata.items() if v is not None}
 
+    # Get project_id from entity if not provided
+    obs_project_id = project_id or (model.project_id if hasattr(model, "project_id") else None)
+
     # Convert observations
     model.observations = [
         ObservationModel(
+            project_id=obs_project_id,
             content=obs.content,
             category=obs.category,
             context=obs.context,
@@ -76,18 +85,33 @@ async def schema_to_markdown(schema: Any) -> Post:
     """
     # Extract content and metadata
     content = schema.content or ""
-    frontmatter_metadata = dict(schema.entity_metadata or {})
+    entity_metadata = dict(schema.entity_metadata or {})
+
+    # if the content contains frontmatter, remove it and merge
+    if has_frontmatter(content):
+        content_frontmatter = parse_frontmatter(content)
+        content = remove_frontmatter(content)
+
+        # Merge content frontmatter with entity metadata
+        # (entity_metadata takes precedence for conflicts)
+        content_frontmatter.update(entity_metadata)
+        entity_metadata = content_frontmatter
 
     # Remove special fields for ordered frontmatter
     for field in ["type", "title", "permalink"]:
-        frontmatter_metadata.pop(field, None)
+        entity_metadata.pop(field, None)
 
-    # Create Post with ordered fields
+    # Create Post with fields ordered by insert order
     post = Post(
         content,
         title=schema.title,
         type=schema.entity_type,
-        permalink=schema.permalink,
-        **frontmatter_metadata,
     )
+    # set the permalink if passed in
+    if schema.permalink:
+        post.metadata["permalink"] = schema.permalink
+
+    if entity_metadata:
+        post.metadata.update(entity_metadata)
+
     return post