basic-memory 0.7.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/importers/utils.py

@@ -0,0 +1,58 @@
+"""Utility functions for import services."""
+
+import re
+from datetime import datetime
+from typing import Any
+
+
+def clean_filename(name: str) -> str:  # pragma: no cover
+    """Clean a string to be used as a filename.
+
+    Args:
+        name: The string to clean.
+
+    Returns:
+        A cleaned string suitable for use as a filename.
+    """
+    # Replace common punctuation and whitespace with underscores
+    name = re.sub(r"[\s\-,.:/\\\[\]\(\)]+", "_", name)
+    # Remove any non-alphanumeric or underscore characters
+    name = re.sub(r"[^\w]+", "", name)
+    # Ensure the name isn't too long
+    if len(name) > 100:  # pragma: no cover
+        name = name[:100]
+    # Ensure the name isn't empty
+    if not name:  # pragma: no cover
+        name = "untitled"
+    return name
+
+
+def format_timestamp(timestamp: Any) -> str:  # pragma: no cover
+    """Format a timestamp for use in a filename or title.
+
+    Args:
+        timestamp: A timestamp in various formats.
+
+    Returns:
+        A formatted string representation of the timestamp.
+    """
+    if isinstance(timestamp, str):
+        try:
+            # Try ISO format
+            timestamp = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
+        except ValueError:
+            try:
+                # Try unix timestamp as string
+                timestamp = datetime.fromtimestamp(float(timestamp)).astimezone()
+            except ValueError:
+                # Return as is if we can't parse it
+                return timestamp
+    elif isinstance(timestamp, (int, float)):
+        # Unix timestamp
+        timestamp = datetime.fromtimestamp(timestamp).astimezone()
+
+    if isinstance(timestamp, datetime):
+        return timestamp.strftime("%Y-%m-%d %H:%M:%S")
+
+    # Return as is if we can't format it
+    return str(timestamp)  # pragma: no cover

basic_memory/markdown/entity_parser.py

@@ -4,25 +4,104 @@ Uses markdown-it with plugins to parse structured data from markdown content.
 """
 
 from dataclasses import dataclass, field
+from datetime import date, datetime
 from pathlib import Path
-from datetime import datetime
 from typing import Any, Optional
-import dateparser
 
-from markdown_it import MarkdownIt
+import dateparser
 import frontmatter
+import yaml
+from loguru import logger
+from markdown_it import MarkdownIt
 
 from basic_memory.markdown.plugins import observation_plugin, relation_plugin
 from basic_memory.markdown.schemas import (
-    EntityMarkdown,
     EntityFrontmatter,
+    EntityMarkdown,
     Observation,
     Relation,
 )
+from basic_memory.utils import parse_tags
 
 md = MarkdownIt().use(observation_plugin).use(relation_plugin)
 
 
+def normalize_frontmatter_value(value: Any) -> Any:
+    """Normalize frontmatter values to safe types for processing.
+
+    PyYAML automatically converts various string-like values into native Python types:
+    - Date strings ("2025-10-24") → datetime.date objects
+    - Numbers ("1.0") → int or float
+    - Booleans ("true") → bool
+    - Lists → list objects
+
+    This can cause AttributeError when code expects strings and calls string methods
+    like .strip() on these values (see GitHub issue #236).
+
+    This function normalizes all frontmatter values to safe types:
+    - Dates/datetimes → ISO format strings
+    - Numbers (int/float) → strings
+    - Booleans → strings ("True"/"False")
+    - Lists → preserved as lists, but items are recursively normalized
+    - Dicts → preserved as dicts, but values are recursively normalized
+    - Strings → kept as-is
+    - None → kept as None
+
+    Args:
+        value: The frontmatter value to normalize
+
+    Returns:
+        The normalized value safe for string operations
+
+    Example:
+        >>> normalize_frontmatter_value(datetime.date(2025, 10, 24))
+        '2025-10-24'
+        >>> normalize_frontmatter_value([datetime.date(2025, 10, 24), "tag", 123])
+        ['2025-10-24', 'tag', '123']
+        >>> normalize_frontmatter_value(True)
+        'True'
+    """
+    # Convert date/datetime objects to ISO format strings
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if isinstance(value, date):
+        return value.isoformat()
+
+    # Convert boolean to string (must come before int check since bool is subclass of int)
+    if isinstance(value, bool):
+        return str(value)
+
+    # Convert numbers to strings
+    if isinstance(value, (int, float)):
+        return str(value)
+
+    # Recursively process lists (preserve as list, normalize items)
+    if isinstance(value, list):
+        return [normalize_frontmatter_value(item) for item in value]
+
+    # Recursively process dicts (preserve as dict, normalize values)
+    if isinstance(value, dict):
+        return {key: normalize_frontmatter_value(val) for key, val in value.items()}
+
+    # Keep strings and None as-is
+    return value
+
+
+def normalize_frontmatter_metadata(metadata: dict) -> dict:
+    """Normalize all values in frontmatter metadata dict.
+
+    Converts date/datetime objects to ISO format strings to prevent
+    AttributeError when code expects strings (GitHub issue #236).
+
+    Args:
+        metadata: The frontmatter metadata dictionary
+
+    Returns:
+        A new dictionary with all values normalized
+    """
+    return {key: normalize_frontmatter_value(value) for key, value in metadata.items()}
+
+
 @dataclass
 class EntityContent:
     content: str
@@ -56,11 +135,11 @@ def parse(content: str) -> EntityContent:
     )
 
 
-def parse_tags(tags: Any) -> list[str]:
-    """Parse tags into list of strings."""
-    if isinstance(tags, (list, tuple)):
-        return [str(t).strip() for t in tags if str(t).strip()]
-    return [t.strip() for t in tags.split(",") if t.strip()]
+# def parse_tags(tags: Any) -> list[str]:
+#     """Parse tags into list of strings."""
+#     if isinstance(tags, (list, tuple)):
+#         return [str(t).strip() for t in tags if str(t).strip()]
+#     return [t.strip() for t in tags.split(",") if t.strip()]
 
 
 class EntityParser:
@@ -88,33 +167,74 @@ class EntityParser:
                 return parsed
         return None
 
-    async def parse_file(self, file_path: Path) -> EntityMarkdown:
+    async def parse_file(self, path: Path | str) -> EntityMarkdown:
         """Parse markdown file into EntityMarkdown."""
 
-        absolute_path = self.base_path / file_path
+        # Check if the path is already absolute
+        if (
+            isinstance(path, Path)
+            and path.is_absolute()
+            or (isinstance(path, str) and Path(path).is_absolute())
+        ):
+            absolute_path = Path(path)
+        else:
+            absolute_path = self.get_file_path(path)
+
         # Parse frontmatter and content using python-frontmatter
-        post = frontmatter.load(str(absolute_path))
+        file_content = absolute_path.read_text(encoding="utf-8")
+        return await self.parse_file_content(absolute_path, file_content)
+
+    def get_file_path(self, path):
+        """Get absolute path for a file using the base path for the project."""
+        return self.base_path / path
+
+    async def parse_file_content(self, absolute_path, file_content):
+        # Parse frontmatter with proper error handling for malformed YAML (issue #185)
+        try:
+            post = frontmatter.loads(file_content)
+        except yaml.YAMLError as e:
+            # Log the YAML parsing error with file context
+            logger.warning(
+                f"Failed to parse YAML frontmatter in {absolute_path}: {e}. "
+                f"Treating file as plain markdown without frontmatter."
+            )
+            # Create a post with no frontmatter - treat entire content as markdown
+            post = frontmatter.Post(file_content, metadata={})
 
         # Extract file stat info
         file_stats = absolute_path.stat()
 
-        metadata = post.metadata
-        metadata["title"] = post.metadata.get("title", file_path.name)
-        metadata["type"] = post.metadata.get("type", "note")
-        metadata["tags"] = parse_tags(post.metadata.get("tags", []))
-
-        # frontmatter
+        # Normalize frontmatter values to prevent AttributeError on date objects (issue #236)
+        # PyYAML automatically converts date strings like "2025-10-24" to datetime.date objects
+        # This normalization converts them back to ISO format strings to ensure compatibility
+        # with code that expects string values
+        metadata = normalize_frontmatter_metadata(post.metadata)
+
+        # Ensure required fields have defaults (issue #184, #387)
+        # Handle title - use default if missing, None/null, empty, or string "None"
+        title = metadata.get("title")
+        if not title or title == "None":
+            metadata["title"] = absolute_path.stem
+        else:
+            metadata["title"] = title
+        # Handle type - use default if missing OR explicitly set to None/null
+        entity_type = metadata.get("type")
+        metadata["type"] = entity_type if entity_type is not None else "note"
+
+        tags = parse_tags(metadata.get("tags", []))  # pyright: ignore
+        if tags:
+            metadata["tags"] = tags
+
+        # frontmatter - use metadata with defaults applied
         entity_frontmatter = EntityFrontmatter(
-            metadata=post.metadata,
+            metadata=metadata,
         )
-
         entity_content = parse(post.content)
-
         return EntityMarkdown(
             frontmatter=entity_frontmatter,
             content=post.content,
             observations=entity_content.observations,
             relations=entity_content.relations,
-            created=datetime.fromtimestamp(file_stats.st_ctime),
-            modified=datetime.fromtimestamp(file_stats.st_mtime),
+            created=datetime.fromtimestamp(file_stats.st_ctime).astimezone(),
+            modified=datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
         )

basic_memory/markdown/markdown_processor.py

@@ -2,11 +2,11 @@ from pathlib import Path
 from typing import Optional
 from collections import OrderedDict
 
-import frontmatter
 from frontmatter import Post
 from loguru import logger
 
 from basic_memory import file_utils
+from basic_memory.file_utils import dump_frontmatter
 from basic_memory.markdown.entity_parser import EntityParser
 from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation
 
@@ -83,7 +83,7 @@ class MarkdownProcessor:
         """
         # Dirty check if needed
         if expected_checksum is not None:
-            current_content = path.read_text()
+            current_content = path.read_text(encoding="utf-8")
             current_checksum = await file_utils.compute_checksum(current_content)
             if current_checksum != expected_checksum:
                 raise DirtyFileError(f"File {path} has been modified")
@@ -115,7 +115,7 @@ class MarkdownProcessor:
 
         # Create Post object for frontmatter
         post = Post(content, **frontmatter_dict)
-        final_content = frontmatter.dumps(post, sort_keys=False)
+        final_content = dump_frontmatter(post)
 
         logger.debug(f"writing file {path} with content:\n{final_content}")
 

basic_memory/markdown/plugins.py

@@ -8,34 +8,50 @@ from markdown_it.token import Token
 # Observation handling functions
 def is_observation(token: Token) -> bool:
     """Check if token looks like our observation format."""
+    import re
+
     if token.type != "inline":  # pragma: no cover
         return False
-
-    content = token.content.strip()
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
     if not content:  # pragma: no cover
         return False
-
     # if it's a markdown_task, return false
     if content.startswith("[ ]") or content.startswith("[x]") or content.startswith("[-]"):
         return False
 
-    has_category = content.startswith("[") and "]" in content
+    # Exclude markdown links: [text](url)
+    if re.match(r"^\[.*?\]\(.*?\)$", content):
+        return False
+
+    # Exclude wiki links: [[text]]
+    if re.match(r"^\[\[.*?\]\]$", content):
+        return False
+
+    # Check for proper observation format: [category] content
+    match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
     has_tags = "#" in content
-    return has_category or has_tags
+    return bool(match) or has_tags
 
 
 def parse_observation(token: Token) -> Dict[str, Any]:
     """Extract observation parts from token."""
-    # Strip bullet point if present
-    content = token.content.strip()
+    import re
+
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
 
-    # Parse [category]
+    # Parse [category] with regex
+    match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
     category = None
-    if content.startswith("["):
-        end = content.find("]")
-        if end != -1:
-            category = content[1:end].strip() or None  # Convert empty to None
-            content = content[end + 1 :].strip()
+    if match:
+        category = match.group(1).strip()
+        content = match.group(2).strip()
+    else:
+        # Handle empty brackets [] followed by content
+        empty_match = re.match(r"^\[\]\s+(.+)", content)
+        if empty_match:
+            content = empty_match.group(1).strip()
 
     # Parse (context)
     context = None
@@ -50,9 +66,7 @@ def parse_observation(token: Token) -> Dict[str, Any]:
     parts = content.split()
     for part in parts:
         if part.startswith("#"):
-            # Handle multiple #tags stuck together
             if "#" in part[1:]:
-                # Split on # but keep non-empty tags
                 subtags = [t for t in part.split("#") if t]
                 tags.extend(subtags)
             else:
@@ -72,14 +86,16 @@ def is_explicit_relation(token: Token) -> bool:
     if token.type != "inline":  # pragma: no cover
         return False
 
-    content = token.content.strip()
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
     return "[[" in content and "]]" in content
 
 
 def parse_relation(token: Token) -> Dict[str, Any] | None:
     """Extract relation parts from token."""
     # Remove bullet point if present
-    content = token.content.strip()
+    # Use token.tag which contains the actual content for test tokens, fallback to content
+    content = (token.tag or token.content).strip()
 
     # Extract [[target]]
     target = None
@@ -213,10 +229,12 @@ def relation_plugin(md: MarkdownIt) -> None:
                         token.meta["relations"] = [rel]
 
                 # Always check for inline links in any text
-                elif "[[" in token.content:
-                    rels = parse_inline_relations(token.content)
-                    if rels:
-                        token.meta["relations"] = token.meta.get("relations", []) + rels
+                else:
+                    content = token.tag or token.content
+                    if "[[" in content:
+                        rels = parse_inline_relations(content)
+                        if rels:
+                            token.meta["relations"] = token.meta.get("relations", []) + rels
 
     # Add the rule after inline processing
     md.core.ruler.after("inline", "relations", relation_rule)

basic_memory/markdown/schemas.py

@@ -42,7 +42,7 @@ class EntityFrontmatter(BaseModel):
 
     @property
     def tags(self) -> List[str]:
-        return self.metadata.get("tags") if self.metadata else []  # pyright: ignore
+        return self.metadata.get("tags") if self.metadata else None  # pyright: ignore
 
     @property
     def title(self) -> str:

basic_memory/markdown/utils.py

@@ -1,13 +1,14 @@
 """Utilities for converting between markdown and entity models."""
 
 from pathlib import Path
-from typing import Optional, Any
+from typing import Any, Optional
 
 from frontmatter import Post
 
+from basic_memory.file_utils import has_frontmatter, remove_frontmatter, parse_frontmatter
 from basic_memory.markdown import EntityMarkdown
-from basic_memory.models import Entity, Observation as ObservationModel
-from basic_memory.utils import generate_permalink
+from basic_memory.models import Entity
+from basic_memory.models import Observation as ObservationModel
 
 
 def entity_model_from_markdown(
@@ -31,17 +32,16 @@ def entity_model_from_markdown(
     if not markdown.created or not markdown.modified:  # pragma: no cover
         raise ValueError("Both created and modified dates are required in markdown")
 
-    # Generate permalink if not provided
-    permalink = markdown.frontmatter.permalink or generate_permalink(file_path)
-
     # Create or update entity
     model = entity or Entity()
 
     # Update basic fields
     model.title = markdown.frontmatter.title
     model.entity_type = markdown.frontmatter.type
-    model.permalink = permalink
-    model.file_path = str(file_path)
+    # Only update permalink if it exists in frontmatter, otherwise preserve existing
+    if markdown.frontmatter.permalink is not None:
+        model.permalink = markdown.frontmatter.permalink
+    model.file_path = file_path.as_posix()
     model.content_type = "text/markdown"
     model.created_at = markdown.created
     model.updated_at = markdown.modified
@@ -76,18 +76,33 @@ async def schema_to_markdown(schema: Any) -> Post:
     """
     # Extract content and metadata
     content = schema.content or ""
-    frontmatter_metadata = dict(schema.entity_metadata or {})
+    entity_metadata = dict(schema.entity_metadata or {})
+
+    # if the content contains frontmatter, remove it and merge
+    if has_frontmatter(content):
+        content_frontmatter = parse_frontmatter(content)
+        content = remove_frontmatter(content)
+
+        # Merge content frontmatter with entity metadata
+        # (entity_metadata takes precedence for conflicts)
+        content_frontmatter.update(entity_metadata)
+        entity_metadata = content_frontmatter
 
     # Remove special fields for ordered frontmatter
     for field in ["type", "title", "permalink"]:
-        frontmatter_metadata.pop(field, None)
+        entity_metadata.pop(field, None)
 
-    # Create Post with ordered fields
+    # Create Post with fields ordered by insert order
     post = Post(
         content,
         title=schema.title,
         type=schema.entity_type,
-        permalink=schema.permalink,
-        **frontmatter_metadata,
     )
+    # set the permalink if passed in
+    if schema.permalink:
+        post.metadata["permalink"] = schema.permalink
+
+    if entity_metadata:
+        post.metadata.update(entity_metadata)
+
     return post

basic_memory/mcp/async_client.py

@@ -1,8 +1,138 @@
-from httpx import ASGITransport, AsyncClient
+from contextlib import asynccontextmanager, AbstractAsyncContextManager
+from typing import AsyncIterator, Callable, Optional
+
+from httpx import ASGITransport, AsyncClient, Timeout
+from loguru import logger
 
 from basic_memory.api.app import app as fastapi_app
+from basic_memory.config import ConfigManager
+
+
+# Optional factory override for dependency injection
+_client_factory: Optional[Callable[[], AbstractAsyncContextManager[AsyncClient]]] = None
+
+
+def set_client_factory(factory: Callable[[], AbstractAsyncContextManager[AsyncClient]]) -> None:
+    """Override the default client factory (for cloud app, testing, etc).
+
+    Args:
+        factory: An async context manager that yields an AsyncClient
+
+    Example:
+        @asynccontextmanager
+        async def custom_client_factory():
+            async with AsyncClient(...) as client:
+                yield client
+
+        set_client_factory(custom_client_factory)
+    """
+    global _client_factory
+    _client_factory = factory
+
+
+@asynccontextmanager
+async def get_client() -> AsyncIterator[AsyncClient]:
+    """Get an AsyncClient as a context manager.
+
+    This function provides proper resource management for HTTP clients,
+    ensuring connections are closed after use. It supports three modes:
+
+    1. **Factory injection** (cloud app, tests):
+       If a custom factory is set via set_client_factory(), use that.
+
+    2. **CLI cloud mode**:
+       When cloud_mode_enabled is True, create HTTP client with auth
+       token from CLIAuth for requests to cloud proxy endpoint.
+
+    3. **Local mode** (default):
+       Use ASGI transport for in-process requests to local FastAPI app.
+
+    Usage:
+        async with get_client() as client:
+            response = await client.get("/path")
+
+    Yields:
+        AsyncClient: Configured HTTP client for the current mode
+
+    Raises:
+        RuntimeError: If cloud mode is enabled but user is not authenticated
+    """
+    if _client_factory:
+        # Use injected factory (cloud app, tests)
+        async with _client_factory() as client:
+            yield client
+    else:
+        # Default: create based on config
+        config = ConfigManager().config
+        timeout = Timeout(
+            connect=10.0,  # 10 seconds for connection
+            read=30.0,  # 30 seconds for reading response
+            write=30.0,  # 30 seconds for writing request
+            pool=30.0,  # 30 seconds for connection pool
+        )
+
+        if config.cloud_mode_enabled:
+            # CLI cloud mode: inject auth when creating client
+            from basic_memory.cli.auth import CLIAuth
+
+            auth = CLIAuth(client_id=config.cloud_client_id, authkit_domain=config.cloud_domain)
+            token = await auth.get_valid_token()
+
+            if not token:
+                raise RuntimeError(
+                    "Cloud mode enabled but not authenticated. "
+                    "Run 'basic-memory cloud login' first."
+                )
+
+            # Auth header set ONCE at client creation
+            proxy_base_url = f"{config.cloud_host}/proxy"
+            logger.info(f"Creating HTTP client for cloud proxy at: {proxy_base_url}")
+            async with AsyncClient(
+                base_url=proxy_base_url,
+                headers={"Authorization": f"Bearer {token}"},
+                timeout=timeout,
+            ) as client:
+                yield client
+        else:
+            # Local mode: ASGI transport for in-process calls
+            logger.info("Creating ASGI client for local Basic Memory API")
+            async with AsyncClient(
+                transport=ASGITransport(app=fastapi_app), base_url="http://test", timeout=timeout
+            ) as client:
+                yield client
+
+
+def create_client() -> AsyncClient:
+    """Create an HTTP client based on configuration.
+
+    DEPRECATED: Use get_client() context manager instead for proper resource management.
+
+    This function is kept for backward compatibility but will be removed in a future version.
+    The returned client should be closed manually by calling await client.aclose().
+
+    Returns:
+        AsyncClient configured for either local ASGI or remote proxy
+    """
+    config_manager = ConfigManager()
+    config = config_manager.config
 
-BASE_URL = "memory://"
+    # Configure timeout for longer operations like write_note
+    # Default httpx timeout is 5 seconds which is too short for file operations
+    timeout = Timeout(
+        connect=10.0,  # 10 seconds for connection
+        read=30.0,  # 30 seconds for reading response
+        write=30.0,  # 30 seconds for writing request
+        pool=30.0,  # 30 seconds for connection pool
+    )
 
-# Create shared async client
-client = AsyncClient(transport=ASGITransport(app=fastapi_app), base_url=BASE_URL)
+    if config.cloud_mode_enabled:
+        # Use HTTP transport to proxy endpoint
+        proxy_base_url = f"{config.cloud_host}/proxy"
+        logger.info(f"Creating HTTP client for proxy at: {proxy_base_url}")
+        return AsyncClient(base_url=proxy_base_url, timeout=timeout)
+    else:
+        # Default: use ASGI transport for local API (development mode)
+        logger.info("Creating ASGI client for local Basic Memory API")
+        return AsyncClient(
+            transport=ASGITransport(app=fastapi_app), base_url="http://test", timeout=timeout
+        )