basic-memory 0.14.3__py3-none-any.whl → 0.15.0__py3-none-any.whl

basic_memory/sync/watch_service.py

@@ -5,9 +5,10 @@ import os
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
-from typing import List, Optional, Set
+from typing import List, Optional, Set, Sequence
 
 from basic_memory.config import BasicMemoryConfig, WATCH_STATUS_JSON
+from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
 from basic_memory.models import Project
 from basic_memory.repository import ProjectRepository
 from loguru import logger
@@ -15,6 +16,7 @@ from pydantic import BaseModel
 from rich.console import Console
 from watchfiles import awatch
 from watchfiles.main import FileChange, Change
+import time
 
 
 class WatchEvent(BaseModel):
@@ -81,54 +83,110 @@ class WatchService:
         self.state = WatchServiceState()
         self.status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
         self.status_path.parent.mkdir(parents=True, exist_ok=True)
+        self._ignore_patterns_cache: dict[Path, Set[str]] = {}
 
         # quiet mode for mcp so it doesn't mess up stdout
         self.console = Console(quiet=quiet)
 
+    async def _schedule_restart(self, stop_event: asyncio.Event):
+        """Schedule a restart of the watch service after the configured interval."""
+        await asyncio.sleep(self.app_config.watch_project_reload_interval)
+        stop_event.set()
+
+    def _get_ignore_patterns(self, project_path: Path) -> Set[str]:
+        """Get or load ignore patterns for a project path."""
+        if project_path not in self._ignore_patterns_cache:
+            self._ignore_patterns_cache[project_path] = load_gitignore_patterns(project_path)
+        return self._ignore_patterns_cache[project_path]
+
+    async def _watch_projects_cycle(self, projects: Sequence[Project], stop_event: asyncio.Event):
+        """Run one cycle of watching the given projects until stop_event is set."""
+        project_paths = [project.path for project in projects]
+
+        async for changes in awatch(
+            *project_paths,
+            debounce=self.app_config.sync_delay,
+            watch_filter=self.filter_changes,
+            recursive=True,
+            stop_event=stop_event,
+        ):
+            # group changes by project and filter using ignore patterns
+            project_changes = defaultdict(list)
+            for change, path in changes:
+                for project in projects:
+                    if self.is_project_path(project, path):
+                        # Check if the file should be ignored based on gitignore patterns
+                        project_path = Path(project.path)
+                        file_path = Path(path)
+                        ignore_patterns = self._get_ignore_patterns(project_path)
+
+                        if should_ignore_path(file_path, project_path, ignore_patterns):
+                            logger.trace(
+                                f"Ignoring watched file change: {file_path.relative_to(project_path)}"
+                            )
+                            continue
+
+                        project_changes[project].append((change, path))
+                        break
+
+            # create coroutines to handle changes
+            change_handlers = [
+                self.handle_changes(project, changes)  # pyright: ignore
+                for project, changes in project_changes.items()
+            ]
+
+            # process changes
+            await asyncio.gather(*change_handlers)
+
     async def run(self):  # pragma: no cover
         """Watch for file changes and sync them"""
 
-        projects = await self.project_repository.get_active_projects()
-        project_paths = [project.path for project in projects]
+        self.state.running = True
+        self.state.start_time = datetime.now()
+        await self.write_status()
 
         logger.info(
             "Watch service started",
-            f"directories={project_paths}",
             f"debounce_ms={self.app_config.sync_delay}",
             f"pid={os.getpid()}",
         )
 
-        self.state.running = True
-        self.state.start_time = datetime.now()
-        await self.write_status()
-
         try:
-            async for changes in awatch(
-                *project_paths,
-                debounce=self.app_config.sync_delay,
-                watch_filter=self.filter_changes,
-                recursive=True,
-            ):
-                # group changes by project
-                project_changes = defaultdict(list)
-                for change, path in changes:
-                    for project in projects:
-                        if self.is_project_path(project, path):
-                            project_changes[project].append((change, path))
-                            break
-
-                # create coroutines to handle changes
-                change_handlers = [
-                    self.handle_changes(project, changes)  # pyright: ignore
-                    for project, changes in project_changes.items()
-                ]
-
-                # process changes
-                await asyncio.gather(*change_handlers)
+            while self.state.running:
+                # Clear ignore patterns cache to pick up any .gitignore changes
+                self._ignore_patterns_cache.clear()
+
+                # Reload projects to catch any new/removed projects
+                projects = await self.project_repository.get_active_projects()
+
+                project_paths = [project.path for project in projects]
+                logger.debug(f"Starting watch cycle for directories: {project_paths}")
+
+                # Create stop event for this watch cycle
+                stop_event = asyncio.Event()
+
+                # Schedule restart after configured interval to reload projects
+                timer_task = asyncio.create_task(self._schedule_restart(stop_event))
+
+                try:
+                    await self._watch_projects_cycle(projects, stop_event)
+                except Exception as e:
+                    logger.exception("Watch service error during cycle", error=str(e))
+                    self.state.record_error(str(e))
+                    await self.write_status()
+                    # Continue to next cycle instead of exiting
+                    await asyncio.sleep(5)  # Brief pause before retry
+                finally:
+                    # Cancel timer task if it's still running
+                    if not timer_task.done():
+                        timer_task.cancel()
+                        try:
+                            await timer_task
+                        except asyncio.CancelledError:
+                            pass
 
         except Exception as e:
             logger.exception("Watch service error", error=str(e))
-
             self.state.record_error(str(e))
             await self.write_status()
             raise
@@ -175,11 +233,8 @@ class WatchService:
 
     async def handle_changes(self, project: Project, changes: Set[FileChange]) -> None:
         """Process a batch of file changes"""
-        import time
-        from typing import List, Set
-
-        # Lazily initialize sync service for project changes
-        from basic_memory.cli.commands.sync import get_sync_service
+        # avoid circular imports
+        from basic_memory.sync.sync_service import get_sync_service
 
         sync_service = await get_sync_service(project)
         file_service = sync_service.file_service
@@ -197,7 +252,7 @@ class WatchService:
 
         for change, path in changes:
             # convert to relative path
-            relative_path = str(Path(path).relative_to(directory))
+            relative_path = Path(path).relative_to(directory).as_posix()
 
             # Skip .tmp files - they're temporary and shouldn't be synced
             if relative_path.endswith(".tmp"):
@@ -284,13 +339,42 @@ class WatchService:
         # Process deletes
         for path in deletes:
             if path not in processed:
-                logger.debug("Processing deleted file", path=path)
-                await sync_service.handle_delete(path)
-                self.state.add_event(path=path, action="deleted", status="success")
-                self.console.print(f"[red]✕[/red] {path}")
-                logger.info(f"deleted: {path}")
-                processed.add(path)
-                delete_count += 1
+                # Check if file still exists on disk (vim atomic write edge case)
+                full_path = directory / path
+                if full_path.exists() and full_path.is_file():
+                    # File still exists despite DELETE event - treat as modification
+                    logger.debug(
+                        "File exists despite DELETE event, treating as modification", path=path
+                    )
+                    entity, checksum = await sync_service.sync_file(path, new=False)
+                    self.state.add_event(
+                        path=path, action="modified", status="success", checksum=checksum
+                    )
+                    self.console.print(f"[yellow]✎[/yellow] {path} (atomic write)")
+                    logger.info(f"atomic write detected: {path}")
+                    processed.add(path)
+                    modify_count += 1
+                else:
+                    # Check if this was a directory - skip if so
+                    # (we can't tell if the deleted path was a directory since it no longer exists,
+                    # so we check if there's an entity in the database for it)
+                    entity = await sync_service.entity_repository.get_by_file_path(path)
+                    if entity is None:
+                        # No entity means this was likely a directory - skip it
+                        logger.debug(
+                            f"Skipping deleted path with no entity (likely directory), path={path}"
+                        )
+                        processed.add(path)
+                        continue
+
+                    # File truly deleted
+                    logger.debug("Processing deleted file", path=path)
+                    await sync_service.handle_delete(path)
+                    self.state.add_event(path=path, action="deleted", status="success")
+                    self.console.print(f"[red]✕[/red] {path}")
+                    logger.info(f"deleted: {path}")
+                    processed.add(path)
+                    delete_count += 1
 
         # Process adds
         for path in adds:

basic_memory/utils.py

@@ -4,11 +4,13 @@ import os
 
 import logging
 import re
-import unicodedata
+import sys
+from datetime import datetime
 from pathlib import Path
-from typing import Optional, Protocol, Union, runtime_checkable, List, Any
+from typing import Optional, Protocol, Union, runtime_checkable, List
 
 from loguru import logger
+from unidecode import unidecode
 
 
 @runtime_checkable
@@ -26,9 +28,11 @@ FilePath = Union[Path, str]
 logging.getLogger("opentelemetry.sdk.metrics._internal.instrument").setLevel(logging.ERROR)
 
 
-def generate_permalink(file_path: Union[Path, str, Any]) -> str:
-    """
-    Generate a permalink from a file path.
+def generate_permalink(file_path: Union[Path, str, PathLike], split_extension: bool = True) -> str:
+    """Generate a stable permalink from a file path.
+
+    Args:
+        file_path: Original file path (str, Path, or PathLike)
 
     Returns:
         Normalized permalink that matches validation rules. Converts spaces and underscores
@@ -37,7 +41,7 @@ def generate_permalink(file_path: Union[Path, str, Any]) -> str:
     Examples:
         >>> generate_permalink("docs/My Feature.md")
         'docs/my-feature'
-        >>> generate_permalink("specs/API_v2.md")
+        >>> generate_permalink("specs/API (v2).md")
         'specs/api-v2'
         >>> generate_permalink("design/unified_model_refactor.md")
         'design/unified-model-refactor'
@@ -45,84 +49,99 @@ def generate_permalink(file_path: Union[Path, str, Any]) -> str:
         '中文/测试文档'
     """
     # Convert Path to string if needed
-    path_str = str(file_path)
-
-    # Remove extension
-    base = os.path.splitext(path_str)[0]
-
-    # Create a transliteration mapping for specific characters
-    transliteration_map = {
-        "ø": "o",  # Handle Søren -> soren
-        "å": "a",  # Handle Kierkegård -> kierkegard
-        "ü": "u",  # Handle Müller -> muller
-        "é": "e",  # Handle Café -> cafe
-        "è": "e",  # Handle Mère -> mere
-        "ê": "e",  # Handle Fête -> fete
-        "à": "a",  # Handle À la mode -> a la mode
-        "ç": "c",  # Handle Façade -> facade
-        "ñ": "n",  # Handle Niño -> nino
-        "ö": "o",  # Handle Björk -> bjork
-        "ä": "a",  # Handle Häagen -> haagen
-        # Add more mappings as needed
-    }
+    path_str = Path(str(file_path)).as_posix()
+
+    # Remove extension (for now, possibly)
+    (base, extension) = os.path.splitext(path_str)
+
+    # Check if we have CJK characters that should be preserved
+    # CJK ranges: \u4e00-\u9fff (CJK Unified Ideographs), \u3000-\u303f (CJK symbols),
+    # \u3400-\u4dbf (CJK Extension A), \uff00-\uffef (Fullwidth forms)
+    has_cjk_chars = any(
+        "\u4e00" <= char <= "\u9fff"
+        or "\u3000" <= char <= "\u303f"
+        or "\u3400" <= char <= "\u4dbf"
+        or "\uff00" <= char <= "\uffef"
+        for char in base
+    )
 
-    # Process character by character, transliterating Latin characters with diacritics
-    result = ""
-    for char in base:
-        # Direct mapping for known characters
-        if char.lower() in transliteration_map:
-            result += transliteration_map[char.lower()]
-        # General case using Unicode normalization
-        elif unicodedata.category(char).startswith("L") and ord(char) > 127:
-            # Decompose the character (e.g., ü -> u + combining diaeresis)
-            decomposed = unicodedata.normalize("NFD", char)
-            # If decomposition produced multiple characters and first one is ASCII
-            if len(decomposed) > 1 and ord(decomposed[0]) < 128:
-                # Keep only the base character
-                result += decomposed[0].lower()
-            else:
-                # For non-Latin scripts like Chinese, preserve the character
+    if has_cjk_chars:
+        # For text with CJK characters, selectively transliterate only Latin accented chars
+        result = ""
+        for char in base:
+            if (
+                "\u4e00" <= char <= "\u9fff"
+                or "\u3000" <= char <= "\u303f"
+                or "\u3400" <= char <= "\u4dbf"
+            ):
+                # Preserve CJK ideographs and symbols
                 result += char
-        else:
-            # Add the character as is
-            result += char
+            elif "\uff00" <= char <= "\uffef":
+                # Remove Chinese fullwidth punctuation entirely (like ，！？)
+                continue
+            else:
+                # Transliterate Latin accented characters to ASCII
+                result += unidecode(char)
 
-    # Handle special punctuation cases for apostrophes
-    result = result.replace("'", "")
+        # Insert hyphens between CJK and Latin character transitions
+        # Match: CJK followed by Latin letter/digit, or Latin letter/digit followed by CJK
+        result = re.sub(
+            r"([\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf])([a-zA-Z0-9])", r"\1-\2", result
+        )
+        result = re.sub(
+            r"([a-zA-Z0-9])([\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf])", r"\1-\2", result
+        )
 
-    # Insert dash between camelCase
-    # This regex finds boundaries between lowercase and uppercase letters
-    result = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", result)
+        # Insert dash between camelCase
+        result = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", result)
 
-    # Insert dash between Chinese and Latin character boundaries
-    # This is needed for cases like "中文English" -> "中文-english"
-    result = re.sub(r"([\u4e00-\u9fff])([a-zA-Z])", r"\1-\2", result)
-    result = re.sub(r"([a-zA-Z])([\u4e00-\u9fff])", r"\1-\2", result)
+        # Convert ASCII letters to lowercase, preserve CJK
+        lower_text = "".join(c.lower() if c.isascii() and c.isalpha() else c for c in result)
 
-    # Convert ASCII letters to lowercase, preserve non-ASCII characters
-    lower_text = "".join(c.lower() if c.isascii() and c.isalpha() else c for c in result)
+        # Replace underscores with hyphens
+        text_with_hyphens = lower_text.replace("_", "-")
 
-    # Replace underscores with hyphens
-    text_with_hyphens = lower_text.replace("_", "-")
+        # Remove apostrophes entirely (don't replace with hyphens)
+        text_no_apostrophes = text_with_hyphens.replace("'", "")
 
-    # Replace spaces and unsafe ASCII characters with hyphens, but preserve non-ASCII characters
-    # Include common Chinese character ranges and other non-ASCII characters
-    clean_text = re.sub(
-        r"[^a-z0-9\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf/\-]", "-", text_with_hyphens
-    )
+        # Replace unsafe chars with hyphens, but preserve CJK characters
+        clean_text = re.sub(
+            r"[^a-z0-9\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf/\-]", "-", text_no_apostrophes
+        )
+    else:
+        # Original ASCII-only processing for backward compatibility
+        # Transliterate unicode to ascii
+        ascii_text = unidecode(base)
+
+        # Insert dash between camelCase
+        ascii_text = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", ascii_text)
+
+        # Convert to lowercase
+        lower_text = ascii_text.lower()
+
+        # replace underscores with hyphens
+        text_with_hyphens = lower_text.replace("_", "-")
+
+        # Remove apostrophes entirely (don't replace with hyphens)
+        text_no_apostrophes = text_with_hyphens.replace("'", "")
+
+        # Replace remaining invalid chars with hyphens
+        clean_text = re.sub(r"[^a-z0-9/\-]", "-", text_no_apostrophes)
 
     # Collapse multiple hyphens
     clean_text = re.sub(r"-+", "-", clean_text)
 
-    # Remove hyphens between adjacent Chinese characters only
-    # This handles cases like "你好-世界" -> "你好世界"
-    clean_text = re.sub(r"([\u4e00-\u9fff])-([\u4e00-\u9fff])", r"\1\2", clean_text)
-
     # Clean each path segment
     segments = clean_text.split("/")
     clean_segments = [s.strip("-") for s in segments]
 
-    return "/".join(clean_segments)
+    return_val = "/".join(clean_segments)
+
+    # Append file extension back, if necessary
+    if not split_extension and extension:
+        return_val += extension
+
+    return return_val
 
 
 def setup_logging(
@@ -143,7 +162,7 @@ def setup_logging(
         console: Whether to log to the console
     """
     # Remove default handler and any existing handlers
-    # logger.remove()
+    logger.remove()
 
     # Add file handler if we are not running tests and a log file is specified
     if log_file and env != "test":
@@ -161,8 +180,8 @@ def setup_logging(
         )
 
     # Add console logger if requested or in test mode
-    # if env == "test" or console:
-    #     logger.add(sys.stderr, level=log_level, backtrace=True, diagnose=True, colorize=True)
+    if env == "test" or console:
+        logger.add(sys.stderr, level=log_level, backtrace=True, diagnose=True, colorize=True)
 
     logger.info(f"ENV: '{env}' Log level: '{log_level}' Logging to {log_file}")
 
@@ -172,8 +191,6 @@ def setup_logging(
         "httpx": logging.WARNING,
         # File watching logs
         "watchfiles.main": logging.WARNING,
-        # SQLAlchemy deprecation warnings
-        "sqlalchemy": logging.WARNING,
     }
 
     # Set log levels for noisy loggers
@@ -181,6 +198,66 @@ def setup_logging(
         logging.getLogger(logger_name).setLevel(level)
 
 
+def parse_tags(tags: Union[List[str], str, None]) -> List[str]:
+    """Parse tags from various input formats into a consistent list.
+
+    Args:
+        tags: Can be a list of strings, a comma-separated string, or None
+
+    Returns:
+        A list of tag strings, or an empty list if no tags
+
+    Note:
+        This function strips leading '#' characters from tags to prevent
+        their accumulation when tags are processed multiple times.
+    """
+    if tags is None:
+        return []
+
+    # Process list of tags
+    if isinstance(tags, list):
+        # First strip whitespace, then strip leading '#' characters to prevent accumulation
+        return [tag.strip().lstrip("#") for tag in tags if tag and tag.strip()]
+
+    # Process string input
+    if isinstance(tags, str):
+        # Check if it's a JSON array string (common issue from AI assistants)
+        import json
+
+        if tags.strip().startswith("[") and tags.strip().endswith("]"):
+            try:
+                # Try to parse as JSON array
+                parsed_json = json.loads(tags)
+                if isinstance(parsed_json, list):
+                    # Recursively parse the JSON array as a list
+                    return parse_tags(parsed_json)
+            except json.JSONDecodeError:
+                # Not valid JSON, fall through to comma-separated parsing
+                pass
+
+        # Split by comma, strip whitespace, then strip leading '#' characters
+        return [tag.strip().lstrip("#") for tag in tags.split(",") if tag and tag.strip()]
+
+    # For any other type, try to convert to string and parse
+    try:  # pragma: no cover
+        return parse_tags(str(tags))
+    except (ValueError, TypeError):  # pragma: no cover
+        logger.warning(f"Couldn't parse tags from input of type {type(tags)}: {tags}")
+        return []
+
+
+def normalize_newlines(multiline: str) -> str:
+    """Replace any \r\n, \r, or \n with the native newline.
+
+    Args:
+        multiline: String containing any mixture of newlines.
+
+    Returns:
+        A string with normalized newlines native to the platform.
+    """
+    return re.sub(r"\r\n?|\n", os.linesep, multiline)
+
+
 def normalize_file_path_for_comparison(file_path: str) -> str:
     """Normalize a file path for conflict detection.
 
@@ -254,42 +331,8 @@ def detect_potential_file_conflicts(file_path: str, existing_paths: List[str]) -
     return conflicts
 
 
-def parse_tags(tags: Union[List[str], str, None]) -> List[str]:
-    """Parse tags from various input formats into a consistent list.
-
-    Args:
-        tags: Can be a list of strings, a comma-separated string, or None
-
-    Returns:
-        A list of tag strings, or an empty list if no tags
-
-    Note:
-        This function strips leading '#' characters from tags to prevent
-        their accumulation when tags are processed multiple times.
-    """
-    if tags is None:
-        return []
-
-    # Process list of tags
-    if isinstance(tags, list):
-        # First strip whitespace, then strip leading '#' characters to prevent accumulation
-        return [tag.strip().lstrip("#") for tag in tags if tag and tag.strip()]
-
-    # Process comma-separated string of tags
-    if isinstance(tags, str):
-        # Split by comma, strip whitespace, then strip leading '#' characters
-        return [tag.strip().lstrip("#") for tag in tags.split(",") if tag and tag.strip()]
-
-    # For any other type, try to convert to string and parse
-    try:  # pragma: no cover
-        return parse_tags(str(tags))
-    except (ValueError, TypeError):  # pragma: no cover
-        logger.warning(f"Couldn't parse tags from input of type {type(tags)}: {tags}")
-        return []
-
-
-def validate_project_path(path: str, project_path: Path) -> bool:
-    """Ensure path stays within project boundaries."""
+def valid_project_path_value(path: str):
+    """Ensure project path is valid."""
     # Allow empty strings as they resolve to the project root
     if not path:
         return True
@@ -310,8 +353,37 @@ def validate_project_path(path: str, project_path: Path) -> bool:
     if path.strip() and any(ord(c) < 32 and c not in [" ", "\t"] for c in path):
         return False
 
+    return True
+
+
+def validate_project_path(path: str, project_path: Path) -> bool:
+    """Ensure path is valid and stays within project boundaries."""
+
+    if not valid_project_path_value(path):
+        return False
+
     try:
         resolved = (project_path / path).resolve()
         return resolved.is_relative_to(project_path.resolve())
     except (ValueError, OSError):
         return False
+
+
+def ensure_timezone_aware(dt: datetime) -> datetime:
+    """Ensure a datetime is timezone-aware using system timezone.
+
+    If the datetime is naive, convert it to timezone-aware using the system's local timezone.
+    If it's already timezone-aware, return it unchanged.
+
+    Args:
+        dt: The datetime to ensure is timezone-aware
+
+    Returns:
+        A timezone-aware datetime
+    """
+    if dt.tzinfo is None:
+        # Naive datetime - assume it's in local time and add timezone
+        return dt.astimezone()
+    else:
+        # Already timezone-aware
+        return dt