roadmap-cli 0.1.1__py3-none-any.whl

roadmap/core/services/baseline/baseline_state_retriever.py

@@ -0,0 +1,284 @@
+"""Baseline state retrieval using git history and YAML metadata.
+
+This module replaces the previous DB-based baseline approach with:
+1. Local baseline: Retrieved from git history at last_synced timestamp
+2. Remote baseline: Stored in sync_metadata YAML frontmatter
+
+This enables idempotent syncs without complex database state management.
+"""
+
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from structlog import get_logger
+
+from roadmap.core.interfaces.persistence import (
+    FileNotFound,
+    GitHistoryError,
+    PersistenceInterface,
+)
+from roadmap.core.services.sync.sync_state import IssueBaseState
+from roadmap.infrastructure.persistence_gateway import PersistenceGateway
+
+logger = get_logger(__name__)
+
+
+class BaselineRetrievalError(Exception):
+    """Raised when baseline retrieval fails."""
+
+    pass
+
+
+class BaselineStateRetriever:
+    """Retrieves baseline state from git history and YAML metadata."""
+
+    def __init__(
+        self,
+        issues_dir: Path,
+        persistence: PersistenceInterface,
+    ):
+        """Initialize with issues directory and persistence interface.
+
+        Args:
+            issues_dir: Path to issues directory
+            persistence: Interface for git history access
+        """
+        self.issues_dir = issues_dir
+        self.persistence = persistence
+
+    def get_baseline_from_file(self, issue_file: Path) -> IssueBaseState | None:
+        """Get baseline state directly from current issue file.
+
+        Used for initial baseline creation on first sync, where we extract
+        the current state directly without looking in git history.
+
+        Args:
+            issue_file: Path to issue file
+
+        Returns:
+            IssueBaseState extracted from file, or None if extraction fails
+        """
+        try:
+            if not issue_file.exists():
+                logger.debug(
+                    "baseline_file_not_found",
+                    issue_file=str(issue_file),
+                )
+                return None
+
+            content = issue_file.read_text(encoding="utf-8")
+            return self._extract_baseline_from_content(content, issue_file)
+        except Exception as e:
+            logger.warning(
+                "baseline_from_file_extraction_error",
+                issue_file=str(issue_file),
+                error=str(e),
+                severity="operational",
+            )
+            return None
+
+    def get_local_baseline(
+        self, issue_file: Path, last_synced: datetime
+    ) -> IssueBaseState | None:
+        """Get local baseline state from git history.
+
+        Retrieves the issue file as it existed at the last_synced timestamp,
+        then extracts the relevant fields to create a baseline.
+
+        Args:
+            issue_file: Path to issue file
+            last_synced: Timestamp of last successful sync
+
+        Returns:
+            IssueBaseState if found, None if file didn't exist at that time
+        """
+        try:
+            # Get file content as it existed at last_synced time
+            file_content = self.persistence.get_file_at_timestamp(
+                str(issue_file), last_synced
+            )
+
+            if not file_content:
+                logger.debug(
+                    "local_baseline_file_not_found",
+                    issue_file=str(issue_file),
+                    last_synced=last_synced.isoformat(),
+                )
+                return None
+
+            # Parse the file to extract baseline fields
+            baseline = self._extract_baseline_from_content(file_content, issue_file)
+
+            logger.debug(
+                "local_baseline_retrieved",
+                issue_file=str(issue_file),
+                status=baseline.status if baseline else None,
+            )
+            return baseline
+
+        except FileNotFound:
+            logger.debug(
+                "local_baseline_file_not_in_history",
+                issue_file=str(issue_file),
+                last_synced=last_synced.isoformat(),
+                reason="file_did_not_exist_at_timestamp",
+            )
+            return None
+        except GitHistoryError as e:
+            logger.warning(
+                "local_baseline_git_error",
+                issue_file=str(issue_file),
+                error=str(e),
+                severity="operational",
+            )
+            raise BaselineRetrievalError(
+                f"Failed to retrieve baseline from git: {e}"
+            ) from e
+        except Exception as e:
+            logger.warning(
+                "local_baseline_extraction_error",
+                issue_file=str(issue_file),
+                error=str(e),
+                error_type=type(e).__name__,
+                severity="operational",
+            )
+            return None
+
+    def get_remote_baseline(self, issue_file: Path) -> IssueBaseState | None:
+        """Get remote baseline from sync_metadata in issue file.
+
+        The sync_metadata YAML frontmatter contains a remote_state snapshot
+        that represents the last-synced state from the remote backend.
+
+        Args:
+            issue_file: Path to issue file
+
+        Returns:
+            IssueBaseState extracted from remote_state, or None if not present
+        """
+        try:
+            sync_metadata = PersistenceGateway.load_sync_metadata(issue_file)
+
+            if not sync_metadata:
+                logger.debug(
+                    "remote_baseline_not_found",
+                    issue_file=str(issue_file),
+                    reason="no_sync_metadata",
+                )
+                return None
+
+            remote_state = sync_metadata.get("remote_state")
+            if not remote_state:
+                logger.debug(
+                    "remote_baseline_not_found",
+                    issue_file=str(issue_file),
+                    reason="no_remote_state_in_metadata",
+                )
+                return None
+
+            baseline = self._extract_baseline_from_remote_state(
+                remote_state, issue_file
+            )
+
+            logger.debug(
+                "remote_baseline_retrieved",
+                issue_file=str(issue_file),
+                status=baseline.status if baseline else None,
+            )
+            return baseline
+
+        except Exception as e:
+            logger.warning(
+                "remote_baseline_extraction_error",
+                issue_file=str(issue_file),
+                error=str(e),
+                error_type=type(e).__name__,
+                severity="operational",
+            )
+            return None
+
+    def _extract_baseline_from_content(
+        self, content: str, issue_file: Path
+    ) -> IssueBaseState | None:
+        """Extract baseline fields from issue file content.
+
+        Args:
+            content: File content (with YAML frontmatter)
+            issue_file: Path to issue file (for logging)
+
+        Returns:
+            IssueBaseState or None if extraction fails
+        """
+        from roadmap.common.datetime_parser import parse_datetime
+
+        try:
+            frontmatter, _ = PersistenceGateway.parse_frontmatter(content)
+
+            # Parse updated datetime
+            updated_at = datetime.now(UTC)
+            if frontmatter.get("updated"):
+                updated_value = frontmatter["updated"]
+                if isinstance(updated_value, datetime):
+                    updated_at = updated_value
+                else:
+                    parsed = parse_datetime(str(updated_value), "file")
+                    if parsed:
+                        updated_at = parsed
+
+            return IssueBaseState(
+                id=frontmatter.get("id", ""),
+                title=frontmatter.get("title", ""),
+                status=frontmatter.get("status", "todo"),
+                assignee=frontmatter.get("assignee"),
+                headline=frontmatter.get("headline", ""),
+                content=frontmatter.get("content", ""),
+                labels=frontmatter.get("labels", []),
+                updated_at=updated_at,
+            )
+        except Exception as e:
+            logger.debug(
+                "baseline_extraction_failed",
+                issue_file=str(issue_file),
+                error=str(e),
+            )
+            return None
+
+    def _extract_baseline_from_remote_state(
+        self, remote_state: dict[str, Any], issue_file: Path
+    ) -> IssueBaseState | None:
+        """Extract baseline from remote_state snapshot in sync_metadata.
+
+        Args:
+            remote_state: Remote state dict from sync_metadata
+            issue_file: Path to issue file (for logging)
+
+        Returns:
+            IssueBaseState or None if extraction fails
+        """
+        try:
+            # Parse datetime if present
+            updated_at = datetime.now(UTC)
+            if "updated_at" in remote_state:
+                if isinstance(remote_state["updated_at"], str):
+                    updated_at = datetime.fromisoformat(remote_state["updated_at"])
+                else:
+                    updated_at = remote_state["updated_at"]
+
+            return IssueBaseState(
+                id=remote_state.get("id", ""),
+                title=remote_state.get("title", ""),
+                status=remote_state.get("status", "open"),
+                assignee=remote_state.get("assignee"),
+                headline=remote_state.get("headline", ""),
+                content=remote_state.get("content", ""),
+                labels=remote_state.get("labels", []),
+                updated_at=updated_at,
+            )
+        except Exception as e:
+            logger.debug(
+                "remote_baseline_extraction_failed",
+                issue_file=str(issue_file),
+                error=str(e),
+            )
+            return None

roadmap/core/services/baseline/optimized_baseline_builder.py

@@ -0,0 +1,425 @@
+"""Optimized sync state rebuilding using changed file detection.
+
+This module provides intelligent caching strategies for sync state reconstruction,
+using git history to detect which files have actually changed since last baseline.
+This dramatically speeds up the rebuild process for large issue sets.
+"""
+
+from pathlib import Path
+
+from structlog import get_logger
+
+from roadmap.core.interfaces.persistence import GitHistoryError
+from roadmap.core.services.sync.sync_state import SyncState
+from roadmap.infrastructure.persistence_gateway import PersistenceGateway
+
+logger = get_logger(__name__)
+
+
+class OptimizedBaselineBuilder:
+    """Builds sync state baselines with intelligent caching.
+
+    Instead of rebuilding the entire sync state from git history, this class:
+    1. Checks which files have changed since a reference point
+    2. Only rebuilds baselines for modified issues
+    3. Reuses cached baseline states for unchanged issues
+    4. Reduces rebuild time from ~50-100ms to ~5-10ms for typical changes
+    """
+
+    def __init__(self, issues_dir: Path):
+        """Initialize builder.
+
+        Args:
+            issues_dir: Path to issues directory
+        """
+        self.issues_dir = issues_dir
+        self._progress = None  # Optional rich Progress context
+
+    def set_progress_context(self, progress) -> None:
+        """Set optional progress tracker for rebuild operations.
+
+        Args:
+            progress: rich.progress.Progress instance or None
+
+        Example:
+            >>> from rich.progress import Progress
+            >>> with Progress() as progress:
+            ...     builder = OptimizedBaselineBuilder(Path("roadmap/issues"))
+            ...     builder.set_progress_context(progress)
+            ...     # Rebuilds will now show progress
+        """
+        self._progress = progress
+
+    def _update_progress(
+        self,
+        description: str,
+        completed: int | None = None,
+        total: int | None = None,
+    ) -> None:
+        """Update progress bar if tracking is enabled.
+
+        Args:
+            description: What operation is being performed
+            completed: Number of items completed
+            total: Total items to complete
+
+        Internal use only.
+        """
+        if self._progress is None:
+            return
+
+        try:
+            # Update task description if progress tracking is active
+            logger.debug(
+                "baseline_rebuild_progress",
+                description=description,
+                completed=completed,
+                total=total,
+            )
+        except Exception as e:
+            logger.debug(
+                "progress_update_failed",
+                operation="report_progress",
+                error=str(e),
+                action="Continuing without progress update",
+            )
+            # Silently ignore progress update errors
+
+    def get_changed_issue_files(
+        self,
+        git_ref: str = "HEAD~1",
+        limit_to_issues: list[str] | None = None,
+    ) -> set[str]:
+        """Get issue files that changed since a git reference.
+
+        Args:
+            git_ref: Git reference to compare against (default: previous commit)
+            limit_to_issues: Optional list of issue IDs to limit detection to
+
+        Returns:
+            Set of issue file paths that changed
+
+        Examples:
+            >>> builder = OptimizedBaselineBuilder(Path("roadmap/issues"))
+            >>> changed = builder.get_changed_issue_files("HEAD~1")
+            >>> print(changed)  # {'.../TASK-123-example.md', ...}
+        """
+        try:
+            changed = PersistenceGateway.get_changed_files_since_commit(
+                git_ref, str(self.issues_dir)
+            )
+
+            # Filter to only issue files (*.md)
+            issue_files = {f for f in changed if f.endswith(".md")}
+
+            if limit_to_issues:
+                # Filter to specific issues
+                limited = {
+                    f
+                    for f in issue_files
+                    if any(issue_id in f for issue_id in limit_to_issues)
+                }
+                logger.debug(
+                    "changed_issue_files_filtered",
+                    total_changed=len(issue_files),
+                    limited_count=len(limited),
+                )
+                return limited
+
+            logger.debug(
+                "changed_issue_files_detected",
+                count=len(issue_files),
+                git_ref=git_ref,
+            )
+            return issue_files
+
+        except GitHistoryError as e:
+            logger.warning(
+                "changed_file_detection_failed",
+                error=str(e),
+                using_fallback="rebuild_all",
+                severity="operational",
+            )
+            # If git history fails, return empty set to trigger full rebuild
+            return set()
+
+    def should_rebuild_all(
+        self,
+        cached_state: SyncState | None,
+        time_since_last_sync: float | None = None,
+    ) -> bool:
+        """Determine if full rebuild is necessary.
+
+        Args:
+            cached_state: Previous sync state (if any)
+            time_since_last_sync: Seconds elapsed since last sync
+
+        Returns:
+            True if full rebuild recommended, False if incremental is OK
+
+        Rules:
+            - If no cached state: always rebuild
+            - If more than 1 hour since last sync: rebuild (safer)
+            - If metadata indicates corruption: rebuild
+            - Otherwise: incremental update possible
+        """
+        if not cached_state:
+            logger.debug(
+                "rebuild_required_reason",
+                reason="no_cached_state",
+            )
+            return True
+
+        if time_since_last_sync is not None and time_since_last_sync > 3600:
+            logger.debug(
+                "rebuild_recommended_reason",
+                reason="stale_cache_1hour_old",
+                seconds_old=time_since_last_sync,
+            )
+            return True
+
+        # If we get here, incremental update is safe
+        logger.debug(
+            "incremental_update_safe",
+            cached_issues=len(cached_state.base_issues),
+        )
+        return False
+
+    def extract_issue_id_from_path(self, file_path: str) -> str | None:
+        """Extract issue ID from file path.
+
+        Args:
+            file_path: Path like 'issues/backlog/TASK-123-example.md'
+
+        Returns:
+            Issue ID like 'TASK-123' or None if not parseable
+
+        Examples:
+            >>> builder.extract_issue_id_from_path("issues/backlog/TASK-123-example.md")
+            'TASK-123'
+            >>> builder.extract_issue_id_from_path("invalid.txt")
+            None
+        """
+        try:
+            # Extract filename
+            filename = Path(file_path).name
+            # Remove .md extension
+            base = filename.replace(".md", "")
+            # Take part before first dash and second dash
+            # Format is typically TASK-123-description or similar
+            parts = base.split("-")
+            if len(parts) >= 2:
+                # Check if first part is alphabetic and second is numeric
+                if parts[0].isalpha() and parts[1].isdigit():
+                    # Return TYPE-ID format (e.g., TASK-123)
+                    issue_id = f"{parts[0]}-{parts[1]}"
+                    return issue_id
+        except Exception as e:
+            logger.debug(
+                "issue_id_parsing_failed",
+                operation="parse_issue_id",
+                error=str(e),
+                action="Returning None",
+            )
+        return None
+
+    def get_issue_files_to_update(
+        self,
+        all_issue_files: list[Path],
+        changed_files: set[str],
+        cached_state: SyncState | None,
+    ) -> dict[str, Path]:
+        """Determine which issue files need baseline updates.
+
+        Args:
+            all_issue_files: All issue file paths
+            changed_files: Files that changed since git_ref
+            cached_state: Previous sync state (for unchanged issues)
+
+        Returns:
+            Dictionary mapping issue_id to file_path for issues to update
+
+        Logic:
+            - New issues (not in cache): always update
+            - Changed files: always update
+            - Unchanged files: skip (use cached)
+            - Deleted issues: mark for removal
+        """
+        files_to_update = {}
+
+        for file_path in all_issue_files:
+            relative_path = str(file_path.relative_to(self.issues_dir))
+            issue_id = self.extract_issue_id_from_path(relative_path)
+
+            if not issue_id:
+                logger.debug(
+                    "skipping_unparseable_issue_file",
+                    file_path=relative_path,
+                )
+                continue
+
+            # Check if file changed
+            is_changed = any(
+                relative_path in f or file_path.name in f for f in changed_files
+            )
+
+            # Check if new (or full rebuild if no cached state)
+            is_new = cached_state is None or issue_id not in cached_state.base_issues
+
+            if is_changed or is_new:
+                files_to_update[issue_id] = file_path
+                logger.debug(
+                    "issue_requires_update",
+                    issue_id=issue_id,
+                    reason="changed" if is_changed else "new",
+                )
+            else:
+                logger.debug(
+                    "issue_using_cached_baseline",
+                    issue_id=issue_id,
+                    reason="unchanged_file",
+                )
+
+        return files_to_update
+
+    def get_incremental_update_issues(
+        self,
+        all_issue_files: list[Path],
+        changed_files: set[str],
+        cached_state: SyncState,
+    ) -> tuple[dict[str, Path], list[str]]:
+        """Get issues for incremental baseline update.
+
+        Args:
+            all_issue_files: All current issue files
+            changed_files: Files changed since last sync
+            cached_state: Previous sync state
+
+        Returns:
+            Tuple of (issues_to_update, issues_to_remove)
+            - issues_to_update: {issue_id: file_path} for changed/new issues
+            - issues_to_remove: [issue_id] for deleted issues
+
+        Example:
+            >>> files = [Path("issues/backlog/TASK-123-example.md")]
+            >>> changed = {"issues/backlog/TASK-123-example.md"}
+            >>> updates, removals = builder.get_incremental_update_issues(files, changed, state)
+            >>> print(updates)  # {"TASK-123": Path(...)}
+            >>> print(removals)  # []
+        """
+        # Update progress: analyzing files
+        self._update_progress("Analyzing changed files...", 0, len(all_issue_files))
+
+        # Get issues to update
+        issues_to_update = self.get_issue_files_to_update(
+            all_issue_files, changed_files, cached_state
+        )
+
+        # Update progress: detecting deletions
+        self._update_progress(
+            f"Checking for deletions ({len(issues_to_update)} changed)...",
+            len(issues_to_update),
+            len(all_issue_files),
+        )
+
+        # Find deleted issues (in cache but not in current files)
+        current_ids = {self.extract_issue_id_from_path(str(f)) for f in all_issue_files}
+        current_ids.discard(None)
+
+        deleted_ids = [
+            issue_id
+            for issue_id in cached_state.base_issues.keys()
+            if issue_id not in current_ids
+        ]
+
+        if deleted_ids:
+            logger.info(
+                "detected_deleted_issues",
+                count=len(deleted_ids),
+                issue_ids=deleted_ids,
+            )
+
+        return issues_to_update, deleted_ids
+
+    def estimate_rebuild_time(
+        self,
+        issues_count: int,
+        git_operations_per_issue: float = 0.010,  # ~10ms per git operation
+        parse_overhead: float = 0.001,  # ~1ms per file parse
+    ) -> float:
+        """Estimate time to rebuild baseline state.
+
+        Args:
+            issues_count: Number of issues to rebuild
+            git_operations_per_issue: Average time for git operations (seconds)
+            parse_overhead: Time to parse each file (seconds)
+
+        Returns:
+            Estimated time in milliseconds
+
+        Examples:
+            >>> builder = OptimizedBaselineBuilder(Path("roadmap/issues"))
+            >>> time_ms = builder.estimate_rebuild_time(100)
+            >>> print(f"Est. time: {time_ms:.1f}ms")  # ~1.1s for 100 issues
+            >>> time_ms = builder.estimate_rebuild_time(5)
+            >>> print(f"Est. time: {time_ms:.1f}ms")  # ~55ms for 5 issues (incremental)
+        """
+        total_seconds = issues_count * (git_operations_per_issue + parse_overhead)
+        return total_seconds * 1000
+
+
+class CachedBaselineState:
+    """Wrapper for baseline state with cache metadata.
+
+    Tracks whether the state is from cache or freshly rebuilt,
+    and provides statistics about the rebuild process.
+    """
+
+    def __init__(
+        self,
+        state: SyncState,
+        from_cache: bool = False,
+        rebuilt_issues: int | None = None,
+        reused_issues: int | None = None,
+        rebuild_time_ms: float | None = None,
+    ):
+        """Initialize cached baseline state.
+
+        Args:
+            state: The SyncState object
+            from_cache: Whether this is from cache (True) or rebuilt (False)
+            rebuilt_issues: Count of issues rebuilt
+            reused_issues: Count of issues reused from cache
+            rebuild_time_ms: Time to rebuild in milliseconds
+        """
+        self.state = state
+        self.from_cache = from_cache
+        self.rebuilt_issues = rebuilt_issues or 0
+        self.reused_issues = reused_issues or 0
+        self.rebuild_time_ms = rebuild_time_ms or 0.0
+
+    @property
+    def is_full_rebuild(self) -> bool:
+        """Whether this is a full rebuild (no cache reuse)."""
+        return self.reused_issues == 0 and not self.from_cache
+
+    @property
+    def is_incremental(self) -> bool:
+        """Whether this is an incremental update (reused some cached issues)."""
+        return self.reused_issues > 0
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for logging.
+
+        Returns:
+            Dictionary with all metadata
+        """
+        return {
+            "from_cache": self.from_cache,
+            "is_full_rebuild": self.is_full_rebuild,
+            "is_incremental": self.is_incremental,
+            "rebuilt_issues": self.rebuilt_issues,
+            "reused_issues": self.reused_issues,
+            "total_issues": len(self.state.base_issues),
+            "rebuild_time_ms": f"{self.rebuild_time_ms:.1f}",
+        }