roadmap-cli 0.1.1__py3-none-any.whl

roadmap/core/services/sync/batch_processor.py

@@ -0,0 +1,290 @@
+"""Batch processing for parallel sync operations."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import TYPE_CHECKING, Any
+
+from structlog import get_logger
+
+if TYPE_CHECKING:
+    from roadmap.core.domain.issue import Issue
+
+logger = get_logger(__name__)
+
+
+class BatchProcessor:
+    """Process items in parallel batches for improved performance."""
+
+    def __init__(self, max_workers: int = 5, batch_size: int = 10):
+        """Initialize batch processor.
+
+        Args:
+            max_workers: Maximum number of parallel workers
+            batch_size: Number of items to process per batch
+        """
+        self.max_workers = max_workers
+        self.batch_size = batch_size
+
+    def process_batch(
+        self,
+        items: list[Any],
+        process_func: Callable[[Any], Any],
+        on_progress: Callable[[int, Any], None] | None = None,
+    ) -> tuple[list[Any], list[tuple[Any, Exception]]]:
+        """Process items in parallel batches.
+
+        Args:
+            items: List of items to process
+            process_func: Function to apply to each item
+            on_progress: Optional callback for progress updates (index, result)
+
+        Returns:
+            Tuple of (successful_results, failed_items_with_errors)
+        """
+        results = []
+        errors = []
+
+        # Split into batches
+        batches = [
+            items[i : i + self.batch_size]
+            for i in range(0, len(items), self.batch_size)
+        ]
+
+        logger.info(
+            "batch_processing_started",
+            total_items=len(items),
+            batches=len(batches),
+            batch_size=self.batch_size,
+            max_workers=self.max_workers,
+            action="process_batch",
+        )
+
+        # Process each batch in parallel
+        for batch_idx, batch in enumerate(batches):
+            batch_results, batch_errors = self._process_single_batch(
+                batch, process_func, batch_idx * self.batch_size, on_progress
+            )
+            results.extend(batch_results)
+            errors.extend(batch_errors)
+
+        logger.info(
+            "batch_processing_completed",
+            total_items=len(items),
+            successful=len(results),
+            failed=len(errors),
+            action="process_batch",
+        )
+
+        return results, errors
+
+    def _process_single_batch(
+        self,
+        batch: list[Any],
+        process_func: Callable[[Any], Any],
+        offset: int,
+        on_progress: Callable[[int, Any], None] | None = None,
+    ) -> tuple[list[Any], list[tuple[Any, Exception]]]:
+        """Process a single batch using thread pool.
+
+        Args:
+            batch: Batch of items to process
+            process_func: Function to apply to each item
+            offset: Offset for progress reporting
+            on_progress: Optional callback for progress updates
+
+        Returns:
+            Tuple of (results, errors)
+        """
+        results = []
+        errors = []
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # Submit all tasks
+            future_to_item = {
+                executor.submit(process_func, item): (idx + offset, item)
+                for idx, item in enumerate(batch)
+            }
+
+            # Collect results as they complete
+            for future in as_completed(future_to_item):
+                idx, item = future_to_item[future]
+                try:
+                    result = future.result()
+                    results.append(result)
+
+                    if on_progress:
+                        on_progress(idx, result)
+
+                except Exception as e:
+                    logger.error(
+                        "batch_item_failed",
+                        index=idx,
+                        item=str(item)[:100],
+                        error=str(e),
+                        action="process_item",
+                    )
+                    errors.append((item, e))
+
+        return results, errors
+
+
+class AsyncBatchProcessor:
+    """Async batch processor for async operations."""
+
+    def __init__(self, batch_size: int = 10, max_concurrent: int = 5):
+        """Initialize async batch processor.
+
+        Args:
+            batch_size: Number of items per batch
+            max_concurrent: Maximum concurrent tasks
+        """
+        self.batch_size = batch_size
+        self.max_concurrent = max_concurrent
+
+    async def process_batch(
+        self,
+        items: list[Any],
+        process_func: Callable[[Any], Any],
+        on_progress: Callable[[int, Any], None] | None = None,
+    ) -> tuple[list[Any], list[tuple[Any, Exception]]]:
+        """Process items asynchronously in batches.
+
+        Args:
+            items: List of items to process
+            process_func: Async function to apply to each item
+            on_progress: Optional callback for progress updates
+
+        Returns:
+            Tuple of (results, errors)
+        """
+        results = []
+        errors = []
+
+        # Split into batches
+        batches = [
+            items[i : i + self.batch_size]
+            for i in range(0, len(items), self.batch_size)
+        ]
+
+        logger.info(
+            "async_batch_processing_started",
+            total_items=len(items),
+            batches=len(batches),
+            batch_size=self.batch_size,
+            max_concurrent=self.max_concurrent,
+            action="process_batch",
+        )
+
+        # Process batches
+        for batch_idx, batch in enumerate(batches):
+            batch_results, batch_errors = await self._process_single_batch(
+                batch, process_func, batch_idx * self.batch_size, on_progress
+            )
+            results.extend(batch_results)
+            errors.extend(batch_errors)
+
+        logger.info(
+            "async_batch_processing_completed",
+            total_items=len(items),
+            successful=len(results),
+            failed=len(errors),
+            action="process_batch",
+        )
+
+        return results, errors
+
+    async def _process_single_batch(
+        self,
+        batch: list[Any],
+        process_func: Callable[[Any], Any],
+        offset: int,
+        on_progress: Callable[[int, Any], None] | None = None,
+    ) -> tuple[list[Any], list[tuple[Any, Exception]]]:
+        """Process a single batch asynchronously.
+
+        Args:
+            batch: Batch of items to process
+            process_func: Async function to apply
+            offset: Offset for progress reporting
+            on_progress: Optional callback for progress
+
+        Returns:
+            Tuple of (results, errors)
+        """
+        results = []
+        errors = []
+
+        # Create semaphore to limit concurrency
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+
+        async def process_with_semaphore(idx: int, item: Any):
+            async with semaphore:
+                try:
+                    result = await process_func(item)
+                    results.append(result)
+
+                    if on_progress:
+                        on_progress(idx + offset, result)
+
+                    return result
+                except Exception as e:
+                    logger.error(
+                        "async_batch_item_failed",
+                        index=idx + offset,
+                        item=str(item)[:100],
+                        error=str(e),
+                        action="process_item",
+                    )
+                    errors.append((item, e))
+                    return None
+
+        # Process all items in batch concurrently
+        await asyncio.gather(
+            *[process_with_semaphore(idx, item) for idx, item in enumerate(batch)],
+            return_exceptions=True,
+        )
+
+        return results, errors
+
+
+def chunk_list(items: list[Any], chunk_size: int) -> list[list[Any]]:
+    """Split a list into chunks of specified size.
+
+    Args:
+        items: List to chunk
+        chunk_size: Size of each chunk
+
+    Returns:
+        List of chunks
+    """
+    return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
+
+
+def batch_issues_by_milestone(issues: list[Issue]) -> dict[str, list[Issue]]:
+    """Group issues by milestone for batch processing.
+
+    Args:
+        issues: List of issues
+
+    Returns:
+        Dict mapping milestone names to issue lists
+    """
+    batches: dict[str, list[Issue]] = {}
+
+    for issue in issues:
+        milestone = issue.milestone or "_no_milestone"
+        if milestone not in batches:
+            batches[milestone] = []
+        batches[milestone].append(issue)
+
+    logger.info(
+        "issues_batched_by_milestone",
+        total_issues=len(issues),
+        milestones=len(batches),
+        action="batch_issues",
+    )
+
+    return batches

roadmap/core/services/sync/conflict_resolver.py

@@ -0,0 +1,154 @@
+"""Conflict resolution strategies for handling merge conflicts.
+
+When three-way merge detects a true conflict (both sides changed
+differently), this module applies field-level strategies to decide
+what to do with that conflict.
+"""
+
+from enum import StrEnum
+from typing import Any
+
+
+class ConflictResolutionStrategy(StrEnum):
+    """Strategy for resolving a conflict."""
+
+    FLAG_FOR_REVIEW = "flag_for_review"  # Return None, needs manual review
+    GITHUB_WINS = "github_wins"  # Use remote value
+    LOCAL_WINS = "local_wins"  # Use local value
+    MERGE_UNION = "merge_union"  # Combine lists (for labels)
+    MERGE_APPEND = "merge_append"  # Append both (for comments/description)
+
+
+class ConflictResolver:
+    """Applies field-level strategies to resolve conflicts.
+
+    Each field has a predefined strategy for what to do when
+    the three-way merge detects a conflict on that field.
+
+    Critical fields (status, assignee) are flagged for review.
+    Non-critical fields (labels, comments) are auto-merged.
+    """
+
+    # Field-level rules (from the sync implementation plan)
+    RULES = {
+        # Critical fields - flag for review (user must decide)
+        "status": ConflictResolutionStrategy.FLAG_FOR_REVIEW,
+        "assignee": ConflictResolutionStrategy.FLAG_FOR_REVIEW,
+        "milestone": ConflictResolutionStrategy.FLAG_FOR_REVIEW,
+        # Merge-friendly fields
+        "labels": ConflictResolutionStrategy.MERGE_UNION,
+        "description": ConflictResolutionStrategy.MERGE_APPEND,
+        "comments": ConflictResolutionStrategy.MERGE_APPEND,
+        # Metadata - GitHub is authoritative
+        "created_at": ConflictResolutionStrategy.GITHUB_WINS,
+        "updated_at": ConflictResolutionStrategy.GITHUB_WINS,
+        # Default: flag for review if not in RULES
+    }
+
+    def resolve_conflict(
+        self,
+        field: str,
+        base: Any,
+        local: Any,
+        remote: Any,
+    ) -> tuple[Any, bool]:
+        """Resolve a single field conflict.
+
+        Args:
+            field: Field name
+            base: Base value (from last sync)
+            local: Local value (current local state)
+            remote: Remote value (current remote state)
+
+        Returns:
+            (resolved_value, is_flagged)
+            where is_flagged=True means this needs manual review
+        """
+        strategy = self.RULES.get(field, ConflictResolutionStrategy.FLAG_FOR_REVIEW)
+
+        if strategy == ConflictResolutionStrategy.FLAG_FOR_REVIEW:
+            # Return None to indicate this needs manual review
+            return None, True
+
+        elif strategy == ConflictResolutionStrategy.GITHUB_WINS:
+            return remote, False
+
+        elif strategy == ConflictResolutionStrategy.LOCAL_WINS:
+            return local, False
+
+        elif strategy == ConflictResolutionStrategy.MERGE_UNION:
+            # For labels: combine both lists, deduplicate
+            local_list = local if isinstance(local, list) else [local] if local else []
+            remote_list = (
+                remote if isinstance(remote, list) else [remote] if remote else []
+            )
+            merged = list(set(local_list + remote_list))
+            return merged, False
+
+        elif strategy == ConflictResolutionStrategy.MERGE_APPEND:
+            # For comments/description: append both with source markers
+            local_str = str(local) if local else ""
+            remote_str = str(remote) if remote else ""
+            result = f"{local_str}\n\n--- REMOTE CHANGES ---\n{remote_str}"
+            # Still flag for manual cleanup of markers
+            return result, True
+
+        else:
+            # Unexpected strategy, flag for review
+            return None, True
+
+    def resolve_issue_conflicts(
+        self,
+        issue_id: str,
+        base: dict[str, Any],
+        local: dict[str, Any],
+        remote: dict[str, Any],
+        conflict_fields: list[str],
+    ) -> tuple[dict[str, Any], list[str]]:
+        """Resolve all conflicts in an issue.
+
+        Args:
+            issue_id: Issue identifier (for logging)
+            base: Base version of issue
+            local: Local version of issue
+            remote: Remote version of issue
+            conflict_fields: List of field names that have conflicts
+
+        Returns:
+            (resolved_fields, list_of_fields_still_flagged)
+
+        The resolved_fields dict contains fields that were
+        auto-resolved. Fields still flagged need manual review.
+        """
+        resolved = {}
+        flagged = []
+
+        for field in conflict_fields:
+            value, is_flagged = self.resolve_conflict(
+                field,
+                base.get(field),
+                local.get(field),
+                remote.get(field),
+            )
+
+            if is_flagged:
+                flagged.append(field)
+                # Don't include in resolved yet; needs manual review
+            else:
+                if value is not None:
+                    resolved[field] = value
+
+        return resolved, flagged
+
+    def has_critical_conflicts(self, flagged_fields: list[str]) -> bool:
+        """Check if flagged fields include critical ones.
+
+        Critical fields are status, assignee, milestone.
+        If these have conflicts, user MUST review.
+        """
+        critical = {"status", "assignee", "milestone"}
+        return any(field in critical for field in flagged_fields)
+
+    def get_strategy_for_field(self, field: str) -> ConflictResolutionStrategy:
+        """Get the strategy for a specific field."""
+        return self.RULES.get(field, ConflictResolutionStrategy.FLAG_FOR_REVIEW)