mcp-ticketer 0.2.0__py3-none-any.whl → 2.2.9__py3-none-any.whl

mcp_ticketer/queue/worker.py

@@ -7,17 +7,17 @@ import threading
 import time
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any
 
 from dotenv import load_dotenv
 
+# Import adapters module to trigger registration
+import mcp_ticketer.adapters  # noqa: F401
+
 from ..core import AdapterRegistry, Task
 from .queue import Queue, QueueItem, QueueStatus
 from .ticket_registry import TicketRegistry
 
-# Import adapters module to trigger registration
-import mcp_ticketer.adapters  # noqa: F401
-
 # Load environment variables from .env.local
 env_path = Path.cwd() / ".env.local"
 if env_path.exists():
@@ -58,7 +58,7 @@ class Worker:
 
     def __init__(
         self,
-        queue: Optional[Queue] = None,
+        queue: Queue | None = None,
         batch_size: int = DEFAULT_BATCH_SIZE,
         max_concurrent: int = DEFAULT_MAX_CONCURRENT,
     ):
@@ -97,12 +97,12 @@ class Worker:
             f"Worker initialized with batch_size={batch_size}, max_concurrent={max_concurrent}"
         )
 
-    def _signal_handler(self, signum, frame):
+    def _signal_handler(self, signum: int, frame: Any) -> None:
         """Handle shutdown signals."""
         logger.info(f"Received signal {signum}, shutting down...")
         self.stop()
 
-    def start(self, daemon: bool = True):
+    def start(self, daemon: bool = True) -> None:
         """Start the worker.
 
         Args:
@@ -126,14 +126,14 @@ class Worker:
             # Run in main thread
             self._run_loop()
 
-    def stop(self):
+    def stop(self) -> None:
         """Stop the worker."""
         logger.info("Stopping worker...")
         self.running = False
         self.stop_event.set()
 
-    def _run_loop(self):
-        """Main worker loop with batch processing."""
+    def _run_loop(self) -> None:
+        """Run main worker loop with batch processing."""
         logger.info("Worker loop started")
 
         # Reset any stuck items on startup
@@ -174,7 +174,7 @@ class Worker:
                 break
         return batch
 
-    async def _process_batch(self, batch: list[QueueItem]):
+    async def _process_batch(self, batch: list[QueueItem]) -> None:
         """Process a batch of queue items with concurrency control.
 
         Args:
@@ -184,7 +184,7 @@ class Worker:
         logger.info(f"Processing batch of {len(batch)} items")
 
         # Group items by adapter for concurrent processing
-        adapter_groups = {}
+        adapter_groups: dict[str, list[Any]] = {}
         for item in batch:
             if item.adapter not in adapter_groups:
                 adapter_groups[item.adapter] = []
@@ -199,7 +199,9 @@ class Worker:
         # Wait for all adapter groups to complete
         await asyncio.gather(*tasks, return_exceptions=True)
 
-    async def _process_adapter_group(self, adapter: str, items: list[QueueItem]):
+    async def _process_adapter_group(
+        self, adapter: str, items: list[QueueItem]
+    ) -> None:
         """Process items for a specific adapter with concurrency control.
 
         Args:
@@ -216,7 +218,7 @@ class Worker:
         semaphore = self.adapter_semaphores[adapter]
 
         # Process items with concurrency control
-        async def process_with_semaphore(item):
+        async def process_with_semaphore(item: QueueItem) -> None:
             async with semaphore:
                 await self._process_item(item)
 
@@ -226,7 +228,7 @@ class Worker:
         # Process with concurrency control
         await asyncio.gather(*tasks, return_exceptions=True)
 
-    async def _process_item(self, item: QueueItem):
+    async def _process_item(self, item: QueueItem) -> None:
         """Process a single queue item.
 
         Args:
@@ -263,15 +265,19 @@ class Worker:
 
             # Mark as completed in both queue and registry (atomic)
             success = self.queue.update_status(
-                item.id, QueueStatus.COMPLETED, result=result,
-                expected_status=QueueStatus.PROCESSING
+                item.id,
+                QueueStatus.COMPLETED,
+                result=result,
+                expected_status=QueueStatus.PROCESSING,
             )
             if success:
                 self.ticket_registry.update_ticket_status(
                     item.id, "completed", ticket_id=ticket_id, result_data=result
                 )
             else:
-                logger.warning(f"Failed to update status for {item.id} - item may have been processed by another worker")
+                logger.warning(
+                    f"Failed to update status for {item.id} - item may have been processed by another worker"
+                )
 
             self.stats["items_processed"] += 1
             logger.info(f"Successfully processed {item.id}, ticket ID: {ticket_id}")
@@ -301,26 +307,35 @@ class Worker:
                         item.id, "queued", retry_count=new_retry_count
                     )
                 else:
-                    logger.warning(f"Failed to increment retry for {item.id} - item may have been processed by another worker")
+                    logger.warning(
+                        f"Failed to increment retry for {item.id} - item may have been processed by another worker"
+                    )
 
                 # Wait before retry
                 await asyncio.sleep(retry_delay)
             else:
                 # Max retries exceeded, mark as failed (atomic)
                 success = self.queue.update_status(
-                    item.id, QueueStatus.FAILED, error_message=str(e),
-                    expected_status=QueueStatus.PROCESSING
+                    item.id,
+                    QueueStatus.FAILED,
+                    error_message=str(e),
+                    expected_status=QueueStatus.PROCESSING,
                 )
                 if success:
                     self.ticket_registry.update_ticket_status(
-                        item.id, "failed", error_message=str(e), retry_count=item.retry_count
+                        item.id,
+                        "failed",
+                        error_message=str(e),
+                        retry_count=item.retry_count,
                     )
                 else:
-                    logger.warning(f"Failed to mark {item.id} as failed - item may have been processed by another worker")
+                    logger.warning(
+                        f"Failed to mark {item.id} as failed - item may have been processed by another worker"
+                    )
                 self.stats["items_failed"] += 1
                 logger.error(f"Max retries exceeded for {item.id}, marking as failed")
 
-    async def _check_rate_limit(self, adapter: str):
+    async def _check_rate_limit(self, adapter: str) -> None:
         """Check and enforce rate limits.
 
         Args:
@@ -344,7 +359,7 @@ class Worker:
 
         self.last_request_times[adapter] = datetime.now()
 
-    def _get_adapter(self, item: QueueItem):
+    def _get_adapter(self, item: QueueItem) -> Any:
         """Get adapter instance for item.
 
         Args:
@@ -360,24 +375,31 @@ class Worker:
 
         from ..cli.main import load_config
 
-        # Use item's project_dir if available, otherwise use current directory
-        project_path = Path(item.project_dir) if item.project_dir else None
-
-        # Load environment variables from project directory's .env.local if it exists
-        if project_path:
-            env_file = project_path / ".env.local"
-            if env_file.exists():
-                logger.info(f"Worker loading environment from {env_file}")
-                load_dotenv(env_file)
-
-        logger.info(f"Worker project_path: {project_path}")
-        logger.info(f"Worker current working directory: {os.getcwd()}")
-
-        config = load_config(project_dir=project_path)
-        logger.info(f"Worker loaded config: {config}")
-        adapters_config = config.get("adapters", {})
-        adapter_config = adapters_config.get(item.adapter, {})
-        logger.info(f"Worker adapter config for {item.adapter}: {adapter_config}")
+        # PRIORITY 1: Use adapter_config from queue item if available (explicit config)
+        if item.adapter_config:
+            logger.info("Worker using explicit adapter_config from queue item")
+            adapter_config = item.adapter_config
+            logger.info(f"Worker adapter config for {item.adapter}: {adapter_config}")
+        else:
+            # PRIORITY 2: Load from project config file
+            # Use item's project_dir if available, otherwise use current directory
+            project_path = Path(item.project_dir) if item.project_dir else None
+
+            # Load environment variables from project directory's .env.local if it exists
+            if project_path:
+                env_file = project_path / ".env.local"
+                if env_file.exists():
+                    logger.info(f"Worker loading environment from {env_file}")
+                    load_dotenv(env_file)
+
+            logger.info(f"Worker project_path: {project_path}")
+            logger.info(f"Worker current working directory: {os.getcwd()}")
+
+            config = load_config(project_dir=project_path)
+            logger.info(f"Worker loaded config: {config}")
+            adapters_config = config.get("adapters", {})
+            adapter_config = adapters_config.get(item.adapter, {})
+            logger.info(f"Worker adapter config for {item.adapter}: {adapter_config}")
 
         # Add environment variables for authentication
         if item.adapter == "linear":
@@ -397,21 +419,32 @@ class Worker:
         # Add debugging for Linear adapter specifically
         if item.adapter == "linear":
             import os
+
             linear_api_key = os.getenv("LINEAR_API_KEY", "Not set")
-            logger.info(f"Worker LINEAR_API_KEY: {linear_api_key[:20] if linear_api_key != 'Not set' else 'Not set'}...")
-            logger.info(f"Worker adapter_config api_key: {adapter_config.get('api_key', 'Not set')[:20] if adapter_config.get('api_key') else 'Not set'}...")
+            logger.info(
+                f"Worker LINEAR_API_KEY: {linear_api_key[:20] if linear_api_key != 'Not set' else 'Not set'}..."
+            )
+            logger.info(
+                f"Worker adapter_config api_key: {adapter_config.get('api_key', 'Not set')[:20] if adapter_config.get('api_key') else 'Not set'}..."
+            )
 
         adapter = AdapterRegistry.get_adapter(item.adapter, adapter_config)
-        logger.info(f"Worker created adapter: {type(adapter)} with team_id: {getattr(adapter, 'team_id_config', 'Not set')}")
+        logger.info(
+            f"Worker created adapter: {type(adapter)} with team_id: {getattr(adapter, 'team_id_config', 'Not set')}"
+        )
 
         # Add more debugging for Linear adapter
         if item.adapter == "linear":
-            logger.info(f"Worker Linear adapter api_key: {getattr(adapter, 'api_key', 'Not set')[:20] if getattr(adapter, 'api_key', None) else 'Not set'}...")
-            logger.info(f"Worker Linear adapter team_key: {getattr(adapter, 'team_key', 'Not set')}")
+            logger.info(
+                f"Worker Linear adapter api_key: {getattr(adapter, 'api_key', 'Not set')[:20] if getattr(adapter, 'api_key', None) else 'Not set'}..."
+            )
+            logger.info(
+                f"Worker Linear adapter team_key: {getattr(adapter, 'team_key', 'Not set')}"
+            )
 
         return adapter
 
-    async def _execute_operation(self, adapter, item: QueueItem) -> dict[str, Any]:
+    async def _execute_operation(self, adapter: Any, item: QueueItem) -> dict[str, Any]:
         """Execute the queued operation.
 
         Args:
@@ -461,14 +494,13 @@ class Worker:
             result = await adapter.create_epic(
                 title=data["title"],
                 description=data.get("description"),
-                **{k: v for k, v in data.items()
-                   if k not in ["title", "description"]}
+                **{k: v for k, v in data.items() if k not in ["title", "description"]},
             )
             return {
                 "id": result.id if result else None,
                 "title": result.title if result else None,
                 "type": "epic",
-                "success": bool(result)
+                "success": bool(result),
             }
 
         elif operation == "create_issue":
@@ -476,15 +508,18 @@ class Worker:
                 title=data["title"],
                 description=data.get("description"),
                 epic_id=data.get("epic_id"),
-                **{k: v for k, v in data.items()
-                   if k not in ["title", "description", "epic_id"]}
+                **{
+                    k: v
+                    for k, v in data.items()
+                    if k not in ["title", "description", "epic_id"]
+                },
             )
             return {
                 "id": result.id if result else None,
                 "title": result.title if result else None,
                 "type": "issue",
                 "epic_id": data.get("epic_id"),
-                "success": bool(result)
+                "success": bool(result),
             }
 
         elif operation == "create_task":
@@ -492,15 +527,18 @@ class Worker:
                 title=data["title"],
                 parent_id=data["parent_id"],
                 description=data.get("description"),
-                **{k: v for k, v in data.items()
-                   if k not in ["title", "parent_id", "description"]}
+                **{
+                    k: v
+                    for k, v in data.items()
+                    if k not in ["title", "parent_id", "description"]
+                },
             )
             return {
                 "id": result.id if result else None,
                 "title": result.title if result else None,
                 "type": "task",
                 "parent_id": data["parent_id"],
-                "success": bool(result)
+                "success": bool(result),
             }
 
         else:

mcp_ticketer/utils/__init__.py

@@ -0,0 +1,5 @@
+"""Utility modules for mcp-ticketer."""
+
+from .token_utils import estimate_json_tokens, estimate_tokens, paginate_response
+
+__all__ = ["estimate_tokens", "estimate_json_tokens", "paginate_response"]

mcp_ticketer/utils/token_utils.py

@@ -0,0 +1,246 @@
+"""Token counting and pagination utilities for MCP tool responses.
+
+This module provides utilities for estimating token counts and implementing
+token-aware pagination to ensure responses stay under 20k token limits.
+
+Design Decision: Token estimation vs. exact counting
+- Uses 4-chars-per-token heuristic (conservative)
+- Rationale: Actual tokenization requires tiktoken library and GPT-specific
+  tokenizer, which adds dependency and runtime overhead
+- Trade-off: Approximate (±10%) vs. exact, but fast and dependency-free
+- Extension Point: Can add tiktoken support via optional dependency later
+
+Performance: O(1) for token estimation (string length only)
+Memory: O(1) auxiliary space (no allocations beyond JSON serialization)
+"""
+
+import json
+import logging
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+# Type variable for generic list items
+T = TypeVar("T")
+
+# Conservative token estimation: 1 token ≈ 4 characters
+# Based on OpenAI/Anthropic averages for English text + JSON structure
+CHARS_PER_TOKEN = 4
+
+# Default maximum tokens per MCP response
+DEFAULT_MAX_TOKENS = 20_000
+
+# Overhead estimation for response metadata (status, adapter info, etc.)
+BASE_RESPONSE_OVERHEAD = 100
+
+
+def estimate_tokens(text: str) -> int:
+    """Estimate token count for a text string.
+
+    Uses conservative heuristic: 1 token ≈ 4 characters.
+    This works reasonably well for English text and JSON structures.
+
+    Design Trade-off:
+    - Fast: O(len(text)) string length check
+    - Approximate: ±10% accuracy vs. exact tokenization
+    - Zero dependencies: No tiktoken or model-specific tokenizers needed
+
+    Performance:
+    - Time Complexity: O(n) where n is string length
+    - Space Complexity: O(1)
+
+    Args:
+        text: Input text to estimate token count for
+
+    Returns:
+        Estimated token count (conservative, may overestimate slightly)
+
+    Example:
+        >>> estimate_tokens("Hello world")
+        3  # "Hello world" = 11 chars / 4 ≈ 2.75 → rounds to 3
+        >>> estimate_tokens(json.dumps({"id": "123", "title": "Test"}))
+        8  # JSON structure increases char count
+    """
+    if not text:
+        return 0
+    return max(1, len(text) // CHARS_PER_TOKEN)
+
+
+def estimate_json_tokens(data: dict | list | Any) -> int:
+    """Estimate token count for JSON-serializable data.
+
+    Serializes data to JSON string then estimates tokens.
+    Accounts for JSON structure overhead (brackets, quotes, commas).
+
+    Performance:
+    - Time Complexity: O(n) where n is serialized JSON size
+    - Space Complexity: O(n) for JSON string (temporary)
+
+    Args:
+        data: Any JSON-serializable data (dict, list, primitives)
+
+    Returns:
+        Estimated token count for serialized representation
+
+    Example:
+        >>> estimate_json_tokens({"id": "123", "title": "Test"})
+        8
+        >>> estimate_json_tokens([1, 2, 3])
+        2
+    """
+    try:
+        json_str = json.dumps(data, default=str)  # default=str for non-serializable
+        return estimate_tokens(json_str)
+    except (TypeError, ValueError) as e:
+        logging.warning(f"Failed to serialize data for token estimation: {e}")
+        # Fallback: estimate based on string representation
+        return estimate_tokens(str(data))
+
+
+def paginate_response(
+    items: list[T],
+    limit: int = 20,
+    offset: int = 0,
+    max_tokens: int = DEFAULT_MAX_TOKENS,
+    serialize_fn: Callable[[T], dict] | None = None,
+    compact_fn: Callable[[dict], dict] | None = None,
+    compact: bool = True,
+) -> dict[str, Any]:
+    """Paginate a list of items with token-aware limiting.
+
+    This function implements automatic pagination that:
+    1. Respects explicit limit/offset parameters
+    2. Stops adding items if response would exceed max_tokens
+    3. Optionally applies compact transformation to reduce token usage
+    4. Returns pagination metadata for client-side handling
+
+    Design Decision: Token-aware vs. count-based pagination
+    - Hybrid approach: Uses both item count AND token limits
+    - Rationale: Prevents oversized responses even with small item counts
+    - Example: 10 tickets with huge descriptions could exceed 20k tokens
+    - Trade-off: Slightly more complex but safer for production use
+
+    Performance:
+    - Time Complexity: O(n) where n is min(limit, items until token limit)
+    - Space Complexity: O(n) for result items list
+    - Early termination: Stops as soon as token limit would be exceeded
+
+    Args:
+        items: List of items to paginate
+        limit: Maximum number of items to return (default: 20)
+        offset: Number of items to skip (default: 0)
+        max_tokens: Maximum tokens allowed in response (default: 20,000)
+        serialize_fn: Optional function to convert item to dict (e.g., model.model_dump)
+        compact_fn: Optional function to create compact representation
+        compact: Whether to apply compact_fn if provided (default: True)
+
+    Returns:
+        Dictionary containing:
+        - items: List of paginated items (serialized)
+        - count: Number of items returned
+        - total: Total items available (before pagination)
+        - offset: Offset used for this page
+        - limit: Limit requested
+        - has_more: Boolean indicating if more items exist
+        - truncated_by_tokens: Boolean indicating if token limit caused truncation
+        - estimated_tokens: Approximate token count for response
+
+    Error Conditions:
+        - Invalid limit (<= 0): Returns empty result with error flag
+        - Invalid offset (< 0): Uses offset=0
+        - serialize_fn fails: Logs warning and skips item
+
+    Example:
+        >>> tickets = [Ticket(...), Ticket(...), ...]  # 100 tickets
+        >>> result = paginate_response(
+        ...     tickets,
+        ...     limit=20,
+        ...     offset=0,
+        ...     serialize_fn=lambda t: t.model_dump(),
+        ...     compact_fn=_compact_ticket,
+        ... )
+        >>> result["count"]  # 20 (or less if token limit hit)
+        >>> result["has_more"]  # True
+        >>> result["estimated_tokens"]  # ~2500
+    """
+    # Validate parameters
+    if limit <= 0:
+        logging.warning(f"Invalid limit {limit}, using default 20")
+        limit = 20
+
+    if offset < 0:
+        logging.warning(f"Invalid offset {offset}, using 0")
+        offset = 0
+
+    total_items = len(items)
+
+    # Apply offset
+    items_after_offset = items[offset:]
+
+    # Track token usage
+    estimated_tokens = BASE_RESPONSE_OVERHEAD  # Base response overhead
+    result_items: list[dict] = []
+    truncated_by_tokens = False
+
+    # Process items up to limit or token threshold
+    for idx, item in enumerate(items_after_offset):
+        # Check if we've hit the limit
+        if idx >= limit:
+            break
+
+        # Serialize item
+        try:
+            if serialize_fn:
+                item_dict = serialize_fn(item)
+            elif hasattr(item, "model_dump"):
+                item_dict = item.model_dump()
+            elif isinstance(item, dict):
+                item_dict = item
+            else:
+                item_dict = {"data": str(item)}
+
+            # Apply compact mode if requested and function provided
+            if compact and compact_fn:
+                item_dict = compact_fn(item_dict)
+
+            # Estimate tokens for this item
+            item_tokens = estimate_json_tokens(item_dict)
+
+            # Check if adding this item would exceed token limit
+            if estimated_tokens + item_tokens > max_tokens:
+                logging.info(
+                    f"Token limit reached: {estimated_tokens + item_tokens} > {max_tokens}. "
+                    f"Returning {len(result_items)} items instead of requested {limit}."
+                )
+                truncated_by_tokens = True
+                break
+
+            # Add item to results
+            result_items.append(item_dict)
+            estimated_tokens += item_tokens
+
+        except Exception as e:
+            logging.warning(f"Failed to serialize item at index {idx + offset}: {e}")
+            # Skip this item and continue
+            continue
+
+    # Calculate pagination metadata
+    items_returned = len(result_items)
+    has_more = (offset + items_returned) < total_items
+
+    # Warn if approaching token limit
+    if estimated_tokens > max_tokens * 0.8:
+        logging.warning(
+            f"Response approaching token limit: {estimated_tokens}/{max_tokens} tokens "
+            f"({estimated_tokens/max_tokens*100:.1f}%). Consider using compact mode or reducing limit."
+        )
+
+    return {
+        "items": result_items,
+        "count": items_returned,
+        "total": total_items,
+        "offset": offset,
+        "limit": limit,
+        "has_more": has_more,
+        "truncated_by_tokens": truncated_by_tokens,
+        "estimated_tokens": estimated_tokens,
+    }