charisma-cli 0.1.2__py3-none-any.whl

charisma_cli/uploader.py

@@ -0,0 +1,488 @@
+"""Queue consumer, batch accumulation, HTTP client, and retry logic for streaming uploads."""
+
+import json
+import logging
+import threading
+import time
+from pathlib import Path
+from queue import Empty, PriorityQueue
+from typing import Any
+
+import httpx
+
+from charisma_cli.config import Config
+from charisma_cli.models import (
+    ContainerData,
+    FileCategory,
+    FileEvent,
+    UploadSummary,
+)
+from charisma_cli.parser import extract_attachment_refs, parse_container_file, parse_result_file
+from charisma_cli.retry import MAX_RETRIES, RETRYABLE_STATUS_CODES, exponential_backoff
+
+logger = logging.getLogger(__name__)
+
+_BATCH_SIZE = 50
+_BATCH_TIMEOUT_SECONDS = 2.0
+_CONTAINER_ASSOCIATION_TIMEOUT = 60.0
+
+
+def _epoch_ms_to_iso(epoch_ms: int) -> str:
+    """Convert epoch milliseconds to ISO 8601 UTC string."""
+    from datetime import datetime, timezone
+    dt = datetime.fromtimestamp(epoch_ms / 1000.0, tz=timezone.utc)
+    return dt.isoformat()
+
+
+class Uploader:
+    """Consumes file events from a PriorityQueue, parses them, and streams to the Charisma API.
+
+    Runs in a background thread. Accumulates parsed results into batches
+    (flush at 50 items or 2s timeout). Handles retries, circuit breaking,
+    and 401 detection. All exceptions are caught internally.
+
+    Client lifecycle: The Uploader owns its httpx.Client. Call `ensure_client()`
+    before `open_launch()`/`close_launch()` if calling them outside of start/stop.
+    `start()` creates the client, `stop()` closes it.
+    """
+
+    def __init__(self, config: Config, queue: PriorityQueue[FileEvent]) -> None:
+        self._config = config
+        self._queue = queue
+        self._summary = UploadSummary()
+        self._launch_id: str | None = None
+        self._disabled = False
+        self._auth_failed = False
+        self._stop_event = threading.Event()
+        self._drain_event = threading.Event()
+        self._thread: threading.Thread | None = None
+        self._client: httpx.Client | None = None
+        self._batch: list[dict[str, Any]] = []
+        self._batch_start: float = 0.0
+        self._pending_containers: list[ContainerData] = []
+        self._known_result_uuids: set[str] = set()
+        # Map source filename → result_uuid for attachment uploads
+        self._attachment_result_map: dict[str, str] = {}
+
+    @property
+    def summary(self) -> UploadSummary:
+        """Return current upload statistics."""
+        return self._summary
+
+    def configure_logging(self) -> None:
+        """Configure logging based on silent mode.
+
+        When silent=True, upload-related errors are routed to DEBUG level only
+        (no stderr output). When silent=False, errors are logged at ERROR level.
+        """
+        cli_logger = logging.getLogger("charisma_cli")
+        if self._config.silent:
+            cli_logger.setLevel(logging.CRITICAL)
+        else:
+            cli_logger.setLevel(logging.WARNING)
+            if not cli_logger.handlers:
+                handler = logging.StreamHandler()
+                handler.setLevel(logging.WARNING)
+                cli_logger.addHandler(handler)
+
+    def ensure_client(self) -> None:
+        """Create the httpx client if not already created.
+
+        Idempotent — safe to call multiple times.
+        """
+        if self._client is None:
+            self._client = httpx.Client(
+                base_url=self._config.endpoint,
+                headers={"Authorization": f"Bearer {self._config.token}"},
+                timeout=30.0,
+            )
+
+    def start(self) -> None:
+        """Start the uploader consumer thread."""
+        self.ensure_client()
+        self._thread = threading.Thread(target=self._run, daemon=True, name="charisma-uploader")
+        self._thread.start()
+
+    def stop(self) -> None:
+        """Signal the uploader thread to exit and wait for it."""
+        self._stop_event.set()
+        if self._thread is not None:
+            self._thread.join(timeout=10.0)
+        if self._client is not None:
+            self._client.close()
+            self._client = None
+
+    def drain(self, timeout: float) -> None:
+        """Signal drain and wait for queue to empty or timeout to expire.
+
+        Args:
+            timeout: Maximum seconds to wait for the queue to drain.
+        """
+        self._drain_event.set()
+        deadline = time.monotonic() + timeout
+        while not self._queue.empty() and time.monotonic() < deadline:
+            time.sleep(0.1)
+        if not self._queue.empty():
+            logger.warning("Drain timeout expired with %d items remaining in queue", self._queue.qsize())
+
+    def open_launch(self) -> str | None:
+        """Open a streaming launch via the API.
+
+        Returns:
+            The launch ID if successful, None if the open failed.
+        """
+        self.ensure_client()
+
+        payload: dict[str, Any] = {
+            "projectAlias": self._config.project,
+            "expectedTests": 1000,
+        }
+        if self._config.build_id:
+            payload["buildId"] = self._config.build_id
+        if self._config.commit_sha:
+            payload["commitSha"] = self._config.commit_sha
+        if self._config.branch:
+            payload["branch"] = self._config.branch
+
+        for attempt in range(MAX_RETRIES + 1):
+            try:
+                response = self._client.post("/api/v1/launches", json=payload)
+                if response.status_code == 201:
+                    data = response.json()
+                    self._launch_id = data["launchId"]
+                    return self._launch_id
+                if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES:
+                    time.sleep(exponential_backoff(attempt))
+                    continue
+                logger.error("Failed to open launch: HTTP %d", response.status_code)
+                return None
+            except (httpx.ConnectError, httpx.TimeoutException) as e:
+                if attempt < MAX_RETRIES:
+                    time.sleep(exponential_backoff(attempt))
+                    continue
+                logger.error("Failed to open launch after retries: %s", e)
+                return None
+
+        return None
+
+    def close_launch(self) -> None:
+        """Close the streaming launch via the API."""
+        self.ensure_client()
+        if self._launch_id is None:
+            return
+
+        for attempt in range(MAX_RETRIES + 1):
+            try:
+                response = self._client.post(f"/api/v1/launches/{self._launch_id}/close")
+                if response.status_code in (200, 201, 204):
+                    return
+                if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES:
+                    time.sleep(exponential_backoff(attempt))
+                    continue
+                logger.warning("Failed to close launch: HTTP %d", response.status_code)
+                return
+            except (httpx.ConnectError, httpx.TimeoutException) as e:
+                if attempt < MAX_RETRIES:
+                    time.sleep(exponential_backoff(attempt))
+                    continue
+                logger.warning("Failed to close launch after retries: %s", e)
+                return
+
+    def _run(self) -> None:
+        """Main consumer loop running in the background thread."""
+        try:
+            while not self._stop_event.is_set():
+                self._process_queue_item()
+                self._check_batch_timeout()
+            # On stop, flush remaining
+            self._flush_remaining()
+        except Exception:
+            logger.exception("Unexpected error in uploader thread")
+
+    def _process_queue_item(self) -> None:
+        """Try to dequeue and process one item."""
+        try:
+            event: FileEvent = self._queue.get(timeout=0.1)
+        except Empty:
+            return
+
+        if self._disabled or self._auth_failed:
+            self._queue.task_done()
+            return
+
+        try:
+            self._handle_event(event)
+        except Exception:
+            logger.exception("Error processing file: %s", event.path)
+        finally:
+            self._queue.task_done()
+
+    def _handle_event(self, event: FileEvent) -> None:
+        """Route a file event to the appropriate handler."""
+        if event.category == FileCategory.RESULT:
+            self._handle_result(event.path)
+        elif event.category == FileCategory.CONTAINER:
+            self._handle_container(event.path)
+        elif event.category == FileCategory.ATTACHMENT:
+            self._handle_attachment(event.path)
+
+    def _handle_result(self, path: Path) -> None:
+        """Parse a result file and add to the current batch.
+
+        Reads the file once, uses parsed data for both the batch payload
+        and attachment reference extraction.
+        """
+        # Read file once
+        try:
+            raw_text = path.read_text(encoding="utf-8")
+        except (OSError, FileNotFoundError):
+            logger.warning("Cannot read result file: %s", path)
+            self._summary.results_failed += 1
+            return
+
+        try:
+            raw_data = json.loads(raw_text)
+        except (json.JSONDecodeError, ValueError):
+            logger.warning("Malformed JSON in result file: %s", path)
+            self._summary.results_failed += 1
+            return
+
+        result = parse_result_file(path)
+        if result is None:
+            self._summary.results_failed += 1
+            return
+
+        # Track UUID for container association
+        if result.uuid:
+            self._known_result_uuids.add(result.uuid)
+
+        # Build API payload
+        payload: dict[str, Any] = {
+            "testId": result.testId,
+            "outcome": result.outcome,
+            "duration_ms": result.duration_ms,
+        }
+        if result.name:
+            payload["name"] = result.name
+        if result.full_name:
+            payload["fullName"] = result.full_name
+        if result.error_message:
+            payload["error_message"] = result.error_message
+        if result.stack_trace:
+            payload["stack_trace"] = result.stack_trace
+        if result.labels:
+            # Convert Allure's [{name, value}] array to {name: value} dict
+            # (matches batch ingestion format expected by frontend)
+            labels_dict: dict[str, str] = {}
+            for label in result.labels:
+                name_key = label.get("name", "")
+                value_str = label.get("value", "")
+                if name_key and value_str:
+                    labels_dict[name_key] = value_str
+            payload["labels"] = labels_dict
+        if result.parameters:
+            # Convert Allure's [{name, value}] array to {name: value} dict
+            params_dict: dict[str, str] = {}
+            for param in result.parameters:
+                name_key = param.get("name", "")
+                value_str = param.get("value", "")
+                if name_key:
+                    params_dict[name_key] = value_str
+            payload["parameters"] = params_dict
+        if result.started_at is not None:
+            payload["started_at"] = _epoch_ms_to_iso(result.started_at)
+        if result.ended_at is not None:
+            payload["ended_at"] = _epoch_ms_to_iso(result.ended_at)
+
+        # Serialize Allure steps and links into labels dict (per design field mapping)
+        steps = raw_data.get("steps")
+        if steps:
+            # Normalize Allure step format to match the schema _parse_steps expects
+            normalized_steps = []
+            for step in steps:
+                status_details = step.get("statusDetails") or {}
+                start = step.get("start") or 0
+                stop = step.get("stop") or 0
+                normalized_steps.append({
+                    "name": step.get("name", ""),
+                    "status": step.get("status", ""),
+                    "duration_ms": (stop - start) if (start and stop) else None,
+                    "error_message": status_details.get("message"),
+                })
+            labels_d = payload.get("labels", {})
+            labels_d["steps"] = json.dumps(normalized_steps)
+            payload["labels"] = labels_d
+
+        links = raw_data.get("links")
+        if links:
+            labels_d = payload.get("labels", {})
+            labels_d["links"] = json.dumps(links)
+            payload["labels"] = labels_d
+
+        # Add description to labels if present
+        description = raw_data.get("description") or raw_data.get("descriptionHtml")
+        if description:
+            labels_d = payload.get("labels", {})
+            labels_d["description"] = description
+            payload["labels"] = labels_d
+
+        self._batch.append(payload)
+        if not self._batch_start:
+            self._batch_start = time.monotonic()
+
+        # Flush if batch is full
+        if len(self._batch) >= _BATCH_SIZE:
+            self._flush_batch()
+
+        # Extract attachment refs and register them for upload with correct result_uuid
+        try:
+            refs = extract_attachment_refs(raw_data, result.uuid or "")
+            for ref in refs:
+                # Register source → result_uuid mapping so _handle_attachment knows the owner
+                self._attachment_result_map[ref.source] = ref.result_uuid
+        except Exception:
+            logger.debug("Failed to extract attachment refs from %s", path, exc_info=True)
+
+    def _handle_container(self, path: Path) -> None:
+        """Parse a container file and buffer for deferred association."""
+        container = parse_container_file(path)
+        if container is None:
+            return
+
+        container.received_at = time.monotonic()
+        self._pending_containers.append(container)
+        self._try_associate_containers()
+
+    def _handle_attachment(self, path: Path) -> None:
+        """Upload an attachment file to the Charisma attachment API."""
+        if not self._launch_id or not path.exists():
+            self._summary.attachments_skipped += 1
+            return
+
+        # Look up the result_uuid for this attachment
+        result_uuid = self._attachment_result_map.get(path.name, "")
+        if not result_uuid:
+            # Attachment not referenced by any result — skip
+            self._summary.attachments_skipped += 1
+            logger.debug("Skipping attachment with no result reference: %s", path.name)
+            return
+
+        try:
+            content = path.read_bytes()
+            files = {"file": (path.name, content)}
+            data = {"name": path.name, "resultId": result_uuid}
+
+            response = self._client.post(
+                f"/api/v1/launches/{self._launch_id}/attachments",
+                files=files,
+                data=data,
+            )
+            if response.status_code == 201:
+                self._summary.attachments_sent += 1
+            else:
+                self._summary.attachments_skipped += 1
+                logger.debug("Attachment upload failed: HTTP %d for %s", response.status_code, path.name)
+        except (httpx.ConnectError, httpx.TimeoutException) as e:
+            self._summary.attachments_skipped += 1
+            logger.debug("Attachment upload connection error: %s for %s", e, path.name)
+        except Exception:
+            self._summary.attachments_skipped += 1
+            logger.debug("Attachment upload unexpected error for %s", path.name, exc_info=True)
+
+    def _try_associate_containers(self) -> None:
+        """Try to associate buffered containers with known results."""
+        now = time.monotonic()
+        still_pending: list[ContainerData] = []
+        for container in self._pending_containers:
+            if now - container.received_at > _CONTAINER_ASSOCIATION_TIMEOUT:
+                self._summary.containers_sent += 1
+                continue
+            if all(child in self._known_result_uuids for child in container.children):
+                self._summary.containers_sent += 1
+            else:
+                still_pending.append(container)
+        self._pending_containers = still_pending
+
+    def _check_batch_timeout(self) -> None:
+        """Flush batch if timeout has elapsed."""
+        if self._batch and self._batch_start:
+            elapsed = time.monotonic() - self._batch_start
+            if elapsed >= _BATCH_TIMEOUT_SECONDS:
+                self._flush_batch()
+
+    def _flush_batch(self) -> None:
+        """Send the current batch to the API."""
+        if not self._batch or not self._launch_id:
+            return
+
+        batch = self._batch[:]
+        self._batch = []
+        self._batch_start = 0.0
+
+        success = self._send_batch(batch)
+        if success:
+            self._summary.results_sent += len(batch)
+        else:
+            self._summary.results_failed += len(batch)
+
+    def _send_batch(self, batch: list[dict[str, Any]]) -> bool:
+        """Send a batch of results to the API with retry logic.
+
+        Retries on transient errors (5xx, 429, timeout, ConnectError).
+        Engages circuit breaker only after all retries are exhausted for ConnectError.
+
+        Returns:
+            True if the batch was accepted, False otherwise.
+        """
+        if self._client is None:
+            return False
+
+        for attempt in range(MAX_RETRIES + 1):
+            try:
+                response = self._client.post(
+                    f"/api/v1/launches/{self._launch_id}/results",
+                    json={"results": batch},
+                )
+                if response.status_code in (200, 201):
+                    return True
+                if response.status_code == 401:
+                    logger.error("Authentication failed (401) — disabling uploads")
+                    self._auth_failed = True
+                    return False
+                if response.status_code in RETRYABLE_STATUS_CODES and attempt < MAX_RETRIES:
+                    time.sleep(exponential_backoff(attempt))
+                    continue
+                # Non-retryable failure
+                logger.error(
+                    "Batch upload failed: HTTP %d — %s",
+                    response.status_code,
+                    response.text[:500],
+                )
+                return False
+            except (httpx.ConnectError, httpx.TimeoutException) as e:
+                if attempt < MAX_RETRIES:
+                    time.sleep(exponential_backoff(attempt))
+                    continue
+                # All retries exhausted — engage circuit breaker
+                logger.error("API unreachable after %d retries: %s — engaging circuit breaker", MAX_RETRIES, e)
+                self._disabled = True
+                return False
+
+        return False
+
+    def _flush_remaining(self) -> None:
+        """Flush any remaining batch and process remaining queue items."""
+        while not self._queue.empty():
+            try:
+                event = self._queue.get_nowait()
+                if not self._disabled and not self._auth_failed:
+                    self._handle_event(event)
+                self._queue.task_done()
+            except Empty:
+                break
+
+        if self._batch:
+            self._flush_batch()
+
+        self._summary.containers_sent += len(self._pending_containers)
+        self._pending_containers.clear()