emergent-translator 1.1.0__py3-none-any.whl

emergent_translator/chunk_collector.py

@@ -0,0 +1,978 @@
+"""
+Chunk Collector - High-Performance Distributed File Processing
+
+Combines the sidecar collector pattern with chunk coordination for
+handling many files simultaneously with optimal batching.
+
+Architecture:
+    ┌─────────┐ ┌─────────┐ ┌─────────┐
+    │ File 1  │ │ File 2  │ │ File N  │
+    └────┬────┘ └────┬────┘ └────┬────┘
+         │          │          │
+         └──────────┼──────────┘
+                    ▼
+    ┌───────────────────────────────────┐
+    │         CHUNK COLLECTOR           │
+    │  ┌─────────────────────────────┐  │
+    │  │      Chunking Queue         │  │
+    │  │  (split files into chunks)  │  │
+    │  └──────────────┬──────────────┘  │
+    │                 ▼                 │
+    │  ┌─────────────────────────────┐  │
+    │  │    Compression Queue        │  │
+    │  │  (batch 75 chunks → θ)      │  │
+    │  └──────────────┬──────────────┘  │
+    │                 ▼                 │
+    │  ┌─────────────────────────────┐  │
+    │  │      Process Queue          │  │
+    │  │  (dispatch to workers)      │  │
+    │  └──────────────┬──────────────┘  │
+    │                 ▼                 │
+    │  ┌─────────────────────────────┐  │
+    │  │    Result Collector         │  │
+    │  │  (reassemble per job)       │  │
+    │  └─────────────────────────────┘  │
+    └───────────────────────────────────┘
+
+Performance targets:
+- 10,000+ chunks/second throughput
+- 75-chunk optimal batching
+- Zero-copy where possible
+- Backpressure handling
+
+Usage:
+    collector = ChunkCollector(workers=[...])
+    await collector.start()
+
+    # Submit files (returns immediately)
+    job_id = await collector.submit_file(data, "transform")
+
+    # Check status
+    status = collector.get_job_status(job_id)
+
+    # Get result when ready
+    result = await collector.get_result(job_id)
+"""
+
+import asyncio
+import hashlib
+import time
+import uuid
+import base64
+import json
+import logging
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple
+from datetime import datetime
+from collections import defaultdict
+import struct
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Configuration
+# =============================================================================
+
+DEFAULT_COMPRESSION_BATCH_SIZE = 10  # Smaller batches for compression (avoid 413)
+DEFAULT_DISPATCH_BATCH_SIZE = 50  # Larger batches for dispatch
+DEFAULT_CHUNK_SIZE_MB = 1  # 1MB chunks
+DEFAULT_NUM_COMPRESSORS = 4  # Parallel compression workers
+DEFAULT_NUM_DISPATCHERS = 8  # Parallel dispatch workers
+DEFAULT_BATCH_TIMEOUT_MS = 25  # Max wait before sending partial batch
+DEFAULT_MAX_QUEUE_SIZE = 10000  # Backpressure threshold
+
+
+# =============================================================================
+# Data Structures
+# =============================================================================
+
+class JobStatus(Enum):
+    QUEUED = "queued"
+    CHUNKING = "chunking"
+    COMPRESSING = "compressing"
+    PROCESSING = "processing"
+    COLLECTING = "collecting"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+@dataclass
+class ChunkMeta:
+    """Lightweight chunk metadata for queue processing."""
+    chunk_id: str
+    job_id: str
+    sequence: int
+    offset: int
+    size: int
+    checksum: str
+
+
+@dataclass
+class QueuedChunk:
+    """A chunk waiting in the compression queue."""
+    meta: ChunkMeta
+    data: bytes
+    timestamp: float = field(default_factory=time.time)
+
+
+@dataclass
+class CompressedChunk:
+    """A compressed chunk ready for dispatch."""
+    meta: ChunkMeta
+    compressed_data: bytes
+    compression_ratio: float
+    timestamp: float = field(default_factory=time.time)
+
+
+@dataclass
+class ProcessedChunk:
+    """A chunk that has been processed by a worker."""
+    meta: ChunkMeta
+    result_data: bytes
+    success: bool
+    error: Optional[str] = None
+    latency_ms: float = 0.0
+
+
+@dataclass
+class JobState:
+    """State tracking for a distributed job."""
+    job_id: str
+    operation: str
+    status: JobStatus = JobStatus.QUEUED
+    total_chunks: int = 0
+    chunks_compressed: int = 0
+    chunks_dispatched: int = 0
+    chunks_completed: int = 0
+    chunks_failed: int = 0
+    bytes_raw: int = 0
+    bytes_compressed: int = 0
+    created_at: float = field(default_factory=time.time)
+    completed_at: Optional[float] = None
+    results: Dict[int, bytes] = field(default_factory=dict)  # sequence -> result
+    errors: List[str] = field(default_factory=list)
+    operation_params: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class CollectorStats:
+    """Global collector statistics."""
+    files_received: int = 0
+    chunks_created: int = 0
+    chunks_compressed: int = 0
+    chunks_dispatched: int = 0
+    chunks_completed: int = 0
+    batches_sent: int = 0
+    bytes_in: int = 0
+    bytes_compressed: int = 0
+    bytes_out: int = 0
+    errors: int = 0
+    start_time: float = field(default_factory=time.time)
+
+    @property
+    def compression_ratio(self) -> float:
+        if self.bytes_in == 0:
+            return 0.0
+        return self.bytes_compressed / self.bytes_in
+
+    @property
+    def bandwidth_saved_pct(self) -> float:
+        return (1 - self.compression_ratio) * 100
+
+    @property
+    def throughput_chunks_per_sec(self) -> float:
+        elapsed = time.time() - self.start_time
+        if elapsed == 0:
+            return 0.0
+        return self.chunks_completed / elapsed
+
+    @property
+    def throughput_mb_per_sec(self) -> float:
+        elapsed = time.time() - self.start_time
+        if elapsed == 0:
+            return 0.0
+        return (self.bytes_in / 1024 / 1024) / elapsed
+
+
+# =============================================================================
+# Chunk Collector
+# =============================================================================
+
+class ChunkCollector:
+    """
+    High-performance collector for distributed chunk processing.
+
+    Handles multiple concurrent files with optimal batching at every stage.
+    """
+
+    def __init__(
+        self,
+        workers: List[str],
+        compression_batch_size: int = DEFAULT_COMPRESSION_BATCH_SIZE,
+        chunk_size_mb: float = DEFAULT_CHUNK_SIZE_MB,
+        num_compressors: int = DEFAULT_NUM_COMPRESSORS,
+        num_dispatchers: int = DEFAULT_NUM_DISPATCHERS,
+        batch_timeout_ms: float = DEFAULT_BATCH_TIMEOUT_MS,
+        max_queue_size: int = DEFAULT_MAX_QUEUE_SIZE,
+        auth_token: Optional[str] = None
+    ):
+        self.workers = workers
+        self.compression_batch_size = compression_batch_size
+        self.chunk_size_bytes = int(chunk_size_mb * 1024 * 1024)
+        self.num_compressors = num_compressors
+        self.num_dispatchers = num_dispatchers
+        self.batch_timeout_ms = batch_timeout_ms
+        self.max_queue_size = max_queue_size
+        self.auth_token = auth_token or "eudaimonia-translator-demo"
+
+        # Queues
+        self.chunking_queue: asyncio.Queue[Tuple[str, bytes, str, Dict]] = asyncio.Queue()
+        self.compression_queue: asyncio.Queue[QueuedChunk] = asyncio.Queue(maxsize=max_queue_size)
+        self.dispatch_queue: asyncio.Queue[CompressedChunk] = asyncio.Queue(maxsize=max_queue_size)
+        self.result_queue: asyncio.Queue[ProcessedChunk] = asyncio.Queue()
+
+        # State
+        self.jobs: Dict[str, JobState] = {}
+        self.stats = CollectorStats()
+        self._running = False
+        self._tasks: List[asyncio.Task] = []
+        self._http_client: Optional[httpx.AsyncClient] = None
+
+        # Worker selection (round-robin with tracking)
+        self._worker_idx = 0
+        self._worker_active: Dict[str, int] = {w: 0 for w in workers}
+
+    async def start(self):
+        """Start all collector workers."""
+        self._running = True
+        self._http_client = httpx.AsyncClient(timeout=60.0)
+
+        # Start chunking worker (single - CPU bound)
+        self._tasks.append(asyncio.create_task(self._chunking_worker()))
+
+        # Start compression workers (parallel batchers)
+        for i in range(self.num_compressors):
+            self._tasks.append(asyncio.create_task(self._compression_worker(i)))
+
+        # Start dispatch workers (parallel)
+        for i in range(self.num_dispatchers):
+            self._tasks.append(asyncio.create_task(self._dispatch_worker(i)))
+
+        # Start result collector
+        self._tasks.append(asyncio.create_task(self._result_collector()))
+
+        logger.info(
+            f"ChunkCollector started: {self.num_compressors} compressors, "
+            f"{self.num_dispatchers} dispatchers, {len(self.workers)} workers"
+        )
+
+    async def stop(self):
+        """Stop all workers gracefully."""
+        self._running = False
+
+        # Cancel all tasks
+        for task in self._tasks:
+            task.cancel()
+
+        # Wait for tasks to finish
+        await asyncio.gather(*self._tasks, return_exceptions=True)
+
+        # Close HTTP client
+        if self._http_client:
+            await self._http_client.aclose()
+
+        logger.info("ChunkCollector stopped")
+
+    async def submit_file(
+        self,
+        data: bytes,
+        operation: str,
+        operation_params: Optional[Dict] = None,
+        metadata: Optional[Dict] = None
+    ) -> str:
+        """
+        Submit a file for distributed processing.
+
+        Returns immediately with job_id. Use get_job_status() to track progress.
+
+        Args:
+            data: Raw file bytes
+            operation: Operation to perform ("transform", "analyze", etc.)
+            operation_params: Parameters for the operation
+            metadata: Optional metadata to attach to job
+
+        Returns:
+            job_id for tracking
+        """
+        job_id = str(uuid.uuid4())[:8]
+
+        # Create job state
+        job = JobState(
+            job_id=job_id,
+            operation=operation,
+            bytes_raw=len(data),
+            operation_params=operation_params or {}
+        )
+        self.jobs[job_id] = job
+        self.stats.files_received += 1
+        self.stats.bytes_in += len(data)
+
+        # Queue for chunking
+        await self.chunking_queue.put((job_id, data, operation, operation_params or {}))
+
+        logger.info(f"Job {job_id} submitted: {len(data):,} bytes, operation={operation}")
+        return job_id
+
+    async def submit_files(
+        self,
+        files: List[Tuple[bytes, str, Optional[Dict]]],
+    ) -> List[str]:
+        """
+        Submit multiple files at once.
+
+        Args:
+            files: List of (data, operation, params) tuples
+
+        Returns:
+            List of job_ids
+        """
+        job_ids = []
+        for data, operation, params in files:
+            job_id = await self.submit_file(data, operation, params)
+            job_ids.append(job_id)
+        return job_ids
+
+    def get_job_status(self, job_id: str) -> Dict[str, Any]:
+        """Get detailed status for a job."""
+        job = self.jobs.get(job_id)
+        if not job:
+            return {"error": f"Job {job_id} not found"}
+
+        elapsed = time.time() - job.created_at
+
+        return {
+            "job_id": job.job_id,
+            "status": job.status.value,
+            "operation": job.operation,
+            "progress": {
+                "total_chunks": job.total_chunks,
+                "compressed": job.chunks_compressed,
+                "dispatched": job.chunks_dispatched,
+                "completed": job.chunks_completed,
+                "failed": job.chunks_failed,
+                "pct_complete": (job.chunks_completed / job.total_chunks * 100) if job.total_chunks > 0 else 0
+            },
+            "bytes": {
+                "raw": job.bytes_raw,
+                "compressed": job.bytes_compressed,
+                "compression_ratio": f"{job.bytes_compressed / job.bytes_raw:.2%}" if job.bytes_raw > 0 else "0%"
+            },
+            "timing": {
+                "elapsed_seconds": elapsed,
+                "throughput_mbps": (job.bytes_raw / 1024 / 1024) / elapsed if elapsed > 0 else 0
+            },
+            "errors": job.errors
+        }
+
+    async def get_result(self, job_id: str, timeout: float = 300.0) -> Optional[bytes]:
+        """
+        Wait for and return job result.
+
+        Args:
+            job_id: Job to wait for
+            timeout: Maximum seconds to wait
+
+        Returns:
+            Reassembled result bytes, or None if failed/timeout
+        """
+        start = time.time()
+
+        while time.time() - start < timeout:
+            job = self.jobs.get(job_id)
+            if not job:
+                return None
+
+            if job.status == JobStatus.COMPLETED:
+                # Reassemble results in order
+                result_parts = []
+                for seq in sorted(job.results.keys()):
+                    result_parts.append(job.results[seq])
+                return b''.join(result_parts)
+
+            if job.status == JobStatus.FAILED:
+                return None
+
+            await asyncio.sleep(0.1)
+
+        return None
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Get global collector statistics."""
+        s = self.stats
+        return {
+            "files_received": s.files_received,
+            "chunks": {
+                "created": s.chunks_created,
+                "compressed": s.chunks_compressed,
+                "dispatched": s.chunks_dispatched,
+                "completed": s.chunks_completed
+            },
+            "batches_sent": s.batches_sent,
+            "bytes": {
+                "in": s.bytes_in,
+                "compressed": s.bytes_compressed,
+                "out": s.bytes_out,
+                "compression_ratio": f"{s.compression_ratio:.2%}",
+                "bandwidth_saved": f"{s.bandwidth_saved_pct:.1f}%"
+            },
+            "throughput": {
+                "chunks_per_sec": f"{s.throughput_chunks_per_sec:.0f}",
+                "mb_per_sec": f"{s.throughput_mb_per_sec:.2f}"
+            },
+            "errors": s.errors,
+            "queues": {
+                "chunking": self.chunking_queue.qsize(),
+                "compression": self.compression_queue.qsize(),
+                "dispatch": self.dispatch_queue.qsize(),
+                "results": self.result_queue.qsize()
+            },
+            "workers": {
+                "total": len(self.workers),
+                "active_connections": dict(self._worker_active)
+            },
+            "uptime_seconds": time.time() - s.start_time
+        }
+
+    # =========================================================================
+    # Internal Workers
+    # =========================================================================
+
+    async def _chunking_worker(self):
+        """Worker that splits files into chunks."""
+        while self._running:
+            try:
+                job_id, data, operation, params = await asyncio.wait_for(
+                    self.chunking_queue.get(),
+                    timeout=1.0
+                )
+
+                job = self.jobs.get(job_id)
+                if not job:
+                    continue
+
+                job.status = JobStatus.CHUNKING
+
+                # Split into chunks
+                offset = 0
+                sequence = 0
+
+                while offset < len(data):
+                    chunk_data = data[offset:offset + self.chunk_size_bytes]
+                    chunk_size = len(chunk_data)
+
+                    meta = ChunkMeta(
+                        chunk_id=f"{job_id}-{sequence:04d}",
+                        job_id=job_id,
+                        sequence=sequence,
+                        offset=offset,
+                        size=chunk_size,
+                        checksum=hashlib.sha256(chunk_data).hexdigest()[:16]
+                    )
+
+                    # Queue for compression
+                    await self.compression_queue.put(QueuedChunk(meta=meta, data=chunk_data))
+                    self.stats.chunks_created += 1
+
+                    offset += chunk_size
+                    sequence += 1
+
+                job.total_chunks = sequence
+                job.status = JobStatus.COMPRESSING
+
+                logger.debug(f"Job {job_id}: created {sequence} chunks")
+
+            except asyncio.TimeoutError:
+                continue
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Chunking worker error: {e}")
+                self.stats.errors += 1
+
+    async def _compression_worker(self, worker_id: int):
+        """Worker that batches and compresses chunks."""
+        batch: List[QueuedChunk] = []
+        last_flush = time.time()
+
+        while self._running:
+            try:
+                # Try to fill batch
+                try:
+                    chunk = await asyncio.wait_for(
+                        self.compression_queue.get(),
+                        timeout=self.batch_timeout_ms / 1000
+                    )
+                    batch.append(chunk)
+                except asyncio.TimeoutError:
+                    pass
+
+                # Check if we should flush
+                should_flush = (
+                    len(batch) >= self.compression_batch_size or
+                    (batch and (time.time() - last_flush) * 1000 >= self.batch_timeout_ms)
+                )
+
+                if should_flush and batch:
+                    await self._compress_batch(batch)
+                    batch = []
+                    last_flush = time.time()
+
+            except asyncio.CancelledError:
+                # Flush remaining
+                if batch:
+                    await self._compress_batch(batch)
+                break
+            except Exception as e:
+                logger.error(f"Compression worker {worker_id} error: {e}")
+                self.stats.errors += 1
+
+    async def _compress_batch(self, batch: List[QueuedChunk]):
+        """Compress chunks via individual API calls (parallel within batch)."""
+        if not batch:
+            return
+
+        # Compress all chunks in parallel
+        tasks = [self._compress_single(chunk) for chunk in batch]
+        await asyncio.gather(*tasks, return_exceptions=True)
+        self.stats.batches_sent += 1
+
+    async def _compress_single(self, chunk: QueuedChunk, retry: int = 0):
+        """Compress a single chunk via the /compress endpoint (supports large files)."""
+        max_retries = 3
+        # Round-robin worker selection for compression
+        worker_url = self.workers[self._worker_idx % len(self.workers)]
+        self._worker_idx += 1
+
+        try:
+            # Use multipart form upload for larger chunks
+            # Prepend chunk ID to data for tracking
+            chunk_data_with_id = f"CHUNK:{chunk.meta.chunk_id}:".encode('utf-8') + chunk.data
+
+            # Create multipart form data
+            files = {
+                'file': (f'{chunk.meta.chunk_id}.bin', chunk_data_with_id, 'application/octet-stream')
+            }
+
+            response = await self._http_client.post(
+                f"{worker_url}/compress",
+                files=files
+            )
+
+            # Handle rate limiting with exponential backoff
+            if response.status_code == 429 and retry < max_retries:
+                wait_time = (2 ** retry) + (retry * 0.5)  # 1s, 2.5s, 4.5s
+                logger.debug(f"Rate limited, retrying {chunk.meta.chunk_id} in {wait_time}s")
+                await asyncio.sleep(wait_time)
+                return await self._compress_single(chunk, retry + 1)
+
+            if response.status_code == 200:
+                result = response.json()
+                if result.get("success"):
+                    # /compress returns compressed_data in base64
+                    compressed_b64 = result.get("compressed_data", "")
+                    if compressed_b64:
+                        compressed_bytes = base64.b64decode(compressed_b64)
+                    else:
+                        compressed_bytes = b''
+
+                    compressed = CompressedChunk(
+                        meta=chunk.meta,
+                        compressed_data=compressed_bytes,
+                        compression_ratio=result.get("compression_ratio", 0)
+                    )
+
+                    # Update job state
+                    job = self.jobs.get(chunk.meta.job_id)
+                    if job:
+                        job.chunks_compressed += 1
+                        job.bytes_compressed += len(compressed_bytes)
+
+                    # Queue for dispatch
+                    await self.dispatch_queue.put(compressed)
+                    self.stats.chunks_compressed += 1
+                    self.stats.bytes_compressed += len(compressed_bytes)
+                    return
+                else:
+                    error = result.get("errors", ["Unknown error"])
+                    logger.error(f"Compression failed for {chunk.meta.chunk_id}: {error}")
+            else:
+                # Log the actual error response
+                try:
+                    error_detail = response.json().get("detail", response.text[:200])
+                except:
+                    error_detail = response.text[:200]
+                logger.error(f"Compression HTTP {response.status_code} for {chunk.meta.chunk_id}: {error_detail}")
+
+            # Mark as failed
+            job = self.jobs.get(chunk.meta.job_id)
+            if job:
+                job.chunks_failed += 1
+                job.errors.append(f"Compression failed for chunk {chunk.meta.chunk_id}")
+            self.stats.errors += 1
+
+        except Exception as e:
+            logger.error(f"Compression error for {chunk.meta.chunk_id}: {e}")
+            job = self.jobs.get(chunk.meta.job_id)
+            if job:
+                job.chunks_failed += 1
+            self.stats.errors += 1
+
+    async def _dispatch_worker(self, worker_id: int):
+        """Worker that dispatches compressed chunks to processing workers."""
+        while self._running:
+            try:
+                chunk = await asyncio.wait_for(
+                    self.dispatch_queue.get(),
+                    timeout=0.5
+                )
+
+                # Get job for operation info
+                job = self.jobs.get(chunk.meta.job_id)
+                if not job:
+                    continue
+
+                if job.status == JobStatus.COMPRESSING:
+                    job.status = JobStatus.PROCESSING
+
+                # Select worker (round-robin with least connections)
+                worker_url = self._select_worker()
+
+                # Dispatch to worker
+                start_time = time.perf_counter()
+
+                try:
+                    self._worker_active[worker_url] += 1
+
+                    response = await self._http_client.post(
+                        f"{worker_url}/process",
+                        json={
+                            "chunk_id": chunk.meta.chunk_id,
+                            "operation": job.operation,
+                            "data": base64.b64encode(chunk.compressed_data).decode('utf-8'),
+                            "params": job.operation_params
+                        },
+                        headers={"Authorization": f"Bearer {self.auth_token}"}
+                    )
+
+                    latency_ms = (time.perf_counter() - start_time) * 1000
+
+                    if response.status_code == 200:
+                        result = response.json()
+                        if result.get("success"):
+                            result_data = base64.b64decode(result.get("result", ""))
+
+                            processed = ProcessedChunk(
+                                meta=chunk.meta,
+                                result_data=result_data,
+                                success=True,
+                                latency_ms=latency_ms
+                            )
+                            await self.result_queue.put(processed)
+
+                            job.chunks_dispatched += 1
+                            self.stats.chunks_dispatched += 1
+                        else:
+                            error = result.get("error", "Unknown error")
+                            processed = ProcessedChunk(
+                                meta=chunk.meta,
+                                result_data=b'',
+                                success=False,
+                                error=error,
+                                latency_ms=latency_ms
+                            )
+                            await self.result_queue.put(processed)
+                            self.stats.errors += 1
+                    else:
+                        processed = ProcessedChunk(
+                            meta=chunk.meta,
+                            result_data=b'',
+                            success=False,
+                            error=f"HTTP {response.status_code}",
+                            latency_ms=latency_ms
+                        )
+                        await self.result_queue.put(processed)
+                        self.stats.errors += 1
+
+                finally:
+                    self._worker_active[worker_url] -= 1
+
+            except asyncio.TimeoutError:
+                continue
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Dispatch worker {worker_id} error: {e}")
+                self.stats.errors += 1
+
+    async def _result_collector(self):
+        """Collects processed chunks and updates job state."""
+        while self._running:
+            try:
+                result = await asyncio.wait_for(
+                    self.result_queue.get(),
+                    timeout=1.0
+                )
+
+                job = self.jobs.get(result.meta.job_id)
+                if not job:
+                    continue
+
+                if result.success:
+                    job.results[result.meta.sequence] = result.result_data
+                    job.chunks_completed += 1
+                    self.stats.chunks_completed += 1
+                    self.stats.bytes_out += len(result.result_data)
+                else:
+                    job.chunks_failed += 1
+                    job.errors.append(f"Chunk {result.meta.chunk_id}: {result.error}")
+
+                # Check if job is complete
+                total_processed = job.chunks_completed + job.chunks_failed
+                if total_processed >= job.total_chunks and job.total_chunks > 0:
+                    if job.chunks_failed == 0:
+                        job.status = JobStatus.COMPLETED
+                    else:
+                        job.status = JobStatus.COMPLETED  # Partial success
+                    job.completed_at = time.time()
+
+                    elapsed = job.completed_at - job.created_at
+                    logger.info(
+                        f"Job {job.job_id} completed: {job.chunks_completed}/{job.total_chunks} chunks, "
+                        f"{elapsed:.2f}s, {job.bytes_raw/1024/1024/elapsed:.2f} MB/s"
+                    )
+
+            except asyncio.TimeoutError:
+                continue
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Result collector error: {e}")
+
+    def _select_worker(self) -> str:
+        """Select best worker using least-connections."""
+        # Find worker with fewest active connections
+        min_active = min(self._worker_active.values())
+        candidates = [w for w, a in self._worker_active.items() if a == min_active]
+
+        # Round-robin among candidates
+        self._worker_idx = (self._worker_idx + 1) % len(candidates)
+        return candidates[self._worker_idx % len(candidates)]
+
+
+# =============================================================================
+# FastAPI Application
+# =============================================================================
+
+def create_collector_app(
+    workers: List[str],
+    **kwargs
+) -> "FastAPI":
+    """Create a FastAPI app for the chunk collector."""
+    from fastapi import FastAPI, HTTPException, UploadFile, File, Form
+    from fastapi.middleware.cors import CORSMiddleware
+    from pydantic import BaseModel
+    from contextlib import asynccontextmanager
+
+    collector = ChunkCollector(workers=workers, **kwargs)
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        await collector.start()
+        yield
+        await collector.stop()
+
+    app = FastAPI(
+        title="Emergent Language Chunk Collector",
+        description="High-performance distributed file processing with θ-compression",
+        version="1.0.0",
+        lifespan=lifespan
+    )
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    class SubmitRequest(BaseModel):
+        data: str  # Base64 encoded
+        operation: str
+        params: Dict[str, Any] = {}
+
+    class BulkSubmitRequest(BaseModel):
+        files: List[SubmitRequest]
+
+    @app.get("/")
+    async def root():
+        return {
+            "service": "chunk-collector",
+            "version": "1.0.0",
+            "endpoints": {
+                "/submit": "Submit file for processing (POST)",
+                "/submit/upload": "Upload file for processing (POST multipart)",
+                "/submit/bulk": "Submit multiple files (POST)",
+                "/status/{job_id}": "Get job status (GET)",
+                "/result/{job_id}": "Get job result (GET)",
+                "/stats": "Collector statistics (GET)",
+                "/health": "Health check (GET)"
+            }
+        }
+
+    @app.get("/health")
+    async def health():
+        stats = collector.get_stats()
+        return {
+            "status": "healthy",
+            "queues": stats["queues"],
+            "workers": stats["workers"],
+            "uptime": stats["uptime_seconds"]
+        }
+
+    @app.get("/stats")
+    async def stats():
+        return collector.get_stats()
+
+    @app.post("/submit")
+    async def submit(request: SubmitRequest):
+        """Submit base64-encoded file for processing."""
+        try:
+            data = base64.b64decode(request.data)
+        except Exception:
+            raise HTTPException(400, "Invalid base64 data")
+
+        job_id = await collector.submit_file(
+            data=data,
+            operation=request.operation,
+            operation_params=request.params
+        )
+
+        return {"job_id": job_id, "status": "queued"}
+
+    @app.post("/submit/upload")
+    async def submit_upload(
+        file: UploadFile = File(...),
+        operation: str = Form(default="passthrough"),
+        params: str = Form(default="{}")
+    ):
+        """Upload file for processing."""
+        data = await file.read()
+
+        try:
+            operation_params = json.loads(params)
+        except:
+            operation_params = {}
+
+        job_id = await collector.submit_file(
+            data=data,
+            operation=operation,
+            operation_params=operation_params
+        )
+
+        return {
+            "job_id": job_id,
+            "status": "queued",
+            "file_name": file.filename,
+            "file_size": len(data)
+        }
+
+    @app.post("/submit/bulk")
+    async def submit_bulk(request: BulkSubmitRequest):
+        """Submit multiple files at once."""
+        job_ids = []
+        for f in request.files:
+            try:
+                data = base64.b64decode(f.data)
+                job_id = await collector.submit_file(
+                    data=data,
+                    operation=f.operation,
+                    operation_params=f.params
+                )
+                job_ids.append({"job_id": job_id, "status": "queued"})
+            except Exception as e:
+                job_ids.append({"error": str(e)})
+
+        return {"jobs": job_ids}
+
+    @app.get("/status/{job_id}")
+    async def status(job_id: str):
+        return collector.get_job_status(job_id)
+
+    @app.get("/result/{job_id}")
+    async def result(job_id: str, timeout: float = 60.0):
+        """Get job result (waits for completion up to timeout)."""
+        result_bytes = await collector.get_result(job_id, timeout=timeout)
+
+        if result_bytes is None:
+            job = collector.jobs.get(job_id)
+            if job and job.status == JobStatus.FAILED:
+                raise HTTPException(500, f"Job failed: {job.errors}")
+            raise HTTPException(408, "Timeout waiting for result")
+
+        return {
+            "job_id": job_id,
+            "status": "completed",
+            "result": base64.b64encode(result_bytes).decode('utf-8'),
+            "result_size": len(result_bytes)
+        }
+
+    return app
+
+
+# =============================================================================
+# CLI Entry Point
+# =============================================================================
+
+if __name__ == "__main__":
+    import argparse
+    import uvicorn
+
+    parser = argparse.ArgumentParser(description="Chunk Collector Service")
+    parser.add_argument("--port", "-p", type=int, default=8080, help="Port to listen on")
+    parser.add_argument("--workers", "-w", type=str, default="https://emergent-language.fly.dev",
+                       help="Comma-separated worker URLs")
+    parser.add_argument("--batch-size", "-b", type=int, default=10, help="Compression batch size")
+    parser.add_argument("--chunk-size", "-c", type=float, default=1.0, help="Chunk size in MB")
+    parser.add_argument("--compressors", type=int, default=4, help="Number of compression workers")
+    parser.add_argument("--dispatchers", type=int, default=8, help="Number of dispatch workers")
+
+    args = parser.parse_args()
+
+    workers = [w.strip() for w in args.workers.split(",")]
+
+    app = create_collector_app(
+        workers=workers,
+        compression_batch_size=args.batch_size,
+        chunk_size_mb=args.chunk_size,
+        num_compressors=args.compressors,
+        num_dispatchers=args.dispatchers
+    )
+
+    print(f"""
+╔═══════════════════════════════════════════════════════════════════╗
+║              Emergent Language Chunk Collector                    ║
+╠═══════════════════════════════════════════════════════════════════╣
+║  Port:         {args.port:<6}                                        ║
+║  Workers:      {len(workers):<6}                                        ║
+║  Batch size:   {args.batch_size:<6}                                        ║
+║  Chunk size:   {args.chunk_size}MB                                        ║
+║  Compressors:  {args.compressors:<6}                                        ║
+║  Dispatchers:  {args.dispatchers:<6}                                        ║
+╚═══════════════════════════════════════════════════════════════════╝
+""")
+
+    uvicorn.run(app, host="0.0.0.0", port=args.port)