genarena 0.1.0__tar.gz → 0.1.2__tar.gz

{genarena-0.1.0 → genarena-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: genarena
-Version: 0.1.0
+Version: 0.1.2
 Summary: GenArena Arena Evaluation - VLM-based pairwise image generation evaluation
 Author: GenArena Team
 License: Apache-2.0

{genarena-0.1.0 → genarena-0.1.2}/genarena/deploy/Dockerfile

@@ -7,11 +7,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy project files
-COPY . .
-
 # Install Python dependencies
-RUN pip install --no-cache-dir -e .[web]
+RUN pip install genarena[web]
 
 # Download parquet benchmark data from HuggingFace
 # This dataset contains the prompt/benchmark data (not arena battle results)
@@ -22,4 +19,4 @@ RUN python -c "from huggingface_hub import snapshot_download; snapshot_download(
 EXPOSE 7860
 
 # Start the application
-CMD ["python", "genarena/deploy/app.py"]
+CMD ["python", "-m", "genarena.deploy.app"]

{genarena-0.1.0 → genarena-0.1.2}/genarena/visualize/data_loader.py

@@ -2267,14 +2267,15 @@ class HFArenaDataLoader(ArenaDataLoader):
             preload: If True, preload all data at initialization
         """
         self.hf_repo = hf_repo
-        self._image_url_index = self._build_image_index(image_files)
+        # Build both indexes at once
+        self._image_url_index, self._subset_models_index = self._build_image_index(image_files)
         super().__init__(arena_dir, data_dir, preload=preload)
 
     def _build_image_index(
         self, image_files: list[str]
-    ) -> dict[tuple[str, str, int], str]:
+    ) -> tuple[dict[tuple[str, str, int], str], dict[str, list[str]]]:
         """
-        Build index: (subset, model, sample_index) -> hf_file_path
+        Build indexes from HF image file list.
 
         Expected path format: {subset}/models/{exp_name}/{model}/{index}.png
 
@@ -2282,11 +2283,14 @@ class HFArenaDataLoader(ArenaDataLoader):
             image_files: List of image file paths from HF repo
 
         Returns:
-            Dict mapping (subset, model, sample_index) to HF file path
+            Tuple of:
+            - Dict mapping (subset, model, sample_index) to HF file path
+            - Dict mapping subset to sorted list of model names
         """
         from genarena.models import parse_image_index
 
         index: dict[tuple[str, str, int], str] = {}
+        subset_models: dict[str, set[str]] = {}
 
         for path in image_files:
             parts = path.split("/")
@@ -2300,9 +2304,18 @@ class HFArenaDataLoader(ArenaDataLoader):
                 if idx is not None:
                     # If duplicate, later entries overwrite earlier ones
                     index[(subset, model, idx)] = path
+                    # Also track subset -> models mapping
+                    if subset not in subset_models:
+                        subset_models[subset] = set()
+                    subset_models[subset].add(model)
+
+        # Convert sets to sorted lists
+        subset_models_sorted: dict[str, list[str]] = {
+            subset: sorted(models) for subset, models in subset_models.items()
+        }
 
-        logger.info(f"Built image URL index with {len(index)} entries")
-        return index
+        logger.info(f"Built image URL index with {len(index)} entries across {len(subset_models_sorted)} subsets")
+        return index, subset_models_sorted
 
     def get_model_image_url(
         self, subset: str, model: str, sample_index: int
@@ -2333,3 +2346,159 @@ class HFArenaDataLoader(ArenaDataLoader):
         """
         # Return None to indicate image should be fetched via CDN
         return None
+
+    def _get_available_models_for_subset(self, subset: str) -> list[str]:
+        """
+        Get list of models that have images in the HF CDN for this subset.
+
+        Uses pre-built index for O(1) lookup.
+
+        Returns:
+            List of model names (sorted)
+        """
+        return self._subset_models_index.get(subset, [])
+
+    def _has_model_image(self, subset: str, model: str, sample_index: int) -> bool:
+        """
+        Check if a model has an image for a specific sample in the HF CDN.
+
+        Args:
+            subset: Subset name
+            model: Model name
+            sample_index: Sample index
+
+        Returns:
+            True if image exists in CDN index
+        """
+        return (subset, model, sample_index) in self._image_url_index
+
+    def get_subset_info(self, subset: str) -> Optional[SubsetInfo]:
+        """
+        Get information about a subset.
+
+        Override for HF deployment to use CDN image index for models list.
+
+        Args:
+            subset: Subset name
+
+        Returns:
+            SubsetInfo or None if subset doesn't exist
+        """
+        if subset in self._subset_info_cache:
+            return self._subset_info_cache[subset]
+
+        subset_path = os.path.join(self.arena_dir, subset)
+        if not os.path.isdir(subset_path):
+            return None
+
+        # Get models from CDN index instead of local file system
+        models = self._get_available_models_for_subset(subset)
+
+        # Get experiments
+        pk_logs_dir = os.path.join(subset_path, "pk_logs")
+        experiments = []
+        if os.path.isdir(pk_logs_dir):
+            for name in os.listdir(pk_logs_dir):
+                exp_path = os.path.join(pk_logs_dir, name)
+                if os.path.isdir(exp_path):
+                    # Check for battle logs
+                    has_logs = any(
+                        f.endswith(".jsonl")
+                        for f in os.listdir(exp_path)
+                        if os.path.isfile(os.path.join(exp_path, f))
+                    )
+                    if has_logs:
+                        experiments.append(name)
+        experiments.sort()
+
+        # Load state
+        state_path = os.path.join(subset_path, "arena", "state.json")
+        state = load_state(state_path)
+
+        # Get image count range
+        img_range = self._image_count_range.get(subset, (1, 1))
+
+        # Get prompt sources
+        prompt_sources = self._prompt_sources.get(subset, [])
+
+        info = SubsetInfo(
+            name=subset,
+            models=models,
+            experiments=experiments,
+            total_battles=state.total_battles,
+            state=state,
+            min_input_images=img_range[0],
+            max_input_images=img_range[1],
+            prompt_sources=prompt_sources,
+        )
+
+        self._subset_info_cache[subset] = info
+        return info
+
+    def get_sample_all_models(
+        self, subset: str, exp_name: str, sample_index: int,
+        filter_models: Optional[list[str]] = None,
+        stats_scope: str = "filtered"
+    ) -> dict[str, Any]:
+        """
+        Get all model outputs for a specific sample, sorted by win rate.
+
+        Override for HF deployment to use CDN image index instead of local files.
+
+        Args:
+            subset: Subset name
+            exp_name: Experiment name
+            sample_index: Sample index
+            filter_models: Optional list of models to filter (show only these models)
+            stats_scope: 'filtered' = only count battles between filtered models,
+                        'all' = count all battles (but show only filtered models)
+
+        Returns:
+            Dict with sample info and all model outputs sorted by win rate
+        """
+        # Get sample metadata
+        sample_meta = self._get_sample_data(subset, sample_index)
+
+        # Determine which models to use for stats calculation
+        stats_filter = filter_models if stats_scope == "filtered" else None
+        model_stats = self.get_model_win_stats(subset, exp_name, sample_index, stats_filter)
+
+        # Get all models that have outputs in CDN (O(1) lookup)
+        available_models_list = self._get_available_models_for_subset(subset)
+
+        # Apply filter if specified
+        if filter_models:
+            filter_set = set(filter_models)
+            available_models_list = [m for m in available_models_list if m in filter_set]
+
+        # Build model info for models that have images for this sample
+        available_models = []
+        for model in available_models_list:
+            # Check if model has image for this sample in CDN index
+            if self._has_model_image(subset, model, sample_index):
+                stats = model_stats.get(model, {
+                    "wins": 0, "losses": 0, "ties": 0, "total": 0, "win_rate": 0
+                })
+                available_models.append({
+                    "model": model,
+                    "wins": stats["wins"],
+                    "losses": stats["losses"],
+                    "ties": stats["ties"],
+                    "total": stats["total"],
+                    "win_rate": stats["win_rate"],
+                })
+
+        # Sort by win rate (descending), then by wins (descending), then by model name
+        available_models.sort(key=lambda x: (-x["win_rate"], -x["wins"], x["model"]))
+
+        return {
+            "subset": subset,
+            "exp_name": exp_name,
+            "sample_index": sample_index,
+            "instruction": sample_meta.get("instruction", ""),
+            "task_type": sample_meta.get("task_type", ""),
+            "input_image_count": sample_meta.get("input_image_count", 1),
+            "prompt_source": sample_meta.get("prompt_source"),
+            "original_metadata": sample_meta.get("original_metadata"),
+            "models": available_models,
+        }

{genarena-0.1.0 → genarena-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "genarena"
-version = "0.1.0"
+version = "0.1.2"
 description = "GenArena Arena Evaluation - VLM-based pairwise image generation evaluation"
 readme = "README.md"
 requires-python = ">=3.10"