langvision 0.0.1__py3-none-any.whl → 0.1.0__py3-none-any.whl

langvision/model_zoo.py

@@ -1,2 +1,169 @@
-class VisualModelLoader:
-    pass
+"""
+Model Zoo for Langvision - Pre-trained Vision Transformer models.
+"""
+
+import os
+import json
+import logging
+from typing import Dict, List, Optional, Any
+from pathlib import Path
+
+# Default model configurations
+DEFAULT_MODELS = {
+    "vit_tiny_patch16_224": {
+        "name": "vit_tiny_patch16_224",
+        "type": "vision_transformer",
+        "size": "5.4M",
+        "description": "Vision Transformer Tiny (16x16 patches, 224x224 input)",
+        "config": {
+            "img_size": 224,
+            "patch_size": 16,
+            "embed_dim": 192,
+            "depth": 12,
+            "num_heads": 3,
+            "mlp_ratio": 4.0,
+            "num_classes": 1000
+        }
+    },
+    "vit_small_patch16_224": {
+        "name": "vit_small_patch16_224",
+        "type": "vision_transformer",
+        "size": "22.1M",
+        "description": "Vision Transformer Small (16x16 patches, 224x224 input)",
+        "config": {
+            "img_size": 224,
+            "patch_size": 16,
+            "embed_dim": 384,
+            "depth": 12,
+            "num_heads": 6,
+            "mlp_ratio": 4.0,
+            "num_classes": 1000
+        }
+    },
+    "vit_base_patch16_224": {
+        "name": "vit_base_patch16_224",
+        "type": "vision_transformer",
+        "size": "86.4M",
+        "description": "Vision Transformer Base (16x16 patches, 224x224 input)",
+        "config": {
+            "img_size": 224,
+            "patch_size": 16,
+            "embed_dim": 768,
+            "depth": 12,
+            "num_heads": 12,
+            "mlp_ratio": 4.0,
+            "num_classes": 1000
+        }
+    },
+    "vit_large_patch16_224": {
+        "name": "vit_large_patch16_224",
+        "type": "vision_transformer",
+        "size": "304.3M",
+        "description": "Vision Transformer Large (16x16 patches, 224x224 input)",
+        "config": {
+            "img_size": 224,
+            "patch_size": 16,
+            "embed_dim": 1024,
+            "depth": 24,
+            "num_heads": 16,
+            "mlp_ratio": 4.0,
+            "num_classes": 1000
+        }
+    }
+}
+
+
+def get_available_models() -> List[Dict[str, Any]]:
+    """Get list of all available models."""
+    return list(DEFAULT_MODELS.values())
+
+
+def get_model_info(model_name: str) -> Dict[str, Any]:
+    """Get detailed information about a specific model."""
+    if model_name not in DEFAULT_MODELS:
+        raise ValueError(f"Model '{model_name}' not found. Available models: {list(DEFAULT_MODELS.keys())}")
+    
+    return DEFAULT_MODELS[model_name]
+
+
+def download_model(model_name: str, output_dir: str = "./models", force: bool = False) -> str:
+    """Download a pre-trained model (placeholder implementation)."""
+    if model_name not in DEFAULT_MODELS:
+        raise ValueError(f"Model '{model_name}' not found. Available models: {list(DEFAULT_MODELS.keys())}")
+    
+    # Create output directory
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # For now, just save the model configuration
+    # In a real implementation, this would download actual model weights
+    model_info = DEFAULT_MODELS[model_name]
+    output_path = os.path.join(output_dir, f"{model_name}.json")
+    
+    if os.path.exists(output_path) and not force:
+        raise FileExistsError(f"Model already exists at {output_path}. Use --force to overwrite.")
+    
+    with open(output_path, 'w') as f:
+        json.dump(model_info, f, indent=2)
+    
+    return output_path
+
+
+def list_models_by_type(model_type: str = None) -> List[Dict[str, Any]]:
+    """List models filtered by type."""
+    models = get_available_models()
+    if model_type:
+        models = [m for m in models if m.get('type') == model_type]
+    return models
+
+
+def search_models(query: str) -> List[Dict[str, Any]]:
+    """Search models by name or description."""
+    models = get_available_models()
+    query_lower = query.lower()
+    
+    results = []
+    for model in models:
+        if (query_lower in model.get('name', '').lower() or 
+            query_lower in model.get('description', '').lower()):
+            results.append(model)
+    
+    return results
+
+
+class ModelZoo:
+    """Model Zoo manager for Langvision."""
+    
+    def __init__(self, cache_dir: str = "./models"):
+        self.cache_dir = Path(cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.logger = logging.getLogger(__name__)
+    
+    def list_models(self, model_type: str = None) -> List[Dict[str, Any]]:
+        """List available models."""
+        return list_models_by_type(model_type)
+    
+    def get_model(self, model_name: str) -> Dict[str, Any]:
+        """Get model information."""
+        return get_model_info(model_name)
+    
+    def download(self, model_name: str, force: bool = False) -> str:
+        """Download a model."""
+        return download_model(model_name, str(self.cache_dir), force)
+    
+    def search(self, query: str) -> List[Dict[str, Any]]:
+        """Search for models."""
+        return search_models(query)
+    
+    def is_downloaded(self, model_name: str) -> bool:
+        """Check if model is downloaded."""
+        model_path = self.cache_dir / f"{model_name}.json"
+        return model_path.exists()
+    
+    def get_downloaded_models(self) -> List[str]:
+        """Get list of downloaded models."""
+        downloaded = []
+        for model_file in self.cache_dir.glob("*.json"):
+            model_name = model_file.stem
+            if model_name in DEFAULT_MODELS:
+                downloaded.append(model_name)
+        return downloaded

langvision/models/lora.py

@@ -1,30 +1,202 @@
 import torch
 import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Dict, Any
+import math
+from dataclasses import dataclass
+
+@dataclass
+class LoRAConfig:
+    """Configuration class for LoRA parameters."""
+    r: int = 4
+    alpha: float = 1.0
+    dropout: float = 0.0
+    target_modules: Optional[list] = None
+    bias: str = "none"  # "none", "all", "lora_only"
+    task_type: str = "FEATURE_EXTRACTION"
+    inference_mode: bool = False
+    
+    def __post_init__(self):
+        if self.target_modules is None:
+            self.target_modules = ["q_proj", "v_proj", "k_proj", "o_proj"]
 
 class LoRALinear(nn.Module):
-    def __init__(self, in_features, out_features, r=4, alpha=1.0, dropout=0.0):
+    """Enhanced LoRA Linear layer with better initialization and features."""
+    
+    def __init__(self, 
+                 in_features: int, 
+                 out_features: int, 
+                 r: int = 4, 
+                 alpha: float = 1.0, 
+                 dropout: float = 0.0,
+                 bias: bool = True,
+                 fan_in_fan_out: bool = False):
         super().__init__()
         self.r = r
         self.alpha = alpha
-        self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
+        self.fan_in_fan_out = fan_in_fan_out
+        self.in_features = in_features
+        self.out_features = out_features
+        
+        # Original linear layer (frozen)
+        self.linear = nn.Linear(in_features, out_features, bias=bias)
+        
+        # LoRA parameters
         if r > 0:
-            self.lora_A = nn.Parameter(torch.randn(in_features, r) * 0.01)
-            self.lora_B = nn.Parameter(torch.randn(r, out_features) * 0.01)
+            self.lora_A = nn.Parameter(torch.zeros(r, in_features))
+            self.lora_B = nn.Parameter(torch.zeros(out_features, r))
+            self.scaling = alpha / r
+            self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
+            self.reset_parameters()
         else:
             self.lora_A = None
             self.lora_B = None
-        self.scale = alpha / r if r > 0 else 1.0
+            self.scaling = 1.0
+            self.dropout = nn.Identity()
+    
+    def reset_parameters(self):
+        """Initialize LoRA parameters using Kaiming uniform initialization."""
+        if hasattr(self, 'lora_A') and self.lora_A is not None:
+            # Initialize A with random values and B with zeros
+            nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
+            nn.init.zeros_(self.lora_B)
+    
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """Forward pass with LoRA adaptation."""
+        # Original linear transformation
+        result = self.linear(x)
+        
+        # Add LoRA adaptation if enabled
+        if self.r > 0 and self.lora_A is not None:
+            # Apply dropout to input
+            x_dropped = self.dropout(x)
+            
+            # LoRA computation: x @ A^T @ B^T
+            if self.fan_in_fan_out:
+                lora_result = F.linear(x_dropped, self.lora_A.T) @ self.lora_B.T
+            else:
+                lora_result = F.linear(F.linear(x_dropped, self.lora_A.T), self.lora_B.T)
+            
+            result = result + lora_result * self.scaling
+        
+        return result
+    
+    def merge_weights(self):
+        """Merge LoRA weights into the original linear layer."""
+        if self.r > 0 and self.lora_A is not None:
+            # Compute LoRA weight update
+            delta_w = self.lora_B @ self.lora_A * self.scaling
+            
+            # Merge with original weights
+            if self.fan_in_fan_out:
+                self.linear.weight.data += delta_w.T
+            else:
+                self.linear.weight.data += delta_w
+            
+            # Reset LoRA parameters
+            self.lora_A.data.zero_()
+            self.lora_B.data.zero_()
+    
+    def unmerge_weights(self):
+        """Unmerge LoRA weights from the original linear layer."""
+        if self.r > 0 and self.lora_A is not None:
+            # Compute LoRA weight update
+            delta_w = self.lora_B @ self.lora_A * self.scaling
+            
+            # Remove from original weights
+            if self.fan_in_fan_out:
+                self.linear.weight.data -= delta_w.T
+            else:
+                self.linear.weight.data -= delta_w
+
+class AdaLoRALinear(LoRALinear):
+    """Adaptive LoRA with dynamic rank adjustment."""
+    
+    def __init__(self, *args, **kwargs):
+        self.target_rank = kwargs.pop('target_rank', None)
+        self.rank_pattern = kwargs.pop('rank_pattern', None)
+        super().__init__(*args, **kwargs)
+        
+        if self.target_rank is not None:
+            self.rank_scheduler = self._create_rank_scheduler()
+    
+    def _create_rank_scheduler(self):
+        """Create a rank scheduler for adaptive rank adjustment."""
+        # Placeholder for rank scheduling logic
+        return None
+    
+    def update_rank(self, new_rank: int):
+        """Dynamically update the LoRA rank."""
+        if new_rank != self.r and new_rank > 0:
+            old_r = self.r
+            self.r = new_rank
+            self.scaling = self.alpha / new_rank
+            
+            # Resize parameters
+            if old_r > 0:
+                # Preserve existing weights up to min(old_r, new_rank)
+                min_r = min(old_r, new_rank)
+                old_A = self.lora_A.data[:min_r, :]
+                old_B = self.lora_B.data[:, :min_r]
+            
+            # Create new parameters
+            self.lora_A = nn.Parameter(torch.zeros(new_rank, self.in_features))
+            self.lora_B = nn.Parameter(torch.zeros(self.out_features, new_rank))
+            
+            if old_r > 0:
+                # Copy preserved weights
+                self.lora_A.data[:min_r, :] = old_A
+                self.lora_B.data[:, :min_r] = old_B
+            
+            # Initialize new parameters
+            if new_rank > old_r:
+                nn.init.kaiming_uniform_(self.lora_A.data[old_r:, :], a=math.sqrt(5))
+                nn.init.zeros_(self.lora_B.data[:, old_r:])
 
-    def forward(self, x):
+class QLoRALinear(nn.Module):
+    """Quantized LoRA implementation for memory efficiency."""
+    
+    def __init__(self, 
+                 in_features: int, 
+                 out_features: int, 
+                 r: int = 4, 
+                 alpha: float = 1.0, 
+                 dropout: float = 0.0,
+                 compute_dtype: torch.dtype = torch.float16,
+                 quant_type: str = "nf4"):
+        super().__init__()
+        self.r = r
+        self.alpha = alpha
+        self.scaling = alpha / r if r > 0 else 1.0
+        self.compute_dtype = compute_dtype
+        self.quant_type = quant_type
+        
+        # Quantized base layer (placeholder - would need actual quantization library)
+        self.base_layer = nn.Linear(in_features, out_features, bias=False)
+        
+        # LoRA adapters in full precision
+        if r > 0:
+            self.lora_A = nn.Parameter(torch.zeros(r, in_features, dtype=compute_dtype))
+            self.lora_B = nn.Parameter(torch.zeros(out_features, r, dtype=compute_dtype))
+            self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
+            self.reset_parameters()
+    
+    def reset_parameters(self):
+        if hasattr(self, 'lora_A') and self.lora_A is not None:
+            nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
+            nn.init.zeros_(self.lora_B)
+    
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # Convert to compute dtype
+        x = x.to(self.compute_dtype)
+        
+        # Base layer forward (quantized)
+        result = self.base_layer(x)
+        
+        # LoRA adaptation
         if self.r > 0:
-            orig_shape = x.shape
-            # Flatten all but last dim
-            x_2d = x.reshape(-1, x.shape[-1])
-            # Apply LoRA: (N, in_features) @ (in_features, r) @ (r, out_features) = (N, out_features)
-            lora_out = self.dropout(x_2d) @ self.lora_A @ self.lora_B * self.scale
-            # Reshape back to original except last dim is out_features
-            out_shape = list(orig_shape[:-1]) + [self.lora_B.shape[1]]
-            lora_out = lora_out.view(*out_shape)
-            return lora_out
-        else:
-            return 0.0 
+            x_dropped = self.dropout(x)
+            lora_result = F.linear(F.linear(x_dropped, self.lora_A.T), self.lora_B.T)
+            result = result + lora_result * self.scaling
+        
+        return result 

langvision/models/multimodal.py

@@ -0,0 +1,297 @@
+"""
+Multimodal Vision-Language Models with LoRA fine-tuning support.
+"""
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Dict, Any, Tuple, List
+from transformers import AutoTokenizer, AutoModel
+from .vision_transformer import VisionTransformer
+from .lora import LoRALinear, LoRAConfig
+import math
+
+
+class CrossAttention(nn.Module):
+    """Cross-attention mechanism for vision-language fusion."""
+    
+    def __init__(self, 
+                 vision_dim: int, 
+                 text_dim: int, 
+                 hidden_dim: int = 512,
+                 num_heads: int = 8,
+                 dropout: float = 0.1,
+                 lora_config: Optional[LoRAConfig] = None):
+        super().__init__()
+        self.hidden_dim = hidden_dim
+        self.num_heads = num_heads
+        self.head_dim = hidden_dim // num_heads
+        self.scale = self.head_dim ** -0.5
+        
+        # Query, Key, Value projections
+        if lora_config and lora_config.r > 0:
+            self.q_proj = LoRALinear(vision_dim, hidden_dim, 
+                                   r=lora_config.r, alpha=lora_config.alpha, dropout=lora_config.dropout)
+            self.k_proj = LoRALinear(text_dim, hidden_dim,
+                                   r=lora_config.r, alpha=lora_config.alpha, dropout=lora_config.dropout)
+            self.v_proj = LoRALinear(text_dim, hidden_dim,
+                                   r=lora_config.r, alpha=lora_config.alpha, dropout=lora_config.dropout)
+            self.out_proj = LoRALinear(hidden_dim, vision_dim,
+                                     r=lora_config.r, alpha=lora_config.alpha, dropout=lora_config.dropout)
+        else:
+            self.q_proj = nn.Linear(vision_dim, hidden_dim)
+            self.k_proj = nn.Linear(text_dim, hidden_dim)
+            self.v_proj = nn.Linear(text_dim, hidden_dim)
+            self.out_proj = nn.Linear(hidden_dim, vision_dim)
+        
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = nn.LayerNorm(vision_dim)
+    
+    def forward(self, 
+                vision_features: torch.Tensor, 
+                text_features: torch.Tensor,
+                attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+        """
+        Args:
+            vision_features: (B, N_v, D_v) vision tokens
+            text_features: (B, N_t, D_t) text tokens
+            attention_mask: (B, N_t) mask for text tokens
+        """
+        B, N_v, D_v = vision_features.shape
+        B, N_t, D_t = text_features.shape
+        
+        # Project to query, key, value
+        Q = self.q_proj(vision_features)  # (B, N_v, hidden_dim)
+        K = self.k_proj(text_features)    # (B, N_t, hidden_dim)
+        V = self.v_proj(text_features)    # (B, N_t, hidden_dim)
+        
+        # Reshape for multi-head attention
+        Q = Q.view(B, N_v, self.num_heads, self.head_dim).transpose(1, 2)  # (B, num_heads, N_v, head_dim)
+        K = K.view(B, N_t, self.num_heads, self.head_dim).transpose(1, 2)  # (B, num_heads, N_t, head_dim)
+        V = V.view(B, N_t, self.num_heads, self.head_dim).transpose(1, 2)  # (B, num_heads, N_t, head_dim)
+        
+        # Compute attention scores
+        attn_scores = torch.matmul(Q, K.transpose(-2, -1)) * self.scale  # (B, num_heads, N_v, N_t)
+        
+        # Apply attention mask if provided
+        if attention_mask is not None:
+            attn_mask = attention_mask.unsqueeze(1).unsqueeze(2)  # (B, 1, 1, N_t)
+            attn_scores = attn_scores.masked_fill(attn_mask == 0, float('-inf'))
+        
+        # Apply softmax
+        attn_weights = F.softmax(attn_scores, dim=-1)
+        attn_weights = self.dropout(attn_weights)
+        
+        # Apply attention to values
+        attn_output = torch.matmul(attn_weights, V)  # (B, num_heads, N_v, head_dim)
+        
+        # Reshape and project
+        attn_output = attn_output.transpose(1, 2).contiguous().view(B, N_v, self.hidden_dim)
+        output = self.out_proj(attn_output)
+        
+        # Residual connection and layer norm
+        output = self.layer_norm(vision_features + output)
+        
+        return output
+
+
+class VisionLanguageModel(nn.Module):
+    """Vision-Language Model with cross-modal attention and LoRA fine-tuning."""
+    
+    def __init__(self,
+                 vision_model: str = "vit_base",
+                 text_model: str = "bert-base-uncased",
+                 vision_dim: int = 768,
+                 text_dim: int = 768,
+                 hidden_dim: int = 512,
+                 num_classes: int = 1000,
+                 max_text_length: int = 77,
+                 lora_config: Optional[LoRAConfig] = None):
+        super().__init__()
+        
+        self.vision_dim = vision_dim
+        self.text_dim = text_dim
+        self.hidden_dim = hidden_dim
+        self.max_text_length = max_text_length
+        
+        # Vision encoder
+        if vision_model == "vit_base":
+            self.vision_encoder = VisionTransformer(
+                embed_dim=vision_dim,
+                lora_config=lora_config
+            )
+        else:
+            raise ValueError(f"Unsupported vision model: {vision_model}")
+        
+        # Text encoder
+        self.tokenizer = AutoTokenizer.from_pretrained(text_model)
+        self.text_encoder = AutoModel.from_pretrained(text_model)
+        
+        # Freeze text encoder if using LoRA
+        if lora_config and lora_config.r > 0:
+            for param in self.text_encoder.parameters():
+                param.requires_grad = False
+        
+        # Cross-modal fusion
+        self.cross_attention = CrossAttention(
+            vision_dim=vision_dim,
+            text_dim=text_dim,
+            hidden_dim=hidden_dim,
+            lora_config=lora_config
+        )
+        
+        # Classification head
+        if lora_config and lora_config.r > 0:
+            self.classifier = LoRALinear(vision_dim, num_classes,
+                                       r=lora_config.r, alpha=lora_config.alpha, dropout=lora_config.dropout)
+        else:
+            self.classifier = nn.Linear(vision_dim, num_classes)
+        
+        # Text projection for contrastive learning
+        if lora_config and lora_config.r > 0:
+            self.text_projection = LoRALinear(text_dim, hidden_dim,
+                                            r=lora_config.r, alpha=lora_config.alpha, dropout=lora_config.dropout)
+            self.vision_projection = LoRALinear(vision_dim, hidden_dim,
+                                              r=lora_config.r, alpha=lora_config.alpha, dropout=lora_config.dropout)
+        else:
+            self.text_projection = nn.Linear(text_dim, hidden_dim)
+            self.vision_projection = nn.Linear(vision_dim, hidden_dim)
+        
+        self.logit_scale = nn.Parameter(torch.ones([]) * math.log(1 / 0.07))
+    
+    def encode_image(self, images: torch.Tensor) -> torch.Tensor:
+        """Encode images to feature representations."""
+        return self.vision_encoder(images)
+    
+    def encode_text(self, texts: List[str]) -> Tuple[torch.Tensor, torch.Tensor]:
+        """Encode texts to feature representations."""
+        # Tokenize texts
+        tokens = self.tokenizer(
+            texts,
+            padding=True,
+            truncation=True,
+            max_length=self.max_text_length,
+            return_tensors="pt"
+        )
+        
+        # Get text features
+        with torch.no_grad() if hasattr(self, 'text_encoder') else torch.enable_grad():
+            text_outputs = self.text_encoder(**tokens)
+            text_features = text_outputs.last_hidden_state  # (B, seq_len, text_dim)
+        
+        return text_features, tokens['attention_mask']
+    
+    def forward(self, 
+                images: torch.Tensor, 
+                texts: Optional[List[str]] = None,
+                return_features: bool = False) -> Dict[str, torch.Tensor]:
+        """
+        Forward pass for vision-language model.
+        
+        Args:
+            images: (B, C, H, W) input images
+            texts: List of text descriptions
+            return_features: Whether to return intermediate features
+        """
+        # Encode images
+        vision_features = self.encode_image(images)  # (B, N_patches, vision_dim)
+        
+        outputs = {"vision_features": vision_features}
+        
+        if texts is not None:
+            # Encode texts
+            text_features, attention_mask = self.encode_text(texts)  # (B, seq_len, text_dim)
+            outputs["text_features"] = text_features
+            
+            # Cross-modal fusion
+            fused_features = self.cross_attention(
+                vision_features, text_features, attention_mask
+            )  # (B, N_patches, vision_dim)
+            outputs["fused_features"] = fused_features
+            
+            # Global pooling for classification
+            pooled_vision = fused_features.mean(dim=1)  # (B, vision_dim)
+            pooled_text = text_features.mean(dim=1)     # (B, text_dim)
+            
+            # Classification
+            logits = self.classifier(pooled_vision)
+            outputs["logits"] = logits
+            
+            # Contrastive learning projections
+            vision_proj = F.normalize(self.vision_projection(pooled_vision), dim=-1)
+            text_proj = F.normalize(self.text_projection(pooled_text), dim=-1)
+            
+            # Compute contrastive logits
+            logit_scale = self.logit_scale.exp()
+            contrastive_logits = logit_scale * vision_proj @ text_proj.T
+            
+            outputs.update({
+                "vision_proj": vision_proj,
+                "text_proj": text_proj,
+                "contrastive_logits": contrastive_logits,
+                "logit_scale": logit_scale
+            })
+        else:
+            # Image-only classification
+            pooled_vision = vision_features.mean(dim=1)
+            logits = self.classifier(pooled_vision)
+            outputs["logits"] = logits
+        
+        if not return_features:
+            # Return only essential outputs for training
+            essential_keys = ["logits", "contrastive_logits"] if texts else ["logits"]
+            outputs = {k: v for k, v in outputs.items() if k in essential_keys}
+        
+        return outputs
+
+
+class CLIPLoss(nn.Module):
+    """CLIP-style contrastive loss for vision-language learning."""
+    
+    def __init__(self, temperature: float = 0.07):
+        super().__init__()
+        self.temperature = temperature
+        self.cross_entropy = nn.CrossEntropyLoss()
+    
+    def forward(self, vision_proj: torch.Tensor, text_proj: torch.Tensor) -> torch.Tensor:
+        """
+        Compute contrastive loss between vision and text projections.
+        
+        Args:
+            vision_proj: (B, D) normalized vision projections
+            text_proj: (B, D) normalized text projections
+        """
+        batch_size = vision_proj.shape[0]
+        
+        # Compute similarity matrix
+        logits = vision_proj @ text_proj.T / self.temperature
+        
+        # Labels are diagonal (each image matches its corresponding text)
+        labels = torch.arange(batch_size, device=logits.device)
+        
+        # Symmetric loss (image-to-text and text-to-image)
+        loss_i2t = self.cross_entropy(logits, labels)
+        loss_t2i = self.cross_entropy(logits.T, labels)
+        
+        return (loss_i2t + loss_t2i) / 2
+
+
+def create_multimodal_model(model_config: Dict[str, Any]) -> VisionLanguageModel:
+    """Factory function to create multimodal models with different configurations."""
+    
+    # Default configuration
+    default_config = {
+        "vision_model": "vit_base",
+        "text_model": "bert-base-uncased",
+        "vision_dim": 768,
+        "text_dim": 768,
+        "hidden_dim": 512,
+        "num_classes": 1000,
+        "max_text_length": 77,
+        "lora_config": None
+    }
+    
+    # Update with user config
+    config = {**default_config, **model_config}
+    
+    return VisionLanguageModel(**config)