langvio 0.0.1__py3-none-any.whl

langvio/__init__.py

@@ -0,0 +1,132 @@
+"""
+langvio: Connect language models to vision models for natural language visual analysis
+"""
+
+__version__ = "0.0.1"
+
+# === Imports ===
+
+# Standard library
+import sys
+
+# Third-party
+import cv2
+import torch
+from dotenv import load_dotenv
+
+from langvio.core.pipeline import Pipeline
+
+# langvio modules
+from langvio.core.registry import ModelRegistry
+from langvio.llm.base import BaseLLMProcessor
+from langvio.llm.factory import register_llm_processors
+from langvio.vision.base import BaseVisionProcessor
+from langvio.vision.yolo.detector import YOLOProcessor
+
+# === Initialization ===
+
+# Load environment variables
+load_dotenv()
+
+# OpenCV optimizations
+cv2.setNumThreads(4)
+cv2.setUseOptimized(True)
+
+# PyTorch optimizations
+if torch.cuda.is_available():
+    torch.backends.cudnn.benchmark = True
+
+# Initialize model registry
+registry = ModelRegistry()
+
+# Register YOLO and YOLOe vision processors
+registry.register_vision_processor(
+    "yolo", YOLOProcessor, model_path="yolo11n.pt", confidence=0.5
+)
+
+registry.register_vision_processor(
+    "yoloe",
+    YOLOProcessor,
+    model_path="yoloe-11s-seg-pf.pt",
+    confidence=0.5,
+    model_type="yoloe",
+)
+
+registry.register_vision_processor(
+    "yoloe_medium",
+    YOLOProcessor,
+    model_path="yoloe-11m-seg-pf.pt",
+    confidence=0.5,
+    model_type="yoloe",
+)
+
+registry.register_vision_processor(
+    "yoloe_large",
+    YOLOProcessor,
+    model_path="yoloe-11l-seg-pf.pt",
+    confidence=0.5,
+    model_type="yoloe",
+)
+
+# Register LLM processors
+register_llm_processors(registry)
+
+
+# === Pipeline Creator ===
+
+
+def create_pipeline(config_path=None, llm_name=None, vision_name=None):
+    """
+    Create a pipeline with optional configuration.
+
+    Args:
+        config_path: Path to a configuration file
+        llm_name: Name of LLM processor to use
+        vision_name: Name of vision processor to use (default: "yoloe")
+
+    Returns:
+        A configured Pipeline instance
+    """
+    pipeline = Pipeline(config_path)
+
+    if vision_name:
+        pipeline.set_vision_processor(vision_name)
+    else:
+        try:
+            pipeline.set_vision_processor("yoloe_large")
+        except Exception:
+            pipeline.set_vision_processor("yolo")
+
+    if llm_name:
+        pipeline.set_llm_processor(llm_name)
+    else:
+        try:
+            default_llm = pipeline.config.config["llm"]["default"]
+            pipeline.set_llm_processor(default_llm)
+        except Exception:
+            if len(registry.list_llm_processors()) == 0:
+                error_msg = (
+                    "ERROR: No LLM providers are installed. "
+                    "Please install at least one provider:\n"
+                    "- For OpenAI: pip install langvio[openai]\n"
+                    "- For Google Gemini: pip install langvio[google]\n"
+                    "- For all providers: pip install langvio[all-llm]"
+                )
+                print(error_msg, file=sys.stderr)
+                sys.exit(1)
+            else:
+                available_llm = next(iter(registry.list_llm_processors()))
+                pipeline.set_llm_processor(available_llm)
+
+    return pipeline
+
+
+# === Public Exports ===
+
+__all__ = [
+    "Pipeline",
+    "create_pipeline",
+    "registry",
+    "BaseLLMProcessor",
+    "BaseVisionProcessor",
+]

langvio/cli.py

@@ -0,0 +1,120 @@
+"""
+Command-line interface for langvio
+"""
+
+import argparse
+import logging
+import os
+import sys
+
+from langvio import create_pipeline
+from langvio.utils.file_utils import is_image_file, is_video_file
+from langvio.utils.logging import setup_logging
+
+
+def main():
+    """Main entry point for the langvio CLI"""
+    # Parse arguments
+    parser = argparse.ArgumentParser(
+        description="langvio: Connect LLMs with video models"
+    )
+
+    # Required arguments
+    parser.add_argument("--query", "-q", required=True, help="Natural language query")
+    parser.add_argument(
+        "--media", "-m", required=True, help="Path to image or video file"
+    )
+
+    # Optional arguments
+    parser.add_argument("--config", "-c", help="Path to configuration file")
+    parser.add_argument("--llm", "-l", help="LLM processor to use")
+    parser.add_argument("--vision", "-v", help="Vision processor to use")
+    parser.add_argument("--output", "-o", help="Output directory for processed media")
+    parser.add_argument(
+        "--log-level",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        default="INFO",
+        help="Logging level",
+    )
+    parser.add_argument("--log-file", help="Path to log file")
+    parser.add_argument(
+        "--list-models", action="store_true", help="List available models and exit"
+    )
+
+    args = parser.parse_args()
+
+    # Set up logging
+    setup_logging({"level": args.log_level, "file": args.log_file})
+
+    logger = logging.getLogger(__name__)
+
+    # List models if requested
+    if args.list_models:
+        list_available_models()
+        return 0
+
+    # Check if media file exists
+    if not os.path.exists(args.media):
+        logger.error("Media file not found: %s", args.media)
+        return 1
+
+    # Check if media file is supported
+    if not is_image_file(args.media) and not is_video_file(args.media):
+        logger.error("Unsupported media file format: %s", args.media)
+        return 1
+
+    try:
+        # Create pipeline
+        pipeline = create_pipeline(
+            config_path=args.config, llm_name=args.llm, vision_name=args.vision
+        )
+
+        # Update output directory if specified
+        if args.output:
+            pipeline.config.config["media"]["output_dir"] = args.output
+            os.makedirs(args.output, exist_ok=True)
+
+        # Process query
+        result = pipeline.process(args.query, args.media)
+
+        # Print results
+        print("\n===== langvio Results =====")
+        print("Query: {}".format(result["query"]))
+        print("Media: {} ({})".format(result["media_path"], result["media_type"]))
+        print("Output: {}".format(result["output_path"]))
+        print("\nExplanation: {}".format(result["explanation"]))
+        print("\nDetection summary:")
+
+        # Count detections by label
+        counts = {}
+        for frame_dets in result["detections"].values():
+            for det in frame_dets:
+                label = det["label"]
+                counts[label] = counts.get(label, 0) + 1
+
+        for label, count in counts.items():
+            print("- {}: {}".format(label, count))
+
+        print("\nProcessing complete!")
+        return 0
+
+    except Exception as e:
+        logger.error("Error processing query: %s", e, exc_info=True)
+        return 1
+
+
+def list_available_models():
+    """List all available models in the registry"""
+    from langvio import registry
+
+    print("\n===== Available LLM Processors =====")
+    for name, cls in registry.list_llm_processors().items():
+        print("- {}".format(name))
+
+    print("\n===== Available Vision Processors =====")
+    for name, cls in registry.list_vision_processors().items():
+        print("- {}".format(name))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

langvio/config.py

@@ -0,0 +1,190 @@
+"""
+Configuration management for langvio
+"""
+
+import os
+from typing import Any, Dict, Optional
+
+import yaml
+from dotenv import load_dotenv
+
+load_dotenv()  # Load environment variables from .env
+
+
+class Config:
+    """Configuration manager for langvio"""
+
+    DEFAULT_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "default_config.yaml")
+
+    def __init__(self, config_path: Optional[str] = None):
+        """
+        Initialize configuration.
+
+        Args:
+            config_path: Path to a YAML configuration file
+        """
+        # Initialize empty config
+        self.config = {}
+
+        # First load default config
+        self._load_default_config()
+
+        # Then load user config if provided
+        if config_path and os.path.exists(config_path):
+            self.load_config(config_path)
+
+    def _load_default_config(self) -> None:
+        """Load the default configuration from default_config.yaml"""
+        try:
+            if os.path.exists(self.DEFAULT_CONFIG_PATH):
+                with open(self.DEFAULT_CONFIG_PATH, "r") as f:
+                    self.config = yaml.safe_load(f)
+                    if self.config is None:  # Handle empty file case
+                        self.config = {}
+            else:
+                # Fallback default configuration if file doesn't exist
+                self.config = {
+                    "llm": {
+                        "default": "gemini",
+                        "models": {
+                            "gemini": {
+                                "model_name": "gemini-pro",
+                                "model_kwargs": {"temperature": 0.2},
+                            },
+                            "gpt": {
+                                "model_name": "gpt-3.5-turbo",
+                                "model_kwargs": {"temperature": 0.0},
+                            },
+                        },
+                    },
+                    "vision": {
+                        "default": "yolo",
+                        "models": {
+                            "yolo": {
+                                "type": "yolo",
+                                "model_path": "yolov11n.pt",
+                                "confidence": 0.25,
+                            }
+                        },
+                    },
+                    "media": {
+                        "output_dir": "./output",
+                        "temp_dir": "./temp",
+                        "visualization": {
+                            "box_color": [0, 255, 0],
+                            "text_color": [255, 255, 255],
+                            "line_thickness": 2,
+                        },
+                    },
+                    "logging": {"level": "INFO", "file": None},
+                }
+        except Exception as e:
+            raise ValueError(f"Error loading default configuration: {e}")
+
+    def load_config(self, config_path: str) -> None:
+        """
+        Load configuration from a YAML file.
+
+        Args:
+            config_path: Path to a YAML configuration file
+        """
+        try:
+            with open(config_path, "r") as f:
+                user_config = yaml.safe_load(f)
+                if user_config is None:  # Handle empty file case
+                    return
+
+            # Update configuration
+            self._update_config(self.config, user_config)
+        except Exception as e:
+            raise ValueError(f"Error loading configuration from {config_path}: {e}")
+
+    def _update_config(
+        self, base_config: Dict[str, Any], new_config: Dict[str, Any]
+    ) -> None:
+        """Recursively update base config with new config."""
+        for key, value in new_config.items():
+            if (
+                isinstance(value, dict)
+                and key in base_config
+                and isinstance(base_config[key], dict)
+            ):
+                self._update_config(base_config[key], value)
+            else:
+                base_config[key] = value
+
+    def get_llm_config(self, model_name: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get configuration for an LLM model.
+
+        Args:
+            model_name: Name of the model to get config for
+
+        Returns:
+            Model configuration dictionary
+        """
+        if not model_name:
+            model_name = self.config["llm"]["default"]
+
+        if model_name not in self.config["llm"]["models"]:
+            raise ValueError(f"LLM model '{model_name}' not found in configuration")
+
+        return self.config["llm"]["models"][model_name]
+
+    def get_vision_config(self, model_name: Optional[str] = None) -> Dict[str, Any]:
+        """
+        Get configuration for a vision model.
+
+        Args:
+            model_name: Name of the model to get config for
+
+        Returns:
+            Model configuration dictionary
+        """
+        if not model_name:
+            model_name = self.config["vision"]["default"]
+
+        if model_name not in self.config["vision"]["models"]:
+            raise ValueError(f"Vision model '{model_name}' not found in configuration")
+
+        return self.config["vision"]["models"][model_name]
+
+    def get_media_config(self) -> Dict[str, Any]:
+        """
+        Get media processing configuration.
+
+        Returns:
+            Media configuration dictionary
+        """
+        return self.config["media"]
+
+    def get_logging_config(self) -> Dict[str, Any]:
+        """
+        Get logging configuration.
+
+        Returns:
+            Logging configuration dictionary
+        """
+        return self.config["logging"]
+
+    def get_langsmith_config(self) -> Dict[str, Any]:
+        """
+        Get LangSmith configuration if available.
+
+        Returns:
+            LangSmith configuration dictionary
+        """
+        return self.config.get("langsmith", {})
+
+    def save_config(self, config_path: str) -> None:
+        """
+        Save current configuration to a YAML file.
+
+        Args:
+            config_path: Path to save the configuration
+        """
+        try:
+            with open(config_path, "w") as f:
+                yaml.dump(self.config, f, default_flow_style=False)
+        except Exception as e:
+            raise ValueError(f"Error saving configuration to {config_path}: {e}")

langvio/core/base.py

@@ -0,0 +1,46 @@
+"""
+Base classes for langvio components
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+
+
+class Processor(ABC):
+    """Base class for all processors in langvio"""
+
+    def __init__(self, name: str, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize processor.
+
+        Args:
+            name: Processor name
+            config: Configuration parameters
+        """
+        self.name = name
+        self.config = config or {}
+
+    @abstractmethod
+    def initialize(self) -> bool:
+        """
+        Initialize the processor with its configuration.
+
+        Returns:
+            True if initialization was successful
+        """
+
+    @classmethod
+    def from_config(cls, name: str, config: Dict[str, Any]) -> "Processor":
+        """
+        Create a processor from configuration.
+
+        Args:
+            name: Processor name
+            config: Configuration parameters
+
+        Returns:
+            Initialized processor
+        """
+        processor = cls(name, config)
+        processor.initialize()
+        return processor

langvio/core/pipeline.py

@@ -0,0 +1,154 @@
+"""
+Enhanced core pipeline for connecting LLMs with vision models - refactored
+"""
+
+import logging
+import os
+from typing import Any, Dict, Optional
+
+from langvio.config import Config
+from langvio.core.processor_manager import ProcessorManager
+from langvio.core.visualization_manager import VisualizationManager
+from langvio.utils.file_utils import is_video_file
+from langvio.utils.logging import setup_logging
+
+
+class Pipeline:
+    """Main pipeline for processing queries with LLMs and vision models"""
+
+    def __init__(self, config_path: Optional[str] = None):
+        """Initialize pipeline"""
+        # Initialize configuration
+        self.config = Config(config_path)
+
+        # Set up logging
+        setup_logging(self.config.get_logging_config())
+        self.logger = logging.getLogger(__name__)
+
+        # Initialize managers
+        self.processor_manager = ProcessorManager(self.config)
+        self.visualization_manager = VisualizationManager(self.config)
+
+        self.logger.info("Enhanced Pipeline initialized")
+
+    def load_config(self, config_path: str) -> None:
+        """Load configuration from file"""
+        self.config.load_config(config_path)
+        self.logger.info(f"Loaded configuration from {config_path}")
+
+        # Update managers with new config
+        self.processor_manager.config = self.config
+        self.visualization_manager.config = self.config
+
+    def set_llm_processor(self, processor_name: str) -> None:
+        """Set the LLM processor"""
+        self.processor_manager.set_llm_processor(processor_name)
+
+    def set_vision_processor(self, processor_name: str) -> None:
+        """Set the vision processor"""
+        self.processor_manager.set_vision_processor(processor_name)
+
+    def process(self, query: str, media_path: str) -> Dict[str, Any]:
+        """Process a query on media with enhanced capabilities"""
+        self.logger.info(f"Processing query: {query}")
+
+        # Check if processors are set
+        if not self.processor_manager.has_processors():
+            if not self.processor_manager.llm_processor:
+                error_msg = "ERROR: LLM processor not set"
+                self.logger.error(error_msg)
+                raise ValueError(error_msg)
+
+            if not self.processor_manager.vision_processor:
+                error_msg = "ERROR: Vision processor not set"
+                self.logger.error(error_msg)
+                raise ValueError(error_msg)
+
+        # Check if media file exists
+        if not os.path.exists(media_path):
+            error_msg = f"ERROR: Media file not found: {media_path}"
+            self.logger.error(error_msg)
+            raise FileNotFoundError(error_msg)
+
+        # Check media type
+        is_video = is_video_file(media_path)
+        media_type = "video" if is_video else "image"
+
+        try:
+            # 1. Parse query with LLM to get structured parameters
+            query_params = self.processor_manager.parse_query(query)
+            self.logger.info(f"Parsed query params: {query_params}")
+
+            # 2. Run detection with vision processor
+            all_detections = self.processor_manager.process_media(
+                media_path, query_params
+            )
+
+            # Log detection results to JSON
+            self._log_detections_to_json(all_detections, query, media_path)
+
+            # 3. Generate explanation using all detected objects and metrics
+            explanation = self.processor_manager.generate_explanation(
+                query, all_detections
+            )
+
+            # 4. Get highlighted objects from the LLM processor
+            highlighted_objects = self.processor_manager.get_highlighted_objects()
+
+            # 5. Create visualization with highlighted objects
+            output_path = self.visualization_manager.create_visualization(
+                media_path, all_detections, highlighted_objects, query_params
+            )
+
+            # Prepare result
+            result = {
+                "query": query,
+                "media_path": media_path,
+                "media_type": media_type,
+                "output_path": output_path,
+                "explanation": explanation,
+                "detections": all_detections,
+                "query_params": query_params,
+                "highlighted_objects": highlighted_objects,
+            }
+
+            self.logger.info("Processed query successfully")
+            return result
+
+        except Exception as e:
+            self.logger.error(f"Error processing query: {e}")
+            raise
+
+    def _log_detections_to_json(
+        self, all_detections: Dict[str, Any], query: str, media_path: str
+    ):
+        """Log detection results to JSON file"""
+        import json
+        import os
+        from datetime import datetime
+
+        # Create logs directory if it doesn't exist
+        logs_dir = "logs"
+        os.makedirs(logs_dir, exist_ok=True)
+
+        # Create filename with timestamp
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        media_name = os.path.splitext(os.path.basename(media_path))[0]
+        filename = f"detections_{media_name}_{timestamp}.json"
+        filepath = os.path.join(logs_dir, filename)
+
+        # Prepare detection log data
+        detection_log = {
+            "query": query,
+            "media_path": media_path,
+            "timestamp": datetime.now().isoformat(),
+            "detections": all_detections,
+        }
+
+        # Write to JSON file
+        try:
+            with open(filepath, "w", encoding="utf-8") as f:
+                json.dump(detection_log, f, indent=2, ensure_ascii=False)
+            self.logger.info(f"Detection results logged to: {filepath}")
+        except Exception as e:
+            self.logger.error(f"Failed to log detection results to file: {e}")