vibe-aigc 0.6.1__tar.gz → 0.6.3__tar.gz

{vibe_aigc-0.6.1/vibe_aigc.egg-info → vibe_aigc-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vibe-aigc
-Version: 0.6.1
+Version: 0.6.3
 Summary: A New Paradigm for Content Generation via Agentic Orchestration
 Author: Vibe AIGC Contributors
 License-Expression: MIT

{vibe_aigc-0.6.1 → vibe_aigc-0.6.3}/pyproject.toml

@@ -8,7 +8,7 @@ exclude = ["tests*", "docs*", "examples*", "landing*"]
 
 [project]
 name = "vibe-aigc"
-version = "0.6.1"
+version = "0.6.3"
 description = "A New Paradigm for Content Generation via Agentic Orchestration"
 authors = [{name = "Vibe AIGC Contributors"}]
 license = "MIT"

{vibe_aigc-0.6.1 → vibe_aigc-0.6.3}/vibe_aigc/tools.py

@@ -496,8 +496,12 @@ class ToolRegistry:
         return "\n".join(lines)
 
 
-def create_default_registry() -> ToolRegistry:
-    """Create a registry with default tools."""
+def create_default_registry(comfyui_url: str = "http://127.0.0.1:8188") -> ToolRegistry:
+    """Create a registry with default tools.
+    
+    Args:
+        comfyui_url: URL for ComfyUI server (for image/video generation)
+    """
     registry = ToolRegistry()
     
     # Register built-in tools
@@ -505,4 +509,12 @@ def create_default_registry() -> ToolRegistry:
     registry.register(TemplateTool())
     registry.register(CombineTool())
     
+    # Register ComfyUI-based generation tools (Paper Section 5.4)
+    try:
+        from .tools_comfyui import create_comfyui_tools
+        for tool in create_comfyui_tools(comfyui_url):
+            registry.register(tool)
+    except ImportError:
+        pass  # ComfyUI tools optional
+    
     return registry

vibe_aigc-0.6.3/vibe_aigc/tools_comfyui.py

@@ -0,0 +1,271 @@
+"""
+ComfyUI Generation Tools - Image and Video Generation via VibeBackend.
+
+Integrates the VibeBackend into the Paper's tool architecture (Section 5.4):
+"The Planner traverses the system's atomic tool library...to select
+the optimal ensemble of components."
+
+This makes ComfyUI-based generation available to MetaPlanner as atomic tools.
+"""
+
+from typing import Dict, Any, Optional
+from dataclasses import dataclass
+
+from .tools import BaseTool, ToolSpec, ToolResult, ToolCategory
+from .vibe_backend import VibeBackend, GenerationRequest, GenerationResult
+from .discovery import Capability
+
+
+class ImageGenerationTool(BaseTool):
+    """
+    Image generation tool using local ComfyUI.
+    
+    Maps high-level image requests to VibeBackend execution.
+    MetaPlanner can use this for visual content generation.
+    """
+    
+    def __init__(self, comfyui_url: str = "http://127.0.0.1:8188"):
+        self.comfyui_url = comfyui_url
+        self._backend: Optional[VibeBackend] = None
+    
+    @property
+    def spec(self) -> ToolSpec:
+        return ToolSpec(
+            name="image_generation",
+            description="Generate images using local AI models (FLUX, SD, etc.)",
+            category=ToolCategory.IMAGE,
+            input_schema={
+                "type": "object",
+                "required": ["prompt"],
+                "properties": {
+                    "prompt": {"type": "string", "description": "Image description"},
+                    "negative_prompt": {"type": "string", "description": "What to avoid"},
+                    "style": {"type": "string", "description": "Visual style hints"},
+                    "width": {"type": "integer", "default": 768},
+                    "height": {"type": "integer", "default": 512},
+                    "steps": {"type": "integer", "default": 20},
+                    "cfg": {"type": "number", "default": 7.0}
+                }
+            },
+            output_schema={
+                "type": "object",
+                "properties": {
+                    "image_url": {"type": "string"},
+                    "quality_score": {"type": "number"},
+                    "feedback": {"type": "string"}
+                }
+            },
+            examples=[
+                {
+                    "input": {"prompt": "cyberpunk cityscape at night, neon lights, rain"},
+                    "output": {"image_url": "http://...", "quality_score": 7.5}
+                }
+            ]
+        )
+    
+    async def _ensure_backend(self) -> VibeBackend:
+        """Lazily initialize backend."""
+        if self._backend is None:
+            self._backend = VibeBackend(
+                comfyui_url=self.comfyui_url,
+                enable_vlm=True,
+                max_attempts=2,
+                quality_threshold=6.0
+            )
+            await self._backend.initialize()
+        return self._backend
+    
+    async def execute(
+        self,
+        inputs: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ToolResult:
+        """Generate an image based on inputs."""
+        try:
+            backend = await self._ensure_backend()
+            
+            # Build prompt from inputs + context (knowledge base hints)
+            prompt = inputs.get("prompt", "")
+            style = inputs.get("style", "")
+            
+            # If knowledge base provided style hints, incorporate them
+            if context and "technical_specs" in context:
+                specs = context["technical_specs"]
+                if "lighting" in specs:
+                    prompt += f", {', '.join(specs['lighting'])}"
+                if "color_grading" in specs:
+                    prompt += f", {specs['color_grading']}"
+            
+            if style:
+                prompt = f"{prompt}, {style}"
+            
+            result = await backend.generate(GenerationRequest(
+                prompt=prompt,
+                capability=Capability.TEXT_TO_IMAGE,
+                negative_prompt=inputs.get("negative_prompt", ""),
+                width=inputs.get("width", 768),
+                height=inputs.get("height", 512),
+                steps=inputs.get("steps", 20),
+                cfg=inputs.get("cfg", 7.0)
+            ))
+            
+            if result.success:
+                return ToolResult(
+                    success=True,
+                    output={
+                        "image_url": result.output_url,
+                        "quality_score": result.quality_score,
+                        "feedback": result.feedback,
+                        "strengths": result.strengths,
+                        "weaknesses": result.weaknesses
+                    },
+                    metadata={
+                        "model_used": result.model_used,
+                        "attempts": result.attempts
+                    }
+                )
+            else:
+                return ToolResult(
+                    success=False,
+                    output=None,
+                    error=result.error
+                )
+        except Exception as e:
+            return ToolResult(
+                success=False,
+                output=None,
+                error=str(e)
+            )
+
+
+class VideoGenerationTool(BaseTool):
+    """
+    Video generation tool using local ComfyUI.
+    
+    Uses the I2V pipeline: FLUX → Wan I2V animation.
+    MetaPlanner can use this for motion content.
+    """
+    
+    def __init__(self, comfyui_url: str = "http://127.0.0.1:8188"):
+        self.comfyui_url = comfyui_url
+        self._backend: Optional[VibeBackend] = None
+    
+    @property
+    def spec(self) -> ToolSpec:
+        return ToolSpec(
+            name="video_generation",
+            description="Generate videos using local AI models (FLUX + Wan I2V)",
+            category=ToolCategory.VIDEO,
+            input_schema={
+                "type": "object",
+                "required": ["prompt"],
+                "properties": {
+                    "prompt": {"type": "string", "description": "Video description"},
+                    "negative_prompt": {"type": "string", "description": "What to avoid"},
+                    "style": {"type": "string", "description": "Visual style hints"},
+                    "motion": {"type": "string", "description": "Motion description"},
+                    "width": {"type": "integer", "default": 832},
+                    "height": {"type": "integer", "default": 480},
+                    "frames": {"type": "integer", "default": 33},
+                    "fps": {"type": "integer", "default": 16}
+                }
+            },
+            output_schema={
+                "type": "object",
+                "properties": {
+                    "video_url": {"type": "string"},
+                    "duration_seconds": {"type": "number"}
+                }
+            },
+            examples=[
+                {
+                    "input": {"prompt": "samurai walking through rain", "frames": 33},
+                    "output": {"video_url": "http://...", "duration_seconds": 2.0}
+                }
+            ]
+        )
+    
+    async def _ensure_backend(self) -> VibeBackend:
+        """Lazily initialize backend."""
+        if self._backend is None:
+            self._backend = VibeBackend(
+                comfyui_url=self.comfyui_url,
+                enable_vlm=False,  # VLM slower for video
+                max_attempts=1
+            )
+            await self._backend.initialize()
+        return self._backend
+    
+    async def execute(
+        self,
+        inputs: Dict[str, Any],
+        context: Optional[Dict[str, Any]] = None
+    ) -> ToolResult:
+        """Generate a video based on inputs."""
+        try:
+            backend = await self._ensure_backend()
+            
+            # Build prompt
+            prompt = inputs.get("prompt", "")
+            style = inputs.get("style", "")
+            motion = inputs.get("motion", "")
+            
+            # Incorporate knowledge base hints
+            if context and "technical_specs" in context:
+                specs = context["technical_specs"]
+                if "camera" in specs:
+                    prompt += f", {', '.join(specs['camera'][:2])}"
+                if "editing" in specs:
+                    motion = motion or specs["editing"][0] if specs["editing"] else ""
+            
+            if style:
+                prompt = f"{prompt}, {style}"
+            if motion:
+                prompt = f"{prompt}, {motion}"
+            
+            frames = inputs.get("frames", 33)
+            fps = inputs.get("fps", 16)
+            
+            result = await backend.generate(GenerationRequest(
+                prompt=prompt,
+                capability=Capability.TEXT_TO_VIDEO,
+                negative_prompt=inputs.get("negative_prompt", ""),
+                width=inputs.get("width", 832),
+                height=inputs.get("height", 480),
+                frames=frames,
+                steps=20,
+                cfg=5.0
+            ))
+            
+            if result.success:
+                return ToolResult(
+                    success=True,
+                    output={
+                        "video_url": result.output_url,
+                        "duration_seconds": frames / fps
+                    },
+                    metadata={
+                        "frames": frames,
+                        "fps": fps
+                    }
+                )
+            else:
+                return ToolResult(
+                    success=False,
+                    output=None,
+                    error=result.error
+                )
+        except Exception as e:
+            return ToolResult(
+                success=False,
+                output=None,
+                error=str(e)
+            )
+
+
+def create_comfyui_tools(comfyui_url: str = "http://127.0.0.1:8188") -> list:
+    """Create ComfyUI-based generation tools."""
+    return [
+        ImageGenerationTool(comfyui_url=comfyui_url),
+        VideoGenerationTool(comfyui_url=comfyui_url)
+    ]

{vibe_aigc-0.6.1 → vibe_aigc-0.6.3}/vibe_aigc/vibe_backend.py

@@ -150,6 +150,10 @@ class VibeBackend:
             import random
             request.seed = random.randint(0, 2**32 - 1)
         
+        # Special handling for TEXT_TO_VIDEO: use I2V pipeline
+        if request.capability == Capability.TEXT_TO_VIDEO:
+            return await self._generate_video_via_i2v(request)
+        
         # Try to get workflow
         workflow = await self._get_workflow(request)
         if not workflow:
@@ -388,6 +392,199 @@ class VibeBackend:
         except Exception as e:
             return GenerationResult(success=False, error=str(e))
     
+    async def _generate_video_via_i2v(self, request: GenerationRequest) -> GenerationResult:
+        """Generate video via Image-to-Video pipeline.
+        
+        Two-step process:
+        1. Generate base image with TEXT_TO_IMAGE
+        2. Animate with IMAGE_TO_VIDEO
+        """
+        print("\n[1/2] Generating base image...")
+        
+        # Step 1: Generate image
+        image_workflow = self._create_flux_image_workflow(
+            prompt=request.prompt,
+            negative=request.negative_prompt,
+            width=request.width,
+            height=request.height,
+            seed=request.seed
+        )
+        
+        image_result = await self._execute_workflow(image_workflow)
+        if not image_result.success:
+            return GenerationResult(
+                success=False,
+                error=f"Image generation failed: {image_result.error}"
+            )
+        
+        print(f"    Base image: {image_result.output_path}")
+        
+        # Step 2: Upload image and animate
+        print("\n[2/2] Animating with I2V...")
+        
+        # Download image
+        async with aiohttp.ClientSession() as session:
+            async with session.get(image_result.output_url) as resp:
+                image_data = await resp.read()
+            
+            # Upload to ComfyUI
+            form = aiohttp.FormData()
+            form.add_field('image', image_data, filename='input.png', content_type='image/png')
+            
+            async with session.post(f"{self.url}/upload/image", data=form) as resp:
+                upload_result = await resp.json()
+                uploaded_name = upload_result.get("name", "input.png")
+                print(f"    Uploaded: {uploaded_name}")
+        
+        # Create I2V workflow
+        i2v_workflow = self._create_wan_i2v_workflow(
+            uploaded_image=uploaded_name,
+            prompt=request.prompt,
+            negative=request.negative_prompt,
+            width=request.width,
+            height=request.height,
+            frames=request.frames,
+            seed=request.seed
+        )
+        
+        video_result = await self._execute_workflow(i2v_workflow)
+        if not video_result.success:
+            return GenerationResult(
+                success=False,
+                error=f"Animation failed: {video_result.error}"
+            )
+        
+        print(f"    Video: {video_result.output_path}")
+        return video_result
+    
+    def _create_flux_image_workflow(
+        self, prompt: str, negative: str, width: int, height: int, seed: int
+    ) -> Dict[str, Any]:
+        """Create FLUX image generation workflow."""
+        return {
+            "1": {
+                "class_type": "CheckpointLoaderSimple",
+                "inputs": {"ckpt_name": "flux1-dev-fp8.safetensors"}
+            },
+            "2": {
+                "class_type": "CLIPTextEncode",
+                "inputs": {"text": prompt, "clip": ["1", 1]}
+            },
+            "3": {
+                "class_type": "CLIPTextEncode",
+                "inputs": {"text": negative or "blurry, distorted, ugly", "clip": ["1", 1]}
+            },
+            "4": {
+                "class_type": "EmptyLatentImage",
+                "inputs": {"width": width, "height": height, "batch_size": 1}
+            },
+            "5": {
+                "class_type": "KSampler",
+                "inputs": {
+                    "seed": seed,
+                    "steps": 20,
+                    "cfg": 3.5,
+                    "sampler_name": "euler",
+                    "scheduler": "simple",
+                    "denoise": 1.0,
+                    "model": ["1", 0],
+                    "positive": ["2", 0],
+                    "negative": ["3", 0],
+                    "latent_image": ["4", 0]
+                }
+            },
+            "6": {
+                "class_type": "VAEDecode",
+                "inputs": {"samples": ["5", 0], "vae": ["1", 2]}
+            },
+            "7": {
+                "class_type": "SaveImage",
+                "inputs": {"images": ["6", 0], "filename_prefix": "vibe_base"}
+            }
+        }
+    
+    def _create_wan_i2v_workflow(
+        self, uploaded_image: str, prompt: str, negative: str,
+        width: int, height: int, frames: int, seed: int
+    ) -> Dict[str, Any]:
+        """Create Wan 2.1 I2V workflow."""
+        return {
+            "1": {
+                "class_type": "UNETLoader",
+                "inputs": {
+                    "unet_name": "I2V/Wan2_1-I2V-14B-480p_fp8_e4m3fn_scaled_KJ.safetensors",
+                    "weight_dtype": "fp8_e4m3fn"
+                }
+            },
+            "2": {
+                "class_type": "CLIPLoader",
+                "inputs": {
+                    "clip_name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors",
+                    "type": "wan"
+                }
+            },
+            "3": {
+                "class_type": "VAELoader",
+                "inputs": {"vae_name": "wan_2.1_vae.safetensors"}
+            },
+            "4": {
+                "class_type": "LoadImage",
+                "inputs": {"image": uploaded_image}
+            },
+            "5": {
+                "class_type": "CLIPTextEncode",
+                "inputs": {"text": prompt + ", smooth motion, cinematic", "clip": ["2", 0]}
+            },
+            "6": {
+                "class_type": "CLIPTextEncode",
+                "inputs": {"text": negative or "static, frozen, blurry, distorted", "clip": ["2", 0]}
+            },
+            "7": {
+                "class_type": "WanImageToVideo",
+                "inputs": {
+                    "positive": ["5", 0],
+                    "negative": ["6", 0],
+                    "vae": ["3", 0],
+                    "width": width,
+                    "height": height,
+                    "length": frames,
+                    "batch_size": 1,
+                    "start_image": ["4", 0]
+                }
+            },
+            "8": {
+                "class_type": "KSampler",
+                "inputs": {
+                    "seed": seed,
+                    "steps": 30,
+                    "cfg": 5.0,
+                    "sampler_name": "euler",
+                    "scheduler": "normal",
+                    "denoise": 1.0,
+                    "model": ["1", 0],
+                    "positive": ["7", 0],
+                    "negative": ["7", 1],
+                    "latent_image": ["7", 2]
+                }
+            },
+            "9": {
+                "class_type": "VAEDecode",
+                "inputs": {"samples": ["8", 0], "vae": ["3", 0]}
+            },
+            "10": {
+                "class_type": "VHS_VideoCombine",
+                "inputs": {
+                    "images": ["9", 0],
+                    "frame_rate": 16,
+                    "loop_count": 0,
+                    "filename_prefix": "vibe_i2v",
+                    "format": "image/webp",
+                    "pingpong": False,
+                    "save_output": True
+                }
+            }
+        }
+    
     def status(self) -> str:
         """Get backend status."""
         if not self._initialized:

{vibe_aigc-0.6.1 → vibe_aigc-0.6.3/vibe_aigc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vibe-aigc
-Version: 0.6.1
+Version: 0.6.3
 Summary: A New Paradigm for Content Generation via Agentic Orchestration
 Author: Vibe AIGC Contributors
 License-Expression: MIT

{vibe_aigc-0.6.1 → vibe_aigc-0.6.3}/vibe_aigc.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ vibe_aigc/mv_pipeline.py
 vibe_aigc/persistence.py
 vibe_aigc/planner.py
 vibe_aigc/tools.py
+vibe_aigc/tools_comfyui.py
 vibe_aigc/tools_multimodal.py
 vibe_aigc/vibe_backend.py
 vibe_aigc/video.py