pynode-flow 0.2.0__py3-none-any.whl

pynode/nodes/Qwen3VLMNode/qwen3_vlm_node.py

@@ -0,0 +1,519 @@
+"""
+Qwen3-VL Vision Language Model node.
+Performs vision-language inference on images using the Qwen3-VL model.
+
+Supports two modes:
+  - Generic: Answer any text prompt about an image.
+  - Detection: Locate objects and return structured detections compatible with
+    downstream YOLO-style nodes (requires the `supervision` package).
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+from pynode.nodes.base_node import BaseNode, Info, MessageKeys
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Build node info/help content
+# ---------------------------------------------------------------------------
+_info = Info()
+_info.add_text(
+    "Runs the Qwen3-VL vision language model against an incoming image and "
+    "produces either a text response (Generic mode) or structured object "
+    "detections (Detection mode)."
+)
+_info.add_header("Inputs")
+_info.add_bullet("Input 0:", "Image message (same format as CameraNode / UltralyticsNode)")
+_info.add_header("Outputs")
+_info.add_bullet("Output 0 – Generic:", "payload.text with the model's answer")
+_info.add_bullet(
+    "Output 0 – Detection:",
+    "payload.detections, payload.detection_count, payload.bbox_format, "
+    "and optionally payload.image (annotated)"
+)
+_info.add_header("Properties")
+_info.add_bullet("Model:", "Hugging Face model ID (e.g. Qwen/Qwen3-VL-2B-Instruct)")
+_info.add_bullet("Device:", "cpu or cuda:N")
+_info.add_bullet("Mode:", "Generic (free-form Q&A) or Detection (object localisation)")
+_info.add_bullet("Prompt:", "Question in Generic mode, or object-class target in Detection mode")
+_info.add_bullet("Max Tokens:", "Maximum tokens generated by the model")
+_info.add_bullet("Draw Results:", "Draw bounding boxes on output image (Detection mode only)")
+
+
+class Qwen3VLMNode(BaseNode):
+    """
+    Qwen3-VL Vision Language Model node.
+
+    Generic mode  – sends the configured prompt alongside the image and emits
+                    the model's text response as ``payload.text``.
+
+    Detection mode – instructs the model to outline all instances of the
+                     given target class, parses the JSON bounding-box response
+                     via ``supervision`` and emits detections in the same
+                     schema as UltralyticsNode for pipeline compatibility.
+                     Requires the ``supervision`` package (>=0.25.0).
+    """
+
+    # ------------------------------------------------------------------
+    # Visual / palette properties
+    # ------------------------------------------------------------------
+    display_name = "Qwen3-VL"
+    icon = "🧠"
+    category = "vision"
+    color = "#7C3AED"
+    border_color = "#5B21B6"
+    text_color = "#FFFFFF"
+    input_count = 1
+    output_count = 1
+    info = str(_info)
+
+    # ------------------------------------------------------------------
+    # Default configuration
+    # ------------------------------------------------------------------
+    DEFAULT_CONFIG: Dict[str, Any] = {
+        "model_name": "Qwen/Qwen3-VL-2B-Instruct",
+        "hf_token": "",
+        "mode": "generic",
+        "prompt": "Describe this image in detail.",
+        "max_tokens": "512",
+        "draw_results": "true",
+        "include_image": True,
+    }
+
+    # ------------------------------------------------------------------
+    # Dynamic device options
+    # ------------------------------------------------------------------
+    @staticmethod
+    def _get_device_options() -> List[Dict[str, str]]:
+        devices = [{"value": "cpu", "label": "CPU"}]
+        try:
+            import torch  # type: ignore
+
+            if torch.cuda.is_available():
+                for i in range(torch.cuda.device_count()):
+                    props = torch.cuda.get_device_properties(i)
+                    mem_gb = props.total_memory / (1024 ** 3)
+                    devices.append(
+                        {
+                            "value": f"cuda:{i}",
+                            "label": f"CUDA:{i} – {props.name} ({mem_gb:.1f} GB)",
+                        }
+                    )
+        except Exception as exc:
+            logger.warning("Error detecting CUDA devices: %s", exc)
+        return devices
+
+    @staticmethod
+    def _default_device() -> str:
+        try:
+            import torch  # type: ignore
+
+            return "cuda:0" if torch.cuda.is_available() else "cpu"
+        except ImportError:
+            return "cpu"
+
+    # ------------------------------------------------------------------
+    # Properties panel schema
+    # ------------------------------------------------------------------
+    @classmethod
+    def get_properties(cls) -> List[Dict[str, Any]]:
+        return [
+            {
+                "name": "model_name",
+                "label": "Model",
+                "type": "select",
+                "options": [
+                    {"value": "Qwen/Qwen3-VL-2B-Instruct", "label": "Qwen3-VL 2B Instruct (fast)"},
+                    {"value": "Qwen/Qwen3-VL-7B-Instruct", "label": "Qwen3-VL 7B Instruct"},
+                    {"value": "Qwen/Qwen3-VL-72B-Instruct", "label": "Qwen3-VL 72B Instruct (large)"},
+                    {"value": "Qwen/Qwen2.5-VL-3B-Instruct", "label": "Qwen2.5-VL 3B Instruct"},
+                    {"value": "Qwen/Qwen2.5-VL-7B-Instruct", "label": "Qwen2.5-VL 7B Instruct"},
+                ],
+            },
+            {
+                "name": "hf_token",
+                "label": "HuggingFace Token",
+                "type": "password",
+                "placeholder": "hf_…  (required for gated models)",
+            },
+            {
+                "name": "device",
+                "label": "Device",
+                "type": "select",
+                "options": cls._get_device_options(),
+            },
+            {
+                "name": "mode",
+                "label": "Mode",
+                "type": "select",
+                "options": [
+                    {"value": "generic", "label": "Generic (Q&A / description)"},
+                    {"value": "detection", "label": "Detection (object localisation)"},
+                ],
+            },
+            {
+                "name": "prompt",
+                "label": "Prompt / Detection Target",
+                "type": "textarea",
+            },
+            {
+                "name": "max_tokens",
+                "label": "Max Tokens",
+                "type": "text",
+                "placeholder": "512",
+            },
+            {
+                "name": "draw_results",
+                "label": "Draw Detections on Image",
+                "type": "select",
+                "options": [
+                    {"value": "true", "label": "Yes"},
+                    {"value": "false", "label": "No"},
+                ],
+            },
+            {
+                "name": "include_image",
+                "label": "Include Image in Output",
+                "type": "checkbox",
+                "default": True,
+            },
+        ]
+
+    properties = property(lambda self: self.get_properties())  # type: ignore[assignment]
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+    def __init__(self, node_id: Optional[str] = None, name: str = "qwen3-vlm"):
+        super().__init__(node_id, name)
+        cfg = self.DEFAULT_CONFIG.copy()
+        cfg["device"] = self._default_device()
+        self.configure(cfg)
+
+        self._model = None
+        self._processor = None
+        self._loaded_model_name: Optional[str] = None
+
+    # ------------------------------------------------------------------
+    # Configure – reload model when model_name or device changes
+    # ------------------------------------------------------------------
+    def configure(self, config: Dict[str, Any]):
+        old_name = self.config.get("model_name") if hasattr(self, "config") else None
+        old_device = self.config.get("device") if hasattr(self, "config") else None
+        super().configure(config)
+        new_name = self.config.get("model_name")
+        new_device = self.config.get("device")
+
+        if (old_name != new_name or old_device != new_device) and old_name is not None:
+            self._model = None
+            self._processor = None
+            self._loaded_model_name = None
+
+    # ------------------------------------------------------------------
+    # Lazy model loading
+    # ------------------------------------------------------------------
+    def _load_model(self):
+        """Load (or reload) the Qwen3-VL model and processor."""
+        if self._model is not None:
+            return True
+
+        model_name = self.config.get("model_name", "Qwen/Qwen3-VL-2B-Instruct")
+        device = self.config.get("device", "cpu")
+
+        token = self.config.get("hf_token", "").strip() or None
+
+        try:
+            from transformers import AutoModelForImageTextToText, AutoProcessor  # type: ignore
+
+            logger.info("Qwen3VLMNode: loading model '%s' …", model_name)
+            self._model = AutoModelForImageTextToText.from_pretrained(
+                model_name, torch_dtype="auto", device_map=device, token=token
+            )
+            self._processor = AutoProcessor.from_pretrained(model_name, token=token)
+            self._loaded_model_name = model_name
+            logger.info("Qwen3VLMNode: model loaded successfully.")
+            return True
+
+        except ImportError:
+            self.report_error(
+                "transformers package not installed. "
+                "Run: pip install transformers torch Pillow"
+            )
+        except Exception as exc:
+            self.report_error(f"Failed to load model '{model_name}': {exc}")
+
+        self._model = None
+        self._processor = None
+        return False
+
+    # ------------------------------------------------------------------
+    # Core inference helpers
+    # ------------------------------------------------------------------
+    def _run_inference(self, pil_image, prompt: str, max_tokens: int) -> Optional[str]:
+        """Run the model and return the decoded text response."""
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image", "image": pil_image},
+                    {"type": "text", "text": prompt},
+                ],
+            }
+        ]
+
+        try:
+            import torch  # type: ignore
+
+            inputs = self._processor.apply_chat_template(
+                messages,
+                tokenize=True,
+                add_generation_prompt=True,
+                return_dict=True,
+                return_tensors="pt",
+            )
+            inputs = inputs.to(self._model.device)
+
+            with torch.inference_mode():
+                generated_ids = self._model.generate(
+                    **inputs,
+                    max_new_tokens=max_tokens,
+                )
+
+            trimmed = [
+                out_ids[len(in_ids):]
+                for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+            ]
+            return self._processor.batch_decode(
+                trimmed,
+                skip_special_tokens=True,
+                clean_up_tokenization_spaces=False,
+            )[0]
+
+        except Exception as exc:
+            self.report_error(f"Inference error: {exc}")
+            return None
+
+    # ------------------------------------------------------------------
+    # Detection mode
+    # ------------------------------------------------------------------
+    def _run_detection(
+        self, pil_image, target: str, max_tokens: int, draw: bool, include_image: bool
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Run detection-mode inference.
+        Returns a payload dict compatible with downstream YOLO-style nodes,
+        or None on failure.
+        """
+        try:
+            import supervision as sv  # type: ignore
+        except ImportError:
+            self.report_error(
+                "supervision package not installed. "
+                "Run: pip install supervision"
+            )
+            return None
+
+        prompt = (
+            f"Outline the position of {target} and output all the coordinates "
+            "in JSON format."
+        )
+        raw_response = self._run_inference(pil_image, prompt, max_tokens)
+        if raw_response is None:
+            return None
+
+        logger.debug("Qwen3VLMNode detection response: %s", raw_response)
+
+        try:
+            detections = sv.Detections.from_vlm(
+                vlm=sv.VLM.QWEN_3_VL,
+                result=raw_response,
+                resolution_wh=pil_image.size,
+            )
+        except Exception as exc:
+            self.report_error(f"Failed to parse detections: {exc}")
+            return None
+
+        # Convert sv.Detections to serialisable list (compatible with YOLO output)
+        det_list = []
+        for i in range(len(detections)):
+            entry: Dict[str, Any] = {
+                "class_id": int(detections.class_id[i]) if detections.class_id is not None else 0,
+                "class_name": (
+                    detections.data.get("class_name", [target] * len(detections))[i]
+                    if isinstance(detections.data.get("class_name"), (list, np.ndarray))
+                    else target
+                ),
+                "confidence": (
+                    float(detections.confidence[i])
+                    if detections.confidence is not None
+                    else 1.0
+                ),
+                "bbox": detections.xyxy[i].tolist(),
+                "bbox_format": "xyxy",
+            }
+            det_list.append(entry)
+
+        payload: Dict[str, Any] = {
+            "detections": det_list,
+            "detection_count": len(det_list),
+            "bbox_format": "xyxy",
+            "raw_response": raw_response,
+        }
+
+        if include_image:
+            # Annotate the image with supervision annotators
+            annotated_pil = self._annotate(pil_image, detections, draw)
+            # Convert PIL -> BGR numpy for encode_image
+            annotated_np = self._pil_to_bgr(annotated_pil)
+            # We encode to the same format the message arrived in; at this
+            # point we only have the numpy array so use bgr_numpy_dict.
+            encoded = self.encode_image(annotated_np, "bgr_numpy_dict")
+            if encoded is not None:
+                payload[MessageKeys.IMAGE.PATH] = encoded
+
+        return payload
+
+    @staticmethod
+    def _annotate(image, detections, draw: bool):
+        """Return annotated PIL image (or a copy if draw=False)."""
+        import supervision as sv  # type: ignore
+
+        if not draw or len(detections) == 0:
+            return image.copy()
+
+        try:
+            text_scale = sv.calculate_optimal_text_scale(resolution_wh=image.size)
+            thickness = sv.calculate_optimal_line_thickness(resolution_wh=image.size)
+            color_by_class = detections.class_id is not None
+
+            color_palette = sv.ColorPalette.from_hex([
+                "#ffff00", "#ff9b00", "#ff66ff", "#3399ff",
+                "#ff66b2", "#ff8080", "#b266ff", "#9999ff",
+                "#66ffff", "#33ff99", "#66ff66", "#99ff00",
+            ])
+            lookup = sv.ColorLookup.CLASS if color_by_class else sv.ColorLookup.INDEX
+
+            box_ann = sv.BoxAnnotator(color=color_palette, color_lookup=lookup, thickness=thickness)
+            label_ann = sv.LabelAnnotator(
+                color=color_palette,
+                color_lookup=lookup,
+                text_color=sv.Color.BLACK,
+                text_scale=text_scale,
+                text_thickness=max(1, thickness - 1),
+                smart_position=True,
+            )
+
+            annotated = image.copy()
+            annotated = box_ann.annotate(annotated, detections)
+            annotated = label_ann.annotate(annotated, detections)
+            return annotated
+        except Exception as exc:
+            logger.warning("Annotation failed: %s", exc)
+            return image.copy()
+
+    # ------------------------------------------------------------------
+    # Generic mode
+    # ------------------------------------------------------------------
+    def _run_generic(
+        self, pil_image, prompt: str, max_tokens: int, include_image: bool,
+        encode_format: str
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Run generic (Q&A / description) inference.
+        Returns a payload dict with ``text`` and optionally the original image.
+        """
+        response = self._run_inference(pil_image, prompt, max_tokens)
+        if response is None:
+            return None
+
+        payload: Dict[str, Any] = {
+            "text": response,
+            "prompt": prompt,
+        }
+
+        if include_image:
+            bgr = self._pil_to_bgr(pil_image)
+            encoded = self.encode_image(bgr, encode_format)
+            if encoded is not None:
+                payload[MessageKeys.IMAGE.PATH] = encoded
+
+        return payload
+
+    # ------------------------------------------------------------------
+    # Image conversion helpers
+    # ------------------------------------------------------------------
+    @staticmethod
+    def _bgr_to_pil(bgr_array: np.ndarray):
+        """Convert OpenCV BGR numpy array to PIL Image (RGB)."""
+        import cv2  # type: ignore
+        from PIL import Image  # type: ignore
+
+        rgb = cv2.cvtColor(bgr_array, cv2.COLOR_BGR2RGB)
+        return Image.fromarray(rgb)
+
+    @staticmethod
+    def _pil_to_bgr(pil_image) -> np.ndarray:
+        """Convert PIL Image (RGB) to OpenCV BGR numpy array."""
+        import cv2  # type: ignore
+        import numpy as np  # type: ignore
+
+        rgb_array = np.array(pil_image)
+        return cv2.cvtColor(rgb_array, cv2.COLOR_RGB2BGR)
+
+    # ------------------------------------------------------------------
+    # Message handler
+    # ------------------------------------------------------------------
+    def on_input(self, msg: Dict[str, Any], input_index: int = 0):
+        """
+        Process an incoming image message through the Qwen3-VL model.
+        """
+        # --- lazy model load ---
+        if self._model is None and not self._load_model():
+            return
+
+        # --- extract image ---
+        payload = msg.get(MessageKeys.PAYLOAD)
+        if payload is None:
+            self.report_error("No payload in message")
+            return
+
+        bgr_image, input_format = self.decode_image(payload)
+        if bgr_image is None or input_format is None:
+            self.report_error("Failed to decode image from payload")
+            return
+
+        pil_image = self._bgr_to_pil(bgr_image)
+
+        # --- read config ---
+        mode = self.config.get("mode", "generic")
+        prompt = self.config.get("prompt", "Describe this image in detail.")
+        max_tokens = self.get_config_int("max_tokens", 512)
+        draw_results = self.get_config_bool("draw_results", True)
+        include_image = self.config.get("include_image", True)
+        if isinstance(include_image, str):
+            include_image = include_image.lower() == "true"
+
+        # --- run inference ---
+        try:
+            if mode == "detection":
+                out_payload = self._run_detection(
+                    pil_image, prompt, max_tokens, draw_results, include_image
+                )
+            else:
+                out_payload = self._run_generic(
+                    pil_image, prompt, max_tokens, include_image, input_format
+                )
+        except Exception as exc:
+            self.report_error(f"Unexpected error during inference: {exc}")
+            return
+
+        if out_payload is None:
+            return
+
+        msg[MessageKeys.PAYLOAD] = out_payload
+        msg[MessageKeys.TOPIC] = msg.get(MessageKeys.TOPIC, "qwen3-vlm")
+        self.send(msg)

pynode/nodes/Qwen3VLMNode/requirements.txt

@@ -0,0 +1,5 @@
+transformers>=4.50.0
+torch>=2.0.0
+Pillow>=9.0.0
+supervision>=0.25.0
+qwen-vl-utils>=0.0.8

pynode/nodes/RESTEndpointNode/__init__.py

@@ -0,0 +1,3 @@
+from .rest_endpoint_node import RESTEndpointNode
+
+__all__ = ['RESTEndpointNode']