Add vision folder with local or openai vision options

src/reachy_mini_conversation_demo/main.py

@@ -11,22 +11,32 @@ from reachy_mini import ReachyMini
 from reachy_mini.utils import create_head_pose
 
 from reachy_mini_conversation_demo.config import config
-from reachy_mini_conversation_demo.head_tracker import HeadTracker
+from reachy_mini_conversation_demo.vision.head_tracker import HeadTracker
 from reachy_mini_conversation_demo.openai_realtime import OpenAIRealtimeHandler
 from reachy_mini_conversation_demo.prompts import SESSION_INSTRUCTIONS
 from reachy_mini_conversation_demo.tools import (
     ToolDependencies,
 )
-from reachy_mini_conversation_demo.audio_sway import AudioSync, AudioConfig
+from reachy_mini_conversation_demo.audio.audio_sway import AudioSync, AudioConfig
 from reachy_mini_conversation_demo.movement import MovementManager
-from reachy_mini_conversation_demo.gstreamer import GstPlayer, GstRecorder
-from reachy_mini_conversation_demo.vision import VisionManager, init_vision, init_camera
+from reachy_mini_conversation_demo.audio.gstreamer import GstPlayer, GstRecorder
+from reachy_mini_conversation_demo.vision.processors import (
+    VisionManager,
+    init_vision,
+    init_camera,
+)
 
 # Command-line arguments
 parser = argparse.ArgumentParser(description="Reachy Mini Conversation Demo")
 parser.add_argument("--sim", action="store_true", help="Run in simulation mode")
 parser.add_argument("--vision", action="store_true", help="Enable vision")
 parser.add_argument("--head-tracking", action="store_true", help="Enable head tracking")
+parser.add_argument(
+    "--vision-provider",
+    choices=["openai", "local"],
+    default="local",
+    help="Choose vision provider (default: local)",
+)
 parser.add_argument("--debug", action="store_true", help="Enable debug logging")
 args = parser.parse_args()
 
@@ -135,7 +145,9 @@ async def loop():
 
     vision_manager: VisionManager | None = None
     if camera and camera.isOpened() and VISION_ENABLED:
-        vision_manager = init_vision(camera=camera)
+        processor_type = args.vision_provider
+        vision_manager = init_vision(camera=camera, processor_type=processor_type)
+        logger.info(f"Vision processor type: {processor_type}")
 
     current_robot = ReachyMini()
 

src/reachy_mini_conversation_demo/movement.py

@@ -9,7 +9,7 @@ import cv2
 from reachy_mini import ReachyMini
 from reachy_mini.reachy_mini import IMAGE_SIZE
 from reachy_mini.utils import create_head_pose
-from reachy_mini_conversation_demo.head_tracker import HeadTracker
+from reachy_mini_conversation_demo.vision.head_tracker import HeadTracker
 
 logger = logging.getLogger(__name__)
 

src/reachy_mini_conversation_demo/openai_realtime.py

@@ -19,7 +19,7 @@ from reachy_mini_conversation_demo.tools import (
     ALL_TOOL_SPECS,
     dispatch_tool_call,
 )
-from reachy_mini_conversation_demo.audio_sway import AudioSync, pcm_to_b64
+from reachy_mini_conversation_demo.audio.audio_sway import AudioSync, pcm_to_b64
 from reachy_mini_conversation_demo.config import config
 
 logger = logging.getLogger(__name__)

src/reachy_mini_conversation_demo/tools.py

@@ -15,7 +15,7 @@ from typing import Any, Dict, Literal, Optional
 import cv2
 import numpy as np
 
-from reachy_mini_conversation_demo.vision import VisionManager
+from reachy_mini_conversation_demo.vision.processors import VisionManager
 from reachy_mini_conversation_demo.movement import MovementManager
 
 logger = logging.getLogger(__name__)

src/reachy_mini_conversation_demo/vision/openai_vision.py

@@ -0,0 +1,97 @@
+import base64
+import logging
+import os
+import cv2
+from openai import OpenAI
+from .processors import VisionConfig
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAIVisionProcessor:
+    def __init__(self, config: VisionConfig = None):
+        self.config = config or VisionConfig()
+        self._initialized = False
+        self.client = None
+
+    def initialize(self):
+        """Initialize OpenAI client with proper error handling"""
+        try:
+            api_key = os.getenv("OPENAI_API_KEY")
+            if not api_key:
+                logger.error("OPENAI_API_KEY not found in environment variables")
+                return False
+
+            self.client = OpenAI(api_key=api_key)
+
+            # Smoke test the API/key
+            try:
+                _ = self.client.models.list()
+                self._initialized = True
+                logger.info("OpenAI Vision processor initialized successfully")
+                return True
+            except Exception as e:
+                logger.error(f"Failed to connect to OpenAI API: {e}")
+                return False
+
+        except Exception as e:
+            logger.error(f"Failed to initialize OpenAI Vision processor: {e}")
+            return False
+
+    def process_image(
+        self, cv2_image, prompt="Briefly describe what you see in one sentence."
+    ):
+        """Process image using OpenAI (Responses API) with retry logic"""
+        if not self._initialized:
+            return "OpenAI Vision processor not initialized"
+
+        for attempt in range(self.config.max_retries):
+            try:
+                # Convert image to base64
+                rgb_image = cv2.cvtColor(cv2_image, cv2.COLOR_BGR2RGB)
+                ok, jpeg_buffer = cv2.imencode(
+                    ".jpg",
+                    rgb_image,
+                    [cv2.IMWRITE_JPEG_QUALITY, self.config.jpeg_quality],
+                )
+                if not ok:
+                    return "Failed to encode image"
+                image_base64 = base64.b64encode(jpeg_buffer.tobytes()).decode("utf-8")
+
+                # Responses API with input_image
+                response = self.client.responses.create(
+                    model=self.config.openai_model,  # e.g., gpt-4.1 or gpt-4.1-mini
+                    input=[
+                        {
+                            "role": "user",
+                            "content": [
+                                {"type": "input_text", "text": prompt},
+                                {
+                                    "type": "input_image",
+                                    "image_url": f"data:image/jpeg;base64,{image_base64}",
+                                },
+                            ],
+                        }
+                    ],
+                    max_output_tokens=300,
+                )
+
+                # Unified text accessor
+                text = (response.output_text or "").strip()
+                return text if text else "No response"
+
+            except Exception as e:
+                logger.error(f"OpenAI Vision API error (attempt {attempt + 1}): {e}")
+                if attempt < self.config.max_retries - 1:
+                    import time
+
+                    time.sleep(self.config.retry_delay)
+                else:
+                    return f"OpenAI Vision processing failed after {self.config.max_retries} attempts"
+
+    def get_model_info(self):
+        return {
+            "processor_type": "openai",
+            "initialized": self._initialized,
+            "model": self.config.openai_model,
+        }

src/reachy_mini_conversation_demo/{vision.py → vision/processors.py}

@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
 class VisionConfig:
     """Configuration for vision processing"""
 
+    processor_type: str = "local"
+    openai_model: str = os.getenv("OPENAI_VISION_MODEL", "gpt-4.1-mini")
     model_path: str = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
     vision_interval: float = 5.0
     max_new_tokens: int = 64
@@ -66,7 +68,7 @@ class VisionProcessor:
             if self.device == "cuda":
                 dtype = torch.bfloat16
             elif self.device == "mps":
-                dtype = torch.float16  # best for MPS
+                dtype = torch.float32  # best for MPS
             else:
                 dtype = torch.float32
 
@@ -100,13 +102,10 @@ class VisionProcessor:
 
         for attempt in range(self.config.max_retries):
             try:
-                # Convert CV2 BGR to RGB
-                rgb_image = cv2.cvtColor(cv2_image, cv2.COLOR_BGR2RGB)
-
                 # Convert to JPEG bytes
                 success, jpeg_buffer = cv2.imencode(
                     ".jpg",
-                    rgb_image,
+                    cv2_image,
                     [cv2.IMWRITE_JPEG_QUALITY, self.config.jpeg_quality],
                 )
                 if not success:
@@ -136,20 +135,17 @@ class VisionProcessor:
                     return_tensors="pt",
                 )
 
-                # move to device with proper dtype
-                if self.device == "cuda":
-                    inputs = inputs.to(self.device, dtype=torch.bfloat16)
-                elif self.device == "mps":
-                    inputs = inputs.to(self.device, dtype=torch.float16)
-                else:
-                    inputs = inputs.to(self.device, dtype=torch.float32)
+                # Move tensors to device WITHOUT forcing dtype (keeps input_ids as torch.long)
+                inputs = {
+                    k: (v.to(self.device) if hasattr(v, "to") else v)
+                    for k, v in inputs.items()
+                }
 
                 with torch.no_grad():
                     generated_ids = self.model.generate(
                         **inputs,
-                        do_sample=True if self.config.temperature > 0 else False,
+                        do_sample=False,
                         max_new_tokens=self.config.max_new_tokens,
-                        temperature=self.config.temperature,
                         pad_token_id=self.processor.tokenizer.eos_token_id,
                     )
 
@@ -203,6 +199,7 @@ class VisionProcessor:
     def get_model_info(self) -> Dict[str, Any]:
         """Get information about the loaded model"""
         return {
+            "processor_type": "local",
             "initialized": self._initialized,
             "device": self.device,
             "model_path": self.model_path,
@@ -220,7 +217,7 @@ class VisionManager:
         self.camera = camera
         self.config = config or VisionConfig()
         self.vision_interval = self.config.vision_interval
-        self.processor = VisionProcessor(self.config)
+        self.processor = create_vision_processor(self.config)  # Use factory function
 
         self._current_description = ""
         self._last_processed_time = 0
@@ -294,13 +291,11 @@ class VisionManager:
     async def get_status(self) -> Dict[str, Any]:
         """Get comprehensive status information"""
         return {
-            "running": self._running,
             "last_processed": self._last_processed_time,
             "processor_info": self.processor.get_model_info(),
             "config": {
                 "interval": self.vision_interval,
-                "model_path": self.config.model_path,
-                "device": self.processor.device,
+                "processor_type": self.config.processor_type,
             },
         }
 
@@ -311,40 +306,58 @@ def init_camera(camera_index=0, simulation=True):
     if simulation:
         # Default build-in camera in SIM
         # TODO: please, test on Linux and Windows
-        # TODO simulation in find_camera
         camera = cv2.VideoCapture(0, api_preference)
     else:
-        # TODO handle macos in find_camera
+        # TODO handle macos properly
         if sys.platform == "darwin":
             camera = cv2.VideoCapture(camera_index, cv2.CAP_AVFOUNDATION)
         else:
-            camera = find_camera()
+            camera = cv2.VideoCapture(camera_index)
 
     return camera
 
 
-def init_vision(camera: cv2.VideoCapture) -> VisionManager:
+def create_vision_processor(config: VisionConfig):
+    """Factory function to create the appropriate vision processor"""
+    if config.processor_type == "openai":
+        try:
+            from .openai_vision import OpenAIVisionProcessor
+
+            return OpenAIVisionProcessor(config)
+        except ImportError:
+            logger.error("OpenAI vision processor not available, falling back to local")
+            return VisionProcessor(config)
+    else:
+        return VisionProcessor(config)
+
+
+def init_vision(
+    camera: cv2.VideoCapture, processor_type: str = "local"
+) -> VisionManager:
     model_id = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
 
     cache_dir = os.path.expandvars(os.getenv("HF_HOME", "$HOME/.cache/huggingface"))
 
-    try:
-        os.makedirs(cache_dir, exist_ok=True)
-        os.environ["HF_HOME"] = cache_dir
-        logger.info("HF_HOME set to %s", cache_dir)
-    except Exception as e:
-        logger.warning("Failed to prepare HF cache dir %s: %s", cache_dir, e)
-        return
-
-    snapshot_download(
-        repo_id=model_id,
-        repo_type="model",
-        cache_dir=cache_dir,
-    )
-    logger.info(f"Prefetched model_id={model_id} into cache_dir={cache_dir}")
+    # Only download model if using local processor
+    if processor_type == "local":
+        try:
+            os.makedirs(cache_dir, exist_ok=True)
+            os.environ["HF_HOME"] = cache_dir
+            logger.info("HF_HOME set to %s", cache_dir)
+        except Exception as e:
+            logger.warning("Failed to prepare HF cache dir %s: %s", cache_dir, e)
+            return None
+
+        snapshot_download(
+            repo_id=model_id,
+            repo_type="model",
+            cache_dir=cache_dir,
+        )
+        logger.info(f"Prefetched model_id={model_id} into cache_dir={cache_dir}")
 
-    # Configure VLLM processing
+    # Configure vision processing
     vision_config = VisionConfig(
+        processor_type=processor_type,
         model_path=model_id,
         vision_interval=5.0,
         max_new_tokens=64,
@@ -359,7 +372,7 @@ def init_vision(camera: cv2.VideoCapture) -> VisionManager:
 
     device_info = vision_manager.processor.get_model_info()
     logger.info(
-        f"Vision processing enabled: {device_info['model_path']} on {device_info['device']}",
+        f"Vision processing enabled: {device_info.get('model_path', device_info.get('processor_type'))} on {device_info.get('device', 'API')}",
     )
 
     return vision_manager