Alina Lozovskaya
commited on
Commit
·
7054b54
1
Parent(s):
58b3ac3
Improve variable names
Browse files
src/reachy_mini_conversation_demo/console.py
CHANGED
|
@@ -78,9 +78,9 @@ class LocalStream:
|
|
| 78 |
"""Read mic frames from the recorder and forward them to the handler."""
|
| 79 |
logger.info("Starting receive loop")
|
| 80 |
while not self._stop_event.is_set():
|
| 81 |
-
|
| 82 |
-
if
|
| 83 |
-
frame_mono =
|
| 84 |
frame = audio_to_int16(frame_mono)
|
| 85 |
await self.handler.receive((16000, frame))
|
| 86 |
# await asyncio.sleep(0) # yield to event loop
|
|
@@ -90,10 +90,10 @@ class LocalStream:
|
|
| 90 |
async def play_loop(self) -> None:
|
| 91 |
"""Fetch outputs from the handler: log text and play audio frames."""
|
| 92 |
while not self._stop_event.is_set():
|
| 93 |
-
|
| 94 |
|
| 95 |
-
if isinstance(
|
| 96 |
-
for msg in
|
| 97 |
content = msg.get("content", "")
|
| 98 |
if isinstance(content, str):
|
| 99 |
logger.info(
|
|
@@ -102,14 +102,17 @@ class LocalStream:
|
|
| 102 |
content if len(content) < 500 else content[:500] + "…",
|
| 103 |
)
|
| 104 |
|
| 105 |
-
elif isinstance(
|
| 106 |
-
|
| 107 |
device_sample_rate = self._robot.media.get_audio_samplerate()
|
| 108 |
-
|
| 109 |
-
if
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
| 112 |
|
| 113 |
-
|
|
|
|
| 114 |
|
| 115 |
await asyncio.sleep(0) # yield to event loop
|
|
|
|
| 78 |
"""Read mic frames from the recorder and forward them to the handler."""
|
| 79 |
logger.info("Starting receive loop")
|
| 80 |
while not self._stop_event.is_set():
|
| 81 |
+
audio_frame = self._robot.media.get_audio_sample()
|
| 82 |
+
if audio_frame is not None:
|
| 83 |
+
frame_mono = audio_frame.T[0] # both channels are identical
|
| 84 |
frame = audio_to_int16(frame_mono)
|
| 85 |
await self.handler.receive((16000, frame))
|
| 86 |
# await asyncio.sleep(0) # yield to event loop
|
|
|
|
| 90 |
async def play_loop(self) -> None:
|
| 91 |
"""Fetch outputs from the handler: log text and play audio frames."""
|
| 92 |
while not self._stop_event.is_set():
|
| 93 |
+
handler_output = await self.handler.emit()
|
| 94 |
|
| 95 |
+
if isinstance(handler_output, AdditionalOutputs):
|
| 96 |
+
for msg in handler_output.args:
|
| 97 |
content = msg.get("content", "")
|
| 98 |
if isinstance(content, str):
|
| 99 |
logger.info(
|
|
|
|
| 102 |
content if len(content) < 500 else content[:500] + "…",
|
| 103 |
)
|
| 104 |
|
| 105 |
+
elif isinstance(handler_output, tuple):
|
| 106 |
+
input_sample_rate, audio_frame = handler_output
|
| 107 |
device_sample_rate = self._robot.media.get_audio_samplerate()
|
| 108 |
+
audio_frame = audio_to_float32(audio_frame.squeeze())
|
| 109 |
+
if input_sample_rate != device_sample_rate:
|
| 110 |
+
audio_frame = librosa.resample(
|
| 111 |
+
audio_frame, orig_sr=input_sample_rate, target_sr=device_sample_rate
|
| 112 |
+
)
|
| 113 |
+
self._robot.media.push_audio_sample(audio_frame)
|
| 114 |
|
| 115 |
+
else:
|
| 116 |
+
logger.debug("Ignoring output type=%s", type(handler_output).__name__)
|
| 117 |
|
| 118 |
await asyncio.sleep(0) # yield to event loop
|
src/reachy_mini_conversation_demo/openai_realtime.py
CHANGED
|
@@ -136,11 +136,11 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 136 |
# 3) when args done, execute Python tool, send function_call_output, then trigger a new response
|
| 137 |
if event.type == "response.function_call_arguments.done":
|
| 138 |
call_id = getattr(event, "call_id", None)
|
| 139 |
-
|
| 140 |
-
if not
|
| 141 |
continue
|
| 142 |
-
tool_name =
|
| 143 |
-
args_json_str =
|
| 144 |
|
| 145 |
try:
|
| 146 |
tool_result = await dispatch_tool_call(tool_name, args_json_str, self.deps)
|
|
|
|
| 136 |
# 3) when args done, execute Python tool, send function_call_output, then trigger a new response
|
| 137 |
if event.type == "response.function_call_arguments.done":
|
| 138 |
call_id = getattr(event, "call_id", None)
|
| 139 |
+
tool_call_info = self._pending_calls.get(call_id)
|
| 140 |
+
if not tool_call_info:
|
| 141 |
continue
|
| 142 |
+
tool_name = tool_call_info["name"]
|
| 143 |
+
args_json_str = tool_call_info["args_buf"] or "{}"
|
| 144 |
|
| 145 |
try:
|
| 146 |
tool_result = await dispatch_tool_call(tool_name, args_json_str, self.deps)
|
src/reachy_mini_conversation_demo/tools.py
CHANGED
|
@@ -36,14 +36,14 @@ except ImportError as e:
|
|
| 36 |
EMOTION_AVAILABLE = False
|
| 37 |
|
| 38 |
|
| 39 |
-
def
|
| 40 |
"""Recursively find all concrete (non-abstract) subclasses of a base class."""
|
| 41 |
result = []
|
| 42 |
for cls in base.__subclasses__():
|
| 43 |
if not inspect.isabstract(cls):
|
| 44 |
result.append(cls)
|
| 45 |
# recurse into subclasses
|
| 46 |
-
result.extend(
|
| 47 |
return result
|
| 48 |
|
| 49 |
|
|
@@ -157,7 +157,7 @@ class MoveHead(Tool):
|
|
| 157 |
return {"status": f"looking {direction}"}
|
| 158 |
|
| 159 |
except Exception as e:
|
| 160 |
-
logger.
|
| 161 |
return {"error": f"move_head failed: {type(e).__name__}: {e}"}
|
| 162 |
|
| 163 |
|
|
@@ -198,11 +198,15 @@ class Camera(Tool):
|
|
| 198 |
|
| 199 |
# Use vision manager for processing if available
|
| 200 |
if deps.vision_manager is not None:
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
|
|
|
|
|
|
| 204 |
return (
|
| 205 |
-
{"image_description":
|
|
|
|
|
|
|
| 206 |
)
|
| 207 |
else:
|
| 208 |
# Return base64 encoded image like main_works.py camera tool
|
|
@@ -341,12 +345,12 @@ def get_available_emotions_and_descriptions() -> str:
|
|
| 341 |
return "Emotions not available"
|
| 342 |
|
| 343 |
try:
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
for name in
|
| 347 |
description = RECORDED_MOVES.get(name).description
|
| 348 |
-
|
| 349 |
-
return
|
| 350 |
except Exception as e:
|
| 351 |
return f"Error getting emotions: {e}"
|
| 352 |
|
|
@@ -448,15 +452,15 @@ class DoNothing(Tool):
|
|
| 448 |
# Registry & specs (dynamic)
|
| 449 |
|
| 450 |
# List of available tool classes
|
| 451 |
-
ALL_TOOLS: Dict[str, Tool] = {cls.name: cls() for cls in
|
| 452 |
ALL_TOOL_SPECS = [tool.spec() for tool in ALL_TOOLS.values()]
|
| 453 |
|
| 454 |
|
| 455 |
# Dispatcher
|
| 456 |
def _safe_load_obj(args_json: str) -> dict[str, Any]:
|
| 457 |
try:
|
| 458 |
-
|
| 459 |
-
return
|
| 460 |
except Exception:
|
| 461 |
logger.warning("bad args_json=%r", args_json)
|
| 462 |
return {}
|
|
|
|
| 36 |
EMOTION_AVAILABLE = False
|
| 37 |
|
| 38 |
|
| 39 |
+
def get_concrete_subclasses(base):
|
| 40 |
"""Recursively find all concrete (non-abstract) subclasses of a base class."""
|
| 41 |
result = []
|
| 42 |
for cls in base.__subclasses__():
|
| 43 |
if not inspect.isabstract(cls):
|
| 44 |
result.append(cls)
|
| 45 |
# recurse into subclasses
|
| 46 |
+
result.extend(get_concrete_subclasses(cls))
|
| 47 |
return result
|
| 48 |
|
| 49 |
|
|
|
|
| 157 |
return {"status": f"looking {direction}"}
|
| 158 |
|
| 159 |
except Exception as e:
|
| 160 |
+
logger.error("move_head failed")
|
| 161 |
return {"error": f"move_head failed: {type(e).__name__}: {e}"}
|
| 162 |
|
| 163 |
|
|
|
|
| 198 |
|
| 199 |
# Use vision manager for processing if available
|
| 200 |
if deps.vision_manager is not None:
|
| 201 |
+
vision_result = await asyncio.to_thread(
|
| 202 |
+
deps.vision_manager.processor.process_image, frame, image_query
|
| 203 |
+
)
|
| 204 |
+
if isinstance(vision_result, dict) and "error" in vision_result:
|
| 205 |
+
return vision_result
|
| 206 |
return (
|
| 207 |
+
{"image_description": vision_result}
|
| 208 |
+
if isinstance(vision_result, str)
|
| 209 |
+
else {"error": "vision returned non-string"}
|
| 210 |
)
|
| 211 |
else:
|
| 212 |
# Return base64 encoded image like main_works.py camera tool
|
|
|
|
| 345 |
return "Emotions not available"
|
| 346 |
|
| 347 |
try:
|
| 348 |
+
emotion_names = RECORDED_MOVES.list_moves()
|
| 349 |
+
output = "Available emotions:\n"
|
| 350 |
+
for name in emotion_names:
|
| 351 |
description = RECORDED_MOVES.get(name).description
|
| 352 |
+
output += f" - {name}: {description}\n"
|
| 353 |
+
return output
|
| 354 |
except Exception as e:
|
| 355 |
return f"Error getting emotions: {e}"
|
| 356 |
|
|
|
|
| 452 |
# Registry & specs (dynamic)
|
| 453 |
|
| 454 |
# List of available tool classes
|
| 455 |
+
ALL_TOOLS: Dict[str, Tool] = {cls.name: cls() for cls in get_concrete_subclasses(Tool)}
|
| 456 |
ALL_TOOL_SPECS = [tool.spec() for tool in ALL_TOOLS.values()]
|
| 457 |
|
| 458 |
|
| 459 |
# Dispatcher
|
| 460 |
def _safe_load_obj(args_json: str) -> dict[str, Any]:
|
| 461 |
try:
|
| 462 |
+
parsed_args = json.loads(args_json or "{}")
|
| 463 |
+
return parsed_args if isinstance(parsed_args, dict) else {}
|
| 464 |
except Exception:
|
| 465 |
logger.warning("bad args_json=%r", args_json)
|
| 466 |
return {}
|