Alina Lozovskaya
commited on
Commit
·
4f57a60
1
Parent(s):
c423ee0
Add mypy workflow and solve mypy errors
Browse files- .github/workflows/typecheck.yml +29 -0
- .gitignore +1 -0
- pyproject.toml +12 -2
- src/reachy_mini_conversation_demo/audio/head_wobbler.py +7 -5
- src/reachy_mini_conversation_demo/audio/speech_tapper.py +13 -11
- src/reachy_mini_conversation_demo/camera_worker.py +23 -23
- src/reachy_mini_conversation_demo/config.py +4 -4
- src/reachy_mini_conversation_demo/console.py +6 -6
- src/reachy_mini_conversation_demo/dance_emotion_moves.py +16 -15
- src/reachy_mini_conversation_demo/main.py +5 -4
- src/reachy_mini_conversation_demo/moves.py +31 -31
- src/reachy_mini_conversation_demo/openai_realtime.py +54 -43
- src/reachy_mini_conversation_demo/tools.py +36 -34
- src/reachy_mini_conversation_demo/utils.py +8 -6
- src/reachy_mini_conversation_demo/vision/processors.py +17 -16
- src/reachy_mini_conversation_demo/vision/yolo_head_tracker.py +14 -9
- tests/audio/test_head_wobbler.py +6 -5
- uv.lock +72 -7
.github/workflows/typecheck.yml
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Type check
|
| 2 |
+
|
| 3 |
+
on: [push, pull_request]
|
| 4 |
+
|
| 5 |
+
permissions:
|
| 6 |
+
contents: read
|
| 7 |
+
|
| 8 |
+
concurrency:
|
| 9 |
+
group: ${{ github.workflow }}-${{ github.ref }}
|
| 10 |
+
cancel-in-progress: true
|
| 11 |
+
|
| 12 |
+
jobs:
|
| 13 |
+
mypy:
|
| 14 |
+
runs-on: ubuntu-latest
|
| 15 |
+
timeout-minutes: 10
|
| 16 |
+
steps:
|
| 17 |
+
- uses: actions/checkout@v4
|
| 18 |
+
|
| 19 |
+
- uses: actions/setup-python@v5
|
| 20 |
+
with:
|
| 21 |
+
python-version: "3.12"
|
| 22 |
+
|
| 23 |
+
- uses: astral-sh/setup-uv@v5
|
| 24 |
+
|
| 25 |
+
- name: Install dev deps (locked)
|
| 26 |
+
run: uv sync --frozen --group dev
|
| 27 |
+
|
| 28 |
+
- name: Run mypy
|
| 29 |
+
run: uv run mypy --pretty --show-error-codes .
|
.gitignore
CHANGED
|
@@ -29,6 +29,7 @@ coverage.xml
|
|
| 29 |
|
| 30 |
# Linting and formatting
|
| 31 |
.ruff_cache/
|
|
|
|
| 32 |
|
| 33 |
# IDE
|
| 34 |
.vscode/
|
|
|
|
| 29 |
|
| 30 |
# Linting and formatting
|
| 31 |
.ruff_cache/
|
| 32 |
+
.mypy_cache/
|
| 33 |
|
| 34 |
# IDE
|
| 35 |
.vscode/
|
pyproject.toml
CHANGED
|
@@ -23,7 +23,7 @@ dependencies = [
|
|
| 23 |
#OpenAI
|
| 24 |
"openai>=2.1",
|
| 25 |
|
| 26 |
-
#Reachy mini
|
| 27 |
"reachy_mini_dances_library",
|
| 28 |
"reachy_mini_toolbox",
|
| 29 |
"reachy_mini>=1.0.0.rc4",
|
|
@@ -40,7 +40,11 @@ all_vision = [
|
|
| 40 |
]
|
| 41 |
|
| 42 |
[dependency-groups]
|
| 43 |
-
dev = [
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
[project.scripts]
|
| 46 |
reachy-mini-conversation-demo = "reachy_mini_conversation_demo.main:main"
|
|
@@ -88,3 +92,9 @@ quote-style = "double"
|
|
| 88 |
indent-style = "space"
|
| 89 |
skip-magic-trailing-comma = false
|
| 90 |
line-ending = "auto"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
#OpenAI
|
| 24 |
"openai>=2.1",
|
| 25 |
|
| 26 |
+
#Reachy mini
|
| 27 |
"reachy_mini_dances_library",
|
| 28 |
"reachy_mini_toolbox",
|
| 29 |
"reachy_mini>=1.0.0.rc4",
|
|
|
|
| 40 |
]
|
| 41 |
|
| 42 |
[dependency-groups]
|
| 43 |
+
dev = [
|
| 44 |
+
"pytest",
|
| 45 |
+
"ruff==0.12.0",
|
| 46 |
+
"mypy>=1.18.2",
|
| 47 |
+
]
|
| 48 |
|
| 49 |
[project.scripts]
|
| 50 |
reachy-mini-conversation-demo = "reachy_mini_conversation_demo.main:main"
|
|
|
|
| 92 |
indent-style = "space"
|
| 93 |
skip-magic-trailing-comma = false
|
| 94 |
line-ending = "auto"
|
| 95 |
+
|
| 96 |
+
[tool.mypy]
|
| 97 |
+
python_version = "3.12"
|
| 98 |
+
files = ["src/"]
|
| 99 |
+
ignore_missing_imports = true
|
| 100 |
+
strict = true
|
src/reachy_mini_conversation_demo/audio/head_wobbler.py
CHANGED
|
@@ -5,9 +5,11 @@ import queue
|
|
| 5 |
import base64
|
| 6 |
import logging
|
| 7 |
import threading
|
| 8 |
-
from typing import
|
|
|
|
| 9 |
|
| 10 |
import numpy as np
|
|
|
|
| 11 |
|
| 12 |
from reachy_mini_conversation_demo.audio.speech_tapper import HOP_MS, SwayRollRT
|
| 13 |
|
|
@@ -20,13 +22,13 @@ logger = logging.getLogger(__name__)
|
|
| 20 |
class HeadWobbler:
|
| 21 |
"""Converts audio deltas (base64) into head movement offsets."""
|
| 22 |
|
| 23 |
-
def __init__(self, set_speech_offsets):
|
| 24 |
"""Initialize the head wobbler."""
|
| 25 |
self._apply_offsets = set_speech_offsets
|
| 26 |
-
self._base_ts:
|
| 27 |
self._hops_done: int = 0
|
| 28 |
|
| 29 |
-
self.audio_queue: queue.Queue[
|
| 30 |
self.sway = SwayRollRT()
|
| 31 |
|
| 32 |
# Synchronization primitives
|
|
@@ -35,7 +37,7 @@ class HeadWobbler:
|
|
| 35 |
self._generation = 0
|
| 36 |
|
| 37 |
self._stop_event = threading.Event()
|
| 38 |
-
self._thread:
|
| 39 |
|
| 40 |
def feed(self, delta_b64: str) -> None:
|
| 41 |
"""Thread-safe: push audio into the consumer queue."""
|
|
|
|
| 5 |
import base64
|
| 6 |
import logging
|
| 7 |
import threading
|
| 8 |
+
from typing import Any
|
| 9 |
+
from collections.abc import Callable
|
| 10 |
|
| 11 |
import numpy as np
|
| 12 |
+
from numpy.typing import NDArray
|
| 13 |
|
| 14 |
from reachy_mini_conversation_demo.audio.speech_tapper import HOP_MS, SwayRollRT
|
| 15 |
|
|
|
|
| 22 |
class HeadWobbler:
|
| 23 |
"""Converts audio deltas (base64) into head movement offsets."""
|
| 24 |
|
| 25 |
+
def __init__(self, set_speech_offsets: Callable[[tuple[float, float, float, float, float, float]], None]) -> None:
|
| 26 |
"""Initialize the head wobbler."""
|
| 27 |
self._apply_offsets = set_speech_offsets
|
| 28 |
+
self._base_ts: float | None = None
|
| 29 |
self._hops_done: int = 0
|
| 30 |
|
| 31 |
+
self.audio_queue: queue.Queue[tuple[int, int, NDArray[Any]]] = queue.Queue()
|
| 32 |
self.sway = SwayRollRT()
|
| 33 |
|
| 34 |
# Synchronization primitives
|
|
|
|
| 37 |
self._generation = 0
|
| 38 |
|
| 39 |
self._stop_event = threading.Event()
|
| 40 |
+
self._thread: threading.Thread | None = None
|
| 41 |
|
| 42 |
def feed(self, delta_b64: str) -> None:
|
| 43 |
"""Thread-safe: push audio into the consumer queue."""
|
src/reachy_mini_conversation_demo/audio/speech_tapper.py
CHANGED
|
@@ -1,10 +1,11 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
import math
|
| 3 |
-
from typing import
|
| 4 |
from itertools import islice
|
| 5 |
from collections import deque
|
| 6 |
|
| 7 |
import numpy as np
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
# Tunables
|
|
@@ -48,7 +49,7 @@ SWAY_ATTACK_FR = max(1, int(SWAY_ATTACK_MS / HOP_MS))
|
|
| 48 |
SWAY_RELEASE_FR = max(1, int(SWAY_RELEASE_MS / HOP_MS))
|
| 49 |
|
| 50 |
|
| 51 |
-
def _rms_dbfs(x: np.
|
| 52 |
"""Root-mean-square in dBFS for float32 mono array in [-1,1]."""
|
| 53 |
# numerically stable rms (avoid overflow)
|
| 54 |
x = x.astype(np.float32, copy=False)
|
|
@@ -66,7 +67,7 @@ def _loudness_gain(db: float, offset: float = SENS_DB_OFFSET) -> float:
|
|
| 66 |
return t**LOUDNESS_GAMMA if LOUDNESS_GAMMA != 1.0 else t
|
| 67 |
|
| 68 |
|
| 69 |
-
def _to_float32_mono(x:
|
| 70 |
"""Convert arbitrary PCM array to float32 mono in [-1,1].
|
| 71 |
|
| 72 |
Accepts shapes: (N,), (1,N), (N,1), (C,N), (N,C).
|
|
@@ -94,7 +95,7 @@ def _to_float32_mono(x: np.ndarray) -> np.ndarray:
|
|
| 94 |
return a.astype(np.float32) / (scale if scale != 0.0 else 1.0)
|
| 95 |
|
| 96 |
|
| 97 |
-
def _resample_linear(x: np.
|
| 98 |
"""Lightweight linear resampler for short buffers."""
|
| 99 |
if sr_in == sr_out or x.size == 0:
|
| 100 |
return x
|
|
@@ -104,7 +105,7 @@ def _resample_linear(x: np.ndarray, sr_in: int, sr_out: int) -> np.ndarray:
|
|
| 104 |
return np.zeros(0, dtype=np.float32)
|
| 105 |
t_in = np.linspace(0.0, 1.0, num=x.size, dtype=np.float32, endpoint=True)
|
| 106 |
t_out = np.linspace(0.0, 1.0, num=n_out, dtype=np.float32, endpoint=True)
|
| 107 |
-
return np.interp(t_out, t_in, x).astype(np.float32, copy=False)
|
| 108 |
|
| 109 |
|
| 110 |
class SwayRollRT:
|
|
@@ -118,8 +119,8 @@ class SwayRollRT:
|
|
| 118 |
def __init__(self, rng_seed: int = 7):
|
| 119 |
"""Initialize state."""
|
| 120 |
self._seed = int(rng_seed)
|
| 121 |
-
self.samples = deque(maxlen=10 * SR) # sliding window for VAD/env
|
| 122 |
-
self.carry = np.zeros(0, dtype=np.float32)
|
| 123 |
|
| 124 |
self.vad_on = False
|
| 125 |
self.vad_above = 0
|
|
@@ -150,7 +151,7 @@ class SwayRollRT:
|
|
| 150 |
self.sway_down = 0
|
| 151 |
self.t = 0.0
|
| 152 |
|
| 153 |
-
def feed(self, pcm:
|
| 154 |
"""Stream in PCM chunk. Returns a list of sway dicts, one per hop (HOP_MS).
|
| 155 |
|
| 156 |
Args:
|
|
@@ -173,11 +174,12 @@ class SwayRollRT:
|
|
| 173 |
else:
|
| 174 |
self.carry = x
|
| 175 |
|
| 176 |
-
out:
|
| 177 |
|
| 178 |
while self.carry.size >= HOP:
|
| 179 |
hop = self.carry[:HOP]
|
| 180 |
-
|
|
|
|
| 181 |
|
| 182 |
# keep sliding window for VAD/env computation
|
| 183 |
# (deque accepts any iterable; list() for small HOP is fine)
|
|
@@ -260,7 +262,7 @@ class SwayRollRT:
|
|
| 260 |
"x_mm": x_mm,
|
| 261 |
"y_mm": y_mm,
|
| 262 |
"z_mm": z_mm,
|
| 263 |
-
}
|
| 264 |
)
|
| 265 |
|
| 266 |
return out
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
import math
|
| 3 |
+
from typing import Any
|
| 4 |
from itertools import islice
|
| 5 |
from collections import deque
|
| 6 |
|
| 7 |
import numpy as np
|
| 8 |
+
from numpy.typing import NDArray
|
| 9 |
|
| 10 |
|
| 11 |
# Tunables
|
|
|
|
| 49 |
SWAY_RELEASE_FR = max(1, int(SWAY_RELEASE_MS / HOP_MS))
|
| 50 |
|
| 51 |
|
| 52 |
+
def _rms_dbfs(x: NDArray[np.floating[Any]]) -> float:
|
| 53 |
"""Root-mean-square in dBFS for float32 mono array in [-1,1]."""
|
| 54 |
# numerically stable rms (avoid overflow)
|
| 55 |
x = x.astype(np.float32, copy=False)
|
|
|
|
| 67 |
return t**LOUDNESS_GAMMA if LOUDNESS_GAMMA != 1.0 else t
|
| 68 |
|
| 69 |
|
| 70 |
+
def _to_float32_mono(x: NDArray[Any]) -> NDArray[np.floating[Any]]:
|
| 71 |
"""Convert arbitrary PCM array to float32 mono in [-1,1].
|
| 72 |
|
| 73 |
Accepts shapes: (N,), (1,N), (N,1), (C,N), (N,C).
|
|
|
|
| 95 |
return a.astype(np.float32) / (scale if scale != 0.0 else 1.0)
|
| 96 |
|
| 97 |
|
| 98 |
+
def _resample_linear(x: NDArray[np.floating[Any]], sr_in: int, sr_out: int) -> NDArray[np.floating[Any]]:
|
| 99 |
"""Lightweight linear resampler for short buffers."""
|
| 100 |
if sr_in == sr_out or x.size == 0:
|
| 101 |
return x
|
|
|
|
| 105 |
return np.zeros(0, dtype=np.float32)
|
| 106 |
t_in = np.linspace(0.0, 1.0, num=x.size, dtype=np.float32, endpoint=True)
|
| 107 |
t_out = np.linspace(0.0, 1.0, num=n_out, dtype=np.float32, endpoint=True)
|
| 108 |
+
return np.interp(t_out, t_in, x).astype(np.float32, copy=False) # type: ignore[no-any-return]
|
| 109 |
|
| 110 |
|
| 111 |
class SwayRollRT:
|
|
|
|
| 119 |
def __init__(self, rng_seed: int = 7):
|
| 120 |
"""Initialize state."""
|
| 121 |
self._seed = int(rng_seed)
|
| 122 |
+
self.samples: deque[float] = deque(maxlen=10 * SR) # sliding window for VAD/env
|
| 123 |
+
self.carry: NDArray[np.floating[Any]] = np.zeros(0, dtype=np.float32)
|
| 124 |
|
| 125 |
self.vad_on = False
|
| 126 |
self.vad_above = 0
|
|
|
|
| 151 |
self.sway_down = 0
|
| 152 |
self.t = 0.0
|
| 153 |
|
| 154 |
+
def feed(self, pcm: NDArray[Any], sr: int | None) -> list[dict[str, float]]:
|
| 155 |
"""Stream in PCM chunk. Returns a list of sway dicts, one per hop (HOP_MS).
|
| 156 |
|
| 157 |
Args:
|
|
|
|
| 174 |
else:
|
| 175 |
self.carry = x
|
| 176 |
|
| 177 |
+
out: list[dict[str, float]] = []
|
| 178 |
|
| 179 |
while self.carry.size >= HOP:
|
| 180 |
hop = self.carry[:HOP]
|
| 181 |
+
remaining: NDArray[np.floating[Any]] = self.carry[HOP:]
|
| 182 |
+
self.carry = remaining
|
| 183 |
|
| 184 |
# keep sliding window for VAD/env computation
|
| 185 |
# (deque accepts any iterable; list() for small HOP is fine)
|
|
|
|
| 262 |
"x_mm": x_mm,
|
| 263 |
"y_mm": y_mm,
|
| 264 |
"z_mm": z_mm,
|
| 265 |
+
},
|
| 266 |
)
|
| 267 |
|
| 268 |
return out
|
src/reachy_mini_conversation_demo/camera_worker.py
CHANGED
|
@@ -9,10 +9,11 @@ Ported from main_works.py camera_worker() function to provide:
|
|
| 9 |
import time
|
| 10 |
import logging
|
| 11 |
import threading
|
| 12 |
-
from typing import
|
| 13 |
|
| 14 |
import cv2
|
| 15 |
import numpy as np
|
|
|
|
| 16 |
from scipy.spatial.transform import Rotation as R
|
| 17 |
|
| 18 |
from reachy_mini import ReachyMini
|
|
@@ -25,20 +26,20 @@ logger = logging.getLogger(__name__)
|
|
| 25 |
class CameraWorker:
|
| 26 |
"""Thread-safe camera worker with frame buffering and face tracking."""
|
| 27 |
|
| 28 |
-
def __init__(self, reachy_mini: ReachyMini, head_tracker=None):
|
| 29 |
"""Initialize."""
|
| 30 |
self.reachy_mini = reachy_mini
|
| 31 |
self.head_tracker = head_tracker
|
| 32 |
|
| 33 |
# Thread-safe frame storage
|
| 34 |
-
self.latest_frame:
|
| 35 |
self.frame_lock = threading.Lock()
|
| 36 |
self._stop_event = threading.Event()
|
| 37 |
-
self._thread:
|
| 38 |
|
| 39 |
# Face tracking state
|
| 40 |
self.is_head_tracking_enabled = True
|
| 41 |
-
self.face_tracking_offsets = [
|
| 42 |
0.0,
|
| 43 |
0.0,
|
| 44 |
0.0,
|
|
@@ -49,31 +50,31 @@ class CameraWorker:
|
|
| 49 |
self.face_tracking_lock = threading.Lock()
|
| 50 |
|
| 51 |
# Face tracking timing variables (same as main_works.py)
|
| 52 |
-
self.last_face_detected_time:
|
| 53 |
-
self.interpolation_start_time:
|
| 54 |
-
self.interpolation_start_pose:
|
| 55 |
self.face_lost_delay = 2.0 # seconds to wait before starting interpolation
|
| 56 |
self.interpolation_duration = 1.0 # seconds to interpolate back to neutral
|
| 57 |
|
| 58 |
# Track state changes
|
| 59 |
self.previous_head_tracking_state = self.is_head_tracking_enabled
|
| 60 |
|
| 61 |
-
def get_latest_frame(self) ->
|
| 62 |
"""Get the latest frame (thread-safe)."""
|
| 63 |
with self.frame_lock:
|
| 64 |
if self.latest_frame is None:
|
| 65 |
return None
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
return frame
|
| 70 |
|
| 71 |
def get_face_tracking_offsets(
|
| 72 |
self,
|
| 73 |
-
) ->
|
| 74 |
"""Get current face tracking offsets (thread-safe)."""
|
| 75 |
with self.face_tracking_lock:
|
| 76 |
-
|
|
|
|
| 77 |
|
| 78 |
def set_head_tracking_enabled(self, enabled: bool) -> None:
|
| 79 |
"""Enable/disable head tracking."""
|
|
@@ -168,12 +169,11 @@ class CameraWorker:
|
|
| 168 |
rotation[2], # roll, pitch, yaw
|
| 169 |
]
|
| 170 |
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
if
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
pass
|
| 177 |
|
| 178 |
# Handle smooth interpolation (works for both face-lost and tracking-disabled cases)
|
| 179 |
if self.last_face_detected_time is not None:
|
|
@@ -191,7 +191,7 @@ class CameraWorker:
|
|
| 191 |
self.interpolation_start_pose = np.eye(4)
|
| 192 |
self.interpolation_start_pose[:3, 3] = current_translation
|
| 193 |
self.interpolation_start_pose[:3, :3] = R.from_euler(
|
| 194 |
-
"xyz", current_rotation_euler
|
| 195 |
).as_matrix()
|
| 196 |
|
| 197 |
# Calculate interpolation progress (t from 0 to 1)
|
|
@@ -200,7 +200,7 @@ class CameraWorker:
|
|
| 200 |
|
| 201 |
# Interpolate between current pose and neutral pose
|
| 202 |
interpolated_pose = linear_pose_interpolation(
|
| 203 |
-
self.interpolation_start_pose, neutral_pose, t
|
| 204 |
)
|
| 205 |
|
| 206 |
# Extract translation and rotation from interpolated pose
|
|
|
|
| 9 |
import time
|
| 10 |
import logging
|
| 11 |
import threading
|
| 12 |
+
from typing import Any
|
| 13 |
|
| 14 |
import cv2
|
| 15 |
import numpy as np
|
| 16 |
+
from numpy.typing import NDArray
|
| 17 |
from scipy.spatial.transform import Rotation as R
|
| 18 |
|
| 19 |
from reachy_mini import ReachyMini
|
|
|
|
| 26 |
class CameraWorker:
|
| 27 |
"""Thread-safe camera worker with frame buffering and face tracking."""
|
| 28 |
|
| 29 |
+
def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None:
|
| 30 |
"""Initialize."""
|
| 31 |
self.reachy_mini = reachy_mini
|
| 32 |
self.head_tracker = head_tracker
|
| 33 |
|
| 34 |
# Thread-safe frame storage
|
| 35 |
+
self.latest_frame: NDArray[np.uint8] | None = None
|
| 36 |
self.frame_lock = threading.Lock()
|
| 37 |
self._stop_event = threading.Event()
|
| 38 |
+
self._thread: threading.Thread | None = None
|
| 39 |
|
| 40 |
# Face tracking state
|
| 41 |
self.is_head_tracking_enabled = True
|
| 42 |
+
self.face_tracking_offsets: list[float] = [
|
| 43 |
0.0,
|
| 44 |
0.0,
|
| 45 |
0.0,
|
|
|
|
| 50 |
self.face_tracking_lock = threading.Lock()
|
| 51 |
|
| 52 |
# Face tracking timing variables (same as main_works.py)
|
| 53 |
+
self.last_face_detected_time: float | None = None
|
| 54 |
+
self.interpolation_start_time: float | None = None
|
| 55 |
+
self.interpolation_start_pose: NDArray[np.floating[Any]] | None = None
|
| 56 |
self.face_lost_delay = 2.0 # seconds to wait before starting interpolation
|
| 57 |
self.interpolation_duration = 1.0 # seconds to interpolate back to neutral
|
| 58 |
|
| 59 |
# Track state changes
|
| 60 |
self.previous_head_tracking_state = self.is_head_tracking_enabled
|
| 61 |
|
| 62 |
+
def get_latest_frame(self) -> NDArray[np.uint8] | None:
|
| 63 |
"""Get the latest frame (thread-safe)."""
|
| 64 |
with self.frame_lock:
|
| 65 |
if self.latest_frame is None:
|
| 66 |
return None
|
| 67 |
+
frame = self.latest_frame.copy()
|
| 68 |
+
frame_rgb: NDArray[np.uint8] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # type: ignore[assignment]
|
| 69 |
+
return frame_rgb
|
|
|
|
| 70 |
|
| 71 |
def get_face_tracking_offsets(
|
| 72 |
self,
|
| 73 |
+
) -> tuple[float, float, float, float, float, float]:
|
| 74 |
"""Get current face tracking offsets (thread-safe)."""
|
| 75 |
with self.face_tracking_lock:
|
| 76 |
+
offsets = self.face_tracking_offsets
|
| 77 |
+
return (offsets[0], offsets[1], offsets[2], offsets[3], offsets[4], offsets[5])
|
| 78 |
|
| 79 |
def set_head_tracking_enabled(self, enabled: bool) -> None:
|
| 80 |
"""Enable/disable head tracking."""
|
|
|
|
| 169 |
rotation[2], # roll, pitch, yaw
|
| 170 |
]
|
| 171 |
|
| 172 |
+
# No face detected while tracking enabled - set face lost timestamp
|
| 173 |
+
elif self.last_face_detected_time is None or self.last_face_detected_time == current_time:
|
| 174 |
+
# Only update if we haven't already set a face lost time
|
| 175 |
+
# (current_time check prevents overriding the disable-triggered timestamp)
|
| 176 |
+
pass
|
|
|
|
| 177 |
|
| 178 |
# Handle smooth interpolation (works for both face-lost and tracking-disabled cases)
|
| 179 |
if self.last_face_detected_time is not None:
|
|
|
|
| 191 |
self.interpolation_start_pose = np.eye(4)
|
| 192 |
self.interpolation_start_pose[:3, 3] = current_translation
|
| 193 |
self.interpolation_start_pose[:3, :3] = R.from_euler(
|
| 194 |
+
"xyz", current_rotation_euler,
|
| 195 |
).as_matrix()
|
| 196 |
|
| 197 |
# Calculate interpolation progress (t from 0 to 1)
|
|
|
|
| 200 |
|
| 201 |
# Interpolate between current pose and neutral pose
|
| 202 |
interpolated_pose = linear_pose_interpolation(
|
| 203 |
+
self.interpolation_start_pose, neutral_pose, t,
|
| 204 |
)
|
| 205 |
|
| 206 |
# Extract translation and rotation from interpolated pose
|
src/reachy_mini_conversation_demo/config.py
CHANGED
|
@@ -13,13 +13,13 @@ if not env_file.exists():
|
|
| 13 |
raise RuntimeError(
|
| 14 |
".env file not found. Please create one based on .env.example:\n"
|
| 15 |
" cp .env.example .env\n"
|
| 16 |
-
"Then add your OPENAI_API_KEY to the .env file."
|
| 17 |
)
|
| 18 |
|
| 19 |
# Load .env and verify it was loaded successfully
|
| 20 |
if not load_dotenv():
|
| 21 |
raise RuntimeError(
|
| 22 |
-
"Failed to load .env file. Please ensure the file is readable and properly formatted."
|
| 23 |
)
|
| 24 |
|
| 25 |
logger.info("Configuration loaded from .env file")
|
|
@@ -33,11 +33,11 @@ class Config:
|
|
| 33 |
if OPENAI_API_KEY is None:
|
| 34 |
raise RuntimeError(
|
| 35 |
"OPENAI_API_KEY is not set in .env file. Please add it:\n"
|
| 36 |
-
" OPENAI_API_KEY=your_api_key_here"
|
| 37 |
)
|
| 38 |
if not OPENAI_API_KEY.strip():
|
| 39 |
raise RuntimeError(
|
| 40 |
-
"OPENAI_API_KEY is empty in .env file. Please provide a valid API key."
|
| 41 |
)
|
| 42 |
|
| 43 |
# Optional
|
|
|
|
| 13 |
raise RuntimeError(
|
| 14 |
".env file not found. Please create one based on .env.example:\n"
|
| 15 |
" cp .env.example .env\n"
|
| 16 |
+
"Then add your OPENAI_API_KEY to the .env file.",
|
| 17 |
)
|
| 18 |
|
| 19 |
# Load .env and verify it was loaded successfully
|
| 20 |
if not load_dotenv():
|
| 21 |
raise RuntimeError(
|
| 22 |
+
"Failed to load .env file. Please ensure the file is readable and properly formatted.",
|
| 23 |
)
|
| 24 |
|
| 25 |
logger.info("Configuration loaded from .env file")
|
|
|
|
| 33 |
if OPENAI_API_KEY is None:
|
| 34 |
raise RuntimeError(
|
| 35 |
"OPENAI_API_KEY is not set in .env file. Please add it:\n"
|
| 36 |
+
" OPENAI_API_KEY=your_api_key_here",
|
| 37 |
)
|
| 38 |
if not OPENAI_API_KEY.strip():
|
| 39 |
raise RuntimeError(
|
| 40 |
+
"OPENAI_API_KEY is empty in .env file. Please provide a valid API key.",
|
| 41 |
)
|
| 42 |
|
| 43 |
# Optional
|
src/reachy_mini_conversation_demo/console.py
CHANGED
|
@@ -24,9 +24,9 @@ class LocalStream:
|
|
| 24 |
self.handler = handler
|
| 25 |
self._robot = robot
|
| 26 |
self._stop_event = asyncio.Event()
|
| 27 |
-
self._tasks = []
|
| 28 |
# Allow the handler to flush the player queue when appropriate.
|
| 29 |
-
self.handler._clear_queue = self.clear_audio_queue
|
| 30 |
|
| 31 |
def launch(self) -> None:
|
| 32 |
"""Start the recorder/player and run the async processing loops."""
|
|
@@ -105,12 +105,12 @@ class LocalStream:
|
|
| 105 |
elif isinstance(handler_output, tuple):
|
| 106 |
input_sample_rate, audio_frame = handler_output
|
| 107 |
device_sample_rate = self._robot.media.get_audio_samplerate()
|
| 108 |
-
|
| 109 |
if input_sample_rate != device_sample_rate:
|
| 110 |
-
|
| 111 |
-
|
| 112 |
)
|
| 113 |
-
self._robot.media.push_audio_sample(
|
| 114 |
|
| 115 |
else:
|
| 116 |
logger.debug("Ignoring output type=%s", type(handler_output).__name__)
|
|
|
|
| 24 |
self.handler = handler
|
| 25 |
self._robot = robot
|
| 26 |
self._stop_event = asyncio.Event()
|
| 27 |
+
self._tasks: list[asyncio.Task[None]] = []
|
| 28 |
# Allow the handler to flush the player queue when appropriate.
|
| 29 |
+
self.handler._clear_queue = self.clear_audio_queue
|
| 30 |
|
| 31 |
def launch(self) -> None:
|
| 32 |
"""Start the recorder/player and run the async processing loops."""
|
|
|
|
| 105 |
elif isinstance(handler_output, tuple):
|
| 106 |
input_sample_rate, audio_frame = handler_output
|
| 107 |
device_sample_rate = self._robot.media.get_audio_samplerate()
|
| 108 |
+
audio_frame_float = audio_to_float32(audio_frame.squeeze())
|
| 109 |
if input_sample_rate != device_sample_rate:
|
| 110 |
+
audio_frame_float = librosa.resample(
|
| 111 |
+
audio_frame_float, orig_sr=input_sample_rate, target_sr=device_sample_rate,
|
| 112 |
)
|
| 113 |
+
self._robot.media.push_audio_sample(audio_frame_float)
|
| 114 |
|
| 115 |
else:
|
| 116 |
logger.debug("Ignoring output type=%s", type(handler_output).__name__)
|
src/reachy_mini_conversation_demo/dance_emotion_moves.py
CHANGED
|
@@ -6,9 +6,10 @@ and executed sequentially by the MovementManager.
|
|
| 6 |
|
| 7 |
from __future__ import annotations
|
| 8 |
import logging
|
| 9 |
-
from typing import
|
| 10 |
|
| 11 |
import numpy as np
|
|
|
|
| 12 |
|
| 13 |
from reachy_mini.motion.move import Move
|
| 14 |
from reachy_mini.motion.recorded_move import RecordedMoves
|
|
@@ -18,7 +19,7 @@ from reachy_mini_dances_library.dance_move import DanceMove
|
|
| 18 |
logger = logging.getLogger(__name__)
|
| 19 |
|
| 20 |
|
| 21 |
-
class DanceQueueMove(Move):
|
| 22 |
"""Wrapper for dance moves to work with the movement queue system."""
|
| 23 |
|
| 24 |
def __init__(self, move_name: str):
|
|
@@ -29,9 +30,9 @@ class DanceQueueMove(Move):
|
|
| 29 |
@property
|
| 30 |
def duration(self) -> float:
|
| 31 |
"""Duration property required by official Move interface."""
|
| 32 |
-
return self.dance_move.duration
|
| 33 |
|
| 34 |
-
def evaluate(self, t: float) -> tuple[np.
|
| 35 |
"""Evaluate dance move at time t."""
|
| 36 |
try:
|
| 37 |
# Get the pose from the dance move
|
|
@@ -52,7 +53,7 @@ class DanceQueueMove(Move):
|
|
| 52 |
return (neutral_head_pose, np.array([0.0, 0.0]), 0.0)
|
| 53 |
|
| 54 |
|
| 55 |
-
class EmotionQueueMove(Move):
|
| 56 |
"""Wrapper for emotion moves to work with the movement queue system."""
|
| 57 |
|
| 58 |
def __init__(self, emotion_name: str, recorded_moves: RecordedMoves):
|
|
@@ -63,9 +64,9 @@ class EmotionQueueMove(Move):
|
|
| 63 |
@property
|
| 64 |
def duration(self) -> float:
|
| 65 |
"""Duration property required by official Move interface."""
|
| 66 |
-
return self.emotion_move.duration
|
| 67 |
|
| 68 |
-
def evaluate(self, t: float) -> tuple[np.
|
| 69 |
"""Evaluate emotion move at time t."""
|
| 70 |
try:
|
| 71 |
# Get the pose from the emotion move
|
|
@@ -86,17 +87,17 @@ class EmotionQueueMove(Move):
|
|
| 86 |
return (neutral_head_pose, np.array([0.0, 0.0]), 0.0)
|
| 87 |
|
| 88 |
|
| 89 |
-
class GotoQueueMove(Move):
|
| 90 |
"""Wrapper for goto moves to work with the movement queue system."""
|
| 91 |
|
| 92 |
def __init__(
|
| 93 |
self,
|
| 94 |
-
target_head_pose: np.
|
| 95 |
-
start_head_pose: np.
|
| 96 |
-
target_antennas:
|
| 97 |
-
start_antennas:
|
| 98 |
target_body_yaw: float = 0,
|
| 99 |
-
start_body_yaw: float = None,
|
| 100 |
duration: float = 1.0,
|
| 101 |
):
|
| 102 |
"""Initialize a GotoQueueMove."""
|
|
@@ -113,7 +114,7 @@ class GotoQueueMove(Move):
|
|
| 113 |
"""Duration property required by official Move interface."""
|
| 114 |
return self._duration
|
| 115 |
|
| 116 |
-
def evaluate(self, t: float) -> tuple[np.
|
| 117 |
"""Evaluate goto move at time t using linear interpolation."""
|
| 118 |
try:
|
| 119 |
from reachy_mini.utils import create_head_pose
|
|
@@ -136,7 +137,7 @@ class GotoQueueMove(Move):
|
|
| 136 |
[
|
| 137 |
self.start_antennas[0] + (self.target_antennas[0] - self.start_antennas[0]) * t_clamped,
|
| 138 |
self.start_antennas[1] + (self.target_antennas[1] - self.start_antennas[1]) * t_clamped,
|
| 139 |
-
]
|
| 140 |
)
|
| 141 |
|
| 142 |
# Interpolate body yaw
|
|
|
|
| 6 |
|
| 7 |
from __future__ import annotations
|
| 8 |
import logging
|
| 9 |
+
from typing import Any
|
| 10 |
|
| 11 |
import numpy as np
|
| 12 |
+
from numpy.typing import NDArray
|
| 13 |
|
| 14 |
from reachy_mini.motion.move import Move
|
| 15 |
from reachy_mini.motion.recorded_move import RecordedMoves
|
|
|
|
| 19 |
logger = logging.getLogger(__name__)
|
| 20 |
|
| 21 |
|
| 22 |
+
class DanceQueueMove(Move): # type: ignore[misc]
|
| 23 |
"""Wrapper for dance moves to work with the movement queue system."""
|
| 24 |
|
| 25 |
def __init__(self, move_name: str):
|
|
|
|
| 30 |
@property
|
| 31 |
def duration(self) -> float:
|
| 32 |
"""Duration property required by official Move interface."""
|
| 33 |
+
return float(self.dance_move.duration)
|
| 34 |
|
| 35 |
+
def evaluate(self, t: float) -> tuple[NDArray[np.floating[Any]] | None, NDArray[np.floating[Any]] | None, float | None]:
|
| 36 |
"""Evaluate dance move at time t."""
|
| 37 |
try:
|
| 38 |
# Get the pose from the dance move
|
|
|
|
| 53 |
return (neutral_head_pose, np.array([0.0, 0.0]), 0.0)
|
| 54 |
|
| 55 |
|
| 56 |
+
class EmotionQueueMove(Move): # type: ignore[misc]
|
| 57 |
"""Wrapper for emotion moves to work with the movement queue system."""
|
| 58 |
|
| 59 |
def __init__(self, emotion_name: str, recorded_moves: RecordedMoves):
|
|
|
|
| 64 |
@property
|
| 65 |
def duration(self) -> float:
|
| 66 |
"""Duration property required by official Move interface."""
|
| 67 |
+
return float(self.emotion_move.duration)
|
| 68 |
|
| 69 |
+
def evaluate(self, t: float) -> tuple[NDArray[np.floating[Any]] | None, NDArray[np.floating[Any]] | None, float | None]:
|
| 70 |
"""Evaluate emotion move at time t."""
|
| 71 |
try:
|
| 72 |
# Get the pose from the emotion move
|
|
|
|
| 87 |
return (neutral_head_pose, np.array([0.0, 0.0]), 0.0)
|
| 88 |
|
| 89 |
|
| 90 |
+
class GotoQueueMove(Move): # type: ignore[misc]
|
| 91 |
"""Wrapper for goto moves to work with the movement queue system."""
|
| 92 |
|
| 93 |
def __init__(
|
| 94 |
self,
|
| 95 |
+
target_head_pose: NDArray[np.floating[Any]],
|
| 96 |
+
start_head_pose: NDArray[np.floating[Any]] | None = None,
|
| 97 |
+
target_antennas: tuple[float, float] = (0, 0),
|
| 98 |
+
start_antennas: tuple[float, float] | None = None,
|
| 99 |
target_body_yaw: float = 0,
|
| 100 |
+
start_body_yaw: float | None = None,
|
| 101 |
duration: float = 1.0,
|
| 102 |
):
|
| 103 |
"""Initialize a GotoQueueMove."""
|
|
|
|
| 114 |
"""Duration property required by official Move interface."""
|
| 115 |
return self._duration
|
| 116 |
|
| 117 |
+
def evaluate(self, t: float) -> tuple[NDArray[np.floating[Any]] | None, NDArray[np.floating[Any]] | None, float | None]:
|
| 118 |
"""Evaluate goto move at time t using linear interpolation."""
|
| 119 |
try:
|
| 120 |
from reachy_mini.utils import create_head_pose
|
|
|
|
| 137 |
[
|
| 138 |
self.start_antennas[0] + (self.target_antennas[0] - self.start_antennas[0]) * t_clamped,
|
| 139 |
self.start_antennas[1] + (self.target_antennas[1] - self.start_antennas[1]) * t_clamped,
|
| 140 |
+
],
|
| 141 |
)
|
| 142 |
|
| 143 |
# Interpolate body yaw
|
src/reachy_mini_conversation_demo/main.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
import sys
|
|
|
|
| 5 |
|
| 6 |
import gradio as gr
|
| 7 |
from fastapi import FastAPI
|
|
@@ -20,13 +21,13 @@ from reachy_mini_conversation_demo.openai_realtime import OpenaiRealtimeHandler
|
|
| 20 |
from reachy_mini_conversation_demo.audio.head_wobbler import HeadWobbler
|
| 21 |
|
| 22 |
|
| 23 |
-
def update_chatbot(chatbot: list[dict], response: dict):
|
| 24 |
"""Update the chatbot with AdditionalOutputs."""
|
| 25 |
chatbot.append(response)
|
| 26 |
return chatbot
|
| 27 |
|
| 28 |
|
| 29 |
-
def main():
|
| 30 |
"""Entrypoint for the Reachy Mini conversation demo."""
|
| 31 |
args = parse_args()
|
| 32 |
|
|
@@ -41,7 +42,7 @@ def main():
|
|
| 41 |
# Check if running in simulation mode without --gradio
|
| 42 |
if robot.client.get_status()["simulation_enabled"] and not args.gradio:
|
| 43 |
logger.error(
|
| 44 |
-
"Simulation mode requires Gradio interface. Please use --gradio flag when running in simulation mode."
|
| 45 |
)
|
| 46 |
robot.client.disconnect()
|
| 47 |
sys.exit(1)
|
|
@@ -76,7 +77,7 @@ def main():
|
|
| 76 |
|
| 77 |
handler = OpenaiRealtimeHandler(deps)
|
| 78 |
|
| 79 |
-
stream_manager = None
|
| 80 |
|
| 81 |
if args.gradio:
|
| 82 |
stream = Stream(
|
|
|
|
| 2 |
|
| 3 |
import os
|
| 4 |
import sys
|
| 5 |
+
from typing import Any
|
| 6 |
|
| 7 |
import gradio as gr
|
| 8 |
from fastapi import FastAPI
|
|
|
|
| 21 |
from reachy_mini_conversation_demo.audio.head_wobbler import HeadWobbler
|
| 22 |
|
| 23 |
|
| 24 |
+
def update_chatbot(chatbot: list[dict[str, Any]], response: dict[str, Any]) -> list[dict[str, Any]]:
|
| 25 |
"""Update the chatbot with AdditionalOutputs."""
|
| 26 |
chatbot.append(response)
|
| 27 |
return chatbot
|
| 28 |
|
| 29 |
|
| 30 |
+
def main() -> None:
|
| 31 |
"""Entrypoint for the Reachy Mini conversation demo."""
|
| 32 |
args = parse_args()
|
| 33 |
|
|
|
|
| 42 |
# Check if running in simulation mode without --gradio
|
| 43 |
if robot.client.get_status()["simulation_enabled"] and not args.gradio:
|
| 44 |
logger.error(
|
| 45 |
+
"Simulation mode requires Gradio interface. Please use --gradio flag when running in simulation mode.",
|
| 46 |
)
|
| 47 |
robot.client.disconnect()
|
| 48 |
sys.exit(1)
|
|
|
|
| 77 |
|
| 78 |
handler = OpenaiRealtimeHandler(deps)
|
| 79 |
|
| 80 |
+
stream_manager: gr.Blocks | LocalStream | None = None
|
| 81 |
|
| 82 |
if args.gradio:
|
| 83 |
stream = Stream(
|
src/reachy_mini_conversation_demo/moves.py
CHANGED
|
@@ -36,11 +36,12 @@ import time
|
|
| 36 |
import logging
|
| 37 |
import threading
|
| 38 |
from queue import Empty, Queue
|
| 39 |
-
from typing import Any
|
| 40 |
from collections import deque
|
| 41 |
from dataclasses import dataclass
|
| 42 |
|
| 43 |
import numpy as np
|
|
|
|
| 44 |
|
| 45 |
from reachy_mini import ReachyMini
|
| 46 |
from reachy_mini.utils import create_head_pose
|
|
@@ -57,16 +58,16 @@ logger = logging.getLogger(__name__)
|
|
| 57 |
CONTROL_LOOP_FREQUENCY_HZ = 100.0 # Hz - Target frequency for the movement control loop
|
| 58 |
|
| 59 |
# Type definitions
|
| 60 |
-
FullBodyPose =
|
| 61 |
|
| 62 |
|
| 63 |
-
class BreathingMove(Move):
|
| 64 |
"""Breathing move with interpolation to neutral and then continuous breathing patterns."""
|
| 65 |
|
| 66 |
def __init__(
|
| 67 |
self,
|
| 68 |
-
interpolation_start_pose: np.
|
| 69 |
-
interpolation_start_antennas:
|
| 70 |
interpolation_duration: float = 1.0,
|
| 71 |
):
|
| 72 |
"""Initialize breathing move.
|
|
@@ -96,7 +97,7 @@ class BreathingMove(Move):
|
|
| 96 |
"""Duration property required by official Move interface."""
|
| 97 |
return float("inf") # Continuous breathing (never ends naturally)
|
| 98 |
|
| 99 |
-
def evaluate(self, t: float) -> tuple[np.
|
| 100 |
"""Evaluate breathing move at time t."""
|
| 101 |
if t < self.interpolation_duration:
|
| 102 |
# Phase 1: Interpolate to neutral base position
|
|
@@ -104,7 +105,7 @@ class BreathingMove(Move):
|
|
| 104 |
|
| 105 |
# Interpolate head pose
|
| 106 |
head_pose = linear_pose_interpolation(
|
| 107 |
-
self.interpolation_start_pose, self.neutral_head_pose, interpolation_t
|
| 108 |
)
|
| 109 |
|
| 110 |
# Interpolate antennas
|
|
@@ -168,12 +169,12 @@ class MovementState:
|
|
| 168 |
"""State tracking for the movement system."""
|
| 169 |
|
| 170 |
# Primary move state
|
| 171 |
-
current_move:
|
| 172 |
-
move_start_time:
|
| 173 |
last_activity_time: float = 0.0
|
| 174 |
|
| 175 |
# Secondary move state (offsets)
|
| 176 |
-
speech_offsets:
|
| 177 |
0.0,
|
| 178 |
0.0,
|
| 179 |
0.0,
|
|
@@ -181,7 +182,7 @@ class MovementState:
|
|
| 181 |
0.0,
|
| 182 |
0.0,
|
| 183 |
)
|
| 184 |
-
face_tracking_offsets:
|
| 185 |
0.0,
|
| 186 |
0.0,
|
| 187 |
0.0,
|
|
@@ -191,7 +192,7 @@ class MovementState:
|
|
| 191 |
)
|
| 192 |
|
| 193 |
# Status flags
|
| 194 |
-
last_primary_pose:
|
| 195 |
|
| 196 |
def update_activity(self) -> None:
|
| 197 |
"""Update the last activity time."""
|
|
@@ -242,7 +243,7 @@ class MovementManager:
|
|
| 242 |
def __init__(
|
| 243 |
self,
|
| 244 |
current_robot: ReachyMini,
|
| 245 |
-
camera_worker=None,
|
| 246 |
):
|
| 247 |
"""Initialize movement manager."""
|
| 248 |
self.current_robot = current_robot
|
|
@@ -258,7 +259,7 @@ class MovementManager:
|
|
| 258 |
self.state.last_primary_pose = (neutral_pose, (0.0, 0.0), 0.0)
|
| 259 |
|
| 260 |
# Move queue (primary moves)
|
| 261 |
-
self.move_queue = deque()
|
| 262 |
|
| 263 |
# Configuration
|
| 264 |
self.idle_inactivity_delay = 0.3 # seconds
|
|
@@ -266,10 +267,10 @@ class MovementManager:
|
|
| 266 |
self.target_period = 1.0 / self.target_frequency
|
| 267 |
|
| 268 |
self._stop_event = threading.Event()
|
| 269 |
-
self._thread:
|
| 270 |
self._is_listening = False
|
| 271 |
self._last_commanded_pose: FullBodyPose = clone_full_body_pose(self.state.last_primary_pose)
|
| 272 |
-
self._listening_antennas:
|
| 273 |
self._antenna_unfreeze_blend = 1.0
|
| 274 |
self._antenna_blend_duration = 0.4 # seconds to blend back after listening
|
| 275 |
self._last_listening_blend_time = self._now()
|
|
@@ -283,7 +284,7 @@ class MovementManager:
|
|
| 283 |
# Cross-thread signalling
|
| 284 |
self._command_queue: Queue[tuple[str, Any]] = Queue()
|
| 285 |
self._speech_offsets_lock = threading.Lock()
|
| 286 |
-
self._pending_speech_offsets:
|
| 287 |
0.0,
|
| 288 |
0.0,
|
| 289 |
0.0,
|
|
@@ -294,7 +295,7 @@ class MovementManager:
|
|
| 294 |
self._speech_offsets_dirty = False
|
| 295 |
|
| 296 |
self._face_offsets_lock = threading.Lock()
|
| 297 |
-
self._pending_face_offsets:
|
| 298 |
0.0,
|
| 299 |
0.0,
|
| 300 |
0.0,
|
|
@@ -326,7 +327,7 @@ class MovementManager:
|
|
| 326 |
"""
|
| 327 |
self._command_queue.put(("clear_queue", None))
|
| 328 |
|
| 329 |
-
def set_speech_offsets(self, offsets:
|
| 330 |
"""Update speech-induced secondary offsets (x, y, z, roll, pitch, yaw).
|
| 331 |
|
| 332 |
Offsets are interpreted as metres for translation and radians for
|
|
@@ -383,7 +384,7 @@ class MovementManager:
|
|
| 383 |
|
| 384 |
def _apply_pending_offsets(self) -> None:
|
| 385 |
"""Apply the most recent speech/face offset updates."""
|
| 386 |
-
speech_offsets:
|
| 387 |
with self._speech_offsets_lock:
|
| 388 |
if self._speech_offsets_dirty:
|
| 389 |
speech_offsets = self._pending_speech_offsets
|
|
@@ -393,7 +394,7 @@ class MovementManager:
|
|
| 393 |
self.state.speech_offsets = speech_offsets
|
| 394 |
self.state.update_activity()
|
| 395 |
|
| 396 |
-
face_offsets:
|
| 397 |
with self._face_offsets_lock:
|
| 398 |
if self._face_offsets_dirty:
|
| 399 |
face_offsets = self._pending_face_offsets
|
|
@@ -549,14 +550,13 @@ class MovementManager:
|
|
| 549 |
)
|
| 550 |
|
| 551 |
self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose)
|
|
|
|
|
|
|
|
|
|
| 552 |
else:
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
else:
|
| 557 |
-
neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
|
| 558 |
-
primary_full_body_pose = (neutral_head_pose, (0.0, 0.0), 0.0)
|
| 559 |
-
self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose)
|
| 560 |
|
| 561 |
return primary_full_body_pose
|
| 562 |
|
|
@@ -595,7 +595,7 @@ class MovementManager:
|
|
| 595 |
self._manage_move_queue(current_time)
|
| 596 |
self._manage_breathing(current_time)
|
| 597 |
|
| 598 |
-
def _calculate_blended_antennas(self, target_antennas:
|
| 599 |
"""Blend target antennas with listening freeze state and update blending."""
|
| 600 |
now = self._now()
|
| 601 |
listening = self._is_listening
|
|
@@ -631,7 +631,7 @@ class MovementManager:
|
|
| 631 |
|
| 632 |
return antennas_cmd
|
| 633 |
|
| 634 |
-
def _issue_control_command(self, head: np.
|
| 635 |
"""Send the fused pose to the robot with throttled error logging."""
|
| 636 |
try:
|
| 637 |
self.current_robot.set_target(head=head, antennas=antennas, body_yaw=body_yaw)
|
|
@@ -651,7 +651,7 @@ class MovementManager:
|
|
| 651 |
self._last_commanded_pose = clone_full_body_pose((head, antennas, body_yaw))
|
| 652 |
|
| 653 |
def _update_frequency_stats(
|
| 654 |
-
self, loop_start: float, prev_loop_start: float, stats: LoopFrequencyStats
|
| 655 |
) -> LoopFrequencyStats:
|
| 656 |
"""Update frequency statistics based on the current loop start time."""
|
| 657 |
period = loop_start - prev_loop_start
|
|
|
|
| 36 |
import logging
|
| 37 |
import threading
|
| 38 |
from queue import Empty, Queue
|
| 39 |
+
from typing import Any
|
| 40 |
from collections import deque
|
| 41 |
from dataclasses import dataclass
|
| 42 |
|
| 43 |
import numpy as np
|
| 44 |
+
from numpy.typing import NDArray
|
| 45 |
|
| 46 |
from reachy_mini import ReachyMini
|
| 47 |
from reachy_mini.utils import create_head_pose
|
|
|
|
| 58 |
CONTROL_LOOP_FREQUENCY_HZ = 100.0 # Hz - Target frequency for the movement control loop
|
| 59 |
|
| 60 |
# Type definitions
|
| 61 |
+
FullBodyPose = tuple[NDArray[np.floating[Any]], tuple[float, float], float] # (head_pose_4x4, antennas, body_yaw)
|
| 62 |
|
| 63 |
|
| 64 |
+
class BreathingMove(Move): # type: ignore[misc]
|
| 65 |
"""Breathing move with interpolation to neutral and then continuous breathing patterns."""
|
| 66 |
|
| 67 |
def __init__(
|
| 68 |
self,
|
| 69 |
+
interpolation_start_pose: NDArray[np.floating[Any]],
|
| 70 |
+
interpolation_start_antennas: tuple[float, float],
|
| 71 |
interpolation_duration: float = 1.0,
|
| 72 |
):
|
| 73 |
"""Initialize breathing move.
|
|
|
|
| 97 |
"""Duration property required by official Move interface."""
|
| 98 |
return float("inf") # Continuous breathing (never ends naturally)
|
| 99 |
|
| 100 |
+
def evaluate(self, t: float) -> tuple[NDArray[np.floating[Any]] | None, NDArray[np.floating[Any]] | None, float | None]:
|
| 101 |
"""Evaluate breathing move at time t."""
|
| 102 |
if t < self.interpolation_duration:
|
| 103 |
# Phase 1: Interpolate to neutral base position
|
|
|
|
| 105 |
|
| 106 |
# Interpolate head pose
|
| 107 |
head_pose = linear_pose_interpolation(
|
| 108 |
+
self.interpolation_start_pose, self.neutral_head_pose, interpolation_t,
|
| 109 |
)
|
| 110 |
|
| 111 |
# Interpolate antennas
|
|
|
|
| 169 |
"""State tracking for the movement system."""
|
| 170 |
|
| 171 |
# Primary move state
|
| 172 |
+
current_move: Move | None = None
|
| 173 |
+
move_start_time: float | None = None
|
| 174 |
last_activity_time: float = 0.0
|
| 175 |
|
| 176 |
# Secondary move state (offsets)
|
| 177 |
+
speech_offsets: tuple[float, float, float, float, float, float] = (
|
| 178 |
0.0,
|
| 179 |
0.0,
|
| 180 |
0.0,
|
|
|
|
| 182 |
0.0,
|
| 183 |
0.0,
|
| 184 |
)
|
| 185 |
+
face_tracking_offsets: tuple[float, float, float, float, float, float] = (
|
| 186 |
0.0,
|
| 187 |
0.0,
|
| 188 |
0.0,
|
|
|
|
| 192 |
)
|
| 193 |
|
| 194 |
# Status flags
|
| 195 |
+
last_primary_pose: FullBodyPose | None = None
|
| 196 |
|
| 197 |
def update_activity(self) -> None:
|
| 198 |
"""Update the last activity time."""
|
|
|
|
| 243 |
def __init__(
|
| 244 |
self,
|
| 245 |
current_robot: ReachyMini,
|
| 246 |
+
camera_worker: Any = None,
|
| 247 |
):
|
| 248 |
"""Initialize movement manager."""
|
| 249 |
self.current_robot = current_robot
|
|
|
|
| 259 |
self.state.last_primary_pose = (neutral_pose, (0.0, 0.0), 0.0)
|
| 260 |
|
| 261 |
# Move queue (primary moves)
|
| 262 |
+
self.move_queue: deque[Move] = deque()
|
| 263 |
|
| 264 |
# Configuration
|
| 265 |
self.idle_inactivity_delay = 0.3 # seconds
|
|
|
|
| 267 |
self.target_period = 1.0 / self.target_frequency
|
| 268 |
|
| 269 |
self._stop_event = threading.Event()
|
| 270 |
+
self._thread: threading.Thread | None = None
|
| 271 |
self._is_listening = False
|
| 272 |
self._last_commanded_pose: FullBodyPose = clone_full_body_pose(self.state.last_primary_pose)
|
| 273 |
+
self._listening_antennas: tuple[float, float] = self._last_commanded_pose[1]
|
| 274 |
self._antenna_unfreeze_blend = 1.0
|
| 275 |
self._antenna_blend_duration = 0.4 # seconds to blend back after listening
|
| 276 |
self._last_listening_blend_time = self._now()
|
|
|
|
| 284 |
# Cross-thread signalling
|
| 285 |
self._command_queue: Queue[tuple[str, Any]] = Queue()
|
| 286 |
self._speech_offsets_lock = threading.Lock()
|
| 287 |
+
self._pending_speech_offsets: tuple[float, float, float, float, float, float] = (
|
| 288 |
0.0,
|
| 289 |
0.0,
|
| 290 |
0.0,
|
|
|
|
| 295 |
self._speech_offsets_dirty = False
|
| 296 |
|
| 297 |
self._face_offsets_lock = threading.Lock()
|
| 298 |
+
self._pending_face_offsets: tuple[float, float, float, float, float, float] = (
|
| 299 |
0.0,
|
| 300 |
0.0,
|
| 301 |
0.0,
|
|
|
|
| 327 |
"""
|
| 328 |
self._command_queue.put(("clear_queue", None))
|
| 329 |
|
| 330 |
+
def set_speech_offsets(self, offsets: tuple[float, float, float, float, float, float]) -> None:
|
| 331 |
"""Update speech-induced secondary offsets (x, y, z, roll, pitch, yaw).
|
| 332 |
|
| 333 |
Offsets are interpreted as metres for translation and radians for
|
|
|
|
| 384 |
|
| 385 |
def _apply_pending_offsets(self) -> None:
|
| 386 |
"""Apply the most recent speech/face offset updates."""
|
| 387 |
+
speech_offsets: tuple[float, float, float, float, float, float] | None = None
|
| 388 |
with self._speech_offsets_lock:
|
| 389 |
if self._speech_offsets_dirty:
|
| 390 |
speech_offsets = self._pending_speech_offsets
|
|
|
|
| 394 |
self.state.speech_offsets = speech_offsets
|
| 395 |
self.state.update_activity()
|
| 396 |
|
| 397 |
+
face_offsets: tuple[float, float, float, float, float, float] | None = None
|
| 398 |
with self._face_offsets_lock:
|
| 399 |
if self._face_offsets_dirty:
|
| 400 |
face_offsets = self._pending_face_offsets
|
|
|
|
| 550 |
)
|
| 551 |
|
| 552 |
self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose)
|
| 553 |
+
# Otherwise reuse the last primary pose so we avoid jumps between moves
|
| 554 |
+
elif self.state.last_primary_pose is not None:
|
| 555 |
+
primary_full_body_pose = clone_full_body_pose(self.state.last_primary_pose)
|
| 556 |
else:
|
| 557 |
+
neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
|
| 558 |
+
primary_full_body_pose = (neutral_head_pose, (0.0, 0.0), 0.0)
|
| 559 |
+
self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 560 |
|
| 561 |
return primary_full_body_pose
|
| 562 |
|
|
|
|
| 595 |
self._manage_move_queue(current_time)
|
| 596 |
self._manage_breathing(current_time)
|
| 597 |
|
| 598 |
+
def _calculate_blended_antennas(self, target_antennas: tuple[float, float]) -> tuple[float, float]:
|
| 599 |
"""Blend target antennas with listening freeze state and update blending."""
|
| 600 |
now = self._now()
|
| 601 |
listening = self._is_listening
|
|
|
|
| 631 |
|
| 632 |
return antennas_cmd
|
| 633 |
|
| 634 |
+
def _issue_control_command(self, head: NDArray[np.floating[Any]], antennas: tuple[float, float], body_yaw: float) -> None:
|
| 635 |
"""Send the fused pose to the robot with throttled error logging."""
|
| 636 |
try:
|
| 637 |
self.current_robot.set_target(head=head, antennas=antennas, body_yaw=body_yaw)
|
|
|
|
| 651 |
self._last_commanded_pose = clone_full_body_pose((head, antennas, body_yaw))
|
| 652 |
|
| 653 |
def _update_frequency_stats(
|
| 654 |
+
self, loop_start: float, prev_loop_start: float, stats: LoopFrequencyStats,
|
| 655 |
) -> LoopFrequencyStats:
|
| 656 |
"""Update frequency statistics based on the current loop start time."""
|
| 657 |
period = loop_start - prev_loop_start
|
src/reachy_mini_conversation_demo/openai_realtime.py
CHANGED
|
@@ -2,12 +2,14 @@ import json
|
|
| 2 |
import base64
|
| 3 |
import asyncio
|
| 4 |
import logging
|
|
|
|
| 5 |
from datetime import datetime
|
| 6 |
|
| 7 |
import numpy as np
|
| 8 |
import gradio as gr
|
| 9 |
from openai import AsyncOpenAI
|
| 10 |
from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item
|
|
|
|
| 11 |
|
| 12 |
from reachy_mini_conversation_demo.tools import (
|
| 13 |
ALL_TOOL_SPECS,
|
|
@@ -33,18 +35,18 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 33 |
)
|
| 34 |
self.deps = deps
|
| 35 |
|
| 36 |
-
self.connection = None
|
| 37 |
-
self.output_queue = asyncio.Queue()
|
| 38 |
|
| 39 |
self.last_activity_time = asyncio.get_event_loop().time()
|
| 40 |
self.start_time = asyncio.get_event_loop().time()
|
| 41 |
self.is_idle_tool_call = False
|
| 42 |
|
| 43 |
-
def copy(self):
|
| 44 |
"""Create a copy of the handler."""
|
| 45 |
return OpenaiRealtimeHandler(self.deps)
|
| 46 |
|
| 47 |
-
async def start_up(self):
|
| 48 |
"""Start the handler."""
|
| 49 |
self.client = AsyncOpenAI(api_key=config.OPENAI_API_KEY)
|
| 50 |
async with self.client.beta.realtime.connect(model=config.MODEL_NAME) as conn:
|
|
@@ -59,10 +61,10 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 59 |
},
|
| 60 |
"voice": "ballad",
|
| 61 |
"instructions": SESSION_INSTRUCTIONS,
|
| 62 |
-
"tools": ALL_TOOL_SPECS,
|
| 63 |
"tool_choice": "auto",
|
| 64 |
"temperature": 0.7,
|
| 65 |
-
}
|
| 66 |
)
|
| 67 |
|
| 68 |
# Manage event received from the openai server
|
|
@@ -70,9 +72,10 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 70 |
async for event in self.connection:
|
| 71 |
logger.debug(f"OpenAI event: {event.type}")
|
| 72 |
if event.type == "input_audio_buffer.speech_started":
|
| 73 |
-
if hasattr(self,
|
| 74 |
self._clear_queue()
|
| 75 |
-
self.deps.head_wobbler
|
|
|
|
| 76 |
self.deps.movement_manager.set_listening(True)
|
| 77 |
logger.debug("User speech started")
|
| 78 |
|
|
@@ -83,7 +86,8 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 83 |
if event.type in ("response.audio.completed", "response.completed"):
|
| 84 |
# Doesn't seem to be called
|
| 85 |
logger.debug("response completed")
|
| 86 |
-
self.deps.head_wobbler
|
|
|
|
| 87 |
|
| 88 |
if event.type == "response.created":
|
| 89 |
logger.debug("Response created")
|
|
@@ -91,7 +95,6 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 91 |
if event.type == "response.done":
|
| 92 |
# Doesn't mean the audio is done playing
|
| 93 |
logger.debug("Response done")
|
| 94 |
-
pass
|
| 95 |
|
| 96 |
if event.type == "conversation.item.input_audio_transcription.completed":
|
| 97 |
logger.debug(f"User transcript: {event.transcript}")
|
|
@@ -102,7 +105,8 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 102 |
await self.output_queue.put(AdditionalOutputs({"role": "assistant", "content": event.transcript}))
|
| 103 |
|
| 104 |
if event.type == "response.audio.delta":
|
| 105 |
-
self.deps.head_wobbler
|
|
|
|
| 106 |
self.last_activity_time = asyncio.get_event_loop().time()
|
| 107 |
logger.debug("last activity time updated to %s", self.last_activity_time)
|
| 108 |
await self.output_queue.put(
|
|
@@ -118,6 +122,10 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 118 |
args_json_str = getattr(event, "arguments", None)
|
| 119 |
call_id = getattr(event, "call_id", None)
|
| 120 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
try:
|
| 122 |
tool_result = await dispatch_tool_call(tool_name, args_json_str, self.deps)
|
| 123 |
logger.debug("Tool '%s' executed successfully", tool_name)
|
|
@@ -127,22 +135,23 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 127 |
tool_result = {"error": str(e)}
|
| 128 |
|
| 129 |
# send the tool result back
|
| 130 |
-
|
| 131 |
-
item
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
|
|
|
| 137 |
|
| 138 |
await self.output_queue.put(
|
| 139 |
AdditionalOutputs(
|
| 140 |
{
|
| 141 |
"role": "assistant",
|
| 142 |
"content": json.dumps(tool_result),
|
| 143 |
-
"metadata": {"title": "🛠️ Used tool "
|
| 144 |
},
|
| 145 |
-
)
|
| 146 |
)
|
| 147 |
|
| 148 |
if tool_name == "camera" and "b64_im" in tool_result:
|
|
@@ -157,37 +166,39 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 157 |
"role": "user",
|
| 158 |
"content": [
|
| 159 |
{
|
| 160 |
-
"type": "input_image",
|
| 161 |
"image_url": f"data:image/jpeg;base64,{b64_im}",
|
| 162 |
-
}
|
| 163 |
],
|
| 164 |
-
}
|
| 165 |
)
|
| 166 |
logger.info("Added camera image to conversation")
|
| 167 |
|
| 168 |
-
|
| 169 |
-
|
|
|
|
| 170 |
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
|
|
|
| 177 |
)
|
| 178 |
-
)
|
| 179 |
|
| 180 |
if not self.is_idle_tool_call:
|
| 181 |
await self.connection.response.create(
|
| 182 |
response={
|
| 183 |
-
"instructions": "Use the tool result just returned and answer concisely in speech."
|
| 184 |
-
}
|
| 185 |
)
|
| 186 |
else:
|
| 187 |
self.is_idle_tool_call = False
|
| 188 |
|
| 189 |
# re synchronize the head wobble after a tool call that may have taken some time
|
| 190 |
-
self.deps.head_wobbler
|
|
|
|
| 191 |
|
| 192 |
# server error
|
| 193 |
if event.type == "error":
|
|
@@ -197,7 +208,7 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 197 |
await self.output_queue.put(AdditionalOutputs({"role": "assistant", "content": f"[error] {msg}"}))
|
| 198 |
|
| 199 |
# Microphone receive
|
| 200 |
-
async def receive(self, frame: tuple[int, np.
|
| 201 |
"""Receive audio frame from the microphone and send it to the openai server."""
|
| 202 |
if not self.connection:
|
| 203 |
return
|
|
@@ -205,9 +216,9 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 205 |
array = array.squeeze()
|
| 206 |
audio_message = base64.b64encode(array.tobytes()).decode("utf-8")
|
| 207 |
# Fills the input audio buffer to be sent to the server
|
| 208 |
-
await self.connection.input_audio_buffer.append(audio=audio_message)
|
| 209 |
|
| 210 |
-
async def emit(self) -> tuple[int, np.
|
| 211 |
"""Emit audio frame to be played by the speaker."""
|
| 212 |
# sends to the stream the stuff put in the output queue by the openai event handler
|
| 213 |
# This is called periodically by the fastrtc Stream
|
|
@@ -219,7 +230,7 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 219 |
|
| 220 |
self.last_activity_time = asyncio.get_event_loop().time() # avoid repeated resets
|
| 221 |
|
| 222 |
-
return await wait_for_item(self.output_queue)
|
| 223 |
|
| 224 |
async def shutdown(self) -> None:
|
| 225 |
"""Shutdown the handler."""
|
|
@@ -227,7 +238,7 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 227 |
await self.connection.close()
|
| 228 |
self.connection = None
|
| 229 |
|
| 230 |
-
def format_timestamp(self):
|
| 231 |
"""Format current timestamp with date, time and elapsed seconds."""
|
| 232 |
current_time = asyncio.get_event_loop().time()
|
| 233 |
elapsed_seconds = current_time - self.start_time
|
|
@@ -236,7 +247,7 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 236 |
|
| 237 |
|
| 238 |
|
| 239 |
-
async def send_idle_signal(self, idle_duration) -> None:
|
| 240 |
"""Send an idle signal to the openai server."""
|
| 241 |
logger.debug("Sending idle signal")
|
| 242 |
self.is_idle_tool_call = True
|
|
@@ -249,12 +260,12 @@ class OpenaiRealtimeHandler(AsyncStreamHandler):
|
|
| 249 |
"type": "message",
|
| 250 |
"role": "user",
|
| 251 |
"content": [{"type": "input_text", "text": timestamp_msg}],
|
| 252 |
-
}
|
| 253 |
)
|
| 254 |
await self.connection.response.create(
|
| 255 |
response={
|
| 256 |
"modalities": ["text"],
|
| 257 |
"instructions": "You MUST respond with function calls only - no speech or text. Choose appropriate actions for idle behavior.",
|
| 258 |
"tool_choice": "required",
|
| 259 |
-
}
|
| 260 |
)
|
|
|
|
| 2 |
import base64
|
| 3 |
import asyncio
|
| 4 |
import logging
|
| 5 |
+
from typing import Any
|
| 6 |
from datetime import datetime
|
| 7 |
|
| 8 |
import numpy as np
|
| 9 |
import gradio as gr
|
| 10 |
from openai import AsyncOpenAI
|
| 11 |
from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item
|
| 12 |
+
from numpy.typing import NDArray
|
| 13 |
|
| 14 |
from reachy_mini_conversation_demo.tools import (
|
| 15 |
ALL_TOOL_SPECS,
|
|
|
|
| 35 |
)
|
| 36 |
self.deps = deps
|
| 37 |
|
| 38 |
+
self.connection: Any | None = None
|
| 39 |
+
self.output_queue: asyncio.Queue[tuple[int, NDArray[np.int16]] | AdditionalOutputs] = asyncio.Queue()
|
| 40 |
|
| 41 |
self.last_activity_time = asyncio.get_event_loop().time()
|
| 42 |
self.start_time = asyncio.get_event_loop().time()
|
| 43 |
self.is_idle_tool_call = False
|
| 44 |
|
| 45 |
+
def copy(self) -> "OpenaiRealtimeHandler":
|
| 46 |
"""Create a copy of the handler."""
|
| 47 |
return OpenaiRealtimeHandler(self.deps)
|
| 48 |
|
| 49 |
+
async def start_up(self) -> None:
|
| 50 |
"""Start the handler."""
|
| 51 |
self.client = AsyncOpenAI(api_key=config.OPENAI_API_KEY)
|
| 52 |
async with self.client.beta.realtime.connect(model=config.MODEL_NAME) as conn:
|
|
|
|
| 61 |
},
|
| 62 |
"voice": "ballad",
|
| 63 |
"instructions": SESSION_INSTRUCTIONS,
|
| 64 |
+
"tools": ALL_TOOL_SPECS, # type: ignore[typeddict-item]
|
| 65 |
"tool_choice": "auto",
|
| 66 |
"temperature": 0.7,
|
| 67 |
+
},
|
| 68 |
)
|
| 69 |
|
| 70 |
# Manage event received from the openai server
|
|
|
|
| 72 |
async for event in self.connection:
|
| 73 |
logger.debug(f"OpenAI event: {event.type}")
|
| 74 |
if event.type == "input_audio_buffer.speech_started":
|
| 75 |
+
if hasattr(self, "_clear_queue") and callable(self._clear_queue):
|
| 76 |
self._clear_queue()
|
| 77 |
+
if self.deps.head_wobbler is not None:
|
| 78 |
+
self.deps.head_wobbler.reset()
|
| 79 |
self.deps.movement_manager.set_listening(True)
|
| 80 |
logger.debug("User speech started")
|
| 81 |
|
|
|
|
| 86 |
if event.type in ("response.audio.completed", "response.completed"):
|
| 87 |
# Doesn't seem to be called
|
| 88 |
logger.debug("response completed")
|
| 89 |
+
if self.deps.head_wobbler is not None:
|
| 90 |
+
self.deps.head_wobbler.reset()
|
| 91 |
|
| 92 |
if event.type == "response.created":
|
| 93 |
logger.debug("Response created")
|
|
|
|
| 95 |
if event.type == "response.done":
|
| 96 |
# Doesn't mean the audio is done playing
|
| 97 |
logger.debug("Response done")
|
|
|
|
| 98 |
|
| 99 |
if event.type == "conversation.item.input_audio_transcription.completed":
|
| 100 |
logger.debug(f"User transcript: {event.transcript}")
|
|
|
|
| 105 |
await self.output_queue.put(AdditionalOutputs({"role": "assistant", "content": event.transcript}))
|
| 106 |
|
| 107 |
if event.type == "response.audio.delta":
|
| 108 |
+
if self.deps.head_wobbler is not None:
|
| 109 |
+
self.deps.head_wobbler.feed(event.delta)
|
| 110 |
self.last_activity_time = asyncio.get_event_loop().time()
|
| 111 |
logger.debug("last activity time updated to %s", self.last_activity_time)
|
| 112 |
await self.output_queue.put(
|
|
|
|
| 122 |
args_json_str = getattr(event, "arguments", None)
|
| 123 |
call_id = getattr(event, "call_id", None)
|
| 124 |
|
| 125 |
+
if not isinstance(tool_name, str) or not isinstance(args_json_str, str):
|
| 126 |
+
logger.error("Invalid tool call: tool_name=%s, args=%s", tool_name, args_json_str)
|
| 127 |
+
continue
|
| 128 |
+
|
| 129 |
try:
|
| 130 |
tool_result = await dispatch_tool_call(tool_name, args_json_str, self.deps)
|
| 131 |
logger.debug("Tool '%s' executed successfully", tool_name)
|
|
|
|
| 135 |
tool_result = {"error": str(e)}
|
| 136 |
|
| 137 |
# send the tool result back
|
| 138 |
+
if isinstance(call_id, str):
|
| 139 |
+
await self.connection.conversation.item.create(
|
| 140 |
+
item={
|
| 141 |
+
"type": "function_call_output",
|
| 142 |
+
"call_id": call_id,
|
| 143 |
+
"output": json.dumps(tool_result),
|
| 144 |
+
},
|
| 145 |
+
)
|
| 146 |
|
| 147 |
await self.output_queue.put(
|
| 148 |
AdditionalOutputs(
|
| 149 |
{
|
| 150 |
"role": "assistant",
|
| 151 |
"content": json.dumps(tool_result),
|
| 152 |
+
"metadata": {"title": f"🛠️ Used tool {tool_name}", "status": "done"},
|
| 153 |
},
|
| 154 |
+
),
|
| 155 |
)
|
| 156 |
|
| 157 |
if tool_name == "camera" and "b64_im" in tool_result:
|
|
|
|
| 166 |
"role": "user",
|
| 167 |
"content": [
|
| 168 |
{
|
| 169 |
+
"type": "input_image", # type: ignore[typeddict-item]
|
| 170 |
"image_url": f"data:image/jpeg;base64,{b64_im}",
|
| 171 |
+
},
|
| 172 |
],
|
| 173 |
+
},
|
| 174 |
)
|
| 175 |
logger.info("Added camera image to conversation")
|
| 176 |
|
| 177 |
+
if self.deps.camera_worker is not None:
|
| 178 |
+
np_img = self.deps.camera_worker.get_latest_frame()
|
| 179 |
+
img = gr.Image(value=np_img)
|
| 180 |
|
| 181 |
+
await self.output_queue.put(
|
| 182 |
+
AdditionalOutputs(
|
| 183 |
+
{
|
| 184 |
+
"role": "assistant",
|
| 185 |
+
"content": img,
|
| 186 |
+
},
|
| 187 |
+
),
|
| 188 |
)
|
|
|
|
| 189 |
|
| 190 |
if not self.is_idle_tool_call:
|
| 191 |
await self.connection.response.create(
|
| 192 |
response={
|
| 193 |
+
"instructions": "Use the tool result just returned and answer concisely in speech.",
|
| 194 |
+
},
|
| 195 |
)
|
| 196 |
else:
|
| 197 |
self.is_idle_tool_call = False
|
| 198 |
|
| 199 |
# re synchronize the head wobble after a tool call that may have taken some time
|
| 200 |
+
if self.deps.head_wobbler is not None:
|
| 201 |
+
self.deps.head_wobbler.reset()
|
| 202 |
|
| 203 |
# server error
|
| 204 |
if event.type == "error":
|
|
|
|
| 208 |
await self.output_queue.put(AdditionalOutputs({"role": "assistant", "content": f"[error] {msg}"}))
|
| 209 |
|
| 210 |
# Microphone receive
|
| 211 |
+
async def receive(self, frame: tuple[int, NDArray[np.int16]]) -> None:
|
| 212 |
"""Receive audio frame from the microphone and send it to the openai server."""
|
| 213 |
if not self.connection:
|
| 214 |
return
|
|
|
|
| 216 |
array = array.squeeze()
|
| 217 |
audio_message = base64.b64encode(array.tobytes()).decode("utf-8")
|
| 218 |
# Fills the input audio buffer to be sent to the server
|
| 219 |
+
await self.connection.input_audio_buffer.append(audio=audio_message)
|
| 220 |
|
| 221 |
+
async def emit(self) -> tuple[int, NDArray[np.int16]] | AdditionalOutputs | None:
|
| 222 |
"""Emit audio frame to be played by the speaker."""
|
| 223 |
# sends to the stream the stuff put in the output queue by the openai event handler
|
| 224 |
# This is called periodically by the fastrtc Stream
|
|
|
|
| 230 |
|
| 231 |
self.last_activity_time = asyncio.get_event_loop().time() # avoid repeated resets
|
| 232 |
|
| 233 |
+
return await wait_for_item(self.output_queue) # type: ignore[no-any-return]
|
| 234 |
|
| 235 |
async def shutdown(self) -> None:
|
| 236 |
"""Shutdown the handler."""
|
|
|
|
| 238 |
await self.connection.close()
|
| 239 |
self.connection = None
|
| 240 |
|
| 241 |
+
def format_timestamp(self) -> str:
|
| 242 |
"""Format current timestamp with date, time and elapsed seconds."""
|
| 243 |
current_time = asyncio.get_event_loop().time()
|
| 244 |
elapsed_seconds = current_time - self.start_time
|
|
|
|
| 247 |
|
| 248 |
|
| 249 |
|
| 250 |
+
async def send_idle_signal(self, idle_duration: float) -> None:
|
| 251 |
"""Send an idle signal to the openai server."""
|
| 252 |
logger.debug("Sending idle signal")
|
| 253 |
self.is_idle_tool_call = True
|
|
|
|
| 260 |
"type": "message",
|
| 261 |
"role": "user",
|
| 262 |
"content": [{"type": "input_text", "text": timestamp_msg}],
|
| 263 |
+
},
|
| 264 |
)
|
| 265 |
await self.connection.response.create(
|
| 266 |
response={
|
| 267 |
"modalities": ["text"],
|
| 268 |
"instructions": "You MUST respond with function calls only - no speech or text. Choose appropriate actions for idle behavior.",
|
| 269 |
"tool_choice": "required",
|
| 270 |
+
},
|
| 271 |
)
|
src/reachy_mini_conversation_demo/tools.py
CHANGED
|
@@ -4,7 +4,7 @@ import json
|
|
| 4 |
import asyncio
|
| 5 |
import inspect
|
| 6 |
import logging
|
| 7 |
-
from typing import Any,
|
| 8 |
from dataclasses import dataclass
|
| 9 |
|
| 10 |
from reachy_mini import ReachyMini
|
|
@@ -36,9 +36,9 @@ except ImportError as e:
|
|
| 36 |
EMOTION_AVAILABLE = False
|
| 37 |
|
| 38 |
|
| 39 |
-
def get_concrete_subclasses(base):
|
| 40 |
"""Recursively find all concrete (non-abstract) subclasses of a base class."""
|
| 41 |
-
result = []
|
| 42 |
for cls in base.__subclasses__():
|
| 43 |
if not inspect.isabstract(cls):
|
| 44 |
result.append(cls)
|
|
@@ -58,9 +58,9 @@ class ToolDependencies:
|
|
| 58 |
reachy_mini: ReachyMini
|
| 59 |
movement_manager: Any # MovementManager from moves.py
|
| 60 |
# Optional deps
|
| 61 |
-
camera_worker:
|
| 62 |
-
vision_manager:
|
| 63 |
-
head_wobbler:
|
| 64 |
motion_duration_s: float = 1.0
|
| 65 |
|
| 66 |
|
|
@@ -76,9 +76,9 @@ class Tool(abc.ABC):
|
|
| 76 |
|
| 77 |
name: str
|
| 78 |
description: str
|
| 79 |
-
parameters_schema:
|
| 80 |
|
| 81 |
-
def spec(self) ->
|
| 82 |
"""Return the function spec for LLM consumption."""
|
| 83 |
return {
|
| 84 |
"type": "function",
|
|
@@ -88,7 +88,7 @@ class Tool(abc.ABC):
|
|
| 88 |
}
|
| 89 |
|
| 90 |
@abc.abstractmethod
|
| 91 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 92 |
"""Async tool execution entrypoint."""
|
| 93 |
raise NotImplementedError
|
| 94 |
|
|
@@ -121,9 +121,12 @@ class MoveHead(Tool):
|
|
| 121 |
"front": (0, 0, 0, 0, 0, 0),
|
| 122 |
}
|
| 123 |
|
| 124 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 125 |
"""Move head in a given direction."""
|
| 126 |
-
|
|
|
|
|
|
|
|
|
|
| 127 |
logger.info("Tool call: move_head direction=%s", direction)
|
| 128 |
|
| 129 |
deltas = self.DELTAS.get(direction, self.DELTAS["front"])
|
|
@@ -177,7 +180,7 @@ class Camera(Tool):
|
|
| 177 |
"required": ["question"],
|
| 178 |
}
|
| 179 |
|
| 180 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 181 |
"""Take a picture with the camera and ask a question about it."""
|
| 182 |
image_query = (kwargs.get("question") or "").strip()
|
| 183 |
if not image_query:
|
|
@@ -199,7 +202,7 @@ class Camera(Tool):
|
|
| 199 |
# Use vision manager for processing if available
|
| 200 |
if deps.vision_manager is not None:
|
| 201 |
vision_result = await asyncio.to_thread(
|
| 202 |
-
deps.vision_manager.processor.process_image, frame, image_query
|
| 203 |
)
|
| 204 |
if isinstance(vision_result, dict) and "error" in vision_result:
|
| 205 |
return vision_result
|
|
@@ -208,17 +211,16 @@ class Camera(Tool):
|
|
| 208 |
if isinstance(vision_result, str)
|
| 209 |
else {"error": "vision returned non-string"}
|
| 210 |
)
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
import base64
|
| 214 |
|
| 215 |
-
|
| 216 |
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
|
| 223 |
|
| 224 |
class HeadTracking(Tool):
|
|
@@ -232,7 +234,7 @@ class HeadTracking(Tool):
|
|
| 232 |
"required": ["start"],
|
| 233 |
}
|
| 234 |
|
| 235 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 236 |
"""Enable or disable head tracking."""
|
| 237 |
enable = bool(kwargs.get("start"))
|
| 238 |
|
|
@@ -288,12 +290,12 @@ class Dance(Tool):
|
|
| 288 |
"required": [],
|
| 289 |
}
|
| 290 |
|
| 291 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 292 |
"""Play a named or random dance move once (or repeat). Non-blocking."""
|
| 293 |
if not DANCE_AVAILABLE:
|
| 294 |
return {"error": "Dance system not available"}
|
| 295 |
|
| 296 |
-
move_name = kwargs.get("move"
|
| 297 |
repeat = int(kwargs.get("repeat", 1))
|
| 298 |
|
| 299 |
logger.info("Tool call: dance move=%s repeat=%d", move_name, repeat)
|
|
@@ -326,12 +328,12 @@ class StopDance(Tool):
|
|
| 326 |
"dummy": {
|
| 327 |
"type": "boolean",
|
| 328 |
"description": "dummy boolean, set it to true",
|
| 329 |
-
}
|
| 330 |
},
|
| 331 |
"required": ["dummy"],
|
| 332 |
}
|
| 333 |
|
| 334 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 335 |
"""Stop the current dance move."""
|
| 336 |
logger.info("Tool call: stop_dance")
|
| 337 |
movement_manager = deps.movement_manager
|
|
@@ -373,7 +375,7 @@ class PlayEmotion(Tool):
|
|
| 373 |
"required": ["emotion"],
|
| 374 |
}
|
| 375 |
|
| 376 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 377 |
"""Play a pre-recorded emotion."""
|
| 378 |
if not EMOTION_AVAILABLE:
|
| 379 |
return {"error": "Emotion system not available"}
|
|
@@ -399,7 +401,7 @@ class PlayEmotion(Tool):
|
|
| 399 |
|
| 400 |
except Exception as e:
|
| 401 |
logger.exception("Failed to play emotion")
|
| 402 |
-
return {"error": f"Failed to play emotion: {
|
| 403 |
|
| 404 |
|
| 405 |
class StopEmotion(Tool):
|
|
@@ -413,12 +415,12 @@ class StopEmotion(Tool):
|
|
| 413 |
"dummy": {
|
| 414 |
"type": "boolean",
|
| 415 |
"description": "dummy boolean, set it to true",
|
| 416 |
-
}
|
| 417 |
},
|
| 418 |
"required": ["dummy"],
|
| 419 |
}
|
| 420 |
|
| 421 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 422 |
"""Stop the current emotion."""
|
| 423 |
logger.info("Tool call: stop_emotion")
|
| 424 |
movement_manager = deps.movement_manager
|
|
@@ -442,7 +444,7 @@ class DoNothing(Tool):
|
|
| 442 |
"required": [],
|
| 443 |
}
|
| 444 |
|
| 445 |
-
async def __call__(self, deps: ToolDependencies, **kwargs) ->
|
| 446 |
"""Do nothing - stay still and silent."""
|
| 447 |
reason = kwargs.get("reason", "just chilling")
|
| 448 |
logger.info("Tool call: do_nothing reason=%s", reason)
|
|
@@ -452,7 +454,7 @@ class DoNothing(Tool):
|
|
| 452 |
# Registry & specs (dynamic)
|
| 453 |
|
| 454 |
# List of available tool classes
|
| 455 |
-
ALL_TOOLS:
|
| 456 |
ALL_TOOL_SPECS = [tool.spec() for tool in ALL_TOOLS.values()]
|
| 457 |
|
| 458 |
|
|
@@ -466,7 +468,7 @@ def _safe_load_obj(args_json: str) -> dict[str, Any]:
|
|
| 466 |
return {}
|
| 467 |
|
| 468 |
|
| 469 |
-
async def dispatch_tool_call(tool_name: str, args_json: str, deps: ToolDependencies) ->
|
| 470 |
"""Dispatch a tool call by name with JSON args and dependencies."""
|
| 471 |
tool = ALL_TOOLS.get(tool_name)
|
| 472 |
|
|
|
|
| 4 |
import asyncio
|
| 5 |
import inspect
|
| 6 |
import logging
|
| 7 |
+
from typing import Any, Literal
|
| 8 |
from dataclasses import dataclass
|
| 9 |
|
| 10 |
from reachy_mini import ReachyMini
|
|
|
|
| 36 |
EMOTION_AVAILABLE = False
|
| 37 |
|
| 38 |
|
| 39 |
+
def get_concrete_subclasses(base: type[Tool]) -> list[type[Tool]]:
|
| 40 |
"""Recursively find all concrete (non-abstract) subclasses of a base class."""
|
| 41 |
+
result: list[type[Tool]] = []
|
| 42 |
for cls in base.__subclasses__():
|
| 43 |
if not inspect.isabstract(cls):
|
| 44 |
result.append(cls)
|
|
|
|
| 58 |
reachy_mini: ReachyMini
|
| 59 |
movement_manager: Any # MovementManager from moves.py
|
| 60 |
# Optional deps
|
| 61 |
+
camera_worker: Any | None = None # CameraWorker for frame buffering
|
| 62 |
+
vision_manager: Any | None = None
|
| 63 |
+
head_wobbler: Any | None = None # HeadWobbler for audio-reactive motion
|
| 64 |
motion_duration_s: float = 1.0
|
| 65 |
|
| 66 |
|
|
|
|
| 76 |
|
| 77 |
name: str
|
| 78 |
description: str
|
| 79 |
+
parameters_schema: dict[str, Any]
|
| 80 |
|
| 81 |
+
def spec(self) -> dict[str, Any]:
|
| 82 |
"""Return the function spec for LLM consumption."""
|
| 83 |
return {
|
| 84 |
"type": "function",
|
|
|
|
| 88 |
}
|
| 89 |
|
| 90 |
@abc.abstractmethod
|
| 91 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 92 |
"""Async tool execution entrypoint."""
|
| 93 |
raise NotImplementedError
|
| 94 |
|
|
|
|
| 121 |
"front": (0, 0, 0, 0, 0, 0),
|
| 122 |
}
|
| 123 |
|
| 124 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 125 |
"""Move head in a given direction."""
|
| 126 |
+
direction_raw = kwargs.get("direction")
|
| 127 |
+
if not isinstance(direction_raw, str):
|
| 128 |
+
return {"error": "direction must be a string"}
|
| 129 |
+
direction: Direction = direction_raw # type: ignore[assignment]
|
| 130 |
logger.info("Tool call: move_head direction=%s", direction)
|
| 131 |
|
| 132 |
deltas = self.DELTAS.get(direction, self.DELTAS["front"])
|
|
|
|
| 180 |
"required": ["question"],
|
| 181 |
}
|
| 182 |
|
| 183 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 184 |
"""Take a picture with the camera and ask a question about it."""
|
| 185 |
image_query = (kwargs.get("question") or "").strip()
|
| 186 |
if not image_query:
|
|
|
|
| 202 |
# Use vision manager for processing if available
|
| 203 |
if deps.vision_manager is not None:
|
| 204 |
vision_result = await asyncio.to_thread(
|
| 205 |
+
deps.vision_manager.processor.process_image, frame, image_query,
|
| 206 |
)
|
| 207 |
if isinstance(vision_result, dict) and "error" in vision_result:
|
| 208 |
return vision_result
|
|
|
|
| 211 |
if isinstance(vision_result, str)
|
| 212 |
else {"error": "vision returned non-string"}
|
| 213 |
)
|
| 214 |
+
# Return base64 encoded image like main_works.py camera tool
|
| 215 |
+
import base64
|
|
|
|
| 216 |
|
| 217 |
+
import cv2
|
| 218 |
|
| 219 |
+
temp_path = "/tmp/camera_frame.jpg"
|
| 220 |
+
cv2.imwrite(temp_path, frame)
|
| 221 |
+
with open(temp_path, "rb") as f:
|
| 222 |
+
b64_encoded = base64.b64encode(f.read()).decode("utf-8")
|
| 223 |
+
return {"b64_im": b64_encoded}
|
| 224 |
|
| 225 |
|
| 226 |
class HeadTracking(Tool):
|
|
|
|
| 234 |
"required": ["start"],
|
| 235 |
}
|
| 236 |
|
| 237 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 238 |
"""Enable or disable head tracking."""
|
| 239 |
enable = bool(kwargs.get("start"))
|
| 240 |
|
|
|
|
| 290 |
"required": [],
|
| 291 |
}
|
| 292 |
|
| 293 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 294 |
"""Play a named or random dance move once (or repeat). Non-blocking."""
|
| 295 |
if not DANCE_AVAILABLE:
|
| 296 |
return {"error": "Dance system not available"}
|
| 297 |
|
| 298 |
+
move_name = kwargs.get("move")
|
| 299 |
repeat = int(kwargs.get("repeat", 1))
|
| 300 |
|
| 301 |
logger.info("Tool call: dance move=%s repeat=%d", move_name, repeat)
|
|
|
|
| 328 |
"dummy": {
|
| 329 |
"type": "boolean",
|
| 330 |
"description": "dummy boolean, set it to true",
|
| 331 |
+
},
|
| 332 |
},
|
| 333 |
"required": ["dummy"],
|
| 334 |
}
|
| 335 |
|
| 336 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 337 |
"""Stop the current dance move."""
|
| 338 |
logger.info("Tool call: stop_dance")
|
| 339 |
movement_manager = deps.movement_manager
|
|
|
|
| 375 |
"required": ["emotion"],
|
| 376 |
}
|
| 377 |
|
| 378 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 379 |
"""Play a pre-recorded emotion."""
|
| 380 |
if not EMOTION_AVAILABLE:
|
| 381 |
return {"error": "Emotion system not available"}
|
|
|
|
| 401 |
|
| 402 |
except Exception as e:
|
| 403 |
logger.exception("Failed to play emotion")
|
| 404 |
+
return {"error": f"Failed to play emotion: {e!s}"}
|
| 405 |
|
| 406 |
|
| 407 |
class StopEmotion(Tool):
|
|
|
|
| 415 |
"dummy": {
|
| 416 |
"type": "boolean",
|
| 417 |
"description": "dummy boolean, set it to true",
|
| 418 |
+
},
|
| 419 |
},
|
| 420 |
"required": ["dummy"],
|
| 421 |
}
|
| 422 |
|
| 423 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 424 |
"""Stop the current emotion."""
|
| 425 |
logger.info("Tool call: stop_emotion")
|
| 426 |
movement_manager = deps.movement_manager
|
|
|
|
| 444 |
"required": [],
|
| 445 |
}
|
| 446 |
|
| 447 |
+
async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> dict[str, Any]:
|
| 448 |
"""Do nothing - stay still and silent."""
|
| 449 |
reason = kwargs.get("reason", "just chilling")
|
| 450 |
logger.info("Tool call: do_nothing reason=%s", reason)
|
|
|
|
| 454 |
# Registry & specs (dynamic)
|
| 455 |
|
| 456 |
# List of available tool classes
|
| 457 |
+
ALL_TOOLS: dict[str, Tool] = {cls.name: cls() for cls in get_concrete_subclasses(Tool)} # type: ignore[type-abstract]
|
| 458 |
ALL_TOOL_SPECS = [tool.spec() for tool in ALL_TOOLS.values()]
|
| 459 |
|
| 460 |
|
|
|
|
| 468 |
return {}
|
| 469 |
|
| 470 |
|
| 471 |
+
async def dispatch_tool_call(tool_name: str, args_json: str, deps: ToolDependencies) -> dict[str, Any]:
|
| 472 |
"""Dispatch a tool call by name with JSON args and dependencies."""
|
| 473 |
tool = ALL_TOOLS.get(tool_name)
|
| 474 |
|
src/reachy_mini_conversation_demo/utils.py
CHANGED
|
@@ -1,11 +1,13 @@
|
|
| 1 |
import logging
|
| 2 |
import argparse
|
| 3 |
import warnings
|
|
|
|
| 4 |
|
|
|
|
| 5 |
from reachy_mini_conversation_demo.camera_worker import CameraWorker
|
| 6 |
|
| 7 |
|
| 8 |
-
def parse_args():
|
| 9 |
"""Parse command line arguments."""
|
| 10 |
parser = argparse.ArgumentParser("Reachy Mini Conversation Demo")
|
| 11 |
parser.add_argument(
|
|
@@ -26,7 +28,7 @@ def parse_args():
|
|
| 26 |
return parser.parse_args()
|
| 27 |
|
| 28 |
|
| 29 |
-
def handle_vision_stuff(args, current_robot):
|
| 30 |
"""Initialize camera, head tracker, camera worker, and vision manager.
|
| 31 |
|
| 32 |
By default, vision is handled by gpt-realtime model when camera tool is used.
|
|
@@ -44,7 +46,7 @@ def handle_vision_stuff(args, current_robot):
|
|
| 44 |
|
| 45 |
head_tracker = HeadTracker()
|
| 46 |
elif args.head_tracker == "mediapipe":
|
| 47 |
-
from reachy_mini_toolbox.vision import HeadTracker
|
| 48 |
|
| 49 |
head_tracker = HeadTracker()
|
| 50 |
|
|
@@ -59,17 +61,17 @@ def handle_vision_stuff(args, current_robot):
|
|
| 59 |
vision_manager = initialize_vision_manager(camera_worker)
|
| 60 |
except ImportError as e:
|
| 61 |
raise ImportError(
|
| 62 |
-
"To use --local-vision, please install the extra dependencies: pip install '.[local_vision]'"
|
| 63 |
) from e
|
| 64 |
else:
|
| 65 |
logging.getLogger(__name__).info(
|
| 66 |
-
"Using gpt-realtime for vision (default). Use --local-vision for local processing."
|
| 67 |
)
|
| 68 |
|
| 69 |
return camera_worker, head_tracker, vision_manager
|
| 70 |
|
| 71 |
|
| 72 |
-
def setup_logger(debug):
|
| 73 |
"""Setups the logger."""
|
| 74 |
log_level = "DEBUG" if debug else "INFO"
|
| 75 |
logging.basicConfig(
|
|
|
|
| 1 |
import logging
|
| 2 |
import argparse
|
| 3 |
import warnings
|
| 4 |
+
from typing import Any
|
| 5 |
|
| 6 |
+
from reachy_mini import ReachyMini
|
| 7 |
from reachy_mini_conversation_demo.camera_worker import CameraWorker
|
| 8 |
|
| 9 |
|
| 10 |
+
def parse_args() -> argparse.Namespace:
|
| 11 |
"""Parse command line arguments."""
|
| 12 |
parser = argparse.ArgumentParser("Reachy Mini Conversation Demo")
|
| 13 |
parser.add_argument(
|
|
|
|
| 28 |
return parser.parse_args()
|
| 29 |
|
| 30 |
|
| 31 |
+
def handle_vision_stuff(args: argparse.Namespace, current_robot: ReachyMini) -> tuple[CameraWorker | None, Any, Any]:
|
| 32 |
"""Initialize camera, head tracker, camera worker, and vision manager.
|
| 33 |
|
| 34 |
By default, vision is handled by gpt-realtime model when camera tool is used.
|
|
|
|
| 46 |
|
| 47 |
head_tracker = HeadTracker()
|
| 48 |
elif args.head_tracker == "mediapipe":
|
| 49 |
+
from reachy_mini_toolbox.vision import HeadTracker # type: ignore[no-redef]
|
| 50 |
|
| 51 |
head_tracker = HeadTracker()
|
| 52 |
|
|
|
|
| 61 |
vision_manager = initialize_vision_manager(camera_worker)
|
| 62 |
except ImportError as e:
|
| 63 |
raise ImportError(
|
| 64 |
+
"To use --local-vision, please install the extra dependencies: pip install '.[local_vision]'",
|
| 65 |
) from e
|
| 66 |
else:
|
| 67 |
logging.getLogger(__name__).info(
|
| 68 |
+
"Using gpt-realtime for vision (default). Use --local-vision for local processing.",
|
| 69 |
)
|
| 70 |
|
| 71 |
return camera_worker, head_tracker, vision_manager
|
| 72 |
|
| 73 |
|
| 74 |
+
def setup_logger(debug: bool) -> logging.Logger:
|
| 75 |
"""Setups the logger."""
|
| 76 |
log_level = "DEBUG" if debug else "INFO"
|
| 77 |
logging.basicConfig(
|
src/reachy_mini_conversation_demo/vision/processors.py
CHANGED
|
@@ -3,7 +3,7 @@ import time
|
|
| 3 |
import base64
|
| 4 |
import logging
|
| 5 |
import threading
|
| 6 |
-
from typing import Any
|
| 7 |
from dataclasses import dataclass
|
| 8 |
|
| 9 |
import cv2
|
|
@@ -34,7 +34,7 @@ class VisionConfig:
|
|
| 34 |
class VisionProcessor:
|
| 35 |
"""Handles SmolVLM2 model loading and inference."""
|
| 36 |
|
| 37 |
-
def __init__(self, vision_config: VisionConfig = None):
|
| 38 |
"""Initialize the vision processor."""
|
| 39 |
self.vision_config = vision_config or VisionConfig()
|
| 40 |
self.model_path = self.vision_config.model_path
|
|
@@ -60,7 +60,7 @@ class VisionProcessor:
|
|
| 60 |
"""Load model and processor onto the selected device."""
|
| 61 |
try:
|
| 62 |
logger.info(f"Loading SmolVLM2 model on {self.device} (HF_HOME={config.HF_HOME})")
|
| 63 |
-
self.processor = AutoProcessor.from_pretrained(self.model_path)
|
| 64 |
|
| 65 |
# Select dtype depending on device
|
| 66 |
if self.device == "cuda":
|
|
@@ -74,12 +74,13 @@ class VisionProcessor:
|
|
| 74 |
|
| 75 |
# flash_attention_2 is CUDA-only; skip on MPS/CPU
|
| 76 |
if self.device == "cuda":
|
| 77 |
-
model_kwargs["_attn_implementation"] = "flash_attention_2"
|
| 78 |
|
| 79 |
# Load model weights
|
| 80 |
-
self.model = AutoModelForImageTextToText.from_pretrained(self.model_path, **model_kwargs).to(self.device)
|
| 81 |
|
| 82 |
-
self.model
|
|
|
|
| 83 |
self._initialized = True
|
| 84 |
return True
|
| 85 |
|
|
@@ -89,11 +90,11 @@ class VisionProcessor:
|
|
| 89 |
|
| 90 |
def process_image(
|
| 91 |
self,
|
| 92 |
-
cv2_image: np.ndarray,
|
| 93 |
prompt: str = "Briefly describe what you see in one sentence.",
|
| 94 |
) -> str:
|
| 95 |
"""Process CV2 image and return description with retry logic."""
|
| 96 |
-
if not self._initialized:
|
| 97 |
return "Vision model not initialized"
|
| 98 |
|
| 99 |
for attempt in range(self.vision_config.max_retries):
|
|
@@ -189,7 +190,7 @@ class VisionProcessor:
|
|
| 189 |
# Fallback: return the full text cleaned up
|
| 190 |
return full_text.strip()
|
| 191 |
|
| 192 |
-
def get_model_info(self) ->
|
| 193 |
"""Get information about the loaded model."""
|
| 194 |
return {
|
| 195 |
"initialized": self._initialized,
|
|
@@ -205,16 +206,16 @@ class VisionProcessor:
|
|
| 205 |
class VisionManager:
|
| 206 |
"""Manages periodic vision processing and scene understanding."""
|
| 207 |
|
| 208 |
-
def __init__(self, camera, vision_config: VisionConfig = None):
|
| 209 |
"""Initialize vision manager with camera and configuration."""
|
| 210 |
self.camera = camera
|
| 211 |
self.vision_config = vision_config or VisionConfig()
|
| 212 |
self.vision_interval = self.vision_config.vision_interval
|
| 213 |
self.processor = VisionProcessor(self.vision_config)
|
| 214 |
|
| 215 |
-
self._last_processed_time = 0
|
| 216 |
self._stop_event = threading.Event()
|
| 217 |
-
self._thread:
|
| 218 |
|
| 219 |
# Initialize processor
|
| 220 |
if not self.processor.initialize():
|
|
@@ -245,7 +246,7 @@ class VisionManager:
|
|
| 245 |
frame = self.camera.get_latest_frame()
|
| 246 |
if frame is not None:
|
| 247 |
description = self.processor.process_image(
|
| 248 |
-
frame, "Briefly describe what you see in one sentence."
|
| 249 |
)
|
| 250 |
|
| 251 |
# Only update if we got a valid response
|
|
@@ -263,7 +264,7 @@ class VisionManager:
|
|
| 263 |
|
| 264 |
logger.info("Vision loop finished")
|
| 265 |
|
| 266 |
-
def get_status(self) ->
|
| 267 |
"""Get comprehensive status information."""
|
| 268 |
return {
|
| 269 |
"last_processed": self._last_processed_time,
|
|
@@ -274,7 +275,7 @@ class VisionManager:
|
|
| 274 |
}
|
| 275 |
|
| 276 |
|
| 277 |
-
def initialize_vision_manager(camera_worker) ->
|
| 278 |
"""Initialize vision manager with model download and configuration.
|
| 279 |
|
| 280 |
Args:
|
|
@@ -318,7 +319,7 @@ def initialize_vision_manager(camera_worker) -> Optional[VisionManager]:
|
|
| 318 |
# Log device info
|
| 319 |
device_info = vision_manager.processor.get_model_info()
|
| 320 |
logger.info(
|
| 321 |
-
f"Vision processing enabled: {device_info.get('model_path')} on {device_info.get('device')}"
|
| 322 |
)
|
| 323 |
|
| 324 |
return vision_manager
|
|
|
|
| 3 |
import base64
|
| 4 |
import logging
|
| 5 |
import threading
|
| 6 |
+
from typing import Any
|
| 7 |
from dataclasses import dataclass
|
| 8 |
|
| 9 |
import cv2
|
|
|
|
| 34 |
class VisionProcessor:
|
| 35 |
"""Handles SmolVLM2 model loading and inference."""
|
| 36 |
|
| 37 |
+
def __init__(self, vision_config: VisionConfig | None = None):
|
| 38 |
"""Initialize the vision processor."""
|
| 39 |
self.vision_config = vision_config or VisionConfig()
|
| 40 |
self.model_path = self.vision_config.model_path
|
|
|
|
| 60 |
"""Load model and processor onto the selected device."""
|
| 61 |
try:
|
| 62 |
logger.info(f"Loading SmolVLM2 model on {self.device} (HF_HOME={config.HF_HOME})")
|
| 63 |
+
self.processor = AutoProcessor.from_pretrained(self.model_path) # type: ignore[no-untyped-call]
|
| 64 |
|
| 65 |
# Select dtype depending on device
|
| 66 |
if self.device == "cuda":
|
|
|
|
| 74 |
|
| 75 |
# flash_attention_2 is CUDA-only; skip on MPS/CPU
|
| 76 |
if self.device == "cuda":
|
| 77 |
+
model_kwargs["_attn_implementation"] = "flash_attention_2" # type: ignore[assignment]
|
| 78 |
|
| 79 |
# Load model weights
|
| 80 |
+
self.model = AutoModelForImageTextToText.from_pretrained(self.model_path, **model_kwargs).to(self.device) # type: ignore[arg-type]
|
| 81 |
|
| 82 |
+
if self.model is not None:
|
| 83 |
+
self.model.eval()
|
| 84 |
self._initialized = True
|
| 85 |
return True
|
| 86 |
|
|
|
|
| 90 |
|
| 91 |
def process_image(
|
| 92 |
self,
|
| 93 |
+
cv2_image: np.ndarray[Any, Any],
|
| 94 |
prompt: str = "Briefly describe what you see in one sentence.",
|
| 95 |
) -> str:
|
| 96 |
"""Process CV2 image and return description with retry logic."""
|
| 97 |
+
if not self._initialized or self.processor is None or self.model is None:
|
| 98 |
return "Vision model not initialized"
|
| 99 |
|
| 100 |
for attempt in range(self.vision_config.max_retries):
|
|
|
|
| 190 |
# Fallback: return the full text cleaned up
|
| 191 |
return full_text.strip()
|
| 192 |
|
| 193 |
+
def get_model_info(self) -> dict[str, Any]:
|
| 194 |
"""Get information about the loaded model."""
|
| 195 |
return {
|
| 196 |
"initialized": self._initialized,
|
|
|
|
| 206 |
class VisionManager:
|
| 207 |
"""Manages periodic vision processing and scene understanding."""
|
| 208 |
|
| 209 |
+
def __init__(self, camera: Any, vision_config: VisionConfig | None = None):
|
| 210 |
"""Initialize vision manager with camera and configuration."""
|
| 211 |
self.camera = camera
|
| 212 |
self.vision_config = vision_config or VisionConfig()
|
| 213 |
self.vision_interval = self.vision_config.vision_interval
|
| 214 |
self.processor = VisionProcessor(self.vision_config)
|
| 215 |
|
| 216 |
+
self._last_processed_time = 0.0
|
| 217 |
self._stop_event = threading.Event()
|
| 218 |
+
self._thread: threading.Thread | None = None
|
| 219 |
|
| 220 |
# Initialize processor
|
| 221 |
if not self.processor.initialize():
|
|
|
|
| 246 |
frame = self.camera.get_latest_frame()
|
| 247 |
if frame is not None:
|
| 248 |
description = self.processor.process_image(
|
| 249 |
+
frame, "Briefly describe what you see in one sentence.",
|
| 250 |
)
|
| 251 |
|
| 252 |
# Only update if we got a valid response
|
|
|
|
| 264 |
|
| 265 |
logger.info("Vision loop finished")
|
| 266 |
|
| 267 |
+
def get_status(self) -> dict[str, Any]:
|
| 268 |
"""Get comprehensive status information."""
|
| 269 |
return {
|
| 270 |
"last_processed": self._last_processed_time,
|
|
|
|
| 275 |
}
|
| 276 |
|
| 277 |
|
| 278 |
+
def initialize_vision_manager(camera_worker: Any) -> VisionManager | None:
|
| 279 |
"""Initialize vision manager with model download and configuration.
|
| 280 |
|
| 281 |
Args:
|
|
|
|
| 319 |
# Log device info
|
| 320 |
device_info = vision_manager.processor.get_model_info()
|
| 321 |
logger.info(
|
| 322 |
+
f"Vision processing enabled: {device_info.get('model_path')} on {device_info.get('device')}",
|
| 323 |
)
|
| 324 |
|
| 325 |
return vision_manager
|
src/reachy_mini_conversation_demo/vision/yolo_head_tracker.py
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
import logging
|
| 3 |
-
from typing import
|
| 4 |
|
| 5 |
import numpy as np
|
| 6 |
|
| 7 |
|
| 8 |
try:
|
| 9 |
from supervision import Detections
|
| 10 |
-
from ultralytics import YOLO
|
| 11 |
except ImportError as e:
|
| 12 |
raise ImportError(
|
| 13 |
-
"To use YOLO head tracker, please install the extra dependencies: pip install '.[yolo_vision]'"
|
| 14 |
) from e
|
| 15 |
from huggingface_hub import hf_hub_download
|
| 16 |
|
|
@@ -48,7 +48,7 @@ class HeadTracker:
|
|
| 48 |
logger.error(f"Failed to load YOLO model: {e}")
|
| 49 |
raise
|
| 50 |
|
| 51 |
-
def _select_best_face(self, detections: Detections) ->
|
| 52 |
"""Select the best face based on confidence and area (largest face with highest confidence).
|
| 53 |
|
| 54 |
Args:
|
|
@@ -61,6 +61,10 @@ class HeadTracker:
|
|
| 61 |
if detections.xyxy.shape[0] == 0:
|
| 62 |
return None
|
| 63 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
# Filter by confidence threshold
|
| 65 |
valid_mask = detections.confidence >= self.confidence_threshold
|
| 66 |
if not np.any(valid_mask):
|
|
@@ -78,9 +82,9 @@ class HeadTracker:
|
|
| 78 |
|
| 79 |
# Return index of best face
|
| 80 |
best_idx = valid_indices[np.argmax(scores)]
|
| 81 |
-
return best_idx
|
| 82 |
|
| 83 |
-
def _bbox_to_mp_coords(self, bbox: np.ndarray, w: int, h: int) -> np.ndarray:
|
| 84 |
"""Convert bounding box center to MediaPipe-style coordinates [-1, 1].
|
| 85 |
|
| 86 |
Args:
|
|
@@ -101,7 +105,7 @@ class HeadTracker:
|
|
| 101 |
|
| 102 |
return np.array([norm_x, norm_y], dtype=np.float32)
|
| 103 |
|
| 104 |
-
def get_head_position(self, img: np.ndarray) ->
|
| 105 |
"""Get head position from face detection.
|
| 106 |
|
| 107 |
Args:
|
|
@@ -125,9 +129,10 @@ class HeadTracker:
|
|
| 125 |
return None, None
|
| 126 |
|
| 127 |
bbox = detections.xyxy[face_idx]
|
| 128 |
-
confidence = detections.confidence[face_idx]
|
| 129 |
|
| 130 |
-
|
|
|
|
|
|
|
| 131 |
|
| 132 |
# Get face center in [-1, 1] coordinates
|
| 133 |
face_center = self._bbox_to_mp_coords(bbox, w, h)
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
import logging
|
| 3 |
+
from typing import Any
|
| 4 |
|
| 5 |
import numpy as np
|
| 6 |
|
| 7 |
|
| 8 |
try:
|
| 9 |
from supervision import Detections
|
| 10 |
+
from ultralytics import YOLO # type: ignore[attr-defined]
|
| 11 |
except ImportError as e:
|
| 12 |
raise ImportError(
|
| 13 |
+
"To use YOLO head tracker, please install the extra dependencies: pip install '.[yolo_vision]'",
|
| 14 |
) from e
|
| 15 |
from huggingface_hub import hf_hub_download
|
| 16 |
|
|
|
|
| 48 |
logger.error(f"Failed to load YOLO model: {e}")
|
| 49 |
raise
|
| 50 |
|
| 51 |
+
def _select_best_face(self, detections: Detections) -> int | None:
|
| 52 |
"""Select the best face based on confidence and area (largest face with highest confidence).
|
| 53 |
|
| 54 |
Args:
|
|
|
|
| 61 |
if detections.xyxy.shape[0] == 0:
|
| 62 |
return None
|
| 63 |
|
| 64 |
+
# Check if confidence is available
|
| 65 |
+
if detections.confidence is None:
|
| 66 |
+
return None
|
| 67 |
+
|
| 68 |
# Filter by confidence threshold
|
| 69 |
valid_mask = detections.confidence >= self.confidence_threshold
|
| 70 |
if not np.any(valid_mask):
|
|
|
|
| 82 |
|
| 83 |
# Return index of best face
|
| 84 |
best_idx = valid_indices[np.argmax(scores)]
|
| 85 |
+
return int(best_idx)
|
| 86 |
|
| 87 |
+
def _bbox_to_mp_coords(self, bbox: np.ndarray[Any, Any], w: int, h: int) -> np.ndarray[Any, Any]:
|
| 88 |
"""Convert bounding box center to MediaPipe-style coordinates [-1, 1].
|
| 89 |
|
| 90 |
Args:
|
|
|
|
| 105 |
|
| 106 |
return np.array([norm_x, norm_y], dtype=np.float32)
|
| 107 |
|
| 108 |
+
def get_head_position(self, img: np.ndarray[Any, Any]) -> tuple[np.ndarray[Any, Any] | None, float | None]:
|
| 109 |
"""Get head position from face detection.
|
| 110 |
|
| 111 |
Args:
|
|
|
|
| 129 |
return None, None
|
| 130 |
|
| 131 |
bbox = detections.xyxy[face_idx]
|
|
|
|
| 132 |
|
| 133 |
+
if detections.confidence is not None:
|
| 134 |
+
confidence = detections.confidence[face_idx]
|
| 135 |
+
logger.debug(f"Face detected with confidence: {confidence:.2f}")
|
| 136 |
|
| 137 |
# Get face center in [-1, 1] coordinates
|
| 138 |
face_center = self._bbox_to_mp_coords(bbox, w, h)
|
tests/audio/test_head_wobbler.py
CHANGED
|
@@ -4,7 +4,8 @@ import math
|
|
| 4 |
import time
|
| 5 |
import base64
|
| 6 |
import threading
|
| 7 |
-
from typing import
|
|
|
|
| 8 |
|
| 9 |
import numpy as np
|
| 10 |
|
|
@@ -31,10 +32,10 @@ def _wait_for(predicate: Callable[[], bool], timeout: float = 0.6) -> bool:
|
|
| 31 |
return False
|
| 32 |
|
| 33 |
|
| 34 |
-
def _start_wobbler() ->
|
| 35 |
-
captured:
|
| 36 |
|
| 37 |
-
def capture(offsets:
|
| 38 |
captured.append((time.time(), offsets))
|
| 39 |
|
| 40 |
wobbler = HeadWobbler(set_speech_offsets=capture)
|
|
@@ -74,7 +75,7 @@ def test_reset_allows_future_offsets() -> None:
|
|
| 74 |
wobbler.stop()
|
| 75 |
|
| 76 |
|
| 77 |
-
def test_reset_during_inflight_chunk_keeps_worker(monkeypatch) -> None:
|
| 78 |
"""Simulate reset during chunk processing to ensure the worker survives."""
|
| 79 |
wobbler, captured = _start_wobbler()
|
| 80 |
ready = threading.Event()
|
|
|
|
| 4 |
import time
|
| 5 |
import base64
|
| 6 |
import threading
|
| 7 |
+
from typing import Any
|
| 8 |
+
from collections.abc import Callable
|
| 9 |
|
| 10 |
import numpy as np
|
| 11 |
|
|
|
|
| 32 |
return False
|
| 33 |
|
| 34 |
|
| 35 |
+
def _start_wobbler() -> tuple[HeadWobbler, list[tuple[float, tuple[float, float, float, float, float, float]]]]:
|
| 36 |
+
captured: list[tuple[float, tuple[float, float, float, float, float, float]]] = []
|
| 37 |
|
| 38 |
+
def capture(offsets: tuple[float, float, float, float, float, float]) -> None:
|
| 39 |
captured.append((time.time(), offsets))
|
| 40 |
|
| 41 |
wobbler = HeadWobbler(set_speech_offsets=capture)
|
|
|
|
| 75 |
wobbler.stop()
|
| 76 |
|
| 77 |
|
| 78 |
+
def test_reset_during_inflight_chunk_keeps_worker(monkeypatch: Any) -> None:
|
| 79 |
"""Simulate reset during chunk processing to ensure the worker survives."""
|
| 80 |
wobbler, captured = _start_wobbler()
|
| 81 |
ready = threading.Event()
|
uv.lock
CHANGED
|
@@ -962,7 +962,7 @@ name = "exceptiongroup"
|
|
| 962 |
version = "1.3.0"
|
| 963 |
source = { registry = "https://pypi.org/simple" }
|
| 964 |
dependencies = [
|
| 965 |
-
{ name = "typing-extensions", marker = "python_full_version < '3.
|
| 966 |
]
|
| 967 |
sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
|
| 968 |
wheels = [
|
|
@@ -2320,6 +2320,60 @@ wheels = [
|
|
| 2320 |
{ url = "https://files.pythonhosted.org/packages/b7/da/7d22601b625e241d4f23ef1ebff8acfc60da633c9e7e7922e24d10f592b3/multidict-6.7.0-py3-none-any.whl", hash = "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", size = 12317, upload-time = "2025-10-06T14:52:29.272Z" },
|
| 2321 |
]
|
| 2322 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2323 |
[[package]]
|
| 2324 |
name = "networkx"
|
| 2325 |
version = "3.4.2"
|
|
@@ -2493,7 +2547,7 @@ name = "nvidia-cudnn-cu12"
|
|
| 2493 |
version = "9.10.2.21"
|
| 2494 |
source = { registry = "https://pypi.org/simple" }
|
| 2495 |
dependencies = [
|
| 2496 |
-
{ name = "nvidia-cublas-cu12" },
|
| 2497 |
]
|
| 2498 |
wheels = [
|
| 2499 |
{ url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" },
|
|
@@ -2504,7 +2558,7 @@ name = "nvidia-cufft-cu12"
|
|
| 2504 |
version = "11.3.3.83"
|
| 2505 |
source = { registry = "https://pypi.org/simple" }
|
| 2506 |
dependencies = [
|
| 2507 |
-
{ name = "nvidia-nvjitlink-cu12" },
|
| 2508 |
]
|
| 2509 |
wheels = [
|
| 2510 |
{ url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" },
|
|
@@ -2531,9 +2585,9 @@ name = "nvidia-cusolver-cu12"
|
|
| 2531 |
version = "11.7.3.90"
|
| 2532 |
source = { registry = "https://pypi.org/simple" }
|
| 2533 |
dependencies = [
|
| 2534 |
-
{ name = "nvidia-cublas-cu12" },
|
| 2535 |
-
{ name = "nvidia-cusparse-cu12" },
|
| 2536 |
-
{ name = "nvidia-nvjitlink-cu12" },
|
| 2537 |
]
|
| 2538 |
wheels = [
|
| 2539 |
{ url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" },
|
|
@@ -2544,7 +2598,7 @@ name = "nvidia-cusparse-cu12"
|
|
| 2544 |
version = "12.5.8.93"
|
| 2545 |
source = { registry = "https://pypi.org/simple" }
|
| 2546 |
dependencies = [
|
| 2547 |
-
{ name = "nvidia-nvjitlink-cu12" },
|
| 2548 |
]
|
| 2549 |
wheels = [
|
| 2550 |
{ url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" },
|
|
@@ -2799,6 +2853,15 @@ wheels = [
|
|
| 2799 |
{ url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" },
|
| 2800 |
]
|
| 2801 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2802 |
[[package]]
|
| 2803 |
name = "pillow"
|
| 2804 |
version = "11.3.0"
|
|
@@ -3600,6 +3663,7 @@ yolo-vision = [
|
|
| 3600 |
|
| 3601 |
[package.dev-dependencies]
|
| 3602 |
dev = [
|
|
|
|
| 3603 |
{ name = "pytest" },
|
| 3604 |
{ name = "ruff" },
|
| 3605 |
]
|
|
@@ -3630,6 +3694,7 @@ provides-extras = ["local-vision", "yolo-vision", "mediapipe-vision", "all-visio
|
|
| 3630 |
|
| 3631 |
[package.metadata.requires-dev]
|
| 3632 |
dev = [
|
|
|
|
| 3633 |
{ name = "pytest" },
|
| 3634 |
{ name = "ruff", specifier = "==0.12.0" },
|
| 3635 |
]
|
|
|
|
| 962 |
version = "1.3.0"
|
| 963 |
source = { registry = "https://pypi.org/simple" }
|
| 964 |
dependencies = [
|
| 965 |
+
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
|
| 966 |
]
|
| 967 |
sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
|
| 968 |
wheels = [
|
|
|
|
| 2320 |
{ url = "https://files.pythonhosted.org/packages/b7/da/7d22601b625e241d4f23ef1ebff8acfc60da633c9e7e7922e24d10f592b3/multidict-6.7.0-py3-none-any.whl", hash = "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", size = 12317, upload-time = "2025-10-06T14:52:29.272Z" },
|
| 2321 |
]
|
| 2322 |
|
| 2323 |
+
[[package]]
|
| 2324 |
+
name = "mypy"
|
| 2325 |
+
version = "1.18.2"
|
| 2326 |
+
source = { registry = "https://pypi.org/simple" }
|
| 2327 |
+
dependencies = [
|
| 2328 |
+
{ name = "mypy-extensions" },
|
| 2329 |
+
{ name = "pathspec" },
|
| 2330 |
+
{ name = "tomli", marker = "python_full_version < '3.11'" },
|
| 2331 |
+
{ name = "typing-extensions" },
|
| 2332 |
+
]
|
| 2333 |
+
sdist = { url = "https://files.pythonhosted.org/packages/c0/77/8f0d0001ffad290cef2f7f216f96c814866248a0b92a722365ed54648e7e/mypy-1.18.2.tar.gz", hash = "sha256:06a398102a5f203d7477b2923dda3634c36727fa5c237d8f859ef90c42a9924b", size = 3448846, upload-time = "2025-09-19T00:11:10.519Z" }
|
| 2334 |
+
wheels = [
|
| 2335 |
+
{ url = "https://files.pythonhosted.org/packages/03/6f/657961a0743cff32e6c0611b63ff1c1970a0b482ace35b069203bf705187/mypy-1.18.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eab0cf6294dafe397c261a75f96dc2c31bffe3b944faa24db5def4e2b0f77c", size = 12807973, upload-time = "2025-09-19T00:10:35.282Z" },
|
| 2336 |
+
{ url = "https://files.pythonhosted.org/packages/10/e9/420822d4f661f13ca8900f5fa239b40ee3be8b62b32f3357df9a3045a08b/mypy-1.18.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7a780ca61fc239e4865968ebc5240bb3bf610ef59ac398de9a7421b54e4a207e", size = 11896527, upload-time = "2025-09-19T00:10:55.791Z" },
|
| 2337 |
+
{ url = "https://files.pythonhosted.org/packages/aa/73/a05b2bbaa7005f4642fcfe40fb73f2b4fb6bb44229bd585b5878e9a87ef8/mypy-1.18.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448acd386266989ef11662ce3c8011fd2a7b632e0ec7d61a98edd8e27472225b", size = 12507004, upload-time = "2025-09-19T00:11:05.411Z" },
|
| 2338 |
+
{ url = "https://files.pythonhosted.org/packages/4f/01/f6e4b9f0d031c11ccbd6f17da26564f3a0f3c4155af344006434b0a05a9d/mypy-1.18.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f9e171c465ad3901dc652643ee4bffa8e9fef4d7d0eece23b428908c77a76a66", size = 13245947, upload-time = "2025-09-19T00:10:46.923Z" },
|
| 2339 |
+
{ url = "https://files.pythonhosted.org/packages/d7/97/19727e7499bfa1ae0773d06afd30ac66a58ed7437d940c70548634b24185/mypy-1.18.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:592ec214750bc00741af1f80cbf96b5013d81486b7bb24cb052382c19e40b428", size = 13499217, upload-time = "2025-09-19T00:09:39.472Z" },
|
| 2340 |
+
{ url = "https://files.pythonhosted.org/packages/9f/4f/90dc8c15c1441bf31cf0f9918bb077e452618708199e530f4cbd5cede6ff/mypy-1.18.2-cp310-cp310-win_amd64.whl", hash = "sha256:7fb95f97199ea11769ebe3638c29b550b5221e997c63b14ef93d2e971606ebed", size = 9766753, upload-time = "2025-09-19T00:10:49.161Z" },
|
| 2341 |
+
{ url = "https://files.pythonhosted.org/packages/88/87/cafd3ae563f88f94eec33f35ff722d043e09832ea8530ef149ec1efbaf08/mypy-1.18.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:807d9315ab9d464125aa9fcf6d84fde6e1dc67da0b6f80e7405506b8ac72bc7f", size = 12731198, upload-time = "2025-09-19T00:09:44.857Z" },
|
| 2342 |
+
{ url = "https://files.pythonhosted.org/packages/0f/e0/1e96c3d4266a06d4b0197ace5356d67d937d8358e2ee3ffac71faa843724/mypy-1.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:776bb00de1778caf4db739c6e83919c1d85a448f71979b6a0edd774ea8399341", size = 11817879, upload-time = "2025-09-19T00:09:47.131Z" },
|
| 2343 |
+
{ url = "https://files.pythonhosted.org/packages/72/ef/0c9ba89eb03453e76bdac5a78b08260a848c7bfc5d6603634774d9cd9525/mypy-1.18.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1379451880512ffce14505493bd9fe469e0697543717298242574882cf8cdb8d", size = 12427292, upload-time = "2025-09-19T00:10:22.472Z" },
|
| 2344 |
+
{ url = "https://files.pythonhosted.org/packages/1a/52/ec4a061dd599eb8179d5411d99775bec2a20542505988f40fc2fee781068/mypy-1.18.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1331eb7fd110d60c24999893320967594ff84c38ac6d19e0a76c5fd809a84c86", size = 13163750, upload-time = "2025-09-19T00:09:51.472Z" },
|
| 2345 |
+
{ url = "https://files.pythonhosted.org/packages/c4/5f/2cf2ceb3b36372d51568f2208c021870fe7834cf3186b653ac6446511839/mypy-1.18.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ca30b50a51e7ba93b00422e486cbb124f1c56a535e20eff7b2d6ab72b3b2e37", size = 13351827, upload-time = "2025-09-19T00:09:58.311Z" },
|
| 2346 |
+
{ url = "https://files.pythonhosted.org/packages/c8/7d/2697b930179e7277529eaaec1513f8de622818696857f689e4a5432e5e27/mypy-1.18.2-cp311-cp311-win_amd64.whl", hash = "sha256:664dc726e67fa54e14536f6e1224bcfce1d9e5ac02426d2326e2bb4e081d1ce8", size = 9757983, upload-time = "2025-09-19T00:10:09.071Z" },
|
| 2347 |
+
{ url = "https://files.pythonhosted.org/packages/07/06/dfdd2bc60c66611dd8335f463818514733bc763e4760dee289dcc33df709/mypy-1.18.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33eca32dd124b29400c31d7cf784e795b050ace0e1f91b8dc035672725617e34", size = 12908273, upload-time = "2025-09-19T00:10:58.321Z" },
|
| 2348 |
+
{ url = "https://files.pythonhosted.org/packages/81/14/6a9de6d13a122d5608e1a04130724caf9170333ac5a924e10f670687d3eb/mypy-1.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3c47adf30d65e89b2dcd2fa32f3aeb5e94ca970d2c15fcb25e297871c8e4764", size = 11920910, upload-time = "2025-09-19T00:10:20.043Z" },
|
| 2349 |
+
{ url = "https://files.pythonhosted.org/packages/5f/a9/b29de53e42f18e8cc547e38daa9dfa132ffdc64f7250e353f5c8cdd44bee/mypy-1.18.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d6c838e831a062f5f29d11c9057c6009f60cb294fea33a98422688181fe2893", size = 12465585, upload-time = "2025-09-19T00:10:33.005Z" },
|
| 2350 |
+
{ url = "https://files.pythonhosted.org/packages/77/ae/6c3d2c7c61ff21f2bee938c917616c92ebf852f015fb55917fd6e2811db2/mypy-1.18.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01199871b6110a2ce984bde85acd481232d17413868c9807e95c1b0739a58914", size = 13348562, upload-time = "2025-09-19T00:10:11.51Z" },
|
| 2351 |
+
{ url = "https://files.pythonhosted.org/packages/4d/31/aec68ab3b4aebdf8f36d191b0685d99faa899ab990753ca0fee60fb99511/mypy-1.18.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a2afc0fa0b0e91b4599ddfe0f91e2c26c2b5a5ab263737e998d6817874c5f7c8", size = 13533296, upload-time = "2025-09-19T00:10:06.568Z" },
|
| 2352 |
+
{ url = "https://files.pythonhosted.org/packages/9f/83/abcb3ad9478fca3ebeb6a5358bb0b22c95ea42b43b7789c7fb1297ca44f4/mypy-1.18.2-cp312-cp312-win_amd64.whl", hash = "sha256:d8068d0afe682c7c4897c0f7ce84ea77f6de953262b12d07038f4d296d547074", size = 9828828, upload-time = "2025-09-19T00:10:28.203Z" },
|
| 2353 |
+
{ url = "https://files.pythonhosted.org/packages/5f/04/7f462e6fbba87a72bc8097b93f6842499c428a6ff0c81dd46948d175afe8/mypy-1.18.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:07b8b0f580ca6d289e69209ec9d3911b4a26e5abfde32228a288eb79df129fcc", size = 12898728, upload-time = "2025-09-19T00:10:01.33Z" },
|
| 2354 |
+
{ url = "https://files.pythonhosted.org/packages/99/5b/61ed4efb64f1871b41fd0b82d29a64640f3516078f6c7905b68ab1ad8b13/mypy-1.18.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed4482847168439651d3feee5833ccedbf6657e964572706a2adb1f7fa4dfe2e", size = 11910758, upload-time = "2025-09-19T00:10:42.607Z" },
|
| 2355 |
+
{ url = "https://files.pythonhosted.org/packages/3c/46/d297d4b683cc89a6e4108c4250a6a6b717f5fa96e1a30a7944a6da44da35/mypy-1.18.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3ad2afadd1e9fea5cf99a45a822346971ede8685cc581ed9cd4d42eaf940986", size = 12475342, upload-time = "2025-09-19T00:11:00.371Z" },
|
| 2356 |
+
{ url = "https://files.pythonhosted.org/packages/83/45/4798f4d00df13eae3bfdf726c9244bcb495ab5bd588c0eed93a2f2dd67f3/mypy-1.18.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a431a6f1ef14cf8c144c6b14793a23ec4eae3db28277c358136e79d7d062f62d", size = 13338709, upload-time = "2025-09-19T00:11:03.358Z" },
|
| 2357 |
+
{ url = "https://files.pythonhosted.org/packages/d7/09/479f7358d9625172521a87a9271ddd2441e1dab16a09708f056e97007207/mypy-1.18.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7ab28cc197f1dd77a67e1c6f35cd1f8e8b73ed2217e4fc005f9e6a504e46e7ba", size = 13529806, upload-time = "2025-09-19T00:10:26.073Z" },
|
| 2358 |
+
{ url = "https://files.pythonhosted.org/packages/71/cf/ac0f2c7e9d0ea3c75cd99dff7aec1c9df4a1376537cb90e4c882267ee7e9/mypy-1.18.2-cp313-cp313-win_amd64.whl", hash = "sha256:0e2785a84b34a72ba55fb5daf079a1003a34c05b22238da94fcae2bbe46f3544", size = 9833262, upload-time = "2025-09-19T00:10:40.035Z" },
|
| 2359 |
+
{ url = "https://files.pythonhosted.org/packages/5a/0c/7d5300883da16f0063ae53996358758b2a2df2a09c72a5061fa79a1f5006/mypy-1.18.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:62f0e1e988ad41c2a110edde6c398383a889d95b36b3e60bcf155f5164c4fdce", size = 12893775, upload-time = "2025-09-19T00:10:03.814Z" },
|
| 2360 |
+
{ url = "https://files.pythonhosted.org/packages/50/df/2cffbf25737bdb236f60c973edf62e3e7b4ee1c25b6878629e88e2cde967/mypy-1.18.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8795a039bab805ff0c1dfdb8cd3344642c2b99b8e439d057aba30850b8d3423d", size = 11936852, upload-time = "2025-09-19T00:10:51.631Z" },
|
| 2361 |
+
{ url = "https://files.pythonhosted.org/packages/be/50/34059de13dd269227fb4a03be1faee6e2a4b04a2051c82ac0a0b5a773c9a/mypy-1.18.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ca1e64b24a700ab5ce10133f7ccd956a04715463d30498e64ea8715236f9c9c", size = 12480242, upload-time = "2025-09-19T00:11:07.955Z" },
|
| 2362 |
+
{ url = "https://files.pythonhosted.org/packages/5b/11/040983fad5132d85914c874a2836252bbc57832065548885b5bb5b0d4359/mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb", size = 13326683, upload-time = "2025-09-19T00:09:55.572Z" },
|
| 2363 |
+
{ url = "https://files.pythonhosted.org/packages/e9/ba/89b2901dd77414dd7a8c8729985832a5735053be15b744c18e4586e506ef/mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075", size = 13514749, upload-time = "2025-09-19T00:10:44.827Z" },
|
| 2364 |
+
{ url = "https://files.pythonhosted.org/packages/25/bc/cc98767cffd6b2928ba680f3e5bc969c4152bf7c2d83f92f5a504b92b0eb/mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf", size = 9982959, upload-time = "2025-09-19T00:10:37.344Z" },
|
| 2365 |
+
{ url = "https://files.pythonhosted.org/packages/87/e3/be76d87158ebafa0309946c4a73831974d4d6ab4f4ef40c3b53a385a66fd/mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e", size = 2352367, upload-time = "2025-09-19T00:10:15.489Z" },
|
| 2366 |
+
]
|
| 2367 |
+
|
| 2368 |
+
[[package]]
|
| 2369 |
+
name = "mypy-extensions"
|
| 2370 |
+
version = "1.1.0"
|
| 2371 |
+
source = { registry = "https://pypi.org/simple" }
|
| 2372 |
+
sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" }
|
| 2373 |
+
wheels = [
|
| 2374 |
+
{ url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" },
|
| 2375 |
+
]
|
| 2376 |
+
|
| 2377 |
[[package]]
|
| 2378 |
name = "networkx"
|
| 2379 |
version = "3.4.2"
|
|
|
|
| 2547 |
version = "9.10.2.21"
|
| 2548 |
source = { registry = "https://pypi.org/simple" }
|
| 2549 |
dependencies = [
|
| 2550 |
+
{ name = "nvidia-cublas-cu12", marker = "sys_platform != 'win32'" },
|
| 2551 |
]
|
| 2552 |
wheels = [
|
| 2553 |
{ url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" },
|
|
|
|
| 2558 |
version = "11.3.3.83"
|
| 2559 |
source = { registry = "https://pypi.org/simple" }
|
| 2560 |
dependencies = [
|
| 2561 |
+
{ name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" },
|
| 2562 |
]
|
| 2563 |
wheels = [
|
| 2564 |
{ url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" },
|
|
|
|
| 2585 |
version = "11.7.3.90"
|
| 2586 |
source = { registry = "https://pypi.org/simple" }
|
| 2587 |
dependencies = [
|
| 2588 |
+
{ name = "nvidia-cublas-cu12", marker = "sys_platform != 'win32'" },
|
| 2589 |
+
{ name = "nvidia-cusparse-cu12", marker = "sys_platform != 'win32'" },
|
| 2590 |
+
{ name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" },
|
| 2591 |
]
|
| 2592 |
wheels = [
|
| 2593 |
{ url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" },
|
|
|
|
| 2598 |
version = "12.5.8.93"
|
| 2599 |
source = { registry = "https://pypi.org/simple" }
|
| 2600 |
dependencies = [
|
| 2601 |
+
{ name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" },
|
| 2602 |
]
|
| 2603 |
wheels = [
|
| 2604 |
{ url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" },
|
|
|
|
| 2853 |
{ url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" },
|
| 2854 |
]
|
| 2855 |
|
| 2856 |
+
[[package]]
|
| 2857 |
+
name = "pathspec"
|
| 2858 |
+
version = "0.12.1"
|
| 2859 |
+
source = { registry = "https://pypi.org/simple" }
|
| 2860 |
+
sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" }
|
| 2861 |
+
wheels = [
|
| 2862 |
+
{ url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" },
|
| 2863 |
+
]
|
| 2864 |
+
|
| 2865 |
[[package]]
|
| 2866 |
name = "pillow"
|
| 2867 |
version = "11.3.0"
|
|
|
|
| 3663 |
|
| 3664 |
[package.dev-dependencies]
|
| 3665 |
dev = [
|
| 3666 |
+
{ name = "mypy" },
|
| 3667 |
{ name = "pytest" },
|
| 3668 |
{ name = "ruff" },
|
| 3669 |
]
|
|
|
|
| 3694 |
|
| 3695 |
[package.metadata.requires-dev]
|
| 3696 |
dev = [
|
| 3697 |
+
{ name = "mypy", specifier = ">=1.18.2" },
|
| 3698 |
{ name = "pytest" },
|
| 3699 |
{ name = "ruff", specifier = "==0.12.0" },
|
| 3700 |
]
|