Alina Lozovskaya
commited on
Commit
·
ce5b8b9
1
Parent(s):
77b4d45
Update architecture scheme
Browse files- docs/assets/conversation_demo_arch.svg +2 -2
- docs/scheme.md +40 -19
docs/assets/conversation_demo_arch.svg
CHANGED
|
|
Git LFS Details
|
|
|
Git LFS Details
|
docs/scheme.md
CHANGED
|
@@ -1,37 +1,58 @@
|
|
| 1 |
---
|
| 2 |
config:
|
| 3 |
-
layout:
|
| 4 |
flowchart:
|
| 5 |
htmlLabels: true
|
| 6 |
---
|
| 7 |
flowchart TB
|
| 8 |
-
User["
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
Motion -- commands --> Robot
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
Motion@{ shape: rect}
|
| 24 |
-
Vision@{ shape: rect}
|
| 25 |
User:::userStyle
|
| 26 |
UI:::uiStyle
|
| 27 |
OpenAI:::aiStyle
|
| 28 |
-
Tools:::coreStyle
|
| 29 |
Motion:::coreStyle
|
| 30 |
-
|
| 31 |
Camera:::coreStyle
|
|
|
|
| 32 |
Robot:::hardwareStyle
|
| 33 |
classDef userStyle fill:#e1f5fe,stroke:#01579b,stroke-width:3px
|
| 34 |
classDef uiStyle fill:#b3e5fc,stroke:#0277bd,stroke-width:2px
|
| 35 |
classDef aiStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:3px
|
| 36 |
classDef coreStyle fill:#fff9c4,stroke:#f57f17,stroke-width:2px
|
| 37 |
classDef hardwareStyle fill:#ef9a9a,stroke:#c62828,stroke-width:3px
|
|
|
|
|
|
| 1 |
---
|
| 2 |
config:
|
| 3 |
+
layout: dagre
|
| 4 |
flowchart:
|
| 5 |
htmlLabels: true
|
| 6 |
---
|
| 7 |
flowchart TB
|
| 8 |
+
User(["<span style='font-size:16px;font-weight:bold;'>User</span><br><span style='font-size:13px;color:#01579b;'>Person interacting with system</span>"])
|
| 9 |
+
-- audio stream -->
|
| 10 |
+
UI@{ label: "<span style='font-size:16px;font-weight:bold;'>UI Layer</span><br><span style='font-size:13px;color:#0277bd;'>Gradio/Console</span>" }
|
| 11 |
+
|
| 12 |
+
UI -- audio stream -->
|
| 13 |
+
OpenAI@{ label: "<span style='font-size:17px;font-weight:bold;'>gpt-realtime API</span><br><span style='font-size:13px; color:#7b1fa2;'>Audio+Tool Calls+Vision</span>" }
|
| 14 |
+
|
| 15 |
+
OpenAI -- audio stream -->
|
| 16 |
+
Motion@{ label: "<span style='font-size:16px;font-weight:bold;'>Motion Control</span><br><span style='font-size:13px;color:#f57f17;'>Audio Sync + Tracking</span>" }
|
| 17 |
+
|
| 18 |
+
OpenAI -- tool calls -->
|
| 19 |
+
Handlers@{ label: "<span style='font-size:16px;font-weight:bold;'>Tool Handlers</span><br><span style='font-size:12px;color:#f9a825;'>move_head, camera, head_tracking,<br/>dance, play_emotion, do_nothing</span>" }
|
| 20 |
+
|
| 21 |
+
Handlers -- movement
|
| 22 |
+
requests --> Motion
|
| 23 |
+
|
| 24 |
+
Handlers -- camera frames, face tracking -->
|
| 25 |
+
Camera@{ label: "<span style='font-size:16px;font-weight:bold;'>Camera Worker</span><br><span style='font-size:13px;color:#f57f17;'>Frame Buffer + Face Tracking</span>" }
|
| 26 |
+
|
| 27 |
+
Handlers -. image for
|
| 28 |
+
analysis .-> OpenAI
|
| 29 |
+
|
| 30 |
+
Camera -- face tracking --> Motion
|
| 31 |
+
|
| 32 |
+
Camera -. frames .->
|
| 33 |
+
Vision@{ label: "<span style='font-size:16px;font-weight:bold;'>Vision Processor</span><br><span style='font-size:13px;color:#7b1fa2;'>Local VLM (optional)</span>" }
|
| 34 |
+
|
| 35 |
+
Vision -. description .-> Handlers
|
| 36 |
+
|
| 37 |
+
Robot@{ label: "<span style='font-size:16px;font-weight:bold;'>reachy_mini</span><br><span style='font-size:13px;color:#c62828;'>Robot Control Library</span>" }
|
| 38 |
+
-- camera
|
| 39 |
+
frames --> Camera
|
| 40 |
+
|
| 41 |
Motion -- commands --> Robot
|
| 42 |
+
|
| 43 |
+
Handlers -- results --> OpenAI
|
| 44 |
+
|
|
|
|
|
|
|
| 45 |
User:::userStyle
|
| 46 |
UI:::uiStyle
|
| 47 |
OpenAI:::aiStyle
|
|
|
|
| 48 |
Motion:::coreStyle
|
| 49 |
+
Handlers:::toolStyle
|
| 50 |
Camera:::coreStyle
|
| 51 |
+
Vision:::aiStyle
|
| 52 |
Robot:::hardwareStyle
|
| 53 |
classDef userStyle fill:#e1f5fe,stroke:#01579b,stroke-width:3px
|
| 54 |
classDef uiStyle fill:#b3e5fc,stroke:#0277bd,stroke-width:2px
|
| 55 |
classDef aiStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:3px
|
| 56 |
classDef coreStyle fill:#fff9c4,stroke:#f57f17,stroke-width:2px
|
| 57 |
classDef hardwareStyle fill:#ef9a9a,stroke:#c62828,stroke-width:3px
|
| 58 |
+
classDef toolStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px
|