first commit
This commit is contained in:
David R.
@@ -0,0 +1,18 @@
|
||||
tmp/
|
||||
components/
|
||||
managed_components/
|
||||
build/
|
||||
.vscode/
|
||||
.devcontainer/
|
||||
sdkconfig.old
|
||||
sdkconfig
|
||||
dependencies.lock
|
||||
.env
|
||||
releases/
|
||||
main/assets/lang_config.h
|
||||
main/mmap_generate_emoji.h
|
||||
.DS_Store
|
||||
.cache
|
||||
main/mmap_generate_emoji.h
|
||||
*.pyc
|
||||
*.bin
|
||||
@@ -0,0 +1,14 @@
|
||||
# For more information about build system see
|
||||
# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html
|
||||
# The following five lines of boilerplate have to be in your project's
|
||||
# CMakeLists in this exact order for cmake to work correctly
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
set(PROJECT_VER "1.7.6")
|
||||
|
||||
# Add this line to disable the specific warning
|
||||
add_compile_options(-Wno-missing-field-initializers)
|
||||
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
project(xiaozhi)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Xiaoxia
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,160 @@
|
||||
# An MCP-based Chatbot | 一个基于 MCP 的聊天机器人
|
||||
|
||||
（中文 | [English](README_en.md) | [日本語](README_ja.md)）
|
||||
|
||||
## 视频
|
||||
|
||||
👉 [人类：给 AI 装摄像头 vs AI：当场发现主人三天没洗头【bilibili】](https://www.bilibili.com/video/BV1bpjgzKEhd/)
|
||||
|
||||
👉 [手工打造你的 AI 女友，新手入门教程【bilibili】](https://www.bilibili.com/video/BV1XnmFYLEJN/)
|
||||
|
||||
## 介绍
|
||||
|
||||
这是一个由虾哥开源的 ESP32 项目，以 MIT 许可证发布，允许任何人免费使用，或用于商业用途。
|
||||
|
||||
我们希望通过这个项目，能够帮助大家了解 AI 硬件开发，将当下飞速发展的大语言模型应用到实际的硬件设备中。
|
||||
|
||||
如果你有任何想法或建议，请随时提出 Issues 或加入 QQ 群：575180511
|
||||
|
||||
### 基于 MCP 控制万物
|
||||
|
||||
小智 AI 聊天机器人作为一个语音交互入口，利用 Qwen / DeepSeek 等大模型的 AI 能力，通过 MCP 协议实现多端控制。
|
||||
|
||||
|
||||
|
||||
### 已实现功能
|
||||
|
||||
- Wi-Fi / ML307 Cat.1 4G
|
||||
- 离线语音唤醒 [ESP-SR](https://github.com/espressif/esp-sr)
|
||||
- 支持两种通信协议（[Websocket](docs/websocket.md) 或 MQTT+UDP）
|
||||
- 采用 OPUS 音频编解码
|
||||
- 基于流式 ASR + LLM + TTS 架构的语音交互
|
||||
- 声纹识别，识别当前说话人的身份 [3D Speaker](https://github.com/modelscope/3D-Speaker)
|
||||
- OLED / LCD 显示屏，支持表情显示
|
||||
- 电量显示与电源管理
|
||||
- 支持多语言（中文、英文、日文）
|
||||
- 支持 ESP32-C3、ESP32-S3、ESP32-P4 芯片平台
|
||||
- 通过设备端 MCP 实现设备控制（音量、灯光、电机、GPIO 等）
|
||||
- 通过云端 MCP 扩展大模型能力（智能家居控制、PC桌面操作、知识搜索、邮件收发等）
|
||||
|
||||
## 硬件
|
||||
|
||||
### 面包板手工制作实践
|
||||
|
||||
详见飞书文档教程：
|
||||
|
||||
👉 [《小智 AI 聊天机器人百科全书》](https://ccnphfhqs21z.feishu.cn/wiki/F5krwD16viZoF0kKkvDcrZNYnhb?from=from_copylink)
|
||||
|
||||
面包板效果图如下：
|
||||
|
||||
|
||||
|
||||
### 支持 70 多个开源硬件（仅展示部分）
|
||||
|
||||
- <a href="https://oshwhub.com/li-chuang-kai-fa-ban/li-chuang-shi-zhan-pai-esp32-s3-kai-fa-ban" target="_blank" title="立创·实战派 ESP32-S3 开发板">立创·实战派 ESP32-S3 开发板</a>
|
||||
- <a href="https://github.com/espressif/esp-box" target="_blank" title="乐鑫 ESP32-S3-BOX3">乐鑫 ESP32-S3-BOX3</a>
|
||||
- <a href="https://docs.m5stack.com/zh_CN/core/CoreS3" target="_blank" title="M5Stack CoreS3">M5Stack CoreS3</a>
|
||||
- <a href="https://docs.m5stack.com/en/atom/Atomic%20Echo%20Base" target="_blank" title="AtomS3R + Echo Base">M5Stack AtomS3R + Echo Base</a>
|
||||
- <a href="https://gf.bilibili.com/item/detail/1108782064" target="_blank" title="神奇按钮 2.4">神奇按钮 2.4</a>
|
||||
- <a href="https://www.waveshare.net/shop/ESP32-S3-Touch-AMOLED-1.8.htm" target="_blank" title="微雪电子 ESP32-S3-Touch-AMOLED-1.8">微雪电子 ESP32-S3-Touch-AMOLED-1.8</a>
|
||||
- <a href="https://github.com/Xinyuan-LilyGO/T-Circle-S3" target="_blank" title="LILYGO T-Circle-S3">LILYGO T-Circle-S3</a>
|
||||
- <a href="https://oshwhub.com/tenclass01/xmini_c3" target="_blank" title="虾哥 Mini C3">虾哥 Mini C3</a>
|
||||
- <a href="https://oshwhub.com/movecall/cuican-ai-pendant-lights-up-y" target="_blank" title="Movecall CuiCan ESP32S3">璀璨·AI 吊坠</a>
|
||||
- <a href="https://github.com/WMnologo/xingzhi-ai" target="_blank" title="无名科技Nologo-星智-1.54">无名科技 Nologo-星智-1.54TFT</a>
|
||||
- <a href="https://www.seeedstudio.com/SenseCAP-Watcher-W1-A-p-5979.html" target="_blank" title="SenseCAP Watcher">SenseCAP Watcher</a>
|
||||
- <a href="https://www.bilibili.com/video/BV1BHJtz6E2S/" target="_blank" title="ESP-HI 超低成本机器狗">ESP-HI 超低成本机器狗</a>
|
||||
|
||||
<div style="display: flex; justify-content: space-between;">
|
||||
<a href="docs/v1/lichuang-s3.jpg" target="_blank" title="立创·实战派 ESP32-S3 开发板">
|
||||
<img src="docs/v1/lichuang-s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/espbox3.jpg" target="_blank" title="乐鑫 ESP32-S3-BOX3">
|
||||
<img src="docs/v1/espbox3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/m5cores3.jpg" target="_blank" title="M5Stack CoreS3">
|
||||
<img src="docs/v1/m5cores3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/atoms3r.jpg" target="_blank" title="AtomS3R + Echo Base">
|
||||
<img src="docs/v1/atoms3r.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/magiclick.jpg" target="_blank" title="神奇按钮 2.4">
|
||||
<img src="docs/v1/magiclick.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/waveshare.jpg" target="_blank" title="微雪电子 ESP32-S3-Touch-AMOLED-1.8">
|
||||
<img src="docs/v1/waveshare.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/lilygo-t-circle-s3.jpg" target="_blank" title="LILYGO T-Circle-S3">
|
||||
<img src="docs/v1/lilygo-t-circle-s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/xmini-c3.jpg" target="_blank" title="虾哥 Mini C3">
|
||||
<img src="docs/v1/xmini-c3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/movecall-cuican-esp32s3.jpg" target="_blank" title="CuiCan">
|
||||
<img src="docs/v1/movecall-cuican-esp32s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/wmnologo_xingzhi_1.54.jpg" target="_blank" title="无名科技Nologo-星智-1.54">
|
||||
<img src="docs/v1/wmnologo_xingzhi_1.54.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/sensecap_watcher.jpg" target="_blank" title="SenseCAP Watcher">
|
||||
<img src="docs/v1/sensecap_watcher.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/esp-hi.jpg" target="_blank" title="ESP-HI 超低成本机器狗">
|
||||
<img src="docs/v1/esp-hi.jpg" width="240" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
## 软件
|
||||
|
||||
### 固件烧录
|
||||
|
||||
新手第一次操作建议先不要搭建开发环境，直接使用免开发环境烧录的固件。
|
||||
|
||||
固件默认接入 [xiaozhi.me](https://xiaozhi.me) 官方服务器，个人用户注册账号可以免费使用 Qwen 实时模型。
|
||||
|
||||
👉 [新手烧录固件教程](https://ccnphfhqs21z.feishu.cn/wiki/Zpz4wXBtdimBrLk25WdcXzxcnNS)
|
||||
|
||||
### 开发环境
|
||||
|
||||
- Cursor 或 VSCode
|
||||
- 安装 ESP-IDF 插件，选择 SDK 版本 5.4 或以上
|
||||
- Linux 比 Windows 更好，编译速度快，也免去驱动问题的困扰
|
||||
- 本项目使用 Google C++ 代码风格，提交代码时请确保符合规范
|
||||
|
||||
### 开发者文档
|
||||
|
||||
- [自定义开发板指南](main/boards/README.md) - 学习如何为小智 AI 创建自定义开发板
|
||||
- [MCP 协议物联网控制用法说明](docs/mcp-usage.md) - 了解如何通过 MCP 协议控制物联网设备
|
||||
- [MCP 协议交互流程](docs/mcp-protocol.md) - 设备端 MCP 协议的实现方式
|
||||
- [一份详细的 WebSocket 通信协议文档](docs/websocket.md)
|
||||
|
||||
## 大模型配置
|
||||
|
||||
如果你已经拥有一个的小智 AI 聊天机器人设备，并且已接入官方服务器，可以登录 [xiaozhi.me](https://xiaozhi.me) 控制台进行配置。
|
||||
|
||||
👉 [后台操作视频教程（旧版界面）](https://www.bilibili.com/video/BV1jUCUY2EKM/)
|
||||
|
||||
## 相关开源项目
|
||||
|
||||
在个人电脑上部署服务器，可以参考以下第三方开源的项目：
|
||||
|
||||
- [xinnan-tech/xiaozhi-esp32-server](https://github.com/xinnan-tech/xiaozhi-esp32-server) Python 服务器
|
||||
- [joey-zhou/xiaozhi-esp32-server-java](https://github.com/joey-zhou/xiaozhi-esp32-server-java) Java 服务器
|
||||
- [AnimeAIChat/xiaozhi-server-go](https://github.com/AnimeAIChat/xiaozhi-server-go) Golang 服务器
|
||||
|
||||
使用小智通信协议的第三方客户端项目：
|
||||
|
||||
- [huangjunsen0406/py-xiaozhi](https://github.com/huangjunsen0406/py-xiaozhi) Python 客户端
|
||||
- [TOM88812/xiaozhi-android-client](https://github.com/TOM88812/xiaozhi-android-client) Android 客户端
|
||||
- [100askTeam/xiaozhi-linux](http://github.com/100askTeam/xiaozhi-linux) 百问科技提供的 Linux 客户端
|
||||
- [78/xiaozhi-sf32](https://github.com/78/xiaozhi-sf32) 思澈科技的蓝牙芯片固件
|
||||
- [QuecPython/solution-xiaozhiAI](https://github.com/QuecPython/solution-xiaozhiAI) 移远提供的 QuecPython 固件
|
||||
|
||||
## Star History
|
||||
|
||||
<a href="https://star-history.com/#78/xiaozhi-esp32&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
@@ -0,0 +1,157 @@
|
||||
# An MCP-based Chatbot
|
||||
|
||||
(English | [中文](README.md) | [日本語](README_ja.md))
|
||||
|
||||
## Video
|
||||
|
||||
👉 [Human: Give AI a camera vs AI: Instantly finds out the owner hasn't washed hair for three days【bilibili】](https://www.bilibili.com/video/BV1bpjgzKEhd/)
|
||||
|
||||
👉 [Handcraft your AI girlfriend, beginner's guide【bilibili】](https://www.bilibili.com/video/BV1XnmFYLEJN/)
|
||||
|
||||
## Introduction
|
||||
|
||||
This is an open-source ESP32 project, released under the MIT license, allowing anyone to use it for free, including for commercial purposes.
|
||||
|
||||
We hope this project helps everyone understand AI hardware development and apply rapidly evolving large language models to real hardware devices.
|
||||
|
||||
If you have any ideas or suggestions, please feel free to raise Issues or join the QQ group: 575180511
|
||||
|
||||
### Control Everything with MCP
|
||||
|
||||
As a voice interaction entry, the XiaoZhi AI chatbot leverages the AI capabilities of large models like Qwen / DeepSeek, and achieves multi-terminal control via the MCP protocol.
|
||||
|
||||
|
||||
|
||||
### Features Implemented
|
||||
|
||||
- Wi-Fi / ML307 Cat.1 4G
|
||||
- Offline voice wake-up [ESP-SR](https://github.com/espressif/esp-sr)
|
||||
- Supports two communication protocols ([Websocket](docs/websocket.md) or MQTT+UDP)
|
||||
- Uses OPUS audio codec
|
||||
- Voice interaction based on streaming ASR + LLM + TTS architecture
|
||||
- Speaker recognition, identifies the current speaker [3D Speaker](https://github.com/modelscope/3D-Speaker)
|
||||
- OLED / LCD display, supports emoji display
|
||||
- Battery display and power management
|
||||
- Multi-language support (Chinese, English, Japanese)
|
||||
- Supports ESP32-C3, ESP32-S3, ESP32-P4 chip platforms
|
||||
- Device-side MCP for device control (Speaker, LED, Servo, GPIO, etc.)
|
||||
- Cloud-side MCP to extend large model capabilities (smart home control, PC desktop operation, knowledge search, email, etc.)
|
||||
|
||||
## Hardware
|
||||
|
||||
### Breadboard DIY Practice
|
||||
|
||||
See the Feishu document tutorial:
|
||||
|
||||
👉 ["XiaoZhi AI Chatbot Encyclopedia"](https://ccnphfhqs21z.feishu.cn/wiki/F5krwD16viZoF0kKkvDcrZNYnhb?from=from_copylink)
|
||||
|
||||
Breadboard demo:
|
||||
|
||||
|
||||
|
||||
### Supports 70+ Open Source Hardware (Partial List)
|
||||
|
||||
- <a href="https://oshwhub.com/li-chuang-kai-fa-ban/li-chuang-shi-zhan-pai-esp32-s3-kai-fa-ban" target="_blank" title="LiChuang ESP32-S3 Development Board">LiChuang ESP32-S3 Development Board</a>
|
||||
- <a href="https://github.com/espressif/esp-box" target="_blank" title="Espressif ESP32-S3-BOX3">Espressif ESP32-S3-BOX3</a>
|
||||
- <a href="https://docs.m5stack.com/zh_CN/core/CoreS3" target="_blank" title="M5Stack CoreS3">M5Stack CoreS3</a>
|
||||
- <a href="https://docs.m5stack.com/en/atom/Atomic%20Echo%20Base" target="_blank" title="AtomS3R + Echo Base">M5Stack AtomS3R + Echo Base</a>
|
||||
- <a href="https://gf.bilibili.com/item/detail/1108782064" target="_blank" title="Magic Button 2.4">Magic Button 2.4</a>
|
||||
- <a href="https://www.waveshare.net/shop/ESP32-S3-Touch-AMOLED-1.8.htm" target="_blank" title="Waveshare ESP32-S3-Touch-AMOLED-1.8">Waveshare ESP32-S3-Touch-AMOLED-1.8</a>
|
||||
- <a href="https://github.com/Xinyuan-LilyGO/T-Circle-S3" target="_blank" title="LILYGO T-Circle-S3">LILYGO T-Circle-S3</a>
|
||||
- <a href="https://oshwhub.com/tenclass01/xmini_c3" target="_blank" title="XiaGe Mini C3">XiaGe Mini C3</a>
|
||||
- <a href="https://oshwhub.com/movecall/cuican-ai-pendant-lights-up-y" target="_blank" title="Movecall CuiCan ESP32S3">CuiCan AI Pendant</a>
|
||||
- <a href="https://github.com/WMnologo/xingzhi-ai" target="_blank" title="WMnologo-Xingzhi-1.54">WMnologo-Xingzhi-1.54TFT</a>
|
||||
- <a href="https://www.seeedstudio.com/SenseCAP-Watcher-W1-A-p-5979.html" target="_blank" title="SenseCAP Watcher">SenseCAP Watcher</a>
|
||||
- <a href="https://www.bilibili.com/video/BV1BHJtz6E2S/" target="_blank" title="ESP-HI Low Cost Robot Dog">ESP-HI Low Cost Robot Dog</a>
|
||||
|
||||
<div style="display: flex; justify-content: space-between;">
|
||||
<a href="docs/v1/lichuang-s3.jpg" target="_blank" title="LiChuang ESP32-S3 Development Board">
|
||||
<img src="docs/v1/lichuang-s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/espbox3.jpg" target="_blank" title="Espressif ESP32-S3-BOX3">
|
||||
<img src="docs/v1/espbox3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/m5cores3.jpg" target="_blank" title="M5Stack CoreS3">
|
||||
<img src="docs/v1/m5cores3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/atoms3r.jpg" target="_blank" title="AtomS3R + Echo Base">
|
||||
<img src="docs/v1/atoms3r.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/magiclick.jpg" target="_blank" title="Magic Button 2.4">
|
||||
<img src="docs/v1/magiclick.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/waveshare.jpg" target="_blank" title="Waveshare ESP32-S3-Touch-AMOLED-1.8">
|
||||
<img src="docs/v1/waveshare.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/lilygo-t-circle-s3.jpg" target="_blank" title="LILYGO T-Circle-S3">
|
||||
<img src="docs/v1/lilygo-t-circle-s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/xmini-c3.jpg" target="_blank" title="XiaGe Mini C3">
|
||||
<img src="docs/v1/xmini-c3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/movecall-cuican-esp32s3.jpg" target="_blank" title="CuiCan">
|
||||
<img src="docs/v1/movecall-cuican-esp32s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/wmnologo_xingzhi_1.54.jpg" target="_blank" title="WMnologo-Xingzhi-1.54">
|
||||
<img src="docs/v1/wmnologo_xingzhi_1.54.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/sensecap_watcher.jpg" target="_blank" title="SenseCAP Watcher">
|
||||
<img src="docs/v1/sensecap_watcher.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/esp-hi.jpg" target="_blank" title="ESP-HI Low Cost Robot Dog">
|
||||
<img src="docs/v1/esp-hi.jpg" width="240" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
## Software
|
||||
|
||||
### Firmware Flashing
|
||||
|
||||
For beginners, it is recommended to use the firmware that can be flashed without setting up a development environment.
|
||||
|
||||
The firmware connects to the official [xiaozhi.me](https://xiaozhi.me) server by default. Personal users can register an account to use the Qwen real-time model for free.
|
||||
|
||||
👉 [Beginner's Firmware Flashing Guide](https://ccnphfhqs21z.feishu.cn/wiki/Zpz4wXBtdimBrLk25WdcXzxcnNS)
|
||||
|
||||
### Development Environment
|
||||
|
||||
- Cursor or VSCode
|
||||
- Install ESP-IDF plugin, select SDK version 5.4 or above
|
||||
- Linux is better than Windows for faster compilation and fewer driver issues
|
||||
- This project uses Google C++ code style, please ensure compliance when submitting code
|
||||
|
||||
### Developer Documentation
|
||||
|
||||
- [Custom Board Guide](main/boards/README.md) - Learn how to create custom boards for XiaoZhi AI
|
||||
- [MCP Protocol IoT Control Usage](docs/mcp-usage.md) - Learn how to control IoT devices via MCP protocol
|
||||
- [MCP Protocol Interaction Flow](docs/mcp-protocol.md) - Device-side MCP protocol implementation
|
||||
- [A detailed WebSocket communication protocol document](docs/websocket.md)
|
||||
|
||||
## Large Model Configuration
|
||||
|
||||
If you already have a XiaoZhi AI chatbot device and have connected to the official server, you can log in to the [xiaozhi.me](https://xiaozhi.me) console for configuration.
|
||||
|
||||
👉 [Backend Operation Video Tutorial (Old Interface)](https://www.bilibili.com/video/BV1jUCUY2EKM/)
|
||||
|
||||
## Related Open Source Projects
|
||||
|
||||
For server deployment on personal computers, refer to the following open-source projects:
|
||||
|
||||
- [xinnan-tech/xiaozhi-esp32-server](https://github.com/xinnan-tech/xiaozhi-esp32-server) Python server
|
||||
- [joey-zhou/xiaozhi-esp32-server-java](https://github.com/joey-zhou/xiaozhi-esp32-server-java) Java server
|
||||
- [AnimeAIChat/xiaozhi-server-go](https://github.com/AnimeAIChat/xiaozhi-server-go) Golang server
|
||||
|
||||
Other client projects using the XiaoZhi communication protocol:
|
||||
|
||||
- [huangjunsen0406/py-xiaozhi](https://github.com/huangjunsen0406/py-xiaozhi) Python client
|
||||
- [TOM88812/xiaozhi-android-client](https://github.com/TOM88812/xiaozhi-android-client) Android client
|
||||
|
||||
## Star History
|
||||
|
||||
<a href="https://star-history.com/#78/xiaozhi-esp32&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
@@ -0,0 +1,157 @@
|
||||
# MCP ベースのチャットボット
|
||||
|
||||
（日本語 | [中文](README.md) | [English](README_en.md)）
|
||||
|
||||
## 動画
|
||||
|
||||
👉 [人間：AIにカメラを装着 vs AI：その場で飼い主が3日間髪を洗っていないことを発見【bilibili】](https://www.bilibili.com/video/BV1bpjgzKEhd/)
|
||||
|
||||
👉 [手作りでAIガールフレンドを作る、初心者入門チュートリアル【bilibili】](https://www.bilibili.com/video/BV1XnmFYLEJN/)
|
||||
|
||||
## イントロダクション
|
||||
|
||||
これはエビ兄さんがオープンソースで公開しているESP32プロジェクトで、MITライセンスのもと、誰でも無料で、商用利用も可能です。
|
||||
|
||||
このプロジェクトを通じて、AIハードウェア開発を理解し、急速に進化する大規模言語モデルを実際のハードウェアデバイスに応用できるようになることを目指しています。
|
||||
|
||||
ご意見やご提案があれば、いつでもIssueを提出するか、QQグループ：575180511 にご参加ください。
|
||||
|
||||
### MCPであらゆるものを制御
|
||||
|
||||
シャオジーAIチャットボットは音声インタラクションの入口として、Qwen / DeepSeekなどの大規模モデルのAI能力を活用し、MCPプロトコルを通じてマルチエンド制御を実現します。
|
||||
|
||||
|
||||
|
||||
### 実装済み機能
|
||||
|
||||
- Wi-Fi / ML307 Cat.1 4G
|
||||
- オフライン音声ウェイクアップ [ESP-SR](https://github.com/espressif/esp-sr)
|
||||
- 2種類の通信プロトコルに対応（[Websocket](docs/websocket.md) または MQTT+UDP）
|
||||
- OPUSオーディオコーデックを採用
|
||||
- ストリーミングASR + LLM + TTSアーキテクチャに基づく音声インタラクション
|
||||
- 話者認識、現在話している人を識別 [3D Speaker](https://github.com/modelscope/3D-Speaker)
|
||||
- OLED / LCDディスプレイ、表情表示対応
|
||||
- バッテリー表示と電源管理
|
||||
- 多言語対応（中国語、英語、日本語）
|
||||
- ESP32-C3、ESP32-S3、ESP32-P4チッププラットフォーム対応
|
||||
- デバイス側MCPによるデバイス制御（音量・明るさ調整、アクション制御など）
|
||||
- クラウド側MCPで大規模モデル能力を拡張（スマートホーム制御、PCデスクトップ操作、知識検索、メール送受信など）
|
||||
|
||||
## ハードウェア
|
||||
|
||||
### ブレッドボード手作り実践
|
||||
|
||||
Feishuドキュメントチュートリアルをご覧ください：
|
||||
|
||||
👉 [「シャオジーAIチャットボット百科事典」](https://ccnphfhqs21z.feishu.cn/wiki/F5krwD16viZoF0kKkvDcrZNYnhb?from=from_copylink)
|
||||
|
||||
ブレッドボードのデモ：
|
||||
|
||||
|
||||
|
||||
### 70種類以上のオープンソースハードウェアに対応（一部のみ表示）
|
||||
|
||||
- <a href="https://oshwhub.com/li-chuang-kai-fa-ban/li-chuang-shi-zhan-pai-esp32-s3-kai-fa-ban" target="_blank" title="立創・実戦派 ESP32-S3 開発ボード">立創・実戦派 ESP32-S3 開発ボード</a>
|
||||
- <a href="https://github.com/espressif/esp-box" target="_blank" title="楽鑫 ESP32-S3-BOX3">楽鑫 ESP32-S3-BOX3</a>
|
||||
- <a href="https://docs.m5stack.com/zh_CN/core/CoreS3" target="_blank" title="M5Stack CoreS3">M5Stack CoreS3</a>
|
||||
- <a href="https://docs.m5stack.com/en/atom/Atomic%20Echo%20Base" target="_blank" title="AtomS3R + Echo Base">M5Stack AtomS3R + Echo Base</a>
|
||||
- <a href="https://gf.bilibili.com/item/detail/1108782064" target="_blank" title="マジックボタン2.4">マジックボタン2.4</a>
|
||||
- <a href="https://www.waveshare.net/shop/ESP32-S3-Touch-AMOLED-1.8.htm" target="_blank" title="微雪電子 ESP32-S3-Touch-AMOLED-1.8">微雪電子 ESP32-S3-Touch-AMOLED-1.8</a>
|
||||
- <a href="https://github.com/Xinyuan-LilyGO/T-Circle-S3" target="_blank" title="LILYGO T-Circle-S3">LILYGO T-Circle-S3</a>
|
||||
- <a href="https://oshwhub.com/tenclass01/xmini_c3" target="_blank" title="エビ兄さん Mini C3">エビ兄さん Mini C3</a>
|
||||
- <a href="https://oshwhub.com/movecall/cuican-ai-pendant-lights-up-y" target="_blank" title="Movecall CuiCan ESP32S3">CuiCan AIペンダント</a>
|
||||
- <a href="https://github.com/WMnologo/xingzhi-ai" target="_blank" title="無名科技Nologo-星智-1.54">無名科技Nologo-星智-1.54TFT</a>
|
||||
- <a href="https://www.seeedstudio.com/SenseCAP-Watcher-W1-A-p-5979.html" target="_blank" title="SenseCAP Watcher">SenseCAP Watcher</a>
|
||||
- <a href="https://www.bilibili.com/video/BV1BHJtz6E2S/" target="_blank" title="ESP-HI 超低コストロボット犬">ESP-HI 超低コストロボット犬</a>
|
||||
|
||||
<div style="display: flex; justify-content: space-between;">
|
||||
<a href="docs/v1/lichuang-s3.jpg" target="_blank" title="立創・実戦派 ESP32-S3 開発ボード">
|
||||
<img src="docs/v1/lichuang-s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/espbox3.jpg" target="_blank" title="楽鑫 ESP32-S3-BOX3">
|
||||
<img src="docs/v1/espbox3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/m5cores3.jpg" target="_blank" title="M5Stack CoreS3">
|
||||
<img src="docs/v1/m5cores3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/atoms3r.jpg" target="_blank" title="AtomS3R + Echo Base">
|
||||
<img src="docs/v1/atoms3r.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/magiclick.jpg" target="_blank" title="マジックボタン2.4">
|
||||
<img src="docs/v1/magiclick.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/waveshare.jpg" target="_blank" title="微雪電子 ESP32-S3-Touch-AMOLED-1.8">
|
||||
<img src="docs/v1/waveshare.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/lilygo-t-circle-s3.jpg" target="_blank" title="LILYGO T-Circle-S3">
|
||||
<img src="docs/v1/lilygo-t-circle-s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/xmini-c3.jpg" target="_blank" title="エビ兄さん Mini C3">
|
||||
<img src="docs/v1/xmini-c3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/movecall-cuican-esp32s3.jpg" target="_blank" title="CuiCan">
|
||||
<img src="docs/v1/movecall-cuican-esp32s3.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/wmnologo_xingzhi_1.54.jpg" target="_blank" title="無名科技Nologo-星智-1.54">
|
||||
<img src="docs/v1/wmnologo_xingzhi_1.54.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/sensecap_watcher.jpg" target="_blank" title="SenseCAP Watcher">
|
||||
<img src="docs/v1/sensecap_watcher.jpg" width="240" />
|
||||
</a>
|
||||
<a href="docs/v1/esp-hi.jpg" target="_blank" title="ESP-HI 超低コストロボット犬">
|
||||
<img src="docs/v1/esp-hi.jpg" width="240" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
## ソフトウェア
|
||||
|
||||
### ファームウェア書き込み
|
||||
|
||||
初心者の方は、まず開発環境を構築せずに書き込み可能なファームウェアを使用することをおすすめします。
|
||||
|
||||
ファームウェアはデフォルトで公式 [xiaozhi.me](https://xiaozhi.me) サーバーに接続します。個人ユーザーはアカウント登録でQwenリアルタイムモデルを無料で利用できます。
|
||||
|
||||
👉 [初心者向けファームウェア書き込みガイド](https://ccnphfhqs21z.feishu.cn/wiki/Zpz4wXBtdimBrLk25WdcXzxcnNS)
|
||||
|
||||
### 開発環境
|
||||
|
||||
- Cursor または VSCode
|
||||
- ESP-IDFプラグインをインストールし、SDKバージョン5.4以上を選択
|
||||
- LinuxはWindowsよりも優れており、コンパイルが速く、ドライバの問題も少ない
|
||||
- 本プロジェクトはGoogle C++コードスタイルを採用、コード提出時は準拠を確認してください
|
||||
|
||||
### 開発者ドキュメント
|
||||
|
||||
- [カスタム開発ボードガイド](main/boards/README.md) - シャオジーAI用のカスタム開発ボード作成方法
|
||||
- [MCPプロトコルIoT制御使用法](docs/mcp-usage.md) - MCPプロトコルでIoTデバイスを制御する方法
|
||||
- [MCPプロトコルインタラクションフロー](docs/mcp-protocol.md) - デバイス側MCPプロトコルの実装方法
|
||||
- [詳細なWebSocket通信プロトコルドキュメント](docs/websocket.md)
|
||||
|
||||
## 大規模モデル設定
|
||||
|
||||
すでにシャオジーAIチャットボットデバイスをお持ちで、公式サーバーに接続済みの場合は、[xiaozhi.me](https://xiaozhi.me) コンソールで設定できます。
|
||||
|
||||
👉 [バックエンド操作ビデオチュートリアル（旧インターフェース）](https://www.bilibili.com/video/BV1jUCUY2EKM/)
|
||||
|
||||
## 関連オープンソースプロジェクト
|
||||
|
||||
個人PCでサーバーをデプロイする場合は、以下のオープンソースプロジェクトを参照してください：
|
||||
|
||||
- [xinnan-tech/xiaozhi-esp32-server](https://github.com/xinnan-tech/xiaozhi-esp32-server) Pythonサーバー
|
||||
- [joey-zhou/xiaozhi-esp32-server-java](https://github.com/joey-zhou/xiaozhi-esp32-server-java) Javaサーバー
|
||||
- [AnimeAIChat/xiaozhi-server-go](https://github.com/AnimeAIChat/xiaozhi-server-go) Golangサーバー
|
||||
|
||||
シャオジー通信プロトコルを利用した他のクライアントプロジェクト：
|
||||
|
||||
- [huangjunsen0406/py-xiaozhi](https://github.com/huangjunsen0406/py-xiaozhi) Pythonクライアント
|
||||
- [TOM88812/xiaozhi-android-client](https://github.com/TOM88812/xiaozhi-android-client) Androidクライアント
|
||||
|
||||
## スター履歴
|
||||
|
||||
<a href="https://star-history.com/#78/xiaozhi-esp32&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=78/xiaozhi-esp32&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 96 KiB |
@@ -0,0 +1,269 @@
|
||||
# MCP (Model Context Protocol) 交互流程
|
||||
|
||||
NOTICE: AI 辅助生成, 在实现后台服务时, 请参照代码确认细节!!
|
||||
|
||||
本项目中的 MCP 协议用于后台 API（MCP 客户端）与 ESP32 设备（MCP 服务器）之间的通信，以便后台能够发现和调用设备提供的功能（工具）。
|
||||
|
||||
## 协议格式
|
||||
|
||||
根据代码 (`main/protocols/protocol.cc`, `main/mcp_server.cc`)，MCP 消息是封装在基础通信协议（如 WebSocket 或 MQTT）的消息体中的。其内部结构遵循 [JSON-RPC 2.0](https://www.jsonrpc.org/specification) 规范。
|
||||
|
||||
整体消息结构示例：
|
||||
|
||||
```json
|
||||
{
|
||||
"session_id": "...", // 会话 ID
|
||||
"type": "mcp", // 消息类型，固定为 "mcp"
|
||||
"payload": { // JSON-RPC 2.0 负载
|
||||
"jsonrpc": "2.0",
|
||||
"method": "...", // 方法名 (如 "initialize", "tools/list", "tools/call")
|
||||
"params": { ... }, // 方法参数 (对于 request)
|
||||
"id": ..., // 请求 ID (对于 request 和 response)
|
||||
"result": { ... }, // 方法执行结果 (对于 success response)
|
||||
"error": { ... } // 错误信息 (对于 error response)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
其中，`payload` 部分是标准的 JSON-RPC 2.0 消息：
|
||||
|
||||
- `jsonrpc`: 固定的字符串 "2.0"。
|
||||
- `method`: 要调用的方法名称 (对于 Request)。
|
||||
- `params`: 方法的参数，一个结构化值，通常为对象 (对于 Request)。
|
||||
- `id`: 请求的标识符，客户端发送请求时提供，服务器响应时原样返回。用于匹配请求和响应。
|
||||
- `result`: 方法成功执行时的结果 (对于 Success Response)。
|
||||
- `error`: 方法执行失败时的错误信息 (对于 Error Response)。
|
||||
|
||||
## 交互流程及发送时机
|
||||
|
||||
MCP 的交互主要围绕客户端（后台 API）发现和调用设备上的“工具”（Tool）进行。
|
||||
|
||||
1. **连接建立与能力通告**
|
||||
|
||||
- **时机：** 设备启动并成功连接到后台 API 后。
|
||||
- **发送方：** 设备。
|
||||
- **消息：** 设备发送基础协议的 "hello" 消息给后台 API，消息中包含设备支持的能力列表，例如通过 `CONFIG_IOT_PROTOCOL_MCP` 配置来表明支持 MCP 协议 (`"mcp": true`)。
|
||||
- **示例 (非 MCP 负载，而是基础协议消息):**
|
||||
```json
|
||||
{
|
||||
"type": "hello",
|
||||
"version": ...,
|
||||
"features": {
|
||||
"mcp": true,
|
||||
...
|
||||
},
|
||||
"transport": "websocket", // 或 "mqtt"
|
||||
"audio_params": { ... },
|
||||
"session_id": "..." // 设备收到服务器hello后可能设置
|
||||
}
|
||||
```
|
||||
|
||||
2. **初始化 MCP 会话**
|
||||
|
||||
- **时机：** 后台 API 收到设备 "hello" 消息，确认设备支持 MCP 后，通常作为 MCP 会话的第一个请求发送。
|
||||
- **发送方：** 后台 API (客户端)。
|
||||
- **方法：** `initialize`
|
||||
- **消息 (MCP payload):**
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"capabilities": {
|
||||
// 客户端能力，可选
|
||||
|
||||
// 摄像头视觉相关
|
||||
"vision": {
|
||||
"url": "...", //摄像头: 图片处理地址(必须是http地址, 不是websocket地址)
|
||||
"token": "..." // url token
|
||||
}
|
||||
|
||||
// ... 其他客户端能力
|
||||
}
|
||||
},
|
||||
"id": 1 // 请求 ID
|
||||
}
|
||||
```
|
||||
|
||||
- **设备响应时机：** 设备收到 `initialize` 请求并处理后。
|
||||
- **设备响应消息 (MCP payload):**
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1, // 匹配请求 ID
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {
|
||||
"tools": {} // 这里的 tools 似乎不列出详细信息，需要 tools/list
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "...", // 设备名称 (BOARD_NAME)
|
||||
"version": "..." // 设备固件版本
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. **发现设备工具列表**
|
||||
|
||||
- **时机：** 后台 API 需要获取设备当前支持的具体功能（工具）列表及其调用方式时。
|
||||
- **发送方：** 后台 API (客户端)。
|
||||
- **方法：** `tools/list`
|
||||
- **消息 (MCP payload):**
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "tools/list",
|
||||
"params": {
|
||||
"cursor": "" // 用于分页，首次请求为空字符串
|
||||
},
|
||||
"id": 2 // 请求 ID
|
||||
}
|
||||
```
|
||||
- **设备响应时机：** 设备收到 `tools/list` 请求并生成工具列表后。
|
||||
- **设备响应消息 (MCP payload):**
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2, // 匹配请求 ID
|
||||
"result": {
|
||||
"tools": [ // 工具对象列表
|
||||
{
|
||||
"name": "self.get_device_status",
|
||||
"description": "...",
|
||||
"inputSchema": { ... } // 参数 schema
|
||||
},
|
||||
{
|
||||
"name": "self.audio_speaker.set_volume",
|
||||
"description": "...",
|
||||
"inputSchema": { ... } // 参数 schema
|
||||
}
|
||||
// ... 更多工具
|
||||
],
|
||||
"nextCursor": "..." // 如果列表很大需要分页，这里会包含下一个请求的 cursor 值
|
||||
}
|
||||
}
|
||||
```
|
||||
- **分页处理：** 如果 `nextCursor` 字段非空，客户端需要再次发送 `tools/list` 请求，并在 `params` 中带上这个 `cursor` 值以获取下一页工具。
|
||||
|
||||
4. **调用设备工具**
|
||||
|
||||
- **时机：** 后台 API 需要执行设备上的某个具体功能时。
|
||||
- **发送方：** 后台 API (客户端)。
|
||||
- **方法：** `tools/call`
|
||||
- **消息 (MCP payload):**
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "self.audio_speaker.set_volume", // 要调用的工具名称
|
||||
"arguments": {
|
||||
// 工具参数，对象格式
|
||||
"volume": 50 // 参数名及其值
|
||||
}
|
||||
},
|
||||
"id": 3 // 请求 ID
|
||||
}
|
||||
```
|
||||
- **设备响应时机：** 设备收到 `tools/call` 请求，执行相应的工具函数后。
|
||||
- **设备成功响应消息 (MCP payload):**
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 3, // 匹配请求 ID
|
||||
"result": {
|
||||
"content": [
|
||||
// 工具执行结果内容
|
||||
{ "type": "text", "text": "true" } // 示例：set_volume 返回 bool
|
||||
],
|
||||
"isError": false // 表示成功
|
||||
}
|
||||
}
|
||||
```
|
||||
- **设备失败响应消息 (MCP payload):**
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 3, // 匹配请求 ID
|
||||
"error": {
|
||||
"code": -32601, // JSON-RPC 错误码，例如 Method not found (-32601)
|
||||
"message": "Unknown tool: self.non_existent_tool" // 错误描述
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
5. **设备主动发送消息 (Notifications)**
|
||||
- **时机：** 设备内部发生需要通知后台 API 的事件时（例如，状态变化，虽然代码示例中没有明确的工具发送此类消息，但 `Application::SendMcpMessage` 的存在暗示了设备可能主动发送 MCP 消息）。
|
||||
- **发送方：** 设备 (服务器)。
|
||||
- **方法：** 可能是以 `notifications/` 开头的方法名，或者其他自定义方法。
|
||||
- **消息 (MCP payload):** 遵循 JSON-RPC Notification 格式，没有 `id` 字段。
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/state_changed", // 示例方法名
|
||||
"params": {
|
||||
"newState": "idle",
|
||||
"oldState": "connecting"
|
||||
}
|
||||
// 没有 id 字段
|
||||
}
|
||||
```
|
||||
- **后台 API 处理：** 接收到 Notification 后，后台 API 进行相应的处理，但不回复。
|
||||
|
||||
## 交互图
|
||||
|
||||
下面是一个简化的交互序列图，展示了主要的 MCP 消息流程：
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Device as ESP32 Device
|
||||
participant BackendAPI as 后台 API (Client)
|
||||
|
||||
Note over Device, BackendAPI: 建立 WebSocket / MQTT 连接
|
||||
|
||||
Device->>BackendAPI: Hello Message (包含 "mcp": true)
|
||||
|
||||
BackendAPI->>Device: MCP Initialize Request
|
||||
Note over BackendAPI: method: initialize
|
||||
Note over BackendAPI: params: { capabilities: ... }
|
||||
|
||||
Device->>BackendAPI: MCP Initialize Response
|
||||
Note over Device: result: { protocolVersion: ..., serverInfo: ... }
|
||||
|
||||
BackendAPI->>Device: MCP Get Tools List Request
|
||||
Note over BackendAPI: method: tools/list
|
||||
Note over BackendAPI: params: { cursor: "" }
|
||||
|
||||
Device->>BackendAPI: MCP Get Tools List Response
|
||||
Note over Device: result: { tools: [...], nextCursor: ... }
|
||||
|
||||
loop Optional Pagination
|
||||
BackendAPI->>Device: MCP Get Tools List Request
|
||||
Note over BackendAPI: method: tools/list
|
||||
Note over BackendAPI: params: { cursor: "..." }
|
||||
Device->>BackendAPI: MCP Get Tools List Response
|
||||
Note over Device: result: { tools: [...], nextCursor: "" }
|
||||
end
|
||||
|
||||
BackendAPI->>Device: MCP Call Tool Request
|
||||
Note over BackendAPI: method: tools/call
|
||||
Note over BackendAPI: params: { name: "...", arguments: { ... } }
|
||||
|
||||
alt Tool Call Successful
|
||||
Device->>BackendAPI: MCP Tool Call Success Response
|
||||
Note over Device: result: { content: [...], isError: false }
|
||||
else Tool Call Failed
|
||||
Device->>BackendAPI: MCP Tool Call Error Response
|
||||
Note over Device: error: { code: ..., message: ... }
|
||||
end
|
||||
|
||||
opt Device Notification
|
||||
Device->>BackendAPI: MCP Notification
|
||||
Note over Device: method: notifications/...
|
||||
Note over Device: params: { ... }
|
||||
end
|
||||
```
|
||||
|
||||
这份文档概述了该项目中 MCP 协议的主要交互流程。具体的参数细节和工具功能需要参考 `main/mcp_server.cc` 中 `McpServer::AddCommonTools` 以及各个工具的实现。
|
||||
@@ -0,0 +1,115 @@
|
||||
# MCP 协议物联网控制用法说明
|
||||
|
||||
> 本文档介绍如何基于 MCP 协议实现 ESP32 设备的物联网控制。详细协议流程请参考 [`mcp-protocol.md`](./mcp-protocol.md)。
|
||||
|
||||
## 简介
|
||||
|
||||
MCP（Model Context Protocol）是新一代推荐用于物联网控制的协议，通过标准 JSON-RPC 2.0 格式在后台与设备间发现和调用"工具"（Tool），实现灵活的设备控制。
|
||||
|
||||
## 典型使用流程
|
||||
|
||||
1. 设备启动后通过基础协议（如 WebSocket/MQTT）与后台建立连接。
|
||||
2. 后台通过 MCP 协议的 `initialize` 方法初始化会话。
|
||||
3. 后台通过 `tools/list` 获取设备支持的所有工具（功能）及参数说明。
|
||||
4. 后台通过 `tools/call` 调用具体工具，实现对设备的控制。
|
||||
|
||||
详细协议格式与交互请见 [`mcp-protocol.md`](./mcp-protocol.md)。
|
||||
|
||||
## 设备端工具注册方法说明
|
||||
|
||||
设备通过 `McpServer::AddTool` 方法注册可被后台调用的"工具"。其常用函数签名如下：
|
||||
|
||||
```cpp
|
||||
void AddTool(
|
||||
const std::string& name, // 工具名称，建议唯一且有层次感，如 self.dog.forward
|
||||
const std::string& description, // 工具描述，简明说明功能，便于大模型理解
|
||||
const PropertyList& properties, // 输入参数列表（可为空），支持类型：布尔、整数、字符串
|
||||
std::function<ReturnValue(const PropertyList&)> callback // 工具被调用时的回调实现
|
||||
);
|
||||
```
|
||||
- name：工具唯一标识，建议用"模块.功能"命名风格。
|
||||
- description：自然语言描述，便于 AI/用户理解。
|
||||
- properties：参数列表，支持类型有布尔、整数、字符串，可指定范围和默认值。
|
||||
- callback：收到调用请求时的实际执行逻辑，返回值可为 bool/int/string。
|
||||
|
||||
## 典型注册示例（以 ESP-Hi 为例）
|
||||
|
||||
```cpp
|
||||
void InitializeTools() {
|
||||
auto& mcp_server = McpServer::GetInstance();
|
||||
// 例1：无参数，控制机器人前进
|
||||
mcp_server.AddTool("self.dog.forward", "机器人向前移动", PropertyList(), [this](const PropertyList&) -> ReturnValue {
|
||||
servo_dog_ctrl_send(DOG_STATE_FORWARD, NULL);
|
||||
return true;
|
||||
});
|
||||
// 例2：带参数，设置灯光 RGB 颜色
|
||||
mcp_server.AddTool("self.light.set_rgb", "设置RGB颜色", PropertyList({
|
||||
Property("r", kPropertyTypeInteger, 0, 255),
|
||||
Property("g", kPropertyTypeInteger, 0, 255),
|
||||
Property("b", kPropertyTypeInteger, 0, 255)
|
||||
}), [this](const PropertyList& properties) -> ReturnValue {
|
||||
int r = properties["r"].value<int>();
|
||||
int g = properties["g"].value<int>();
|
||||
int b = properties["b"].value<int>();
|
||||
led_on_ = true;
|
||||
SetLedColor(r, g, b);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
## 常见工具调用 JSON-RPC 示例
|
||||
|
||||
### 1. 获取工具列表
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "tools/list",
|
||||
"params": { "cursor": "" },
|
||||
"id": 1
|
||||
}
|
||||
```
|
||||
|
||||
### 2. 控制底盘前进
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "self.chassis.go_forward",
|
||||
"arguments": {}
|
||||
},
|
||||
"id": 2
|
||||
}
|
||||
```
|
||||
|
||||
### 3. 切换灯光模式
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "self.chassis.switch_light_mode",
|
||||
"arguments": { "light_mode": 3 }
|
||||
},
|
||||
"id": 3
|
||||
}
|
||||
```
|
||||
|
||||
### 4. 摄像头翻转
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "self.camera.set_camera_flipped",
|
||||
"arguments": {}
|
||||
},
|
||||
"id": 4
|
||||
}
|
||||
```
|
||||
|
||||
## 备注
|
||||
- 工具名称、参数及返回值请以设备端 `AddTool` 注册为准。
|
||||
- 推荐所有新项目统一采用 MCP 协议进行物联网控制。
|
||||
- 详细协议与进阶用法请查阅 [`mcp-protocol.md`](./mcp-protocol.md)。
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 94 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 35 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 47 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 50 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 45 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 51 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
@@ -0,0 +1,430 @@
|
||||
以下是一份基于代码实现整理的 WebSocket 通信协议文档，概述设备端与服务器之间如何通过 WebSocket 进行交互。
|
||||
|
||||
该文档仅基于所提供的代码推断，实际部署时可能需要结合服务器端实现进行进一步确认或补充。
|
||||
|
||||
---
|
||||
|
||||
## 1. 总体流程概览
|
||||
|
||||
1. **设备端初始化**
|
||||
- 设备上电、初始化 `Application`：
|
||||
- 初始化音频编解码器、显示屏、LED 等
|
||||
- 连接网络
|
||||
- 创建并初始化实现 `Protocol` 接口的 WebSocket 协议实例（`WebsocketProtocol`）
|
||||
- 进入主循环等待事件（音频输入、音频输出、调度任务等）。
|
||||
|
||||
2. **建立 WebSocket 连接**
|
||||
- 当设备需要开始语音会话时（例如用户唤醒、手动按键触发等），调用 `OpenAudioChannel()`：
|
||||
- 根据配置获取 WebSocket URL
|
||||
- 设置若干请求头（`Authorization`, `Protocol-Version`, `Device-Id`, `Client-Id`）
|
||||
- 调用 `Connect()` 与服务器建立 WebSocket 连接
|
||||
|
||||
3. **设备端发送 "hello" 消息**
|
||||
- 连接成功后，设备会发送一条 JSON 消息，示例结构如下：
|
||||
```json
|
||||
{
|
||||
"type": "hello",
|
||||
"version": 1,
|
||||
"features": {
|
||||
"mcp": true
|
||||
},
|
||||
"transport": "websocket",
|
||||
"audio_params": {
|
||||
"format": "opus",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1,
|
||||
"frame_duration": 60
|
||||
}
|
||||
}
|
||||
```
|
||||
- 其中 `features` 字段为可选，内容根据设备编译配置自动生成。例如：`"mcp": true` 表示支持 MCP 协议。
|
||||
- `frame_duration` 的值对应 `OPUS_FRAME_DURATION_MS`（例如 60ms）。
|
||||
|
||||
4. **服务器回复 "hello"**
|
||||
- 设备等待服务器返回一条包含 `"type": "hello"` 的 JSON 消息，并检查 `"transport": "websocket"` 是否匹配。
|
||||
- 服务器可选下发 `session_id` 字段，设备端收到后会自动记录。
|
||||
- 示例：
|
||||
```json
|
||||
{
|
||||
"type": "hello",
|
||||
"transport": "websocket",
|
||||
"session_id": "xxx",
|
||||
"audio_params": {
|
||||
"format": "opus",
|
||||
"sample_rate": 24000,
|
||||
"channels": 1,
|
||||
"frame_duration": 60
|
||||
}
|
||||
}
|
||||
```
|
||||
- 如果匹配，则认为服务器已就绪，标记音频通道打开成功。
|
||||
- 如果在超时时间（默认 10 秒）内未收到正确回复，认为连接失败并触发网络错误回调。
|
||||
|
||||
5. **后续消息交互**
|
||||
- 设备端和服务器端之间可发送两种主要类型的数据：
|
||||
1. **二进制音频数据**（Opus 编码）
|
||||
2. **文本 JSON 消息**（用于传输聊天状态、TTS/STT 事件、MCP 协议消息等）
|
||||
|
||||
- 在代码里，接收回调主要分为：
|
||||
- `OnData(...)`:
|
||||
- 当 `binary` 为 `true` 时，认为是音频帧；设备会将其当作 Opus 数据进行解码。
|
||||
- 当 `binary` 为 `false` 时，认为是 JSON 文本，需要在设备端用 cJSON 进行解析并做相应业务逻辑处理（如聊天、TTS、MCP 协议消息等）。
|
||||
|
||||
- 当服务器或网络出现断连，回调 `OnDisconnected()` 被触发：
|
||||
- 设备会调用 `on_audio_channel_closed_()`，并最终回到空闲状态。
|
||||
|
||||
6. **关闭 WebSocket 连接**
|
||||
- 设备在需要结束语音会话时，会调用 `CloseAudioChannel()` 主动断开连接，并回到空闲状态。
|
||||
- 或者如果服务器端主动断开，也会引发同样的回调流程。
|
||||
|
||||
---
|
||||
|
||||
## 2. 通用请求头
|
||||
|
||||
在建立 WebSocket 连接时，代码示例中设置了以下请求头：
|
||||
|
||||
- `Authorization`: 用于存放访问令牌，形如 `"Bearer <token>"`
|
||||
- `Protocol-Version`: 固定示例中为 `"1"`，与 hello 消息体内的 `version` 字段保持一致
|
||||
- `Device-Id`: 设备物理网卡 MAC 地址
|
||||
- `Client-Id`: 软件生成的 UUID（擦除 NVS 或重新烧录完整固件会重置）
|
||||
|
||||
这些头会随着 WebSocket 握手一起发送到服务器，服务器可根据需求进行校验、认证等。
|
||||
|
||||
---
|
||||
|
||||
## 3. JSON 消息结构
|
||||
|
||||
WebSocket 文本帧以 JSON 方式传输，以下为常见的 `"type"` 字段及其对应业务逻辑。若消息里包含未列出的字段，可能为可选或特定实现细节。
|
||||
|
||||
### 3.1 设备端→服务器
|
||||
|
||||
1. **Hello**
|
||||
- 连接成功后，由设备端发送，告知服务器基本参数。
|
||||
- 例：
|
||||
```json
|
||||
{
|
||||
"type": "hello",
|
||||
"version": 1,
|
||||
"features": {
|
||||
"mcp": true
|
||||
},
|
||||
"transport": "websocket",
|
||||
"audio_params": {
|
||||
"format": "opus",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1,
|
||||
"frame_duration": 60
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. **Listen**
|
||||
- 表示设备端开始或停止录音监听。
|
||||
- 常见字段：
|
||||
- `"session_id"`：会话标识
|
||||
- `"type": "listen"`
|
||||
- `"state"`：`"start"`, `"stop"`, `"detect"`（唤醒检测已触发）
|
||||
- `"mode"`：`"auto"`, `"manual"` 或 `"realtime"`，表示识别模式。
|
||||
- 例：开始监听
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "listen",
|
||||
"state": "start",
|
||||
"mode": "manual"
|
||||
}
|
||||
```
|
||||
|
||||
3. **Abort**
|
||||
- 终止当前说话（TTS 播放）或语音通道。
|
||||
- 例：
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "abort",
|
||||
"reason": "wake_word_detected"
|
||||
}
|
||||
```
|
||||
- `reason` 值可为 `"wake_word_detected"` 或其他。
|
||||
|
||||
4. **Wake Word Detected**
|
||||
- 用于设备端向服务器告知检测到唤醒词。
|
||||
- 在发送该消息之前，可提前发送唤醒词的 Opus 音频数据，用于服务器进行声纹检测。
|
||||
- 例：
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "listen",
|
||||
"state": "detect",
|
||||
"text": "你好小明"
|
||||
}
|
||||
```
|
||||
|
||||
5. **MCP**
|
||||
- 推荐用于物联网控制的新一代协议。所有设备能力发现、工具调用等均通过 type: "mcp" 的消息进行，payload 内部为标准 JSON-RPC 2.0（详见 [MCP 协议文档](./mcp-protocol.md)）。
|
||||
|
||||
- **设备端到服务器发送 result 的例子：**
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"result": {
|
||||
"content": [
|
||||
{ "type": "text", "text": "true" }
|
||||
],
|
||||
"isError": false
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3.2 服务器→设备端
|
||||
|
||||
1. **Hello**
|
||||
- 服务器端返回的握手确认消息。
|
||||
- 必须包含 `"type": "hello"` 和 `"transport": "websocket"`。
|
||||
- 可能会带有 `audio_params`，表示服务器期望的音频参数，或与设备端对齐的配置。
|
||||
- 服务器可选下发 `session_id` 字段，设备端收到后会自动记录。
|
||||
- 成功接收后设备端会设置事件标志，表示 WebSocket 通道就绪。
|
||||
|
||||
2. **STT**
|
||||
- `{"session_id": "xxx", "type": "stt", "text": "..."}`
|
||||
- 表示服务器端识别到了用户语音。（例如语音转文本结果）
|
||||
- 设备可能将此文本显示到屏幕上，后续再进入回答等流程。
|
||||
|
||||
3. **LLM**
|
||||
- `{"session_id": "xxx", "type": "llm", "emotion": "happy", "text": "😀"}`
|
||||
- 服务器指示设备调整表情动画 / UI 表达。
|
||||
|
||||
4. **TTS**
|
||||
- `{"session_id": "xxx", "type": "tts", "state": "start"}`：服务器准备下发 TTS 音频，设备端进入 "speaking" 播放状态。
|
||||
- `{"session_id": "xxx", "type": "tts", "state": "stop"}`：表示本次 TTS 结束。
|
||||
- `{"session_id": "xxx", "type": "tts", "state": "sentence_start", "text": "..."}`
|
||||
- 让设备在界面上显示当前要播放或朗读的文本片段（例如用于显示给用户）。
|
||||
|
||||
5. **MCP**
|
||||
- 服务器通过 type: "mcp" 的消息下发物联网相关的控制指令或返回调用结果，payload 结构同上。
|
||||
|
||||
- **服务器到设备端发送 tools/call 的例子：**
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "mcp",
|
||||
"payload": {
|
||||
"jsonrpc": "2.0",
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "self.light.set_rgb",
|
||||
"arguments": { "r": 255, "g": 0, "b": 0 }
|
||||
},
|
||||
"id": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
6. **音频数据：二进制帧**
|
||||
- 当服务器发送音频二进制帧（Opus 编码）时，设备端解码并播放。
|
||||
- 若设备端正在处于 "listening" （录音）状态，收到的音频帧会被忽略或清空以防冲突。
|
||||
|
||||
---
|
||||
|
||||
## 4. 音频编解码
|
||||
|
||||
1. **设备端发送录音数据**
|
||||
- 音频输入经过可能的回声消除、降噪或音量增益后，通过 Opus 编码打包为二进制帧发送给服务器。
|
||||
- 如果设备端每次编码生成的二进制帧大小为 N 字节，则会通过 WebSocket 的 **binary** 消息发送这块数据。
|
||||
|
||||
2. **设备端播放收到的音频**
|
||||
- 收到服务器的二进制帧时，同样认定是 Opus 数据。
|
||||
- 设备端会进行解码，然后交由音频输出接口播放。
|
||||
- 如果服务器的音频采样率与设备不一致，会在解码后再进行重采样。
|
||||
|
||||
---
|
||||
|
||||
## 5. 常见状态流转
|
||||
|
||||
以下为常见设备端关键状态流转，与 WebSocket 消息对应：
|
||||
|
||||
1. **Idle** → **Connecting**
|
||||
- 用户触发或唤醒后，设备调用 `OpenAudioChannel()` → 建立 WebSocket 连接 → 发送 `"type":"hello"`。
|
||||
|
||||
2. **Connecting** → **Listening**
|
||||
- 成功建立连接后，若继续执行 `SendStartListening(...)`，则进入录音状态。此时设备会持续编码麦克风数据并发送到服务器。
|
||||
|
||||
3. **Listening** → **Speaking**
|
||||
- 收到服务器 TTS Start 消息 (`{"type":"tts","state":"start"}`) → 停止录音并播放接收到的音频。
|
||||
|
||||
4. **Speaking** → **Idle**
|
||||
- 服务器 TTS Stop (`{"type":"tts","state":"stop"}`) → 音频播放结束。若未继续进入自动监听，则返回 Idle；如果配置了自动循环，则再度进入 Listening。
|
||||
|
||||
5. **Listening** / **Speaking** → **Idle**（遇到异常或主动中断）
|
||||
- 调用 `SendAbortSpeaking(...)` 或 `CloseAudioChannel()` → 中断会话 → 关闭 WebSocket → 状态回到 Idle。
|
||||
|
||||
### 自动模式状态流转图
|
||||
|
||||
```mermaid
|
||||
stateDiagram
|
||||
direction TB
|
||||
[*] --> kDeviceStateUnknown
|
||||
kDeviceStateUnknown --> kDeviceStateStarting:初始化
|
||||
kDeviceStateStarting --> kDeviceStateWifiConfiguring:配置WiFi
|
||||
kDeviceStateStarting --> kDeviceStateActivating:激活设备
|
||||
kDeviceStateActivating --> kDeviceStateUpgrading:检测到新版本
|
||||
kDeviceStateActivating --> kDeviceStateIdle:激活完成
|
||||
kDeviceStateIdle --> kDeviceStateConnecting:开始连接
|
||||
kDeviceStateConnecting --> kDeviceStateIdle:连接失败
|
||||
kDeviceStateConnecting --> kDeviceStateListening:连接成功
|
||||
kDeviceStateListening --> kDeviceStateSpeaking:开始说话
|
||||
kDeviceStateSpeaking --> kDeviceStateListening:结束说话
|
||||
kDeviceStateListening --> kDeviceStateIdle:手动终止
|
||||
kDeviceStateSpeaking --> kDeviceStateIdle:自动终止
|
||||
```
|
||||
|
||||
### 手动模式状态流转图
|
||||
|
||||
```mermaid
|
||||
stateDiagram
|
||||
direction TB
|
||||
[*] --> kDeviceStateUnknown
|
||||
kDeviceStateUnknown --> kDeviceStateStarting:初始化
|
||||
kDeviceStateStarting --> kDeviceStateWifiConfiguring:配置WiFi
|
||||
kDeviceStateStarting --> kDeviceStateActivating:激活设备
|
||||
kDeviceStateActivating --> kDeviceStateUpgrading:检测到新版本
|
||||
kDeviceStateActivating --> kDeviceStateIdle:激活完成
|
||||
kDeviceStateIdle --> kDeviceStateConnecting:开始连接
|
||||
kDeviceStateConnecting --> kDeviceStateIdle:连接失败
|
||||
kDeviceStateConnecting --> kDeviceStateListening:连接成功
|
||||
kDeviceStateIdle --> kDeviceStateListening:开始监听
|
||||
kDeviceStateListening --> kDeviceStateIdle:停止监听
|
||||
kDeviceStateIdle --> kDeviceStateSpeaking:开始说话
|
||||
kDeviceStateSpeaking --> kDeviceStateIdle:结束说话
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. 错误处理
|
||||
|
||||
1. **连接失败**
|
||||
- 如果 `Connect(url)` 返回失败或在等待服务器 "hello" 消息时超时，触发 `on_network_error_()` 回调。设备会提示"无法连接到服务"或类似错误信息。
|
||||
|
||||
2. **服务器断开**
|
||||
- 如果 WebSocket 异常断开，回调 `OnDisconnected()`：
|
||||
- 设备回调 `on_audio_channel_closed_()`
|
||||
- 切换到 Idle 或其他重试逻辑。
|
||||
|
||||
---
|
||||
|
||||
## 7. 其它注意事项
|
||||
|
||||
1. **鉴权**
|
||||
- 设备通过设置 `Authorization: Bearer <token>` 提供鉴权，服务器端需验证是否有效。
|
||||
- 如果令牌过期或无效，服务器可拒绝握手或在后续断开。
|
||||
|
||||
2. **会话控制**
|
||||
- 代码中部分消息包含 `session_id`，用于区分独立的对话或操作。服务端可根据需要对不同会话做分离处理。
|
||||
|
||||
3. **音频负载**
|
||||
- 代码里默认使用 Opus 格式，并设置 `sample_rate = 16000`，单声道。帧时长由 `OPUS_FRAME_DURATION_MS` 控制，一般为 60ms。可根据带宽或性能做适当调整。为了获得更好的音乐播放效果，服务器下行音频可能使用 24000 采样率。
|
||||
|
||||
4. **物联网控制推荐 MCP 协议**
|
||||
- 设备与服务器之间的物联网能力发现、状态同步、控制指令等，建议全部通过 MCP 协议（type: "mcp"）实现。原有的 type: "iot" 方案已废弃。
|
||||
- MCP 协议可在 WebSocket、MQTT 等多种底层协议上传输，具备更好的扩展性和标准化能力。
|
||||
- 详细用法请参考 [MCP 协议文档](./mcp-protocol.md) 及 [MCP 物联网控制用法](./mcp-usage.md)。
|
||||
|
||||
5. **错误或异常 JSON**
|
||||
- 当 JSON 中缺少必要字段，例如 `{"type": ...}`，设备端会记录错误日志（`ESP_LOGE(TAG, "Missing message type, data: %s", data);`），不会执行任何业务。
|
||||
|
||||
---
|
||||
|
||||
## 8. 消息示例
|
||||
|
||||
下面给出一个典型的双向消息示例（流程简化示意）：
|
||||
|
||||
1. **设备端 → 服务器**（握手）
|
||||
```json
|
||||
{
|
||||
"type": "hello",
|
||||
"version": 1,
|
||||
"features": {
|
||||
"mcp": true
|
||||
},
|
||||
"transport": "websocket",
|
||||
"audio_params": {
|
||||
"format": "opus",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1,
|
||||
"frame_duration": 60
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. **服务器 → 设备端**（握手应答）
|
||||
```json
|
||||
{
|
||||
"type": "hello",
|
||||
"transport": "websocket",
|
||||
"session_id": "xxx",
|
||||
"audio_params": {
|
||||
"format": "opus",
|
||||
"sample_rate": 16000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. **设备端 → 服务器**（开始监听）
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "listen",
|
||||
"state": "start",
|
||||
"mode": "auto"
|
||||
}
|
||||
```
|
||||
同时设备端开始发送二进制帧（Opus 数据）。
|
||||
|
||||
4. **服务器 → 设备端**（ASR 结果）
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "stt",
|
||||
"text": "用户说的话"
|
||||
}
|
||||
```
|
||||
|
||||
5. **服务器 → 设备端**（TTS开始）
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "tts",
|
||||
"state": "start"
|
||||
}
|
||||
```
|
||||
接着服务器发送二进制音频帧给设备端播放。
|
||||
|
||||
6. **服务器 → 设备端**（TTS结束）
|
||||
```json
|
||||
{
|
||||
"session_id": "xxx",
|
||||
"type": "tts",
|
||||
"state": "stop"
|
||||
}
|
||||
```
|
||||
设备端停止播放音频，若无更多指令，则回到空闲状态。
|
||||
|
||||
---
|
||||
|
||||
## 9. 总结
|
||||
|
||||
本协议通过在 WebSocket 上层传输 JSON 文本与二进制音频帧，完成功能包括音频流上传、TTS 音频播放、语音识别与状态管理、MCP 指令下发等。其核心特征：
|
||||
|
||||
- **握手阶段**：发送 `"type":"hello"`，等待服务器返回。
|
||||
- **音频通道**：采用 Opus 编码的二进制帧双向传输语音流。
|
||||
- **JSON 消息**：使用 `"type"` 为核心字段标识不同业务逻辑，包括 TTS、STT、MCP、WakeWord 等。
|
||||
- **扩展性**：可根据实际需求在 JSON 消息中添加字段，或在 headers 里进行额外鉴权。
|
||||
|
||||
服务器与设备端需提前约定各类消息的字段含义、时序逻辑以及错误处理规则，方能保证通信顺畅。上述信息可作为基础文档，便于后续对接、开发或扩展。
|
||||
@@ -0,0 +1,326 @@
|
||||
set(SOURCES "audio_codecs/audio_codec.cc"
|
||||
"audio_codecs/no_audio_codec.cc"
|
||||
"audio_codecs/box_audio_codec.cc"
|
||||
"audio_codecs/es8311_audio_codec.cc"
|
||||
"audio_codecs/es8374_audio_codec.cc"
|
||||
"audio_codecs/es8388_audio_codec.cc"
|
||||
"audio_processing/audio_debugger.cc"
|
||||
"led/single_led.cc"
|
||||
"led/circular_strip.cc"
|
||||
"led/gpio_led.cc"
|
||||
"display/display.cc"
|
||||
"display/lcd_display.cc"
|
||||
"display/oled_display.cc"
|
||||
"protocols/protocol.cc"
|
||||
"protocols/mqtt_protocol.cc"
|
||||
"protocols/websocket_protocol.cc"
|
||||
"iot/thing.cc"
|
||||
"iot/thing_manager.cc"
|
||||
"mcp_server.cc"
|
||||
"system_info.cc"
|
||||
"application.cc"
|
||||
"ota.cc"
|
||||
"settings.cc"
|
||||
"background_task.cc"
|
||||
"main.cc"
|
||||
)
|
||||
|
||||
set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols" "audio_processing")
|
||||
|
||||
# 添加 IOT 相关文件
|
||||
file(GLOB IOT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/iot/things/*.cc)
|
||||
list(APPEND SOURCES ${IOT_SOURCES})
|
||||
|
||||
# 添加板级公共文件
|
||||
file(GLOB BOARD_COMMON_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/boards/common/*.cc)
|
||||
list(APPEND SOURCES ${BOARD_COMMON_SOURCES})
|
||||
list(APPEND INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/boards/common)
|
||||
|
||||
# 根据 BOARD_TYPE 配置添加对应的板级文件
|
||||
if(CONFIG_BOARD_TYPE_BREAD_COMPACT_WIFI)
|
||||
set(BOARD_TYPE "bread-compact-wifi")
|
||||
elseif(CONFIG_BOARD_TYPE_BREAD_COMPACT_ML307)
|
||||
set(BOARD_TYPE "bread-compact-ml307")
|
||||
elseif(CONFIG_BOARD_TYPE_BREAD_COMPACT_ESP32)
|
||||
set(BOARD_TYPE "bread-compact-esp32")
|
||||
elseif(CONFIG_BOARD_TYPE_BREAD_COMPACT_ESP32_LCD)
|
||||
set(BOARD_TYPE "bread-compact-esp32-lcd")
|
||||
elseif(CONFIG_BOARD_TYPE_DF_K10)
|
||||
set(BOARD_TYPE "df-k10")
|
||||
elseif(CONFIG_BOARD_TYPE_DF_S3_AI_CAM)
|
||||
set(BOARD_TYPE "df-s3-ai-cam")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP_BOX_3)
|
||||
set(BOARD_TYPE "esp-box-3")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP_BOX)
|
||||
set(BOARD_TYPE "esp-box")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP_BOX_LITE)
|
||||
set(BOARD_TYPE "esp-box-lite")
|
||||
elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_1)
|
||||
set(BOARD_TYPE "kevin-box-1")
|
||||
elseif(CONFIG_BOARD_TYPE_KEVIN_BOX_2)
|
||||
set(BOARD_TYPE "kevin-box-2")
|
||||
elseif(CONFIG_BOARD_TYPE_KEVIN_C3)
|
||||
set(BOARD_TYPE "kevin-c3")
|
||||
elseif(CONFIG_BOARD_TYPE_KEVIN_SP_V3_DEV)
|
||||
set(BOARD_TYPE "kevin-sp-v3-dev")
|
||||
elseif(CONFIG_BOARD_TYPE_KEVIN_SP_V4_DEV)
|
||||
set(BOARD_TYPE "kevin-sp-v4-dev")
|
||||
elseif(CONFIG_BOARD_TYPE_KEVIN_YUYING_313LCD)
|
||||
set(BOARD_TYPE "kevin-yuying-313lcd")
|
||||
elseif(CONFIG_BOARD_TYPE_LICHUANG_DEV)
|
||||
set(BOARD_TYPE "lichuang-dev")
|
||||
elseif(CONFIG_BOARD_TYPE_LICHUANG_C3_DEV)
|
||||
set(BOARD_TYPE "lichuang-c3-dev")
|
||||
elseif(CONFIG_BOARD_TYPE_MAGICLICK_2P4)
|
||||
set(BOARD_TYPE "magiclick-2p4")
|
||||
elseif(CONFIG_BOARD_TYPE_MAGICLICK_2P5)
|
||||
set(BOARD_TYPE "magiclick-2p5")
|
||||
elseif(CONFIG_BOARD_TYPE_MAGICLICK_C3)
|
||||
set(BOARD_TYPE "magiclick-c3")
|
||||
elseif(CONFIG_BOARD_TYPE_MAGICLICK_C3_V2)
|
||||
set(BOARD_TYPE "magiclick-c3-v2")
|
||||
elseif(CONFIG_BOARD_TYPE_M5STACK_CORE_S3)
|
||||
set(BOARD_TYPE "m5stack-core-s3")
|
||||
elseif(CONFIG_BOARD_TYPE_M5STACK_CORE_TAB5)
|
||||
set(BOARD_TYPE "m5stack-tab5")
|
||||
elseif(CONFIG_BOARD_TYPE_ATOMS3_ECHO_BASE)
|
||||
set(BOARD_TYPE "atoms3-echo-base")
|
||||
elseif(CONFIG_BOARD_TYPE_ATOMS3R_ECHO_BASE)
|
||||
set(BOARD_TYPE "atoms3r-echo-base")
|
||||
elseif(CONFIG_BOARD_TYPE_ATOMS3R_CAM_M12_ECHO_BASE)
|
||||
set(BOARD_TYPE "atoms3r-cam-m12-echo-base")
|
||||
elseif(CONFIG_BOARD_TYPE_ATOMMATRIX_ECHO_BASE)
|
||||
set(BOARD_TYPE "atommatrix-echo-base")
|
||||
elseif(CONFIG_BOARD_TYPE_XMINI_C3)
|
||||
set(BOARD_TYPE "xmini-c3")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_KORVO2_V3)
|
||||
set(BOARD_TYPE "esp32s3-korvo2-v3")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP_SPARKBOT)
|
||||
set(BOARD_TYPE "esp-sparkbot")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP_SPOT_S3)
|
||||
set(BOARD_TYPE "esp-spot-s3")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP_HI)
|
||||
set(BOARD_TYPE "esp-hi")
|
||||
elseif(CONFIG_BOARD_TYPE_ECHOEAR)
|
||||
set(BOARD_TYPE "echoear")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_Touch_AMOLED_1_8)
|
||||
set(BOARD_TYPE "esp32-s3-touch-amoled-1.8")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_Touch_AMOLED_1_75)
|
||||
set(BOARD_TYPE "waveshare-s3-touch-amoled-1.75")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_Touch_LCD_1_85C)
|
||||
set(BOARD_TYPE "esp32-s3-touch-lcd-1.85c")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_Touch_LCD_1_85)
|
||||
set(BOARD_TYPE "esp32-s3-touch-lcd-1.85")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_Touch_LCD_1_46)
|
||||
set(BOARD_TYPE "esp32-s3-touch-lcd-1.46")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_Touch_LCD_3_5)
|
||||
set(BOARD_TYPE "esp32-s3-touch-lcd-3.5")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32C6_LCD_1_69)
|
||||
set(BOARD_TYPE "waveshare-c6-lcd-1.69")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32P4_NANO)
|
||||
set(BOARD_TYPE "waveshare-p4-nano")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_4B)
|
||||
set(BOARD_TYPE "waveshare-p4-wifi6-touch-lcd-4b")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_XC)
|
||||
set(BOARD_TYPE "waveshare-p4-wifi6-touch-lcd-xc")
|
||||
elseif(CONFIG_BOARD_TYPE_BREAD_COMPACT_WIFI_LCD)
|
||||
set(BOARD_TYPE "bread-compact-wifi-lcd")
|
||||
elseif(CONFIG_BOARD_TYPE_TUDOUZI)
|
||||
set(BOARD_TYPE "tudouzi")
|
||||
elseif(CONFIG_BOARD_TYPE_LILYGO_T_CIRCLE_S3)
|
||||
set(BOARD_TYPE "lilygo-t-circle-s3")
|
||||
elseif(CONFIG_BOARD_TYPE_LILYGO_T_CAMERAPLUS_S3_V1_0_V1_1)
|
||||
set(BOARD_TYPE "lilygo-t-cameraplus-s3")
|
||||
elseif(CONFIG_BOARD_TYPE_LILYGO_T_CAMERAPLUS_S3_V1_2)
|
||||
set(BOARD_TYPE "lilygo-t-cameraplus-s3")
|
||||
elseif(CONFIG_BOARD_TYPE_LILYGO_T_DISPLAY_S3_PRO_MVSRLORA)
|
||||
set(BOARD_TYPE "lilygo-t-display-s3-pro-mvsrlora")
|
||||
elseif(CONFIG_BOARD_TYPE_LILYGO_T_DISPLAY_S3_PRO_MVSRLORA_NO_BATTERY)
|
||||
set(BOARD_TYPE "lilygo-t-display-s3-pro-mvsrlora")
|
||||
elseif(CONFIG_BOARD_TYPE_MOVECALL_MOJI_ESP32S3)
|
||||
set(BOARD_TYPE "movecall-moji-esp32s3")
|
||||
elseif(CONFIG_BOARD_TYPE_MOVECALL_CUICAN_ESP32S3)
|
||||
set(BOARD_TYPE "movecall-cuican-esp32s3")
|
||||
elseif(CONFIG_BOARD_TYPE_ATK_DNESP32S3)
|
||||
set(BOARD_TYPE "atk-dnesp32s3")
|
||||
elseif(CONFIG_BOARD_TYPE_ATK_DNESP32S3_BOX)
|
||||
set(BOARD_TYPE "atk-dnesp32s3-box")
|
||||
elseif(CONFIG_BOARD_TYPE_ATK_DNESP32S3_BOX0)
|
||||
set(BOARD_TYPE "atk-dnesp32s3-box0")
|
||||
elseif(CONFIG_BOARD_TYPE_ATK_DNESP32S3M_WIFI)
|
||||
set(BOARD_TYPE "atk-dnesp32s3m-wifi")
|
||||
elseif(CONFIG_BOARD_TYPE_ATK_DNESP32S3M_4G)
|
||||
set(BOARD_TYPE "atk-dnesp32s3m-4g")
|
||||
elseif(CONFIG_BOARD_TYPE_DU_CHATX)
|
||||
set(BOARD_TYPE "du-chatx")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32S3_Taiji_Pi)
|
||||
set(BOARD_TYPE "taiji-pi-s3")
|
||||
elseif(CONFIG_BOARD_TYPE_XINGZHI_Cube_0_85TFT_WIFI)
|
||||
set(BOARD_TYPE "xingzhi-cube-0.85tft-wifi")
|
||||
elseif(CONFIG_BOARD_TYPE_XINGZHI_Cube_0_85TFT_ML307)
|
||||
set(BOARD_TYPE "xingzhi-cube-0.85tft-ml307")
|
||||
elseif(CONFIG_BOARD_TYPE_XINGZHI_Cube_0_96OLED_WIFI)
|
||||
set(BOARD_TYPE "xingzhi-cube-0.96oled-wifi")
|
||||
elseif(CONFIG_BOARD_TYPE_XINGZHI_Cube_0_96OLED_ML307)
|
||||
set(BOARD_TYPE "xingzhi-cube-0.96oled-ml307")
|
||||
elseif(CONFIG_BOARD_TYPE_XINGZHI_Cube_1_54TFT_WIFI)
|
||||
set(BOARD_TYPE "xingzhi-cube-1.54tft-wifi")
|
||||
elseif(CONFIG_BOARD_TYPE_XINGZHI_Cube_1_54TFT_ML307)
|
||||
set(BOARD_TYPE "xingzhi-cube-1.54tft-ml307")
|
||||
elseif(CONFIG_BOARD_TYPE_SENSECAP_WATCHER)
|
||||
set(BOARD_TYPE "sensecap-watcher")
|
||||
elseif(CONFIG_BOARD_TYPE_DOIT_S3_AIBOX)
|
||||
set(BOARD_TYPE "doit-s3-aibox")
|
||||
elseif(CONFIG_BOARD_TYPE_MIXGO_NOVA)
|
||||
set(BOARD_TYPE "mixgo-nova")
|
||||
elseif(CONFIG_BOARD_TYPE_GENJUTECH_S3_1_54TFT)
|
||||
set(BOARD_TYPE "genjutech-s3-1.54tft")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32_CGC)
|
||||
set(BOARD_TYPE "esp32-cgc")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32_CGC_144)
|
||||
set(BOARD_TYPE "esp32-cgc-144")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP_S3_LCD_EV_Board)
|
||||
set(BOARD_TYPE "esp-s3-lcd-ev-board")
|
||||
elseif(CONFIG_BOARD_TYPE_ZHENGCHEN_1_54TFT_WIFI)
|
||||
set(BOARD_TYPE "zhengchen-1.54tft-wifi")
|
||||
elseif(CONFIG_BOARD_TYPE_MINSI_K08_DUAL)
|
||||
set(BOARD_TYPE "minsi-k08-dual")
|
||||
elseif(CONFIG_BOARD_TYPE_ZHENGCHEN_1_54TFT_ML307)
|
||||
set(BOARD_TYPE "zhengchen-1.54tft-ml307")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32_S3_1_54_MUMA)
|
||||
set(BOARD_TYPE "sp-esp32-s3-1.54-muma")
|
||||
elseif(CONFIG_BOARD_TYPE_ESP32_S3_1_28_BOX)
|
||||
set(BOARD_TYPE "sp-esp32-s3-1.28-box")
|
||||
elseif(CONFIG_BOARD_TYPE_OTTO_ROBOT)
|
||||
set(BOARD_TYPE "otto-robot")
|
||||
elseif(CONFIG_BOARD_TYPE_ELECTRON_BOT)
|
||||
set(BOARD_TYPE "electron-bot")
|
||||
elseif(CONFIG_BOARD_TYPE_GUITION_JC1060P470)
|
||||
set(BOARD_TYPE "guition-jc1060p470")
|
||||
elseif(CONFIG_BOARD_TYPE_GUITION_JC8012P4A1)
|
||||
set(BOARD_TYPE "guition-jc8012p4a1")
|
||||
elseif(CONFIG_BOARD_TYPE_GUITION_JC_ESP32P4_M3_DEV)
|
||||
set(BOARD_TYPE "guition-jc-esp32p4-m3-dev")
|
||||
elseif(CONFIG_BOARD_TYPE_BREAD_COMPACT_WIFI_CAM)
|
||||
set(BOARD_TYPE "bread-compact-wifi-s3cam")
|
||||
elseif(CONFIG_BOARD_TYPE_JIUCHUAN )
|
||||
set(BOARD_TYPE "jiuchuan-s3")
|
||||
endif()
|
||||
file(GLOB BOARD_SOURCES
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/boards/${BOARD_TYPE}/*.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/boards/${BOARD_TYPE}/*.c
|
||||
)
|
||||
list(APPEND SOURCES ${BOARD_SOURCES})
|
||||
|
||||
if(CONFIG_USE_AUDIO_PROCESSOR)
|
||||
list(APPEND SOURCES "audio_processing/afe_audio_processor.cc")
|
||||
else()
|
||||
list(APPEND SOURCES "audio_processing/no_audio_processor.cc")
|
||||
endif()
|
||||
if(CONFIG_USE_AFE_WAKE_WORD)
|
||||
list(APPEND SOURCES "audio_processing/afe_wake_word.cc")
|
||||
elseif(CONFIG_USE_ESP_WAKE_WORD)
|
||||
list(APPEND SOURCES "audio_processing/esp_wake_word.cc")
|
||||
else()
|
||||
list(APPEND SOURCES "audio_processing/no_wake_word.cc")
|
||||
endif()
|
||||
|
||||
# 根据Kconfig选择语言目录
|
||||
if(CONFIG_LANGUAGE_ZH_CN)
|
||||
set(LANG_DIR "zh-CN")
|
||||
elseif(CONFIG_LANGUAGE_ZH_TW)
|
||||
set(LANG_DIR "zh-TW")
|
||||
elseif(CONFIG_LANGUAGE_EN_US)
|
||||
set(LANG_DIR "en-US")
|
||||
elseif(CONFIG_LANGUAGE_JA_JP)
|
||||
set(LANG_DIR "ja-JP")
|
||||
endif()
|
||||
|
||||
# 定义生成路径
|
||||
set(LANG_JSON "${CMAKE_CURRENT_SOURCE_DIR}/assets/${LANG_DIR}/language.json")
|
||||
set(LANG_HEADER "${CMAKE_CURRENT_SOURCE_DIR}/assets/lang_config.h")
|
||||
file(GLOB LANG_SOUNDS ${CMAKE_CURRENT_SOURCE_DIR}/assets/${LANG_DIR}/*.p3)
|
||||
file(GLOB COMMON_SOUNDS ${CMAKE_CURRENT_SOURCE_DIR}/assets/common/*.p3)
|
||||
|
||||
# 如果目标芯片是 ESP32，则排除特定文件
|
||||
if(CONFIG_IDF_TARGET_ESP32)
|
||||
list(REMOVE_ITEM SOURCES "audio_codecs/box_audio_codec.cc"
|
||||
"audio_codecs/es8388_audio_codec.cc"
|
||||
"led/gpio_led.cc"
|
||||
)
|
||||
endif()
|
||||
|
||||
idf_component_register(SRCS ${SOURCES}
|
||||
EMBED_FILES ${LANG_SOUNDS} ${COMMON_SOUNDS}
|
||||
INCLUDE_DIRS ${INCLUDE_DIRS}
|
||||
WHOLE_ARCHIVE
|
||||
)
|
||||
|
||||
# 使用 target_compile_definitions 来定义 BOARD_TYPE, BOARD_NAME
|
||||
# 如果 BOARD_NAME 为空，则使用 BOARD_TYPE
|
||||
if(NOT BOARD_NAME)
|
||||
set(BOARD_NAME ${BOARD_TYPE})
|
||||
endif()
|
||||
target_compile_definitions(${COMPONENT_LIB}
|
||||
PRIVATE BOARD_TYPE=\"${BOARD_TYPE}\" BOARD_NAME=\"${BOARD_NAME}\"
|
||||
)
|
||||
|
||||
# 添加生成规则
|
||||
add_custom_command(
|
||||
OUTPUT ${LANG_HEADER}
|
||||
COMMAND python ${PROJECT_DIR}/scripts/gen_lang.py
|
||||
--input "${LANG_JSON}"
|
||||
--output "${LANG_HEADER}"
|
||||
DEPENDS
|
||||
${LANG_JSON}
|
||||
${PROJECT_DIR}/scripts/gen_lang.py
|
||||
COMMENT "Generating ${LANG_DIR} language config"
|
||||
)
|
||||
|
||||
# 强制建立生成依赖
|
||||
add_custom_target(lang_header ALL
|
||||
DEPENDS ${LANG_HEADER}
|
||||
)
|
||||
|
||||
if(CONFIG_BOARD_TYPE_ESP_HI)
|
||||
set(URL "https://github.com/espressif2022/image_player/raw/main/test_apps/test_8bit")
|
||||
set(SPIFFS_DIR "${CMAKE_BINARY_DIR}/emoji")
|
||||
file(MAKE_DIRECTORY ${SPIFFS_DIR})
|
||||
|
||||
# List all files to download
|
||||
set(FILES_TO_DOWNLOAD "")
|
||||
list(APPEND FILES_TO_DOWNLOAD "Anger_enter.aaf" "Anger_loop.aaf" "Anger_return.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "happy_enter.aaf" "happy_loop.aaf" "happ_return.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "sad_enter.aaf" "sad_loop.aaf" "sad_return.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "scorn_enter.aaf" "scorn_loop.aaf" "scorn_return.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "left_enter.aaf" "left_loop.aaf" "left_return.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "right_enter.aaf" "right_loop.aaf" "right_return.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "asking.aaf" "blink_once.aaf" "blink_quick.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "connecting.aaf" "panic_enter.aaf" "panic_loop.aaf")
|
||||
list(APPEND FILES_TO_DOWNLOAD "panic_return.aaf" "wake.aaf")
|
||||
|
||||
foreach(FILENAME IN LISTS FILES_TO_DOWNLOAD)
|
||||
set(REMOTE_FILE "${URL}/${FILENAME}")
|
||||
set(LOCAL_FILE "${SPIFFS_DIR}/${FILENAME}")
|
||||
|
||||
# 检查本地文件是否存在
|
||||
if(EXISTS ${LOCAL_FILE})
|
||||
message(STATUS "File ${FILENAME} already exists, skipping download")
|
||||
else()
|
||||
message(STATUS "Downloading ${FILENAME}")
|
||||
file(DOWNLOAD ${REMOTE_FILE} ${LOCAL_FILE}
|
||||
STATUS DOWNLOAD_STATUS)
|
||||
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
|
||||
if(NOT STATUS_CODE EQUAL 0)
|
||||
message(FATAL_ERROR "Failed to download ${FILENAME} from ${URL}")
|
||||
endif()
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
spiffs_create_partition_assets(
|
||||
assets_A
|
||||
${SPIFFS_DIR}
|
||||
FLASH_IN_PROJECT
|
||||
MMAP_FILE_SUPPORT_FORMAT ".aaf"
|
||||
)
|
||||
endif()
|
||||
@@ -0,0 +1,451 @@
|
||||
menu "Xiaozhi Assistant"
|
||||
|
||||
config OTA_URL
|
||||
string "Default OTA URL"
|
||||
default "https://api.tenclass.net/xiaozhi/ota/"
|
||||
help
|
||||
The application will access this URL to check for new firmwares and server address.
|
||||
|
||||
|
||||
choice
|
||||
prompt "Default Language"
|
||||
default LANGUAGE_ZH_CN
|
||||
help
|
||||
Select device display language
|
||||
|
||||
config LANGUAGE_ZH_CN
|
||||
bool "Chinese"
|
||||
config LANGUAGE_ZH_TW
|
||||
bool "Chinese Traditional"
|
||||
config LANGUAGE_EN_US
|
||||
bool "English"
|
||||
config LANGUAGE_JA_JP
|
||||
bool "Japanese"
|
||||
endchoice
|
||||
|
||||
choice BOARD_TYPE
|
||||
prompt "Board Type"
|
||||
default BOARD_TYPE_BREAD_COMPACT_WIFI
|
||||
help
|
||||
Board type. 开发板类型
|
||||
config BOARD_TYPE_BREAD_COMPACT_WIFI
|
||||
bool "面包板新版接线（WiFi）"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_BREAD_COMPACT_WIFI_LCD
|
||||
bool "面包板新版接线（WiFi）+ LCD"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_BREAD_COMPACT_WIFI_CAM
|
||||
bool "面包板新版接线（WiFi）+ LCD + Camera"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_BREAD_COMPACT_ML307
|
||||
bool "面包板新版接线（ML307 AT）"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_BREAD_COMPACT_ESP32
|
||||
bool "面包板（WiFi） ESP32 DevKit"
|
||||
depends on IDF_TARGET_ESP32
|
||||
config BOARD_TYPE_BREAD_COMPACT_ESP32_LCD
|
||||
bool "面包板（WiFi+ LCD） ESP32 DevKit"
|
||||
depends on IDF_TARGET_ESP32
|
||||
config BOARD_TYPE_XMINI_C3
|
||||
bool "虾哥 Mini C3"
|
||||
depends on IDF_TARGET_ESP32C3
|
||||
config BOARD_TYPE_ESP32S3_KORVO2_V3
|
||||
bool "ESP32S3_KORVO2_V3开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP_SPARKBOT
|
||||
bool "ESP-SparkBot开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP_SPOT_S3
|
||||
bool "ESP-Spot-S3"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP_HI
|
||||
bool "ESP-HI"
|
||||
depends on IDF_TARGET_ESP32C3
|
||||
config BOARD_TYPE_ECHOEAR
|
||||
bool "EchoEar"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP_BOX_3
|
||||
bool "ESP BOX 3"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP_BOX
|
||||
bool "ESP BOX"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP_BOX_LITE
|
||||
bool "ESP BOX Lite"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_KEVIN_BOX_1
|
||||
bool "Kevin Box 1"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_KEVIN_BOX_2
|
||||
bool "Kevin Box 2"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_KEVIN_C3
|
||||
bool "Kevin C3"
|
||||
depends on IDF_TARGET_ESP32C3
|
||||
config BOARD_TYPE_KEVIN_SP_V3_DEV
|
||||
bool "Kevin SP V3开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_KEVIN_SP_V4_DEV
|
||||
bool "Kevin SP V4开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32_CGC
|
||||
bool "ESP32 CGC"
|
||||
depends on IDF_TARGET_ESP32
|
||||
config BOARD_TYPE_ESP32_CGC_144
|
||||
bool "ESP32 CGC 144"
|
||||
depends on IDF_TARGET_ESP32
|
||||
config BOARD_TYPE_KEVIN_YUYING_313LCD
|
||||
bool "鱼鹰科技3.13LCD开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_LICHUANG_DEV
|
||||
bool "立创·实战派ESP32-S3开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_LICHUANG_C3_DEV
|
||||
bool "立创·实战派ESP32-C3开发板"
|
||||
depends on IDF_TARGET_ESP32C3
|
||||
config BOARD_TYPE_DF_K10
|
||||
bool "DFRobot 行空板 k10"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_DF_S3_AI_CAM
|
||||
bool "DFRobot ESP32-S3 AI智能摄像头模块"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_MAGICLICK_2P4
|
||||
bool "神奇按钮 Magiclick_2.4"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_MAGICLICK_2P5
|
||||
bool "神奇按钮 Magiclick_2.5"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_MAGICLICK_C3
|
||||
bool "神奇按钮 Magiclick_C3"
|
||||
depends on IDF_TARGET_ESP32C3
|
||||
config BOARD_TYPE_MAGICLICK_C3_V2
|
||||
bool "神奇按钮 Magiclick_C3_v2"
|
||||
depends on IDF_TARGET_ESP32C3
|
||||
config BOARD_TYPE_M5STACK_CORE_S3
|
||||
bool "M5Stack CoreS3"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_M5STACK_CORE_TAB5
|
||||
bool "M5Stack Tab5"
|
||||
depends on IDF_TARGET_ESP32P4
|
||||
config BOARD_TYPE_ATOMS3_ECHO_BASE
|
||||
bool "AtomS3 + Echo Base"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATOMS3R_ECHO_BASE
|
||||
bool "AtomS3R + Echo Base"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATOMS3R_CAM_M12_ECHO_BASE
|
||||
bool "AtomS3R CAM/M12 + Echo Base"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATOMMATRIX_ECHO_BASE
|
||||
bool "AtomMatrix + Echo Base"
|
||||
depends on IDF_TARGET_ESP32
|
||||
config BOARD_TYPE_ESP32S3_Touch_AMOLED_1_8
|
||||
bool "Waveshare ESP32-S3-Touch-AMOLED-1.8"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32S3_Touch_AMOLED_1_75
|
||||
bool "Waveshare ESP32-S3-Touch-AMOLED-1.75"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32S3_Touch_LCD_1_85C
|
||||
bool "Waveshare ESP32-S3-Touch-LCD-1.85C"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32S3_Touch_LCD_1_85
|
||||
bool "Waveshare ESP32-S3-Touch-LCD-1.85"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32S3_Touch_LCD_1_46
|
||||
bool "Waveshare ESP32-S3-Touch-LCD-1.46"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32C6_LCD_1_69
|
||||
bool "Waveshare ESP32-C6-LCD-1.69"
|
||||
depends on IDF_TARGET_ESP32C6
|
||||
config BOARD_TYPE_ESP32S3_Touch_LCD_3_5
|
||||
bool "Waveshare ESP32-S3-Touch-LCD-3.5"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32P4_NANO
|
||||
bool "Waveshare ESP32-P4-NANO"
|
||||
depends on IDF_TARGET_ESP32P4
|
||||
config BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_4B
|
||||
bool "Waveshare ESP32-P4-WIFI6-Touch-LCD-4B"
|
||||
depends on IDF_TARGET_ESP32P4
|
||||
config BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_XC
|
||||
bool "Waveshare ESP32-P4-WIFI6-Touch-LCD-3.4C or ESP32-P4-WIFI6-Touch-LCD-4C"
|
||||
depends on IDF_TARGET_ESP32P4
|
||||
config BOARD_TYPE_TUDOUZI
|
||||
bool "土豆子"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_LILYGO_T_CIRCLE_S3
|
||||
bool "LILYGO T-Circle-S3"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_LILYGO_T_CAMERAPLUS_S3_V1_0_V1_1
|
||||
bool "LILYGO T-CameraPlus-S3_V1_0_V1_1"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_LILYGO_T_CAMERAPLUS_S3_V1_2
|
||||
bool "LILYGO T-CameraPlus-S3_V1_2"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_LILYGO_T_DISPLAY_S3_PRO_MVSRLORA
|
||||
bool "LILYGO T-Display-S3-Pro-MVSRLora"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_LILYGO_T_DISPLAY_S3_PRO_MVSRLORA_NO_BATTERY
|
||||
bool "LILYGO T-Display-S3-Pro-MVSRLora_No_Battery"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_MOVECALL_MOJI_ESP32S3
|
||||
bool "Movecall Moji 小智AI衍生版"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_MOVECALL_CUICAN_ESP32S3
|
||||
bool "Movecall CuiCan 璀璨·AI吊坠"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATK_DNESP32S3
|
||||
bool "正点原子DNESP32S3开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATK_DNESP32S3_BOX
|
||||
bool "正点原子DNESP32S3-BOX"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATK_DNESP32S3_BOX0
|
||||
bool "正点原子DNESP32S3-BOX0"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATK_DNESP32S3M_WIFI
|
||||
bool "正点原子DNESP32S3M-WIFI"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ATK_DNESP32S3M_4G
|
||||
bool "正点原子DNESP32S3M-4G"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_DU_CHATX
|
||||
bool "嘟嘟开发板CHATX(wifi)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32S3_Taiji_Pi
|
||||
bool "太极小派esp32s3"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_XINGZHI_Cube_0_85TFT_WIFI
|
||||
bool "无名科技星智0.85(WIFI)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_XINGZHI_Cube_0_85TFT_ML307
|
||||
bool "无名科技星智0.85(ML307)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_XINGZHI_Cube_0_96OLED_WIFI
|
||||
bool "无名科技星智0.96(WIFI)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_XINGZHI_Cube_0_96OLED_ML307
|
||||
bool "无名科技星智0.96(ML307)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_XINGZHI_Cube_1_54TFT_WIFI
|
||||
bool "无名科技星智1.54(WIFI)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_XINGZHI_Cube_1_54TFT_ML307
|
||||
bool "无名科技星智1.54(ML307)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_SENSECAP_WATCHER
|
||||
bool "SenseCAP Watcher"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_DOIT_S3_AIBOX
|
||||
bool "四博智联AI陪伴盒子"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_MIXGO_NOVA
|
||||
bool "元控·青春"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_GENJUTECH_S3_1_54TFT
|
||||
bool "亘具科技1.54(s3)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP_S3_LCD_EV_Board
|
||||
bool "乐鑫ESP S3 LCD EV Board开发板"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ZHENGCHEN_1_54TFT_WIFI
|
||||
bool "征辰科技1.54(WIFI)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ZHENGCHEN_1_54TFT_ML307
|
||||
bool "征辰科技1.54(ML307)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_MINSI_K08_DUAL
|
||||
bool "敏思科技K08(DUAL)"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32_S3_1_54_MUMA
|
||||
bool "Spotpear ESP32-S3-1.54-MUMA"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_ESP32_S3_1_28_BOX
|
||||
bool "Spotpear ESP32-S3-1.28-BOX"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
config BOARD_TYPE_OTTO_ROBOT
|
||||
bool "ottoRobot"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
select LV_USE_GIF
|
||||
select LV_GIF_CACHE_DECODE_DATA
|
||||
config BOARD_TYPE_ELECTRON_BOT
|
||||
bool "electronBot"
|
||||
depends on IDF_TARGET_ESP32S3
|
||||
select LV_USE_GIF
|
||||
select LV_GIF_CACHE_DECODE_DATA
|
||||
config BOARD_TYPE_GUITION_JC1060P470
|
||||
bool "Guition JC1060P470"
|
||||
depends on IDF_TARGET_ESP32P4
|
||||
config BOARD_TYPE_GUITION_JC8012P4A1
|
||||
bool "Guition JC8012P4A1"
|
||||
depends on IDF_TARGET_ESP32P4
|
||||
config BOARD_TYPE_GUITION_JC_ESP32P4_M3_DEV
|
||||
bool "Guition JC-ESP32P4-M3-DEV"
|
||||
depends on IDF_TARGET_ESP32P4
|
||||
config BOARD_TYPE_JIUCHUAN
|
||||
bool "九川智能"
|
||||
endchoice
|
||||
|
||||
choice ESP_S3_LCD_EV_Board_Version_TYPE
|
||||
depends on BOARD_TYPE_ESP_S3_LCD_EV_Board
|
||||
prompt "EV_BOARD Type"
|
||||
default ESP_S3_LCD_EV_Board_1p4
|
||||
help
|
||||
开发板硬件版本型号选择
|
||||
config ESP_S3_LCD_EV_Board_1p4
|
||||
bool "乐鑫ESP32_S3_LCD_EV_Board-MB_V1.4"
|
||||
config ESP_S3_LCD_EV_Board_1p5
|
||||
bool "乐鑫ESP32_S3_LCD_EV_Board-MB_V1.5"
|
||||
endchoice
|
||||
|
||||
choice DISPLAY_OLED_TYPE
|
||||
depends on BOARD_TYPE_BREAD_COMPACT_WIFI || BOARD_TYPE_BREAD_COMPACT_ML307 || BOARD_TYPE_BREAD_COMPACT_ESP32
|
||||
prompt "OLED Type"
|
||||
default OLED_SSD1306_128X32
|
||||
help
|
||||
OLED 屏幕类型选择
|
||||
config OLED_SSD1306_128X32
|
||||
bool "SSD1306, 分辨率128*32"
|
||||
config OLED_SSD1306_128X64
|
||||
bool "SSD1306, 分辨率128*64"
|
||||
config OLED_SH1106_128X64
|
||||
bool "SH1106, 分辨率128*64"
|
||||
endchoice
|
||||
|
||||
choice DISPLAY_LCD_TYPE
|
||||
depends on BOARD_TYPE_BREAD_COMPACT_WIFI_LCD || BOARD_TYPE_BREAD_COMPACT_ESP32_LCD || BOARD_TYPE_ESP32_CGC || BOARD_TYPE_ESP32P4_NANO || BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_XC || BOARD_TYPE_BREAD_COMPACT_WIFI_CAM
|
||||
prompt "LCD Type"
|
||||
default LCD_ST7789_240X320
|
||||
help
|
||||
屏幕类型选择
|
||||
config LCD_ST7789_240X320
|
||||
bool "ST7789, 分辨率240*320, IPS"
|
||||
config LCD_ST7789_240X320_NO_IPS
|
||||
bool "ST7789, 分辨率240*320, 非IPS"
|
||||
config LCD_ST7789_170X320
|
||||
bool "ST7789, 分辨率170*320"
|
||||
config LCD_ST7789_172X320
|
||||
bool "ST7789, 分辨率172*320"
|
||||
config LCD_ST7789_240X280
|
||||
bool "ST7789, 分辨率240*280"
|
||||
config LCD_ST7789_240X240
|
||||
bool "ST7789, 分辨率240*240"
|
||||
config LCD_ST7789_240X240_7PIN
|
||||
bool "ST7789, 分辨率240*240, 7PIN"
|
||||
config LCD_ST7789_240X135
|
||||
bool "ST7789, 分辨率240*135"
|
||||
config LCD_ST7735_128X160
|
||||
bool "ST7735, 分辨率128*160"
|
||||
config LCD_ST7735_128X128
|
||||
bool "ST7735, 分辨率128*128"
|
||||
config LCD_ST7796_320X480
|
||||
bool "ST7796, 分辨率320*480 IPS"
|
||||
config LCD_ST7796_320X480_NO_IPS
|
||||
bool "ST7796, 分辨率320*480, 非IPS"
|
||||
config LCD_ILI9341_240X320
|
||||
bool "ILI9341, 分辨率240*320"
|
||||
config LCD_ILI9341_240X320_NO_IPS
|
||||
bool "ILI9341, 分辨率240*320, 非IPS"
|
||||
config LCD_GC9A01_240X240
|
||||
bool "GC9A01, 分辨率240*240, 圆屏"
|
||||
config LCD_TYPE_800_1280_10_1_INCH
|
||||
bool "Waveshare 101M-8001280-IPS-CT-K Display"
|
||||
config LCD_TYPE_800_1280_10_1_INCH_A
|
||||
bool "Waveshare 10.1-DSI-TOUCH-A Display"
|
||||
config LCD_TYPE_800_800_3_4_INCH
|
||||
bool "Waveshare ESP32-P4-WIFI6-Touch-LCD-3.4C with 800*800 3.4inch round display"
|
||||
config LCD_TYPE_720_720_4_INCH
|
||||
bool "Waveshare ESP32-P4-WIFI6-Touch-LCD-4C with 720*720 4inch round display"
|
||||
config LCD_CUSTOM
|
||||
bool "自定义屏幕参数"
|
||||
endchoice
|
||||
|
||||
choice DISPLAY_ESP32S3_KORVO2_V3
|
||||
depends on BOARD_TYPE_ESP32S3_KORVO2_V3
|
||||
prompt "ESP32S3_KORVO2_V3 LCD Type"
|
||||
default LCD_ST7789
|
||||
help
|
||||
屏幕类型选择
|
||||
config LCD_ST7789
|
||||
bool "ST7789, 分辨率240*280"
|
||||
config LCD_ILI9341
|
||||
bool "ILI9341, 分辨率240*320"
|
||||
endchoice
|
||||
|
||||
choice DISPLAY_JC_ESP32P4_M3_DEV
|
||||
depends on BOARD_TYPE_GUITION_JC_ESP32P4_M3_DEV
|
||||
prompt "JC-ESP32P4-M3-DEV LCD Type"
|
||||
default NODISPLAY
|
||||
help
|
||||
屏幕类型选择
|
||||
config NODISPLAY
|
||||
bool "No Display"
|
||||
config LCD_TYPE_800_1280_10_1_INCH_A
|
||||
bool "Waveshare 10.1-DSI-TOUCH-A Display"
|
||||
endchoice
|
||||
|
||||
config USE_WECHAT_MESSAGE_STYLE
|
||||
bool "Enable WeChat Message Style"
|
||||
default n
|
||||
help
|
||||
使用微信聊天界面风格
|
||||
|
||||
config USE_ESP_WAKE_WORD
|
||||
bool "Enable Wake Word Detection (without AFE)"
|
||||
default n
|
||||
depends on IDF_TARGET_ESP32C3 || IDF_TARGET_ESP32C5 || IDF_TARGET_ESP32C6 || (IDF_TARGET_ESP32 && SPIRAM)
|
||||
help
|
||||
支持 ESP32 C3、ESP32 C5 与 ESP32 C6，增加ESP32支持（需要开启PSRAM）
|
||||
|
||||
config USE_AFE_WAKE_WORD
|
||||
bool "Enable Wake Word Detection (AFE)"
|
||||
default y
|
||||
depends on (IDF_TARGET_ESP32S3 || IDF_TARGET_ESP32P4) && SPIRAM
|
||||
help
|
||||
需要 ESP32 S3 与 PSRAM 支持
|
||||
|
||||
config USE_AUDIO_PROCESSOR
|
||||
bool "Enable Audio Noise Reduction"
|
||||
default y
|
||||
depends on (IDF_TARGET_ESP32S3 || IDF_TARGET_ESP32P4) && SPIRAM
|
||||
help
|
||||
需要 ESP32 S3 与 PSRAM 支持
|
||||
|
||||
config USE_DEVICE_AEC
|
||||
bool "Enable Device-Side AEC"
|
||||
default n
|
||||
depends on USE_AUDIO_PROCESSOR && (BOARD_TYPE_ESP_BOX_3 || BOARD_TYPE_ESP_BOX || BOARD_TYPE_ESP_BOX_LITE || BOARD_TYPE_LICHUANG_DEV || BOARD_TYPE_ESP32S3_KORVO2_V3 || BOARD_TYPE_ESP32S3_Touch_AMOLED_1_75 || BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_4B || BOARD_TYPE_ESP32P4_WIFI6_Touch_LCD_XC)
|
||||
help
|
||||
因为性能不够，不建议和微信聊天界面风格同时开启
|
||||
|
||||
config USE_SERVER_AEC
|
||||
bool "Enable Server-Side AEC (Unstable)"
|
||||
default n
|
||||
depends on USE_AUDIO_PROCESSOR
|
||||
help
|
||||
启用服务器端 AEC，需要服务器支持
|
||||
|
||||
config USE_AUDIO_DEBUGGER
|
||||
bool "Enable Audio Debugger"
|
||||
default n
|
||||
help
|
||||
启用音频调试功能，通过UDP发送音频数据
|
||||
|
||||
config AUDIO_DEBUG_UDP_SERVER
|
||||
string "Audio Debug UDP Server Address"
|
||||
default "192.168.2.100:8000"
|
||||
depends on USE_AUDIO_DEBUGGER
|
||||
help
|
||||
UDP服务器地址，格式: IP:PORT，用于接收音频调试数据
|
||||
|
||||
choice IOT_PROTOCOL
|
||||
prompt "IoT Protocol"
|
||||
default IOT_PROTOCOL_MCP
|
||||
help
|
||||
IoT 协议，用于获取设备状态与发送控制指令
|
||||
config IOT_PROTOCOL_MCP
|
||||
bool "MCP 2024-11-05"
|
||||
config IOT_PROTOCOL_XIAOZHI
|
||||
bool "Xiaozhi IoT 1.0 (Deprecated)"
|
||||
endchoice
|
||||
|
||||
endmenu
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,144 @@
|
||||
#ifndef _APPLICATION_H_
|
||||
#define _APPLICATION_H_
|
||||
|
||||
#include <freertos/FreeRTOS.h>
|
||||
#include <freertos/event_groups.h>
|
||||
#include <freertos/task.h>
|
||||
#include <esp_timer.h>
|
||||
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
#include <list>
|
||||
#include <vector>
|
||||
#include <condition_variable>
|
||||
#include <memory>
|
||||
|
||||
#include <opus_encoder.h>
|
||||
#include <opus_decoder.h>
|
||||
#include <opus_resampler.h>
|
||||
|
||||
#include "protocol.h"
|
||||
#include "ota.h"
|
||||
#include "background_task.h"
|
||||
#include "audio_processor.h"
|
||||
#include "wake_word.h"
|
||||
#include "audio_debugger.h"
|
||||
|
||||
#define SCHEDULE_EVENT (1 << 0)
|
||||
#define SEND_AUDIO_EVENT (1 << 1)
|
||||
#define CHECK_NEW_VERSION_DONE_EVENT (1 << 2)
|
||||
|
||||
enum AecMode {
|
||||
kAecOff,
|
||||
kAecOnDeviceSide,
|
||||
kAecOnServerSide,
|
||||
};
|
||||
|
||||
enum DeviceState {
|
||||
kDeviceStateUnknown,
|
||||
kDeviceStateStarting,
|
||||
kDeviceStateWifiConfiguring,
|
||||
kDeviceStateIdle,
|
||||
kDeviceStateConnecting,
|
||||
kDeviceStateListening,
|
||||
kDeviceStateSpeaking,
|
||||
kDeviceStateUpgrading,
|
||||
kDeviceStateActivating,
|
||||
kDeviceStateAudioTesting,
|
||||
kDeviceStateFatalError
|
||||
};
|
||||
|
||||
#define OPUS_FRAME_DURATION_MS 60
|
||||
#define MAX_AUDIO_PACKETS_IN_QUEUE (2400 / OPUS_FRAME_DURATION_MS)
|
||||
#define AUDIO_TESTING_MAX_DURATION_MS 10000
|
||||
|
||||
class Application {
|
||||
public:
|
||||
static Application& GetInstance() {
|
||||
static Application instance;
|
||||
return instance;
|
||||
}
|
||||
// 删除拷贝构造函数和赋值运算符
|
||||
Application(const Application&) = delete;
|
||||
Application& operator=(const Application&) = delete;
|
||||
|
||||
void Start();
|
||||
DeviceState GetDeviceState() const { return device_state_; }
|
||||
bool IsVoiceDetected() const { return voice_detected_; }
|
||||
void Schedule(std::function<void()> callback);
|
||||
void SetDeviceState(DeviceState state);
|
||||
void Alert(const char* status, const char* message, const char* emotion = "", const std::string_view& sound = "");
|
||||
void DismissAlert();
|
||||
void AbortSpeaking(AbortReason reason);
|
||||
void ToggleChatState();
|
||||
void StartListening();
|
||||
void StopListening();
|
||||
void UpdateIotStates();
|
||||
void Reboot();
|
||||
void WakeWordInvoke(const std::string& wake_word);
|
||||
void PlaySound(const std::string_view& sound);
|
||||
bool CanEnterSleepMode();
|
||||
void SendMcpMessage(const std::string& payload);
|
||||
void SetAecMode(AecMode mode);
|
||||
AecMode GetAecMode() const { return aec_mode_; }
|
||||
BackgroundTask* GetBackgroundTask() const { return background_task_; }
|
||||
|
||||
private:
|
||||
Application();
|
||||
~Application();
|
||||
|
||||
std::unique_ptr<WakeWord> wake_word_;
|
||||
std::unique_ptr<AudioProcessor> audio_processor_;
|
||||
std::unique_ptr<AudioDebugger> audio_debugger_;
|
||||
std::mutex mutex_;
|
||||
std::list<std::function<void()>> main_tasks_;
|
||||
std::unique_ptr<Protocol> protocol_;
|
||||
EventGroupHandle_t event_group_ = nullptr;
|
||||
esp_timer_handle_t clock_timer_handle_ = nullptr;
|
||||
volatile DeviceState device_state_ = kDeviceStateUnknown;
|
||||
ListeningMode listening_mode_ = kListeningModeAutoStop;
|
||||
AecMode aec_mode_ = kAecOff;
|
||||
|
||||
bool has_server_time_ = false;
|
||||
bool aborted_ = false;
|
||||
bool voice_detected_ = false;
|
||||
bool busy_decoding_audio_ = false;
|
||||
int clock_ticks_ = 0;
|
||||
TaskHandle_t check_new_version_task_handle_ = nullptr;
|
||||
|
||||
// Audio encode / decode
|
||||
TaskHandle_t audio_loop_task_handle_ = nullptr;
|
||||
BackgroundTask* background_task_ = nullptr;
|
||||
std::chrono::steady_clock::time_point last_output_time_;
|
||||
std::list<AudioStreamPacket> audio_send_queue_;
|
||||
std::list<AudioStreamPacket> audio_decode_queue_;
|
||||
std::condition_variable audio_decode_cv_;
|
||||
std::list<AudioStreamPacket> audio_testing_queue_;
|
||||
|
||||
// 新增：用于维护音频包的timestamp队列
|
||||
std::list<uint32_t> timestamp_queue_;
|
||||
std::mutex timestamp_mutex_;
|
||||
|
||||
std::unique_ptr<OpusEncoderWrapper> opus_encoder_;
|
||||
std::unique_ptr<OpusDecoderWrapper> opus_decoder_;
|
||||
|
||||
OpusResampler input_resampler_;
|
||||
OpusResampler reference_resampler_;
|
||||
OpusResampler output_resampler_;
|
||||
|
||||
void MainEventLoop();
|
||||
void OnAudioInput();
|
||||
void OnAudioOutput();
|
||||
bool ReadAudio(std::vector<int16_t>& data, int sample_rate, int samples);
|
||||
void ResetDecoder();
|
||||
void SetDecodeSampleRate(int sample_rate, int frame_duration);
|
||||
void CheckNewVersion(Ota& ota);
|
||||
void ShowActivationCode(const std::string& code, const std::string& message);
|
||||
void OnClockTimer();
|
||||
void SetListeningMode(ListeningMode mode);
|
||||
void AudioLoop();
|
||||
void EnterAudioTestingMode();
|
||||
void ExitAudioTestingMode();
|
||||
};
|
||||
|
||||
#endif // _APPLICATION_H_
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"language": {
|
||||
"type": "en-US"
|
||||
},
|
||||
"strings": {
|
||||
"WARNING": "Warning",
|
||||
"INFO": "Information",
|
||||
"ERROR": "Error",
|
||||
"VERSION": "Ver ",
|
||||
"LOADING_PROTOCOL": "Logging in...",
|
||||
"INITIALIZING": "Initializing...",
|
||||
"PIN_ERROR": "Please insert SIM card",
|
||||
"REG_ERROR": "Unable to access network, please check SIM card status",
|
||||
"DETECTING_MODULE": "Detecting module...",
|
||||
"REGISTERING_NETWORK": "Waiting for network...",
|
||||
"CHECKING_NEW_VERSION": "Checking for new version...",
|
||||
"CHECK_NEW_VERSION_FAILED": "Check for new version failed, will retry in %d seconds: %s",
|
||||
"SWITCH_TO_WIFI_NETWORK": "Switching to Wi-Fi...",
|
||||
"SWITCH_TO_4G_NETWORK": "Switching to 4G...",
|
||||
|
||||
"STANDBY": "Standby",
|
||||
"CONNECT_TO": "Connect to ",
|
||||
"CONNECTING": "Connecting...",
|
||||
"CONNECTION_SUCCESSFUL": "Connection Successful",
|
||||
"CONNECTED_TO": "Connected to ",
|
||||
|
||||
"LISTENING": "Listening...",
|
||||
"SPEAKING": "Speaking...",
|
||||
|
||||
"SERVER_NOT_FOUND": "Looking for available service",
|
||||
"SERVER_NOT_CONNECTED": "Unable to connect to service, please try again later",
|
||||
"SERVER_TIMEOUT": "Waiting for response timeout",
|
||||
"SERVER_ERROR": "Sending failed, please check the network",
|
||||
|
||||
"CONNECT_TO_HOTSPOT": "Hotspot: ",
|
||||
"ACCESS_VIA_BROWSER": " Config URL: ",
|
||||
"WIFI_CONFIG_MODE": "Wi-Fi Configuration Mode",
|
||||
"ENTERING_WIFI_CONFIG_MODE": "Entering Wi-Fi configuration mode...",
|
||||
"SCANNING_WIFI": "Scanning Wi-Fi...",
|
||||
|
||||
"NEW_VERSION": "New version ",
|
||||
"OTA_UPGRADE": "OTA Upgrade",
|
||||
"UPGRADING": "System is upgrading...",
|
||||
"UPGRADE_FAILED": "Upgrade failed",
|
||||
"ACTIVATION": "Activation",
|
||||
|
||||
"BATTERY_LOW": "Low battery",
|
||||
"BATTERY_CHARGING": "Charging",
|
||||
"BATTERY_FULL": "Battery full",
|
||||
"BATTERY_NEED_CHARGE": "Low battery, please charge",
|
||||
|
||||
"VOLUME": "Volume ",
|
||||
"MUTED": "Muted",
|
||||
"MAX_VOLUME": "Max volume",
|
||||
|
||||
"RTC_MODE_OFF": "AEC Off",
|
||||
"RTC_MODE_ON": "AEC On"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"language": {
|
||||
"type": "ja-JP"
|
||||
},
|
||||
"strings": {
|
||||
"WARNING": "警告",
|
||||
"INFO": "情報",
|
||||
"ERROR": "エラー",
|
||||
"VERSION": "バージョン ",
|
||||
"LOADING_PROTOCOL": "サーバーにログイン中...",
|
||||
"INITIALIZING": "初期化中...",
|
||||
"PIN_ERROR": "SIMカードを挿入してください",
|
||||
"REG_ERROR": "ネットワークに接続できません。ネットワーク状態を確認してください",
|
||||
"DETECTING_MODULE": "モジュールを検出中...",
|
||||
"REGISTERING_NETWORK": "ネットワーク接続待機中...",
|
||||
"CHECKING_NEW_VERSION": "新しいバージョンを確認中...",
|
||||
"CHECK_NEW_VERSION_FAILED": "更新確認に失敗しました。%d 秒後に再試行します: %s",
|
||||
"SWITCH_TO_WIFI_NETWORK": "Wi-Fiに切り替え中...",
|
||||
"SWITCH_TO_4G_NETWORK": "4Gに切り替え中...",
|
||||
|
||||
"STANDBY": "待機中",
|
||||
"CONNECT_TO": "接続先 ",
|
||||
"CONNECTING": "接続中...",
|
||||
"CONNECTED_TO": "接続完了 ",
|
||||
|
||||
"LISTENING": "リスニング中...",
|
||||
"SPEAKING": "話しています...",
|
||||
|
||||
"SERVER_NOT_FOUND": "利用可能なサーバーを探しています",
|
||||
"SERVER_NOT_CONNECTED": "サーバーに接続できません。後でもう一度お試しください",
|
||||
"SERVER_TIMEOUT": "応答待機時間が終了しました",
|
||||
"SERVER_ERROR": "送信に失敗しました。ネットワークを確認してください",
|
||||
|
||||
"CONNECT_TO_HOTSPOT": "スマートフォンをWi-Fi ",
|
||||
"ACCESS_VIA_BROWSER": " に接続し、ブラウザでアクセスしてください ",
|
||||
"WIFI_CONFIG_MODE": "ネットワーク設定モード",
|
||||
"ENTERING_WIFI_CONFIG_MODE": "ネットワーク設定中...",
|
||||
"SCANNING_WIFI": "Wi-Fiをスキャン中...",
|
||||
|
||||
"NEW_VERSION": "新しいバージョン ",
|
||||
"OTA_UPGRADE": "OTAアップグレード",
|
||||
"UPGRADING": "システムをアップグレード中...",
|
||||
"UPGRADE_FAILED": "アップグレード失敗",
|
||||
"ACTIVATION": "デバイスをアクティベート",
|
||||
|
||||
"BATTERY_LOW": "バッテリーが少なくなっています",
|
||||
"BATTERY_CHARGING": "充電中",
|
||||
"BATTERY_FULL": "バッテリー満タン",
|
||||
"BATTERY_NEED_CHARGE": "バッテリーが低下しています。充電してください",
|
||||
|
||||
"VOLUME": "音量 ",
|
||||
"MUTED": "ミュートされています",
|
||||
"MAX_VOLUME": "最大音量",
|
||||
|
||||
"RTC_MODE_OFF": "AEC 無効",
|
||||
"RTC_MODE_ON": "AEC 有効"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"language": {
|
||||
"type" :"zh-CN"
|
||||
},
|
||||
"strings": {
|
||||
"WARNING":"警告",
|
||||
"INFO":"信息",
|
||||
"ERROR":"错误",
|
||||
"VERSION": "版本 ",
|
||||
"LOADING_PROTOCOL":"登录服务器...",
|
||||
"INITIALIZING":"正在初始化...",
|
||||
"PIN_ERROR":"请插入 SIM 卡",
|
||||
"REG_ERROR":"无法接入网络，请检查流量卡状态",
|
||||
"DETECTING_MODULE":"检测模组...",
|
||||
"REGISTERING_NETWORK":"等待网络...",
|
||||
"CHECKING_NEW_VERSION":"检查新版本...",
|
||||
"CHECK_NEW_VERSION_FAILED":"检查新版本失败，将在 %d 秒后重试：%s",
|
||||
"SWITCH_TO_WIFI_NETWORK":"切换到 Wi-Fi...",
|
||||
"SWITCH_TO_4G_NETWORK":"切换到 4G...",
|
||||
|
||||
"STANDBY":"待命",
|
||||
"CONNECT_TO":"连接 ",
|
||||
"CONNECTING":"连接中...",
|
||||
"CONNECTED_TO":"已连接 ",
|
||||
|
||||
"LISTENING":"聆听中...",
|
||||
"SPEAKING":"说话中...",
|
||||
|
||||
"SERVER_NOT_FOUND":"正在寻找可用服务",
|
||||
"SERVER_NOT_CONNECTED":"无法连接服务，请稍后再试",
|
||||
"SERVER_TIMEOUT":"等待响应超时",
|
||||
"SERVER_ERROR":"发送失败，请检查网络",
|
||||
|
||||
"CONNECT_TO_HOTSPOT":"手机连接热点 ",
|
||||
"ACCESS_VIA_BROWSER":"，浏览器访问 ",
|
||||
"WIFI_CONFIG_MODE":"配网模式",
|
||||
"ENTERING_WIFI_CONFIG_MODE":"进入配网模式...",
|
||||
"SCANNING_WIFI":"扫描 Wi-Fi...",
|
||||
|
||||
"NEW_VERSION": "新版本 ",
|
||||
"OTA_UPGRADE":"OTA 升级",
|
||||
"UPGRADING":"正在升级系统...",
|
||||
"UPGRADE_FAILED":"升级失败",
|
||||
"ACTIVATION":"激活设备",
|
||||
|
||||
"BATTERY_LOW":"电量不足",
|
||||
"BATTERY_CHARGING":"正在充电",
|
||||
"BATTERY_FULL":"电量已满",
|
||||
"BATTERY_NEED_CHARGE":"电量低，请充电",
|
||||
|
||||
"VOLUME":"音量 ",
|
||||
"MUTED":"已静音",
|
||||
"MAX_VOLUME":"最大音量",
|
||||
|
||||
"RTC_MODE_OFF":"AEC 关闭",
|
||||
"RTC_MODE_ON":"AEC 开启"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user