OmniX-Space/xiaozhi-esp32
1
0
Fork 0
Code Issues Pull Requests Actions Releases Activity

支持频谱显示，感谢哈哈哈群友的代码

Browse Source
This commit is contained in:
苏雨洛
2025-09-05 19:28:01 +08:00
parent 10edbc6402
commit 536ef62412
9 changed files with 1171 additions and 3571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

399
main/mcp_server.cc
View File

@@ -13,125 +13,173 @@
#include "application.h"
#include "display.h"
#include "board.h"
#include "boards/common/esp32_music.h"
#define TAG "MCP"
#define DEFAULT_TOOLCALL_STACK_SIZE 6144
McpServer::McpServer() {
McpServer::McpServer()
{
}
McpServer::~McpServer() {
for (auto tool : tools_) {
McpServer::~McpServer()
{
for (auto tool : tools_)
{
delete tool;
}
tools_.clear();
}
void McpServer::AddCommonTools() {
void McpServer::AddCommonTools()
{
// To speed up the response time, we add the common tools to the beginning of
// the tools list to utilize the prompt cache.
// Backup the original tools list and restore it after adding the common tools.
auto original_tools = std::move(tools_);
auto& board = Board::GetInstance();
auto &board = Board::GetInstance();
AddTool("self.get_device_status",
"Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\n"
"Use this tool for: \n"
"1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n"
"2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
PropertyList(),
[&board](const PropertyList& properties) -> ReturnValue {
return board.GetDeviceStatusJson();
});
"Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\n"
"Use this tool for: \n"
"1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n"
"2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
PropertyList(),
[&board](const PropertyList &properties) -> ReturnValue
{
return board.GetDeviceStatusJson();
});
AddTool("self.audio_speaker.set_volume",
"Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
PropertyList({
Property("volume", kPropertyTypeInteger, 0, 100)
}),
[&board](const PropertyList& properties) -> ReturnValue {
auto codec = board.GetAudioCodec();
codec->SetOutputVolume(properties["volume"].value<int>());
return true;
});
auto backlight = board.GetBacklight();
if (backlight) {
AddTool("self.screen.set_brightness",
"Set the brightness of the screen.",
PropertyList({
Property("brightness", kPropertyTypeInteger, 0, 100)
}),
[backlight](const PropertyList& properties) -> ReturnValue {
uint8_t brightness = static_cast<uint8_t>(properties["brightness"].value<int>());
backlight->SetBrightness(brightness, true);
AddTool("self.audio_speaker.set_volume",
"Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
PropertyList({Property("volume", kPropertyTypeInteger, 0, 100)}),
[&board](const PropertyList &properties) -> ReturnValue
{
auto codec = board.GetAudioCodec();
codec->SetOutputVolume(properties["volume"].value<int>());
return true;
});
auto backlight = board.GetBacklight();
if (backlight)
{
AddTool("self.screen.set_brightness",
"Set the brightness of the screen.",
PropertyList({Property("brightness", kPropertyTypeInteger, 0, 100)}),
[backlight](const PropertyList &properties) -> ReturnValue
{
uint8_t brightness = static_cast<uint8_t>(properties["brightness"].value<int>());
backlight->SetBrightness(brightness, true);
return true;
});
}
auto display = board.GetDisplay();
if (display && !display->GetTheme().empty()) {
if (display && !display->GetTheme().empty())
{
AddTool("self.screen.set_theme",
"Set the theme of the screen. The theme can be `light` or `dark`.",
PropertyList({
Property("theme", kPropertyTypeString)
}),
[display](const PropertyList& properties) -> ReturnValue {
display->SetTheme(properties["theme"].value<std::string>().c_str());
return true;
});
"Set the theme of the screen. The theme can be `light` or `dark`.",
PropertyList({Property("theme", kPropertyTypeString)}),
[display](const PropertyList &properties) -> ReturnValue
{
display->SetTheme(properties["theme"].value<std::string>().c_str());
return true;
});
}
auto camera = board.GetCamera();
if (camera) {
if (camera)
{
AddTool("self.camera.take_photo",
"Take a photo and explain it. Use this tool after the user asks you to see something.\n"
"Args:\n"
" `question`: The question that you want to ask about the photo.\n"
"Return:\n"
" A JSON object that provides the photo information.",
PropertyList({
Property("question", kPropertyTypeString)
}),
[camera](const PropertyList& properties) -> ReturnValue {
if (!camera->Capture()) {
return "{\"success\": false, \"message\": \"Failed to capture photo\"}";
}
auto question = properties["question"].value<std::string>();
return camera->Explain(question);
});
"Take a photo and explain it. Use this tool after the user asks you to see something.\n"
"Args:\n"
" `question`: The question that you want to ask about the photo.\n"
"Return:\n"
" A JSON object that provides the photo information.",
PropertyList({Property("question", kPropertyTypeString)}),
[camera](const PropertyList &properties) -> ReturnValue
{
if (!camera->Capture())
{
return "{\"success\": false, \"message\": \"Failed to capture photo\"}";
}
auto question = properties["question"].value<std::string>();
return camera->Explain(question);
});
}
auto music = board.GetMusic();
if (music) {
if (music)
{
AddTool("self.music.play_song",
"Play the specified song. When users request to play music, this tool will automatically retrieve song details and start streaming.\n"
"parameter:\n"
" `song_name`: The name of the song to be played.\n"
"return:\n"
" Play status information without confirmation, immediately play the song.",
PropertyList({
Property("song_name", kPropertyTypeString)
}),
[music](const PropertyList& properties) -> ReturnValue {
auto song_name = properties["song_name"].value<std::string>();
if (!music->Download(song_name)) {
return "{\"success\": false, \"message\": \"Failed to obtain music resources\"}";
}
auto download_result = music->GetDownloadResult();
ESP_LOGD(TAG, "Music details result: %s", download_result.c_str());
return true;
});
"Play the specified song. When users request to play music, this tool will automatically retrieve song details and start streaming.\n"
"parameter:\n"
" `song_name`: The name of the song to be played.\n"
"return:\n"
" Play status information without confirmation, immediately play the song.",
PropertyList({Property("song_name", kPropertyTypeString)}),
[music](const PropertyList &properties) -> ReturnValue
{
auto song_name = properties["song_name"].value<std::string>();
if (!music->Download(song_name))
{
return "{\"success\": false, \"message\": \"Failed to obtain music resources\"}";
}
auto download_result = music->GetDownloadResult();
ESP_LOGD(TAG, "Music details result: %s", download_result.c_str());
return true;
});
AddTool("self.music.set_display_mode",
"Set the display mode for music playback. You can choose to display the spectrum or lyrics, for example, if the user says' open spectrum 'or' display spectrum ', the corresponding display mode will be set for' open lyrics' or 'display lyrics'.\n"
"parameter:\n"
" `mode`: Display mode, with optional values of 'spectrum' or 'lyrics'.\n"
"return:\n"
" Set result information.",
PropertyList({
Property("mode", kPropertyTypeString) // Display mode: "spectrum" or "lyrics"
}),
[music](const PropertyList &properties) -> ReturnValue
{
auto mode_str = properties["mode"].value<std::string>();
// Convert to lowercase for comparison
std::transform(mode_str.begin(), mode_str.end(), mode_str.begin(), ::tolower);
if (mode_str == "spectrum" || mode_str == "频谱")
{
// Set to spectrum display mode
auto esp32_music = static_cast<Esp32Music *>(music);
esp32_music->SetDisplayMode(Esp32Music::DISPLAY_MODE_SPECTRUM);
return "{\"success\": true, \"message\": \"Switched to spectrum display mode\"}";
}
else if (mode_str == "lyrics" || mode_str == "歌词")
{
// Set to lyrics display mode
auto esp32_music = static_cast<Esp32Music *>(music);
esp32_music->SetDisplayMode(Esp32Music::DISPLAY_MODE_LYRICS);
return "{\"success\": true, \"message\": \"Switched to lyrics display mode\"}";
}
else
{
return "{\"success\": false, \"message\": \"Invalid display mode, please use 'spectrum' or 'lyrics'\"}";
}
return "{\"success\": false, \"message\": \"Failed to set display mode\"}";
});
}
// Restore the original tools list to the end of the tools list
tools_.insert(tools_.end(), original_tools.begin(), original_tools.end());
}
void McpServer::AddTool(McpTool* tool) {
void McpServer::AddTool(McpTool *tool)
{
// Prevent adding duplicate tools
if (std::find_if(tools_.begin(), tools_.end(), [tool](const McpTool* t) { return t->name() == tool->name(); }) != tools_.end()) {
if (std::find_if(tools_.begin(), tools_.end(), [tool](const McpTool *t)
{ return t->name() == tool->name(); }) != tools_.end())
{
ESP_LOGW(TAG, "Tool %s already added", tool->name().c_str());
return;
}
@@ -140,13 +188,16 @@ void McpServer::AddTool(McpTool* tool) {
tools_.push_back(tool);
}
void McpServer::AddTool(const std::string& name, const std::string& description, const PropertyList& properties, std::function<ReturnValue(const PropertyList&)> callback) {
void McpServer::AddTool(const std::string &name, const std::string &description, const PropertyList &properties, std::function<ReturnValue(const PropertyList &)> callback)
{
AddTool(new McpTool(name, description, properties, callback));
}
void McpServer::ParseMessage(const std::string& message) {
cJSON* json = cJSON_Parse(message.c_str());
if (json == nullptr) {
void McpServer::ParseMessage(const std::string &message)
{
cJSON *json = cJSON_Parse(message.c_str());
if (json == nullptr)
{
ESP_LOGE(TAG, "Failed to parse MCP message: %s", message.c_str());
return;
}
@@ -154,17 +205,22 @@ void McpServer::ParseMessage(const std::string& message) {
cJSON_Delete(json);
}
void McpServer::ParseCapabilities(const cJSON* capabilities) {
void McpServer::ParseCapabilities(const cJSON *capabilities)
{
auto vision = cJSON_GetObjectItem(capabilities, "vision");
if (cJSON_IsObject(vision)) {
if (cJSON_IsObject(vision))
{
auto url = cJSON_GetObjectItem(vision, "url");
auto token = cJSON_GetObjectItem(vision, "token");
if (cJSON_IsString(url)) {
if (cJSON_IsString(url))
{
auto camera = Board::GetInstance().GetCamera();
if (camera) {
if (camera)
{
std::string url_str = std::string(url->valuestring);
std::string token_str;
if (cJSON_IsString(token)) {
if (cJSON_IsString(token))
{
token_str = std::string(token->valuestring);
}
camera->SetExplainUrl(url_str, token_str);
@@ -173,44 +229,53 @@ void McpServer::ParseCapabilities(const cJSON* capabilities) {
}
}
void McpServer::ParseMessage(const cJSON* json) {
void McpServer::ParseMessage(const cJSON *json)
{
// Check JSONRPC version
auto version = cJSON_GetObjectItem(json, "jsonrpc");
if (version == nullptr || !cJSON_IsString(version) || strcmp(version->valuestring, "2.0") != 0) {
if (version == nullptr || !cJSON_IsString(version) || strcmp(version->valuestring, "2.0") != 0)
{
ESP_LOGE(TAG, "Invalid JSONRPC version: %s", version ? version->valuestring : "null");
return;
}
// Check method
auto method = cJSON_GetObjectItem(json, "method");
if (method == nullptr || !cJSON_IsString(method)) {
if (method == nullptr || !cJSON_IsString(method))
{
ESP_LOGE(TAG, "Missing method");
return;
}
auto method_str = std::string(method->valuestring);
if (method_str.find("notifications") == 0) {
if (method_str.find("notifications") == 0)
{
return;
}
// Check params
auto params = cJSON_GetObjectItem(json, "params");
if (params != nullptr && !cJSON_IsObject(params)) {
if (params != nullptr && !cJSON_IsObject(params))
{
ESP_LOGE(TAG, "Invalid params for method: %s", method_str.c_str());
return;
}
auto id = cJSON_GetObjectItem(json, "id");
if (id == nullptr || !cJSON_IsNumber(id)) {
if (id == nullptr || !cJSON_IsNumber(id))
{
ESP_LOGE(TAG, "Invalid id for method: %s", method_str.c_str());
return;
}
auto id_int = id->valueint;
if (method_str == "initialize") {
if (cJSON_IsObject(params)) {
if (method_str == "initialize")
{
if (cJSON_IsObject(params))
{
auto capabilities = cJSON_GetObjectItem(params, "capabilities");
if (cJSON_IsObject(capabilities)) {
if (cJSON_IsObject(capabilities))
{
ParseCapabilities(capabilities);
}
}
@@ -219,47 +284,60 @@ void McpServer::ParseMessage(const cJSON* json) {
message += app_desc->version;
message += "\"}}";
ReplyResult(id_int, message);
} else if (method_str == "tools/list") {
}
else if (method_str == "tools/list")
{
std::string cursor_str = "";
if (params != nullptr) {
if (params != nullptr)
{
auto cursor = cJSON_GetObjectItem(params, "cursor");
if (cJSON_IsString(cursor)) {
if (cJSON_IsString(cursor))
{
cursor_str = std::string(cursor->valuestring);
}
}
GetToolsList(id_int, cursor_str);
} else if (method_str == "tools/call") {
if (!cJSON_IsObject(params)) {
}
else if (method_str == "tools/call")
{
if (!cJSON_IsObject(params))
{
ESP_LOGE(TAG, "tools/call: Missing params");
ReplyError(id_int, "Missing params");
return;
}
auto tool_name = cJSON_GetObjectItem(params, "name");
if (!cJSON_IsString(tool_name)) {
if (!cJSON_IsString(tool_name))
{
ESP_LOGE(TAG, "tools/call: Missing name");
ReplyError(id_int, "Missing name");
return;
}
auto tool_arguments = cJSON_GetObjectItem(params, "arguments");
if (tool_arguments != nullptr && !cJSON_IsObject(tool_arguments)) {
if (tool_arguments != nullptr && !cJSON_IsObject(tool_arguments))
{
ESP_LOGE(TAG, "tools/call: Invalid arguments");
ReplyError(id_int, "Invalid arguments");
return;
}
auto stack_size = cJSON_GetObjectItem(params, "stackSize");
if (stack_size != nullptr && !cJSON_IsNumber(stack_size)) {
if (stack_size != nullptr && !cJSON_IsNumber(stack_size))
{
ESP_LOGE(TAG, "tools/call: Invalid stackSize");
ReplyError(id_int, "Invalid stackSize");
return;
}
DoToolCall(id_int, std::string(tool_name->valuestring), tool_arguments, stack_size ? stack_size->valueint : DEFAULT_TOOLCALL_STACK_SIZE);
} else {
}
else
{
ESP_LOGE(TAG, "Method not implemented: %s", method_str.c_str());
ReplyError(id_int, "Method not implemented: " + method_str);
}
}
void McpServer::ReplyResult(int id, const std::string& result) {
void McpServer::ReplyResult(int id, const std::string &result)
{
std::string payload = "{\"jsonrpc\":\"2.0\",\"id\":";
payload += std::to_string(id) + ",\"result\":";
payload += result;
@@ -267,7 +345,8 @@ void McpServer::ReplyResult(int id, const std::string& result) {
Application::GetInstance().SendMcpMessage(payload);
}
void McpServer::ReplyError(int id, const std::string& message) {
void McpServer::ReplyError(int id, const std::string &message)
{
std::string payload = "{\"jsonrpc\":\"2.0\",\"id\":";
payload += std::to_string(id);
payload += ",\"error\":{\"message\":\"";
@@ -276,94 +355,120 @@ void McpServer::ReplyError(int id, const std::string& message) {
Application::GetInstance().SendMcpMessage(payload);
}
void McpServer::GetToolsList(int id, const std::string& cursor) {
void McpServer::GetToolsList(int id, const std::string &cursor)
{
const int max_payload_size = 8000;
std::string json = "{\"tools\":[";
bool found_cursor = cursor.empty();
auto it = tools_.begin();
std::string next_cursor = "";
while (it != tools_.end()) {
while (it != tools_.end())
{
// 如果我们还没有找到起始位置，继续搜索
if (!found_cursor) {
if ((*it)->name() == cursor) {
if (!found_cursor)
{
if ((*it)->name() == cursor)
{
found_cursor = true;
} else {
}
else
{
++it;
continue;
}
}
// 添加tool前检查大小
std::string tool_json = (*it)->to_json() + ",";
if (json.length() + tool_json.length() + 30 > max_payload_size) {
if (json.length() + tool_json.length() + 30 > max_payload_size)
{
// 如果添加这个tool会超出大小限制，设置next_cursor并退出循环
next_cursor = (*it)->name();
break;
}
json += tool_json;
++it;
}
if (json.back() == ',') {
if (json.back() == ',')
{
json.pop_back();
}
if (json.back() == '[' && !tools_.empty()) {
if (json.back() == '[' && !tools_.empty())
{
// 如果没有添加任何tool，返回错误
ESP_LOGE(TAG, "tools/list: Failed to add tool %s because of payload size limit", next_cursor.c_str());
ReplyError(id, "Failed to add tool " + next_cursor + " because of payload size limit");
return;
}
if (next_cursor.empty()) {
if (next_cursor.empty())
{
json += "]}";
} else {
}
else
{
json += "],\"nextCursor\":\"" + next_cursor + "\"}";
}
ReplyResult(id, json);
}
void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments, int stack_size) {
auto tool_iter = std::find_if(tools_.begin(), tools_.end(),
[&tool_name](const McpTool* tool) {
return tool->name() == tool_name;
});
if (tool_iter == tools_.end()) {
void McpServer::DoToolCall(int id, const std::string &tool_name, const cJSON *tool_arguments, int stack_size)
{
auto tool_iter = std::find_if(tools_.begin(), tools_.end(),
[&tool_name](const McpTool *tool)
{
return tool->name() == tool_name;
});
if (tool_iter == tools_.end())
{
ESP_LOGE(TAG, "tools/call: Unknown tool: %s", tool_name.c_str());
ReplyError(id, "Unknown tool: " + tool_name);
return;
}
PropertyList arguments = (*tool_iter)->properties();
try {
for (auto& argument : arguments) {
try
{
for (auto &argument : arguments)
{
bool found = false;
if (cJSON_IsObject(tool_arguments)) {
if (cJSON_IsObject(tool_arguments))
{
auto value = cJSON_GetObjectItem(tool_arguments, argument.name().c_str());
if (argument.type() == kPropertyTypeBoolean && cJSON_IsBool(value)) {
if (argument.type() == kPropertyTypeBoolean && cJSON_IsBool(value))
{
argument.set_value<bool>(value->valueint == 1);
found = true;
} else if (argument.type() == kPropertyTypeInteger && cJSON_IsNumber(value)) {
}
else if (argument.type() == kPropertyTypeInteger && cJSON_IsNumber(value))
{
argument.set_value<int>(value->valueint);
found = true;
} else if (argument.type() == kPropertyTypeString && cJSON_IsString(value)) {
}
else if (argument.type() == kPropertyTypeString && cJSON_IsString(value))
{
argument.set_value<std::string>(value->valuestring);
found = true;
}
}
if (!argument.has_default_value() && !found) {
if (!argument.has_default_value() && !found)
{
ESP_LOGE(TAG, "tools/call: Missing valid argument: %s", argument.name().c_str());
ReplyError(id, "Missing valid argument: " + argument.name());
return;
}
}
} catch (const std::exception& e) {
}
catch (const std::exception &e)
{
ESP_LOGE(TAG, "tools/call: %s", e.what());
ReplyError(id, e.what());
return;
@@ -377,13 +482,13 @@ void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* to
esp_pthread_set_cfg(&cfg);
// Use a thread to call the tool to avoid blocking the main thread
tool_call_thread_ = std::thread([this, id, tool_iter, arguments = std::move(arguments)]() {
tool_call_thread_ = std::thread([this, id, tool_iter, arguments = std::move(arguments)]()
{
try {
ReplyResult(id, (*tool_iter)->Call(arguments));
} catch (const std::exception& e) {
ESP_LOGE(TAG, "tools/call: %s", e.what());
ReplyError(id, e.what());
}
});
} });
tool_call_thread_.detach();
}