OmniX-Space/xiaozhi-esp32
1
0
Fork 0
Code Issues Pull Requests Actions Releases Activity

支持频谱显示，感谢哈哈哈群友的代码

Browse Source
This commit is contained in:
苏雨洛
2025-09-05 19:28:01 +08:00
parent 10edbc6402
commit 536ef62412
9 changed files with 1171 additions and 3571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -60,6 +60,8 @@
- main/boards/common/board.cc
- main/display/display.h
- main/display/display.cc
- main/display/lcd_display.h
- main/display/lcd_display.cc
- main/application.h
- main/application.cc
- main/idf_component.yml

378
main/boards/common/esp32_music.cc
View File

@@ -17,6 +17,7 @@
#include <sstream>
#include <algorithm>
#include <cctype> // 为isdigit函数
#include <thread> // 为线程ID比较
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
@@ -185,12 +186,12 @@ static std::string buildUrlWithParams(const std::string &base_url, const std::st
Esp32Music::Esp32Music() : last_downloaded_data_(), current_music_url_(), current_song_name_(),
song_name_displayed_(false), current_lyric_url_(), lyrics_(),
current_lyric_index_(-1), lyric_thread_(), is_lyric_running_(false),
is_playing_(false), is_downloading_(false),
display_mode_(DISPLAY_MODE_LYRICS), is_playing_(false), is_downloading_(false),
play_thread_(), download_thread_(), audio_buffer_(), buffer_mutex_(),
buffer_cv_(), buffer_size_(0), mp3_decoder_(nullptr), mp3_frame_info_(),
mp3_decoder_initialized_(false)
{
ESP_LOGI(TAG, "Music player initialized");
ESP_LOGI(TAG, "Music player initialized with default spectrum display mode");
InitializeMp3Decoder();
}
@@ -317,7 +318,7 @@ Esp32Music::~Esp32Music()
ESP_LOGI(TAG, "Music player destroyed successfully");
}
bool Esp32Music::Download(const std::string &song_name)
bool Esp32Music::Download(const std::string &song_name, const std::string &artist_name)
{
ESP_LOGI(TAG, "喵波音律QQ交流群:865754861");
ESP_LOGI(TAG, "Starting to get music details for: %s", song_name.c_str());
@@ -329,8 +330,9 @@ bool Esp32Music::Download(const std::string &song_name)
current_song_name_ = song_name;
// 第一步：请求stream_pcm接口获取音频信息
std::string api_url = "https://api.yuafeng.cn/API/ly/mgmusic.php";
std::string full_url = api_url + "?msg=" + url_encode(song_name) + "&n=1";
std::string base_url = "https://music.miao-lab.top";
// std::string full_url = base_url + "/stream_pcm?song=" + url_encode(song_name) + "&artist=" + url_encode(artist_name);
std::string full_url = base_url + "/api?msg=" + url_encode(song_name);
ESP_LOGI(TAG, "Request URL: %s", full_url.c_str());
@@ -368,9 +370,9 @@ bool Esp32Music::Download(const std::string &song_name)
ESP_LOGI(TAG, "HTTP GET Status = %d, content_length = %d", status_code, last_downloaded_data_.length());
ESP_LOGD(TAG, "Complete music details response: %s", last_downloaded_data_.c_str());
// 检查认证响应
if (last_downloaded_data_.find("\"code\": 0") == std::string::npos)
{ // 如果"code"不为0，则为失败
// 简单的认证响应检查（可选）
if (last_downloaded_data_.find("ESP32动态密钥验证失败") != std::string::npos)
{
ESP_LOGE(TAG, "Authentication failed for song: %s", song_name.c_str());
return false;
}
@@ -381,155 +383,118 @@ bool Esp32Music::Download(const std::string &song_name)
cJSON *response_json = cJSON_Parse(last_downloaded_data_.c_str());
if (response_json)
{
cJSON *data = cJSON_GetObjectItem(response_json, "data");
// 提取关键信息
if (data)
// 提取data数组
cJSON *data_array = cJSON_GetObjectItem(response_json, "data");
if (cJSON_IsArray(data_array))
{
// 提取关键信息
cJSON *artist = cJSON_GetObjectItem(data, "singer");
cJSON *title = cJSON_GetObjectItem(data, "song");
cJSON *audio_url = cJSON_GetObjectItem(data, "music");
cJSON *lyric_url = cJSON_GetObjectItem(data, "lyric");
if (cJSON_IsString(artist))
cJSON *item = nullptr;
cJSON *min_item = nullptr;
int min_num = INT_MAX;
// 遍历data数组以找到最小num的项
cJSON_ArrayForEach(item, data_array)
{
ESP_LOGI(TAG, "Artist: %s", artist->valuestring);
}
if (cJSON_IsString(title))
{
ESP_LOGI(TAG, "Title: %s", title->valuestring);
}
// 检查audio_url是否有效
if (cJSON_IsString(audio_url) && audio_url->valuestring && strlen(audio_url->valuestring) > 0)
{
ESP_LOGI(TAG, "Audio URL: %s", audio_url->valuestring);
current_music_url_ = audio_url->valuestring;
song_name_displayed_ = false; // 重置歌名显示标志
StartStreaming(current_music_url_);
// 处理歌词URL
if (cJSON_IsString(lyric_url) && lyric_url->valuestring && strlen(lyric_url->valuestring) > 0)
cJSON *num = cJSON_GetObjectItem(item, "num");
if (cJSON_IsNumber(num) && num->valueint < min_num)
{
ESP_LOGI(TAG, "Loading lyrics for: %s", song_name.c_str());
current_lyric_url_ = lyric_url->valuestring;
min_num = num->valueint;
min_item = item;
}
}
if (min_item)
{
// 提取关键信息
cJSON *artist = cJSON_GetObjectItem(min_item, "singer");
cJSON *title = cJSON_GetObjectItem(min_item, "song");
cJSON *music_url = cJSON_GetObjectItem(min_item, "music_url");
cJSON *audio_url = music_url ? cJSON_GetObjectItem(music_url, "standard") : nullptr;
cJSON *lyric = cJSON_GetObjectItem(min_item, "lyric");
cJSON *lyric_url = lyric ? cJSON_GetObjectItem(lyric, "lrc") : nullptr;
if (cJSON_IsString(artist))
{
ESP_LOGI(TAG, "Artist: %s", artist->valuestring);
}
if (cJSON_IsString(title))
{
ESP_LOGI(TAG, "Title: %s", title->valuestring);
}
// 检查audio_url是否有效
if (cJSON_IsString(audio_url) && audio_url->valuestring && strlen(audio_url->valuestring) > 0)
{
ESP_LOGI(TAG, "Audio URL path: %s", audio_url->valuestring);
// 启动歌词下载和显示
if (is_lyric_running_)
// 第二步：获取audio_url并开始流式播放
current_music_url_ = audio_url->valuestring;
ESP_LOGI(TAG, "喵波音律QQ交流群:865754861");
ESP_LOGI(TAG, "Starting streaming playback for: %s", song_name.c_str());
song_name_displayed_ = false; // 重置歌名显示标志
StartStreaming(current_music_url_);
// 处理歌词URL - 只有在歌词显示模式下才启动歌词
if (cJSON_IsString(lyric_url) && lyric_url->valuestring && strlen(lyric_url->valuestring) > 0)
{
is_lyric_running_ = false;
if (lyric_thread_.joinable())
// 直接返回歌词URL
current_lyric_url_ = lyric_url->valuestring;
// 根据显示模式决定是否启动歌词
if (display_mode_ == DISPLAY_MODE_LYRICS)
{
lyric_thread_.join();
ESP_LOGI(TAG, "Loading lyrics for: %s (lyrics display mode)", song_name.c_str());
// 启动歌词下载和显示
if (is_lyric_running_)
{
is_lyric_running_ = false;
if (lyric_thread_.joinable())
{
lyric_thread_.join();
}
}
is_lyric_running_ = true;
current_lyric_index_ = -1;
lyrics_.clear();
lyric_thread_ = std::thread(&Esp32Music::LyricDisplayThread, this);
}
else
{
ESP_LOGI(TAG, "Lyric URL found but spectrum display mode is active, skipping lyrics");
}
}
else
{
ESP_LOGW(TAG, "No lyric URL found for this song");
}
is_lyric_running_ = true;
current_lyric_index_ = -1;
lyrics_.clear();
lyric_thread_ = std::thread(&Esp32Music::LyricDisplayThread, this);
cJSON_Delete(response_json);
return true;
}
else
{
ESP_LOGW(TAG, "The lyrics URL for this song was not found");
// audio_url为空或无效
ESP_LOGE(TAG, "Audio URL not found or empty for song: %s", song_name.c_str());
ESP_LOGE(TAG, "Failed to find music: 没有找到歌曲 '%s'", song_name.c_str());
cJSON_Delete(response_json);
return false;
}
cJSON_Delete(response_json);
return true;
}
else
{
ESP_LOGE(TAG, "No valid audio URL found, song: %s", song_name.c_str());
ESP_LOGE(TAG, "Unable to find song: '%s'", song_name.c_str());
cJSON_Delete(response_json);
return false;
}
}
else
{
ESP_LOGE(TAG, "The 'data' field was not found in the response data");
cJSON_Delete(response_json);
return false;
}
}
else
{
ESP_LOGE(TAG, "Unable to parse JSON response");
ESP_LOGE(TAG, "Failed to parse JSON response");
}
}
else
{
ESP_LOGE(TAG, "Music API returns empty response");
ESP_LOGE(TAG, "Empty response from music API");
}
return false;
}
bool Esp32Music::Play()
{
if (is_playing_.load())
{ // 使用atomic的load()
ESP_LOGW(TAG, "Music is already playing");
return true;
}
if (last_downloaded_data_.empty())
{
ESP_LOGE(TAG, "No music data to play");
return false;
}
// 清理之前的播放线程
if (play_thread_.joinable())
{
play_thread_.join();
}
// 实际应调用流式播放接口
return StartStreaming(current_music_url_);
}
bool Esp32Music::Stop()
{
if (!is_playing_ && !is_downloading_)
{
ESP_LOGW(TAG, "Music is not playing or downloading");
return true;
}
ESP_LOGI(TAG, "Stopping music playback and download");
// 停止下载和播放
is_downloading_ = false;
is_playing_ = false;
// 重置采样率到原始值
ResetSampleRate();
// 通知所有等待的线程
{
std::lock_guard<std::mutex> lock(buffer_mutex_);
buffer_cv_.notify_all();
}
// 等待线程结束
if (download_thread_.joinable())
{
download_thread_.join();
}
if (play_thread_.joinable())
{
play_thread_.join();
}
// 清空缓冲区
ClearAudioBuffer();
ESP_LOGI(TAG, "Music stopped successfully");
return true;
}
std::string Esp32Music::GetDownloadResult()
{
return last_downloaded_data_;
@@ -587,6 +552,7 @@ bool Esp32Music::StartStreaming(const std::string &music_url)
play_thread_ = std::thread(&Esp32Music::PlayAudioStream, this);
ESP_LOGI(TAG, "Streaming threads started successfully");
return true;
}
@@ -625,6 +591,69 @@ bool Esp32Music::StopStreaming()
buffer_cv_.notify_all();
}
// 等待线程结束（避免重复代码，让StopStreaming也能等待线程完全停止）
if (download_thread_.joinable())
{
download_thread_.join();
ESP_LOGI(TAG, "Download thread joined in StopStreaming");
}
// 等待播放线程结束，使用更安全的方式
if (play_thread_.joinable())
{
// 先设置停止标志
is_playing_ = false;
// 通知条件变量，确保线程能够退出
{
std::lock_guard<std::mutex> lock(buffer_mutex_);
buffer_cv_.notify_all();
}
// 使用超时机制等待线程结束，避免死锁
bool thread_finished = false;
int wait_count = 0;
const int max_wait = 100; // 最多等待1秒
while (!thread_finished && wait_count < max_wait)
{
vTaskDelay(pdMS_TO_TICKS(10));
wait_count++;
// 检查线程是否仍然可join
if (!play_thread_.joinable())
{
thread_finished = true;
break;
}
}
if (play_thread_.joinable())
{
if (wait_count >= max_wait)
{
ESP_LOGW(TAG, "Play thread join timeout, detaching thread");
play_thread_.detach();
}
else
{
play_thread_.join();
ESP_LOGI(TAG, "Play thread joined in StopStreaming");
}
}
}
// 在线程完全结束后，只在频谱模式下停止FFT显示
if (display && display_mode_ == DISPLAY_MODE_SPECTRUM)
{
display->stopFft();
ESP_LOGI(TAG, "Stopped FFT display in StopStreaming (spectrum mode)");
}
else if (display)
{
ESP_LOGI(TAG, "Not in spectrum mode, skipping FFT stop in StopStreaming");
}
ESP_LOGI(TAG, "Music streaming stop signal sent");
return true;
}
@@ -862,21 +891,8 @@ void Esp32Music::PlayAudioStream()
continue;
}
// 设备状态检查通过，显示当前播放的歌名或歌词
if (!song_name_displayed_ && !current_lyric_url_.empty())
{
auto &board = Board::GetInstance();
auto display = board.GetDisplay();
if (display)
{
// 格式化歌词显示
std::string formatted_lyric = current_lyric_url_;
display->SetMusicInfo(formatted_lyric.c_str());
ESP_LOGI(TAG, "Displaying lyric: %s", formatted_lyric.c_str());
song_name_displayed_ = true;
}
}
else if (!song_name_displayed_ && !current_song_name_.empty())
// 设备状态检查通过，显示当前播放的歌名
if (!song_name_displayed_ && !current_song_name_.empty())
{
auto &board = Board::GetInstance();
auto display = board.GetDisplay();
@@ -888,6 +904,20 @@ void Esp32Music::PlayAudioStream()
ESP_LOGI(TAG, "Displaying song name: %s", formatted_song_name.c_str());
song_name_displayed_ = true;
}
// 根据显示模式启动相应的显示功能
if (display)
{
if (display_mode_ == DISPLAY_MODE_SPECTRUM)
{
display->start();
ESP_LOGI(TAG, "Display start() called for spectrum visualization");
}
else
{
ESP_LOGI(TAG, "Lyrics display mode active, FFT visualization disabled");
}
}
}
// 如果需要更多MP3数据，从缓冲区读取
@@ -1060,6 +1090,18 @@ void Esp32Music::PlayAudioStream()
packet.payload.resize(pcm_size_bytes);
memcpy(packet.payload.data(), final_pcm_data, pcm_size_bytes);
if (final_pcm_data_fft == nullptr)
{
final_pcm_data_fft = (int16_t *)heap_caps_malloc(
final_sample_count * sizeof(int16_t),
MALLOC_CAP_SPIRAM);
}
memcpy(
final_pcm_data_fft,
final_pcm_data,
final_sample_count * sizeof(int16_t));
ESP_LOGD(TAG, "Sending %d PCM samples (%d bytes, rate=%d, channels=%d->1) to Application",
final_sample_count, pcm_size_bytes, mp3_frame_info_.samprate, mp3_frame_info_.nChans);
@@ -1098,23 +1140,28 @@ void Esp32Music::PlayAudioStream()
heap_caps_free(mp3_input_buffer);
}
// 播放结束时清空歌名显示
auto &board = Board::GetInstance();
auto display = board.GetDisplay();
if (display)
{
display->SetMusicInfo(""); // 清空歌名显示
ESP_LOGI(TAG, "Cleared song name display on playback end");
}
// 重置采样率到原始值
ResetSampleRate();
// 播放结束时保持音频输出启用状态，让Application管理
// 不在这里禁用音频输出，避免干扰其他音频功能
// 播放结束时进行基本清理，但不调用StopStreaming避免线程自我等待
ESP_LOGI(TAG, "Audio stream playback finished, total played: %d bytes", total_played);
ESP_LOGI(TAG, "Performing basic cleanup from play thread");
// 停止播放标志
is_playing_ = false;
// 只在频谱显示模式下才停止FFT显示
if (display_mode_ == DISPLAY_MODE_SPECTRUM)
{
auto &board = Board::GetInstance();
auto display = board.GetDisplay();
if (display)
{
display->stopFft();
ESP_LOGI(TAG, "Stopped FFT display from play thread (spectrum mode)");
}
}
else
{
ESP_LOGI(TAG, "Not in spectrum mode, skipping FFT stop");
}
}
// 清空音频缓冲区
@@ -1597,4 +1644,15 @@ void Esp32Music::UpdateLyricDisplay(int64_t current_time_ms)
*/
// 删除复杂的AddAuthHeaders方法，使用简单的静态函数
// 删除复杂的认证验证和配置方法，使用简单的静态函数
// 删除复杂的认证验证和配置方法，使用简单的静态函数
// 显示模式控制方法实现
void Esp32Music::SetDisplayMode(DisplayMode mode)
{
DisplayMode old_mode = display_mode_.load();
display_mode_ = mode;
ESP_LOGI(TAG, "Display mode changed from %s to %s",
(old_mode == DISPLAY_MODE_SPECTRUM) ? "SPECTRUM" : "LYRICS",
(mode == DISPLAY_MODE_SPECTRUM) ? "SPECTRUM" : "LYRICS");
}

23
main/boards/common/esp32_music.h
View File

@@ -26,6 +26,13 @@ struct AudioChunk {
};
class Esp32Music : public Music {
public:
// 显示模式控制 - 移动到public区域
enum DisplayMode {
DISPLAY_MODE_SPECTRUM = 0, // 默认显示频谱
DISPLAY_MODE_LYRICS = 1 // 显示歌词
};
private:
std::string last_downloaded_data_;
std::string current_music_url_;
@@ -39,6 +46,8 @@ private:
std::atomic<int> current_lyric_index_;
std::thread lyric_thread_;
std::atomic<bool> is_lyric_running_;
std::atomic<DisplayMode> display_mode_;
std::atomic<bool> is_playing_;
std::atomic<bool> is_downloading_;
std::thread play_thread_;
@@ -77,13 +86,14 @@ private:
// ID3标签处理
size_t SkipId3Tag(uint8_t* data, size_t size);
int16_t* final_pcm_data_fft = nullptr;
public:
Esp32Music();
~Esp32Music();
virtual bool Download(const std::string& song_name) override;
virtual bool Play() override;
virtual bool Stop() override;
virtual bool Download(const std::string& song_name, const std::string& artist_name) override;
virtual std::string GetDownloadResult() override;
// 新增方法
@@ -91,6 +101,11 @@ public:
virtual bool StopStreaming() override; // 停止流式播放
virtual size_t GetBufferSize() const override { return buffer_size_; }
virtual bool IsDownloading() const override { return is_downloading_; }
virtual int16_t* GetAudioData() override { return final_pcm_data_fft; }
// 显示模式控制方法
void SetDisplayMode(DisplayMode mode);
DisplayMode GetDisplayMode() const { return display_mode_.load(); }
};
#endif // ESP32_MUSIC_H
#endif // ESP32_MUSIC_H

5
main/boards/common/music.h
View File

@@ -7,9 +7,7 @@ class Music {
public:
virtual ~Music() = default; // 添加虚析构函数
virtual bool Download(const std::string& song_name) = 0;
virtual bool Play() = 0;
virtual bool Stop() = 0;
virtual bool Download(const std::string& song_name, const std::string& artist_name = "") = 0;
virtual std::string GetDownloadResult() = 0;
// 新增流式播放相关方法
@@ -17,6 +15,7 @@ public:
virtual bool StopStreaming() = 0; // 停止流式播放
virtual size_t GetBufferSize() const = 0;
virtual bool IsDownloading() const = 0;
virtual int16_t* GetAudioData() = 0;
};
#endif // MUSIC_H

5
main/display/display.h
View File

@@ -32,6 +32,9 @@ public:
virtual std::string GetTheme() { return current_theme_name_; }
virtual void UpdateStatusBar(bool update_all = false);
virtual void SetPowerSaveMode(bool on);
virtual void start() {}
virtual void clearScreen() {} // 清除FFT显示，默认为空实现
virtual void stopFft() {} // 停止FFT显示，默认为空实现
inline int width() const { return width_; }
inline int height() const { return height_; }
@@ -90,4 +93,4 @@ private:
virtual void Unlock() override {}
};
#endif
#endif

633
main/display/lcd_display.cc
View File

@@ -4,17 +4,38 @@
#include <algorithm>
#include <font_awesome.h>
#include <esp_log.h>
#include <esp_random.h>
#include <time.h>
#include <esp_err.h>
#include <esp_lvgl_port.h>
#include <esp_heap_caps.h>
#include "assets/lang_config.h"
#include <cstring>
#include <cmath>
#include <math.h>
#include "settings.h"
#include "board.h"
#ifndef M_PI
#define M_PI 3.14159265358979323846
#endif
#define TAG "LcdDisplay"
#define FFT_SIZE 512
static int current_heights[40] = {0};
static float avg_power_spectrum[FFT_SIZE/2]={-25.0f};
#define COLOR_BLACK 0x0000
#define COLOR_RED 0xF800
#define COLOR_GREEN 0x07E0
#define COLOR_BLUE 0x001F
#define COLOR_YELLOW 0xFFE0
#define COLOR_CYAN 0x07FF
#define COLOR_MAGENTA 0xF81F
#define COLOR_WHITE 0xFFFF
// Color definitions for dark theme
#define DARK_BACKGROUND_COLOR lv_color_hex(0x121212) // Dark background
#define DARK_TEXT_COLOR lv_color_white() // White text
@@ -105,7 +126,7 @@ SpiLcdDisplay::SpiLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h
ESP_LOGI(TAG, "Initialize LVGL port");
lvgl_port_cfg_t port_cfg = ESP_LVGL_PORT_INIT_CONFIG();
port_cfg.task_priority = 1;
port_cfg.timer_period_ms = 40;
port_cfg.timer_period_ms = 50;
lvgl_port_init(&port_cfg);
ESP_LOGI(TAG, "Adding LCD display");
@@ -145,6 +166,27 @@ SpiLcdDisplay::SpiLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_h
lv_display_set_offset(display_, offset_x, offset_y);
}
// 初始化 FFT 相关内存
fft_real = (float*)heap_caps_malloc(FFT_SIZE * sizeof(float), MALLOC_CAP_SPIRAM);
fft_imag = (float*)heap_caps_malloc(FFT_SIZE * sizeof(float), MALLOC_CAP_SPIRAM);
hanning_window_float = (float*)heap_caps_malloc(FFT_SIZE * sizeof(float), MALLOC_CAP_SPIRAM);
// 创建窗函数
for (int i = 0; i < FFT_SIZE; i++) {
hanning_window_float[i] = 0.5 * (1.0 - cos(2.0 * M_PI * i / (FFT_SIZE - 1)));
}
if(audio_data==nullptr){
audio_data=(int16_t*)heap_caps_malloc(sizeof(int16_t)*1152, MALLOC_CAP_SPIRAM);
memset(audio_data,0,sizeof(int16_t)*1152);
}
if(frame_audio_data==nullptr){
frame_audio_data=(int16_t*)heap_caps_malloc(sizeof(int16_t)*1152, MALLOC_CAP_SPIRAM);
memset(frame_audio_data,0,sizeof(int16_t)*1152);
}
ESP_LOGI(TAG,"Initialize fft_input, audio_data, frame_audio_data, spectrum_data");
SetupUI();
}
@@ -270,6 +312,24 @@ MipiLcdDisplay::MipiLcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel
}
LcdDisplay::~LcdDisplay() {
// 停止FFT任务
if (fft_task_handle != nullptr) {
ESP_LOGI(TAG, "Stopping FFT task in destructor");
fft_task_should_stop = true;
// 等待任务停止
int wait_count = 0;
while (fft_task_handle != nullptr && wait_count < 100) {
vTaskDelay(pdMS_TO_TICKS(10));
wait_count++;
}
if (fft_task_handle != nullptr) {
vTaskDelete(fft_task_handle);
fft_task_handle = nullptr;
}
}
// 然后再清理 LVGL 对象
if (content_ != nullptr) {
lv_obj_del(content_);
@@ -367,7 +427,7 @@ void LcdDisplay::SetupUI() {
emotion_label_ = lv_label_create(status_bar_);
lv_obj_set_style_text_font(emotion_label_, &font_awesome_30_4, 0);
lv_obj_set_style_text_color(emotion_label_, current_theme_.text, 0);
lv_label_set_text(emotion_label_, FONT_AWESOME_MICROCHIP_AI);
lv_label_set_text(emotion_label_, FONT_AWESOME_AI_CHIP);
lv_obj_set_style_margin_right(emotion_label_, 5, 0); // 添加右边距，与后面的元素分隔
notification_label_ = lv_label_create(status_bar_);
@@ -814,11 +874,11 @@ void LcdDisplay::SetPreviewImage(const lv_img_dsc_t* img_dsc) {
}
if (img_dsc != nullptr) {
// 设置图片源并显示预览图片
lv_image_set_src(preview_image_, img_dsc);
// zoom factor 0.5
lv_image_set_scale(preview_image_, 128 * width_ / img_dsc->header.w);
lv_obj_remove_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
// 设置图片源并显示预览图片
lv_image_set_src(preview_image_, img_dsc);
lv_obj_clear_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
// 隐藏emotion_label_
if (emotion_label_ != nullptr) {
lv_obj_add_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
@@ -827,7 +887,7 @@ void LcdDisplay::SetPreviewImage(const lv_img_dsc_t* img_dsc) {
// 隐藏预览图片并显示emotion_label_
lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
if (emotion_label_ != nullptr) {
lv_obj_remove_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
lv_obj_clear_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
}
}
}
@@ -867,13 +927,6 @@ void LcdDisplay::SetEmotion(const char* emotion) {
std::string_view emotion_view(emotion);
auto it = std::find_if(emotions.begin(), emotions.end(),
[&emotion_view](const Emotion& e) { return e.text == emotion_view; });
if (fonts_.emoji_font == nullptr || it == emotions.end()) {
const char* utf8 = font_awesome_get_utf8(emotion);
if (utf8 != nullptr) {
SetIcon(utf8);
}
return;
}
DisplayLockGuard lock(this);
if (emotion_label_ == nullptr) {
@@ -890,7 +943,7 @@ void LcdDisplay::SetEmotion(const char* emotion) {
#if !CONFIG_USE_WECHAT_MESSAGE_STYLE
// 显示emotion_label_，隐藏preview_image_
lv_obj_remove_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
lv_obj_clear_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
if (preview_image_ != nullptr) {
lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
}
@@ -907,13 +960,43 @@ void LcdDisplay::SetIcon(const char* icon) {
#if !CONFIG_USE_WECHAT_MESSAGE_STYLE
// 显示emotion_label_，隐藏preview_image_
lv_obj_remove_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
lv_obj_clear_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
if (preview_image_ != nullptr) {
lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
}
#endif
}
void LcdDisplay::SetMusicInfo(const char* song_name) {
#if CONFIG_USE_WECHAT_MESSAGE_STYLE
// 微信模式下不显示歌名，保持原有聊天功能
return;
#else
// 非微信模式：在表情下方显示歌名
DisplayLockGuard lock(this);
if (chat_message_label_ == nullptr) {
return;
}
if (song_name != nullptr && strlen(song_name) > 0) {
std::string music_text = "";
music_text += song_name;
lv_label_set_text(chat_message_label_, music_text.c_str());
// 确保显示 emotion_label_ 和 chat_message_label_，隐藏 preview_image_
if (emotion_label_ != nullptr) {
lv_obj_clear_flag(emotion_label_, LV_OBJ_FLAG_HIDDEN);
}
if (preview_image_ != nullptr) {
lv_obj_add_flag(preview_image_, LV_OBJ_FLAG_HIDDEN);
}
} else {
// 清空歌名显示
lv_label_set_text(chat_message_label_, "");
}
#endif
}
void LcdDisplay::SetTheme(const std::string& theme_name) {
DisplayLockGuard lock(this);
@@ -1108,3 +1191,523 @@ void LcdDisplay::SetTheme(const std::string& theme_name) {
// No errors occurred. Save theme to settings
Display::SetTheme(theme_name);
}
void LcdDisplay::create_canvas(){
DisplayLockGuard lock(this);
if (canvas_ != nullptr) {
lv_obj_del(canvas_);
}
if (canvas_buffer_ != nullptr) {
heap_caps_free(canvas_buffer_);
canvas_buffer_ = nullptr;
}
int status_bar_height=lv_obj_get_height(status_bar_);
canvas_width_=width_;
canvas_height_=height_-status_bar_height;
canvas_buffer_=(uint16_t*)heap_caps_malloc(canvas_width_ * canvas_height_ * sizeof(uint16_t), MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
if (canvas_buffer_ == nullptr) {
ESP_LOGE(TAG, "Failed to allocate canvas buffer");
return;
}
ESP_LOGI(TAG, "canvas buffer allocated successfully");
canvas_ = lv_canvas_create(lv_scr_act());
lv_canvas_set_buffer(canvas_, canvas_buffer_, canvas_width_, canvas_height_, LV_COLOR_FORMAT_RGB565);
ESP_LOGI(TAG,"width: %d, height: %d", width_, height_);
lv_obj_set_pos(canvas_, 0, status_bar_height);
lv_obj_set_size(canvas_, canvas_width_, canvas_height_);
lv_canvas_fill_bg(canvas_, lv_color_make(0, 0, 0), LV_OPA_TRANSP);
lv_obj_move_foreground(canvas_);
ESP_LOGI(TAG, "canvas created successfully");
}
void LcdDisplay::start(){
ESP_LOGI(TAG, "Starting LcdDisplay with periodic data updates");
vTaskDelay(pdMS_TO_TICKS(500));
// 创建周期性更新任务
fft_task_should_stop = false; // 重置停止标志
xTaskCreate(
periodicUpdateTaskWrapper,
"display_fft", // 任务名称
4096*2, // 堆栈大小
this, // 参数
1, // 优先级
&fft_task_handle // 保存到成员变量
);
}
void LcdDisplay::drawSpectrumIfReady() {
if (fft_data_ready) {
draw_spectrum(avg_power_spectrum, FFT_SIZE/2);
fft_data_ready = false;
}
}
void LcdDisplay::periodicUpdateTaskWrapper(void* arg) {
auto self = static_cast<LcdDisplay*>(arg);
self->periodicUpdateTask();
}
void LcdDisplay::periodicUpdateTask() {
ESP_LOGI(TAG, "Periodic update task started");
if(canvas_==nullptr){
create_canvas();
}
else{
ESP_LOGI(TAG, "canvas already created");
}
auto music = Board::GetInstance().GetMusic();
const TickType_t displayInterval = pdMS_TO_TICKS(40);
const TickType_t audioProcessInterval = pdMS_TO_TICKS(15);
TickType_t lastDisplayTime = xTaskGetTickCount();
TickType_t lastAudioTime = xTaskGetTickCount();
while (!fft_task_should_stop) {
TickType_t currentTime = xTaskGetTickCount();
if (currentTime - lastAudioTime >= audioProcessInterval) {
if(music->GetAudioData() != nullptr) {
readAudioData(); // 快速处理，不阻塞
} else {
vTaskDelay(pdMS_TO_TICKS(100));
}
lastAudioTime = currentTime;
}
// 显示刷新（30Hz）
if (currentTime - lastDisplayTime >= displayInterval) {
if (fft_data_ready) {
DisplayLockGuard lock(this);
drawSpectrumIfReady();
lv_area_t refresh_area;
refresh_area.x1 = 0;
refresh_area.y1 = height_-100;
refresh_area.x2 = canvas_width_ -1;
refresh_area.y2 = height_ -1; // 只刷新频谱区域
lv_obj_invalidate_area(canvas_, &refresh_area);
//lv_obj_invalidate(canvas_);
fft_data_ready = false;
lastDisplayTime = currentTime;
} // 绘制操作
// 更新FPS计数
//FPS();
}
vTaskDelay(pdMS_TO_TICKS(10));
// 短暂延迟
}
ESP_LOGI(TAG, "FFT display task stopped");
fft_task_handle = nullptr; // 清空任务句柄
vTaskDelete(NULL); // 删除当前任务
}
void LcdDisplay::readAudioData(){
auto music = Board::GetInstance().GetMusic();
if(music->GetAudioData()!=nullptr){
if(audio_display_last_update<=2){
memcpy(audio_data,music->GetAudioData(),sizeof(int16_t)*1152);
for(int i=0;i<1152;i++){
frame_audio_data[i]+=audio_data[i];
}
audio_display_last_update++;
}else{
const int HOP_SIZE = 512;
const int NUM_SEGMENTS = 1 + (1152 - FFT_SIZE) / HOP_SIZE;
for (int seg = 0; seg < NUM_SEGMENTS; seg++) {
int start = seg * HOP_SIZE;
if (start + FFT_SIZE > 1152) break;
// 准备当前段数据
for (int i = 0; i < FFT_SIZE; i++) {
int idx = start + i;
//float sample =frame_audio_data[idx] / 32768.0f;
float sample =frame_audio_data[idx] / 32768.0f;
fft_real[i] = sample * hanning_window_float[i];
fft_imag[i] = 0.0f;
}
compute(fft_real, fft_imag, FFT_SIZE, true);
// 计算功率谱并累加（双边）
for (int i = 0; i < FFT_SIZE/2; i++) {
avg_power_spectrum[i] += fft_real[i] * fft_real[i]+fft_imag[i] * fft_imag[i]; // 功率 = 幅度平方
}
}
// 计算平均值
for (int i = 0; i < FFT_SIZE/2; i++) {
avg_power_spectrum[i] /= NUM_SEGMENTS;
}
audio_display_last_update=0;
//memcpy(spectrum_data, avg_power_spectrum, sizeof(float) * FFT_SIZE/2);
fft_data_ready=true;
//draw_spectrum(avg_power_spectrum, FFT_SIZE/2);
memset(frame_audio_data,0,sizeof(int16_t)*1152);
}
}else{
ESP_LOGI(TAG, "audio_data is nullptr");
vTaskDelay(pdMS_TO_TICKS(500));
}
}
uint16_t LcdDisplay::get_bar_color(int x_pos){
static uint16_t color_table[40];
static bool initialized = false;
if (!initialized) {
// 生成黄绿->黄->黄红的渐变
for (int i = 0; i < 40; i++) {
if (i < 20) {
// 黄绿到黄：增加红色分量
uint8_t r = static_cast<uint8_t>((i / 19.0f) * 31);
color_table[i] = (r << 11) | (0x3F << 5);
} else {
// 黄到黄红：减少绿色分量
uint8_t g = static_cast<uint8_t>((1.0f - (i - 20) / 19.0f * 0.5f) * 63);
color_table[i] = (0x1F << 11) | (g << 5);
}
}
initialized = true;
}
return color_table[x_pos];
}
void LcdDisplay::draw_spectrum(float *power_spectrum,int fft_size){
const int bartotal=40;
int bar_height;
const int bar_max_height=canvas_height_-100;
const int bar_width=240/bartotal;
int x_pos=0;
int y_pos = (canvas_height_) - 1;
float magnitude[bartotal]={0};
float max_magnitude=0;
const float MIN_DB = -25.0f;
const float MAX_DB = 0.0f;
for (int bin = 0; bin < bartotal; bin++) {
int start = bin * (fft_size / bartotal);
int end = (bin+1) * (fft_size / bartotal);
magnitude[bin] = 0;
int count=0;
for (int k = start; k < end; k++) {
magnitude[bin] += sqrt(power_spectrum[k]);
count++;
}
if(count>0){
magnitude[bin] /= count;
}
if (magnitude[bin] > max_magnitude) max_magnitude = magnitude[bin];
}
magnitude[1]=magnitude[1]*0.6;
magnitude[2]=magnitude[2]*0.7;
magnitude[3]=magnitude[3]*0.8;
magnitude[4]=magnitude[4]*0.8;
magnitude[5]=magnitude[5]*0.9;
/*
if (bartotal >= 6) {
magnitude[0] *= 0.3f; // 最低频
magnitude[1] *= 1.1f;
magnitude[2] *= 1.0f;
magnitude[3] *= 0.9f;
magnitude[4] *= 0.8f;
magnitude[5] *= 0.7f;
// 更高频率保持或进一步衰减
for (int i = 6; i < bartotal; i++) {
magnitude[i] *= 0.6f;
}
}
*/
for (int bin = 1; bin < bartotal; bin++) {
if (magnitude[bin] > 0.0f && max_magnitude > 0.0f) {
// 相对dB值：20 * log10(magnitude/ref_level)
magnitude[bin] = 20.0f * log10f(magnitude[bin] / max_magnitude+ 1e-10);
} else {
magnitude[bin] = MIN_DB;
}
if (magnitude[bin] > max_magnitude) max_magnitude = magnitude[bin];
}
clearScreen();
for (int k = 1; k < bartotal; k++) { // 跳过直流分量（k=0）
x_pos=canvas_width_/bartotal*(k-1);
float mag=(magnitude[k] - MIN_DB) / (MAX_DB - MIN_DB);
mag = std::max(0.0f, std::min(1.0f, mag));
bar_height=int(mag*(bar_max_height));
int color=get_bar_color(k);
draw_bar(x_pos,y_pos,bar_width,bar_height, color,k-1);
//printf("x: %d, y: %d,\n", x_pos, bar_height);
}
}
void LcdDisplay::draw_bar(int x,int y,int bar_width,int bar_height,uint16_t color,int bar_index){
const int block_space=2;
const int block_x_size=bar_width-block_space;
const int block_y_size=4;
int blocks_per_col=(bar_height/(block_y_size+block_space));
int start_x=(block_x_size+block_space)/2+x;
if(current_heights[bar_index]<bar_height)
{
current_heights[bar_index]=bar_height;
}
else{
int fall_speed=2;
current_heights[bar_index]=current_heights[bar_index]-fall_speed;
if(current_heights[bar_index]>(block_y_size+block_space))
draw_block(start_x,canvas_height_-current_heights[bar_index],block_x_size,block_y_size,color,bar_index);
}
draw_block(start_x,canvas_height_-1,block_x_size,block_y_size,color,bar_index);
for(int j=1;j<blocks_per_col;j++){
int start_y=j*(block_y_size+block_space);
draw_block(start_x,canvas_height_-start_y,block_x_size,block_y_size,color,bar_index);
}
}
void LcdDisplay::draw_block(int x,int y,int block_x_size,int block_y_size,uint16_t color,int bar_index){
/*
for(int dy=0;dy<block_y_size;dy++){
for(int dx=0;dx<block_x_size;dx++){
canvas_buffer_[(y-dy)*canvas_width_+x+dx]=color;
}
}
*/
for (int row = y; row > y-block_y_size;row--) {
// 一次绘制一行
uint16_t* line_start = &canvas_buffer_[row * canvas_width_ + x];
std::fill_n(line_start, block_x_size, color);
}
}
void LcdDisplay::clearScreen() {
// DisplayLockGuard lock(this);
// 清屏为黑色
//for (int i = 0; i < canvas_width_ * canvas_height_; i++) {
// canvas_buffer_[i] = COLOR_BLACK;
//}
//lv_obj_invalidate(canvas_);
std::fill_n(canvas_buffer_, canvas_width_ * canvas_height_, COLOR_BLACK);
}
void LcdDisplay::stopFft() {
ESP_LOGI(TAG, "Stopping FFT display");
// 停止FFT显示任务
if (fft_task_handle != nullptr) {
ESP_LOGI(TAG, "Stopping FFT display task");
fft_task_should_stop = true; // 设置停止标志
// 等待任务停止（最多等待1秒）
int wait_count = 0;
while (fft_task_handle != nullptr && wait_count < 100) {
vTaskDelay(pdMS_TO_TICKS(10));
wait_count++;
}
if (fft_task_handle != nullptr) {
ESP_LOGW(TAG, "FFT task did not stop gracefully, force deleting");
vTaskDelete(fft_task_handle);
fft_task_handle = nullptr;
} else {
ESP_LOGI(TAG, "FFT display task stopped successfully");
}
}
// 使用显示锁保护所有操作
DisplayLockGuard lock(this);
// 重置FFT状态变量
fft_data_ready = false;
audio_display_last_update = 0;
// 重置频谱条高度
memset(current_heights, 0, sizeof(current_heights));
// 重置平均功率谱数据
for (int i = 0; i < FFT_SIZE/2; i++) {
avg_power_spectrum[i] = -25.0f;
}
// 删除FFT画布对象，让原始UI重新显示
if (canvas_ != nullptr) {
lv_obj_del(canvas_);
canvas_ = nullptr;
ESP_LOGI(TAG, "FFT canvas deleted");
}
// 释放画布缓冲区内存
if (canvas_buffer_ != nullptr) {
heap_caps_free(canvas_buffer_);
canvas_buffer_ = nullptr;
ESP_LOGI(TAG, "FFT canvas buffer freed");
}
// 重置画布尺寸变量
canvas_width_ = 0;
canvas_height_ = 0;
ESP_LOGI(TAG, "FFT display stopped, original UI restored");
}
void LcdDisplay::MyUI(){
DisplayLockGuard lock(this);
auto screen = lv_screen_active();
lv_obj_set_style_text_font(screen, fonts_.text_font, 0);
lv_obj_set_style_text_color(screen, lv_color_white(), 0);
lv_obj_set_style_bg_color(screen, lv_color_black(), 0);
/* Container */
container_ = lv_obj_create(screen);
lv_obj_set_size(container_, LV_HOR_RES, LV_VER_RES);
lv_obj_set_flex_flow(container_, LV_FLEX_FLOW_COLUMN);
lv_obj_set_style_pad_all(container_, 0, 0);
lv_obj_set_style_border_width(container_, 0, 0);
lv_obj_set_style_pad_row(container_, 0, 0);
lv_obj_set_style_bg_color(container_, lv_color_black(), 0);
lv_obj_set_style_border_color(container_, lv_color_white(), 0);
/* Status bar */
status_bar_ = lv_obj_create(container_);
lv_obj_set_size(status_bar_, LV_HOR_RES, fonts_.text_font->line_height);
lv_obj_set_style_radius(status_bar_, 0, 0);
lv_obj_set_style_bg_color(status_bar_, lv_color_black(), 0);
lv_obj_set_style_text_color(status_bar_, lv_color_white(), 0);
/* Status bar */
lv_obj_set_flex_flow(status_bar_, LV_FLEX_FLOW_ROW);
lv_obj_set_style_pad_all(status_bar_, 0, 0);
lv_obj_set_style_border_width(status_bar_, 0, 0);
lv_obj_set_style_pad_column(status_bar_, 0, 0);
lv_obj_set_style_pad_left(status_bar_, 2, 0);
lv_obj_set_style_pad_right(status_bar_, 2, 0);
}
void LcdDisplay::compute(float* real, float* imag, int n, bool forward) {
// 位反转排序
int j = 0;
for (int i = 0; i < n; i++) {
if (j > i) {
std::swap(real[i], real[j]);
std::swap(imag[i], imag[j]);
}
int m = n >> 1;
while (m >= 1 && j >= m) {
j -= m;
m >>= 1;
}
j += m;
}
// FFT计算
for (int s = 1; s <= (int)log2(n); s++) {
int m = 1 << s;
int m2 = m >> 1;
float w_real = 1.0f;
float w_imag = 0.0f;
float angle = (forward ? -2.0f : 2.0f) * M_PI / m;
float wm_real = cosf(angle);
float wm_imag = sinf(angle);
for (int j = 0; j < m2; j++) {
for (int k = j; k < n; k += m) {
int k2 = k + m2;
float t_real = w_real * real[k2] - w_imag * imag[k2];
float t_imag = w_real * imag[k2] + w_imag * real[k2];
real[k2] = real[k] - t_real;
imag[k2] = imag[k] - t_imag;
real[k] += t_real;
imag[k] += t_imag;
}
float w_temp = w_real;
w_real = w_real * wm_real - w_imag * wm_imag;
w_imag = w_temp * wm_imag + w_imag * wm_real;
}
}
// 正向变换需要缩放
if (forward) {
for (int i = 0; i < n; i++) {
real[i] /= n;
imag[i] /= n;
}
}
}

57
main/display/lcd_display.h
View File

@@ -8,6 +8,11 @@
#include <font_emoji.h>
#include <atomic>
#include <vector>
#include <esp_timer.h>
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
#include <freertos/event_groups.h>
// Theme color structure
struct ThemeColors {
@@ -42,6 +47,53 @@ protected:
virtual bool Lock(int timeout_ms = 0) override;
virtual void Unlock() override;
// FFT 绘制方法
void readAudioData();
virtual void clearScreen() override;
virtual void stopFft() override; // 停止FFT显示
// 定时任务方法
void periodicUpdateTask();
static void periodicUpdateTaskWrapper(void* arg);
// LVGL变量
lv_obj_t* canvas_ = nullptr;
uint16_t* canvas_buffer_ = nullptr;
void create_canvas();
uint16_t get_bar_color(int x_pos);
void draw_spectrum(float *power_spectrum,int fft_size);
void draw_bar(int x,int y,int bar_width,int bar_height,uint16_t color,int bar_index);
void draw_block(int x,int y,int block_x_size,int block_y_size,uint16_t color,int bar_index);
int canvas_width_;
int canvas_height_;
int16_t* audio_data=nullptr;
int16_t* frame_audio_data=nullptr;
uint32_t last_fft_update = 0;
bool fft_data_ready = false;
float* spectrum_data=nullptr;
// FFT 相关变量
int audio_display_last_update = 0;
std::atomic<bool> fft_task_should_stop = false; // FFT任务停止标志
TaskHandle_t fft_task_handle = nullptr; // FFT任务句柄
float* fft_real;
float* fft_imag;
float* hanning_window_float;
void compute(float* real, float* imag, int n, bool forward);
// 添加缺少的方法声明
void drawSpectrumIfReady();
void MyUI();
protected:
// 添加protected构造函数
LcdDisplay(esp_lcd_panel_io_handle_t panel_io, esp_lcd_panel_handle_t panel, DisplayFonts fonts, int width, int height);
@@ -50,6 +102,7 @@ public:
~LcdDisplay();
virtual void SetEmotion(const char* emotion) override;
virtual void SetIcon(const char* icon) override;
virtual void SetMusicInfo(const char* song_name) override;
virtual void SetPreviewImage(const lv_img_dsc_t* img_dsc) override;
#if CONFIG_USE_WECHAT_MESSAGE_STYLE
virtual void SetChatMessage(const char* role, const char* content) override;
@@ -57,6 +110,8 @@ public:
// Add theme switching function
virtual void SetTheme(const std::string& theme_name) override;
virtual void start() override;
};
// RGB LCD显示器
@@ -103,4 +158,4 @@ public:
bool mirror_x, bool mirror_y, bool swap_xy,
DisplayFonts fonts);
};
#endif // LCD_DISPLAY_H
#endif // LCD_DISPLAY_H

399
main/mcp_server.cc
View File

@@ -13,125 +13,173 @@
#include "application.h"
#include "display.h"
#include "board.h"
#include "boards/common/esp32_music.h"
#define TAG "MCP"
#define DEFAULT_TOOLCALL_STACK_SIZE 6144
McpServer::McpServer() {
McpServer::McpServer()
{
}
McpServer::~McpServer() {
for (auto tool : tools_) {
McpServer::~McpServer()
{
for (auto tool : tools_)
{
delete tool;
}
tools_.clear();
}
void McpServer::AddCommonTools() {
void McpServer::AddCommonTools()
{
// To speed up the response time, we add the common tools to the beginning of
// the tools list to utilize the prompt cache.
// Backup the original tools list and restore it after adding the common tools.
auto original_tools = std::move(tools_);
auto& board = Board::GetInstance();
auto &board = Board::GetInstance();
AddTool("self.get_device_status",
"Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\n"
"Use this tool for: \n"
"1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n"
"2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
PropertyList(),
[&board](const PropertyList& properties) -> ReturnValue {
return board.GetDeviceStatusJson();
});
"Provides the real-time information of the device, including the current status of the audio speaker, screen, battery, network, etc.\n"
"Use this tool for: \n"
"1. Answering questions about current condition (e.g. what is the current volume of the audio speaker?)\n"
"2. As the first step to control the device (e.g. turn up / down the volume of the audio speaker, etc.)",
PropertyList(),
[&board](const PropertyList &properties) -> ReturnValue
{
return board.GetDeviceStatusJson();
});
AddTool("self.audio_speaker.set_volume",
"Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
PropertyList({
Property("volume", kPropertyTypeInteger, 0, 100)
}),
[&board](const PropertyList& properties) -> ReturnValue {
auto codec = board.GetAudioCodec();
codec->SetOutputVolume(properties["volume"].value<int>());
return true;
});
auto backlight = board.GetBacklight();
if (backlight) {
AddTool("self.screen.set_brightness",
"Set the brightness of the screen.",
PropertyList({
Property("brightness", kPropertyTypeInteger, 0, 100)
}),
[backlight](const PropertyList& properties) -> ReturnValue {
uint8_t brightness = static_cast<uint8_t>(properties["brightness"].value<int>());
backlight->SetBrightness(brightness, true);
AddTool("self.audio_speaker.set_volume",
"Set the volume of the audio speaker. If the current volume is unknown, you must call `self.get_device_status` tool first and then call this tool.",
PropertyList({Property("volume", kPropertyTypeInteger, 0, 100)}),
[&board](const PropertyList &properties) -> ReturnValue
{
auto codec = board.GetAudioCodec();
codec->SetOutputVolume(properties["volume"].value<int>());
return true;
});
auto backlight = board.GetBacklight();
if (backlight)
{
AddTool("self.screen.set_brightness",
"Set the brightness of the screen.",
PropertyList({Property("brightness", kPropertyTypeInteger, 0, 100)}),
[backlight](const PropertyList &properties) -> ReturnValue
{
uint8_t brightness = static_cast<uint8_t>(properties["brightness"].value<int>());
backlight->SetBrightness(brightness, true);
return true;
});
}
auto display = board.GetDisplay();
if (display && !display->GetTheme().empty()) {
if (display && !display->GetTheme().empty())
{
AddTool("self.screen.set_theme",
"Set the theme of the screen. The theme can be `light` or `dark`.",
PropertyList({
Property("theme", kPropertyTypeString)
}),
[display](const PropertyList& properties) -> ReturnValue {
display->SetTheme(properties["theme"].value<std::string>().c_str());
return true;
});
"Set the theme of the screen. The theme can be `light` or `dark`.",
PropertyList({Property("theme", kPropertyTypeString)}),
[display](const PropertyList &properties) -> ReturnValue
{
display->SetTheme(properties["theme"].value<std::string>().c_str());
return true;
});
}
auto camera = board.GetCamera();
if (camera) {
if (camera)
{
AddTool("self.camera.take_photo",
"Take a photo and explain it. Use this tool after the user asks you to see something.\n"
"Args:\n"
" `question`: The question that you want to ask about the photo.\n"
"Return:\n"
" A JSON object that provides the photo information.",
PropertyList({
Property("question", kPropertyTypeString)
}),
[camera](const PropertyList& properties) -> ReturnValue {
if (!camera->Capture()) {
return "{\"success\": false, \"message\": \"Failed to capture photo\"}";
}
auto question = properties["question"].value<std::string>();
return camera->Explain(question);
});
"Take a photo and explain it. Use this tool after the user asks you to see something.\n"
"Args:\n"
" `question`: The question that you want to ask about the photo.\n"
"Return:\n"
" A JSON object that provides the photo information.",
PropertyList({Property("question", kPropertyTypeString)}),
[camera](const PropertyList &properties) -> ReturnValue
{
if (!camera->Capture())
{
return "{\"success\": false, \"message\": \"Failed to capture photo\"}";
}
auto question = properties["question"].value<std::string>();
return camera->Explain(question);
});
}
auto music = board.GetMusic();
if (music) {
if (music)
{
AddTool("self.music.play_song",
"Play the specified song. When users request to play music, this tool will automatically retrieve song details and start streaming.\n"
"parameter:\n"
" `song_name`: The name of the song to be played.\n"
"return:\n"
" Play status information without confirmation, immediately play the song.",
PropertyList({
Property("song_name", kPropertyTypeString)
}),
[music](const PropertyList& properties) -> ReturnValue {
auto song_name = properties["song_name"].value<std::string>();
if (!music->Download(song_name)) {
return "{\"success\": false, \"message\": \"Failed to obtain music resources\"}";
}
auto download_result = music->GetDownloadResult();
ESP_LOGD(TAG, "Music details result: %s", download_result.c_str());
return true;
});
"Play the specified song. When users request to play music, this tool will automatically retrieve song details and start streaming.\n"
"parameter:\n"
" `song_name`: The name of the song to be played.\n"
"return:\n"
" Play status information without confirmation, immediately play the song.",
PropertyList({Property("song_name", kPropertyTypeString)}),
[music](const PropertyList &properties) -> ReturnValue
{
auto song_name = properties["song_name"].value<std::string>();
if (!music->Download(song_name))
{
return "{\"success\": false, \"message\": \"Failed to obtain music resources\"}";
}
auto download_result = music->GetDownloadResult();
ESP_LOGD(TAG, "Music details result: %s", download_result.c_str());
return true;
});
AddTool("self.music.set_display_mode",
"Set the display mode for music playback. You can choose to display the spectrum or lyrics, for example, if the user says' open spectrum 'or' display spectrum ', the corresponding display mode will be set for' open lyrics' or 'display lyrics'.\n"
"parameter:\n"
" `mode`: Display mode, with optional values of 'spectrum' or 'lyrics'.\n"
"return:\n"
" Set result information.",
PropertyList({
Property("mode", kPropertyTypeString) // Display mode: "spectrum" or "lyrics"
}),
[music](const PropertyList &properties) -> ReturnValue
{
auto mode_str = properties["mode"].value<std::string>();
// Convert to lowercase for comparison
std::transform(mode_str.begin(), mode_str.end(), mode_str.begin(), ::tolower);
if (mode_str == "spectrum" || mode_str == "频谱")
{
// Set to spectrum display mode
auto esp32_music = static_cast<Esp32Music *>(music);
esp32_music->SetDisplayMode(Esp32Music::DISPLAY_MODE_SPECTRUM);
return "{\"success\": true, \"message\": \"Switched to spectrum display mode\"}";
}
else if (mode_str == "lyrics" || mode_str == "歌词")
{
// Set to lyrics display mode
auto esp32_music = static_cast<Esp32Music *>(music);
esp32_music->SetDisplayMode(Esp32Music::DISPLAY_MODE_LYRICS);
return "{\"success\": true, \"message\": \"Switched to lyrics display mode\"}";
}
else
{
return "{\"success\": false, \"message\": \"Invalid display mode, please use 'spectrum' or 'lyrics'\"}";
}
return "{\"success\": false, \"message\": \"Failed to set display mode\"}";
});
}
// Restore the original tools list to the end of the tools list
tools_.insert(tools_.end(), original_tools.begin(), original_tools.end());
}
void McpServer::AddTool(McpTool* tool) {
void McpServer::AddTool(McpTool *tool)
{
// Prevent adding duplicate tools
if (std::find_if(tools_.begin(), tools_.end(), [tool](const McpTool* t) { return t->name() == tool->name(); }) != tools_.end()) {
if (std::find_if(tools_.begin(), tools_.end(), [tool](const McpTool *t)
{ return t->name() == tool->name(); }) != tools_.end())
{
ESP_LOGW(TAG, "Tool %s already added", tool->name().c_str());
return;
}
@@ -140,13 +188,16 @@ void McpServer::AddTool(McpTool* tool) {
tools_.push_back(tool);
}
void McpServer::AddTool(const std::string& name, const std::string& description, const PropertyList& properties, std::function<ReturnValue(const PropertyList&)> callback) {
void McpServer::AddTool(const std::string &name, const std::string &description, const PropertyList &properties, std::function<ReturnValue(const PropertyList &)> callback)
{
AddTool(new McpTool(name, description, properties, callback));
}
void McpServer::ParseMessage(const std::string& message) {
cJSON* json = cJSON_Parse(message.c_str());
if (json == nullptr) {
void McpServer::ParseMessage(const std::string &message)
{
cJSON *json = cJSON_Parse(message.c_str());
if (json == nullptr)
{
ESP_LOGE(TAG, "Failed to parse MCP message: %s", message.c_str());
return;
}
@@ -154,17 +205,22 @@ void McpServer::ParseMessage(const std::string& message) {
cJSON_Delete(json);
}
void McpServer::ParseCapabilities(const cJSON* capabilities) {
void McpServer::ParseCapabilities(const cJSON *capabilities)
{
auto vision = cJSON_GetObjectItem(capabilities, "vision");
if (cJSON_IsObject(vision)) {
if (cJSON_IsObject(vision))
{
auto url = cJSON_GetObjectItem(vision, "url");
auto token = cJSON_GetObjectItem(vision, "token");
if (cJSON_IsString(url)) {
if (cJSON_IsString(url))
{
auto camera = Board::GetInstance().GetCamera();
if (camera) {
if (camera)
{
std::string url_str = std::string(url->valuestring);
std::string token_str;
if (cJSON_IsString(token)) {
if (cJSON_IsString(token))
{
token_str = std::string(token->valuestring);
}
camera->SetExplainUrl(url_str, token_str);
@@ -173,44 +229,53 @@ void McpServer::ParseCapabilities(const cJSON* capabilities) {
}
}
void McpServer::ParseMessage(const cJSON* json) {
void McpServer::ParseMessage(const cJSON *json)
{
// Check JSONRPC version
auto version = cJSON_GetObjectItem(json, "jsonrpc");
if (version == nullptr || !cJSON_IsString(version) || strcmp(version->valuestring, "2.0") != 0) {
if (version == nullptr || !cJSON_IsString(version) || strcmp(version->valuestring, "2.0") != 0)
{
ESP_LOGE(TAG, "Invalid JSONRPC version: %s", version ? version->valuestring : "null");
return;
}
// Check method
auto method = cJSON_GetObjectItem(json, "method");
if (method == nullptr || !cJSON_IsString(method)) {
if (method == nullptr || !cJSON_IsString(method))
{
ESP_LOGE(TAG, "Missing method");
return;
}
auto method_str = std::string(method->valuestring);
if (method_str.find("notifications") == 0) {
if (method_str.find("notifications") == 0)
{
return;
}
// Check params
auto params = cJSON_GetObjectItem(json, "params");
if (params != nullptr && !cJSON_IsObject(params)) {
if (params != nullptr && !cJSON_IsObject(params))
{
ESP_LOGE(TAG, "Invalid params for method: %s", method_str.c_str());
return;
}
auto id = cJSON_GetObjectItem(json, "id");
if (id == nullptr || !cJSON_IsNumber(id)) {
if (id == nullptr || !cJSON_IsNumber(id))
{
ESP_LOGE(TAG, "Invalid id for method: %s", method_str.c_str());
return;
}
auto id_int = id->valueint;
if (method_str == "initialize") {
if (cJSON_IsObject(params)) {
if (method_str == "initialize")
{
if (cJSON_IsObject(params))
{
auto capabilities = cJSON_GetObjectItem(params, "capabilities");
if (cJSON_IsObject(capabilities)) {
if (cJSON_IsObject(capabilities))
{
ParseCapabilities(capabilities);
}
}
@@ -219,47 +284,60 @@ void McpServer::ParseMessage(const cJSON* json) {
message += app_desc->version;
message += "\"}}";
ReplyResult(id_int, message);
} else if (method_str == "tools/list") {
}
else if (method_str == "tools/list")
{
std::string cursor_str = "";
if (params != nullptr) {
if (params != nullptr)
{
auto cursor = cJSON_GetObjectItem(params, "cursor");
if (cJSON_IsString(cursor)) {
if (cJSON_IsString(cursor))
{
cursor_str = std::string(cursor->valuestring);
}
}
GetToolsList(id_int, cursor_str);
} else if (method_str == "tools/call") {
if (!cJSON_IsObject(params)) {
}
else if (method_str == "tools/call")
{
if (!cJSON_IsObject(params))
{
ESP_LOGE(TAG, "tools/call: Missing params");
ReplyError(id_int, "Missing params");
return;
}
auto tool_name = cJSON_GetObjectItem(params, "name");
if (!cJSON_IsString(tool_name)) {
if (!cJSON_IsString(tool_name))
{
ESP_LOGE(TAG, "tools/call: Missing name");
ReplyError(id_int, "Missing name");
return;
}
auto tool_arguments = cJSON_GetObjectItem(params, "arguments");
if (tool_arguments != nullptr && !cJSON_IsObject(tool_arguments)) {
if (tool_arguments != nullptr && !cJSON_IsObject(tool_arguments))
{
ESP_LOGE(TAG, "tools/call: Invalid arguments");
ReplyError(id_int, "Invalid arguments");
return;
}
auto stack_size = cJSON_GetObjectItem(params, "stackSize");
if (stack_size != nullptr && !cJSON_IsNumber(stack_size)) {
if (stack_size != nullptr && !cJSON_IsNumber(stack_size))
{
ESP_LOGE(TAG, "tools/call: Invalid stackSize");
ReplyError(id_int, "Invalid stackSize");
return;
}
DoToolCall(id_int, std::string(tool_name->valuestring), tool_arguments, stack_size ? stack_size->valueint : DEFAULT_TOOLCALL_STACK_SIZE);
} else {
}
else
{
ESP_LOGE(TAG, "Method not implemented: %s", method_str.c_str());
ReplyError(id_int, "Method not implemented: " + method_str);
}
}
void McpServer::ReplyResult(int id, const std::string& result) {
void McpServer::ReplyResult(int id, const std::string &result)
{
std::string payload = "{\"jsonrpc\":\"2.0\",\"id\":";
payload += std::to_string(id) + ",\"result\":";
payload += result;
@@ -267,7 +345,8 @@ void McpServer::ReplyResult(int id, const std::string& result) {
Application::GetInstance().SendMcpMessage(payload);
}
void McpServer::ReplyError(int id, const std::string& message) {
void McpServer::ReplyError(int id, const std::string &message)
{
std::string payload = "{\"jsonrpc\":\"2.0\",\"id\":";
payload += std::to_string(id);
payload += ",\"error\":{\"message\":\"";
@@ -276,94 +355,120 @@ void McpServer::ReplyError(int id, const std::string& message) {
Application::GetInstance().SendMcpMessage(payload);
}
void McpServer::GetToolsList(int id, const std::string& cursor) {
void McpServer::GetToolsList(int id, const std::string &cursor)
{
const int max_payload_size = 8000;
std::string json = "{\"tools\":[";
bool found_cursor = cursor.empty();
auto it = tools_.begin();
std::string next_cursor = "";
while (it != tools_.end()) {
while (it != tools_.end())
{
// 如果我们还没有找到起始位置，继续搜索
if (!found_cursor) {
if ((*it)->name() == cursor) {
if (!found_cursor)
{
if ((*it)->name() == cursor)
{
found_cursor = true;
} else {
}
else
{
++it;
continue;
}
}
// 添加tool前检查大小
std::string tool_json = (*it)->to_json() + ",";
if (json.length() + tool_json.length() + 30 > max_payload_size) {
if (json.length() + tool_json.length() + 30 > max_payload_size)
{
// 如果添加这个tool会超出大小限制，设置next_cursor并退出循环
next_cursor = (*it)->name();
break;
}
json += tool_json;
++it;
}
if (json.back() == ',') {
if (json.back() == ',')
{
json.pop_back();
}
if (json.back() == '[' && !tools_.empty()) {
if (json.back() == '[' && !tools_.empty())
{
// 如果没有添加任何tool，返回错误
ESP_LOGE(TAG, "tools/list: Failed to add tool %s because of payload size limit", next_cursor.c_str());
ReplyError(id, "Failed to add tool " + next_cursor + " because of payload size limit");
return;
}
if (next_cursor.empty()) {
if (next_cursor.empty())
{
json += "]}";
} else {
}
else
{
json += "],\"nextCursor\":\"" + next_cursor + "\"}";
}
ReplyResult(id, json);
}
void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* tool_arguments, int stack_size) {
auto tool_iter = std::find_if(tools_.begin(), tools_.end(),
[&tool_name](const McpTool* tool) {
return tool->name() == tool_name;
});
if (tool_iter == tools_.end()) {
void McpServer::DoToolCall(int id, const std::string &tool_name, const cJSON *tool_arguments, int stack_size)
{
auto tool_iter = std::find_if(tools_.begin(), tools_.end(),
[&tool_name](const McpTool *tool)
{
return tool->name() == tool_name;
});
if (tool_iter == tools_.end())
{
ESP_LOGE(TAG, "tools/call: Unknown tool: %s", tool_name.c_str());
ReplyError(id, "Unknown tool: " + tool_name);
return;
}
PropertyList arguments = (*tool_iter)->properties();
try {
for (auto& argument : arguments) {
try
{
for (auto &argument : arguments)
{
bool found = false;
if (cJSON_IsObject(tool_arguments)) {
if (cJSON_IsObject(tool_arguments))
{
auto value = cJSON_GetObjectItem(tool_arguments, argument.name().c_str());
if (argument.type() == kPropertyTypeBoolean && cJSON_IsBool(value)) {
if (argument.type() == kPropertyTypeBoolean && cJSON_IsBool(value))
{
argument.set_value<bool>(value->valueint == 1);
found = true;
} else if (argument.type() == kPropertyTypeInteger && cJSON_IsNumber(value)) {
}
else if (argument.type() == kPropertyTypeInteger && cJSON_IsNumber(value))
{
argument.set_value<int>(value->valueint);
found = true;
} else if (argument.type() == kPropertyTypeString && cJSON_IsString(value)) {
}
else if (argument.type() == kPropertyTypeString && cJSON_IsString(value))
{
argument.set_value<std::string>(value->valuestring);
found = true;
}
}
if (!argument.has_default_value() && !found) {
if (!argument.has_default_value() && !found)
{
ESP_LOGE(TAG, "tools/call: Missing valid argument: %s", argument.name().c_str());
ReplyError(id, "Missing valid argument: " + argument.name());
return;
}
}
} catch (const std::exception& e) {
}
catch (const std::exception &e)
{
ESP_LOGE(TAG, "tools/call: %s", e.what());
ReplyError(id, e.what());
return;
@@ -377,13 +482,13 @@ void McpServer::DoToolCall(int id, const std::string& tool_name, const cJSON* to
esp_pthread_set_cfg(&cfg);
// Use a thread to call the tool to avoid blocking the main thread
tool_call_thread_ = std::thread([this, id, tool_iter, arguments = std::move(arguments)]() {
tool_call_thread_ = std::thread([this, id, tool_iter, arguments = std::move(arguments)]()
{
try {
ReplyResult(id, (*tool_iter)->Call(arguments));
} catch (const std::exception& e) {
ESP_LOGE(TAG, "tools/call: %s", e.what());
ReplyError(id, e.what());
}
});
} });
tool_call_thread_.detach();
}

3240
sdkconfig
View File

File diff suppressed because it is too large Load Diff