add some code

main/audio/wake_words/custom_wake_word.cc

@@ -0,0 +1,185 @@
+#include "custom_wake_word.h"
+#include "audio_service.h"
+#include "system_info.h"
+
+#include <esp_log.h>
+#include "esp_mn_iface.h"
+#include "esp_mn_models.h"
+#include "esp_mn_speech_commands.h"
+
+
+#define TAG "CustomWakeWord"
+
+
+CustomWakeWord::CustomWakeWord()
+    : wake_word_pcm_(), wake_word_opus_() {
+}
+
+CustomWakeWord::~CustomWakeWord() {
+    if (multinet_model_data_ != nullptr && multinet_ != nullptr) {
+        multinet_->destroy(multinet_model_data_);
+        multinet_model_data_ = nullptr;
+    }
+
+    if (wake_word_encode_task_stack_ != nullptr) {
+        heap_caps_free(wake_word_encode_task_stack_);
+    }
+
+    if (wake_word_encode_task_buffer_ != nullptr) {
+        heap_caps_free(wake_word_encode_task_buffer_);
+    }
+
+    if (models_ != nullptr) {
+        esp_srmodel_deinit(models_);
+    }
+}
+
+bool CustomWakeWord::Initialize(AudioCodec* codec) {
+    codec_ = codec;
+
+    models_ = esp_srmodel_init("model");
+    if (models_ == nullptr || models_->num == -1) {
+        ESP_LOGE(TAG, "Failed to initialize wakenet model");
+        return false;
+    }
+
+    // 初始化 multinet (命令词识别)
+    mn_name_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, ESP_MN_CHINESE);
+    if (mn_name_ == nullptr) {
+        ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr");
+        ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word");
+        return false;
+    }
+
+    ESP_LOGI(TAG, "multinet: %s", mn_name_);
+    multinet_ = esp_mn_handle_from_name(mn_name_);
+    multinet_model_data_ = multinet_->create(mn_name_, 3000);  // 3 秒超时
+    multinet_->set_det_threshold(multinet_model_data_, CONFIG_CUSTOM_WAKE_WORD_THRESHOLD / 100.0f);
+    esp_mn_commands_clear();
+    esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD);
+    esp_mn_commands_update();
+    
+    multinet_->print_active_speech_commands(multinet_model_data_);
+    return true;
+}
+
+void CustomWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
+    wake_word_detected_callback_ = callback;
+}
+
+void CustomWakeWord::Start() {
+    running_ = true;
+}
+
+void CustomWakeWord::Stop() {
+    running_ = false;
+}
+
+void CustomWakeWord::Feed(const std::vector<int16_t>& data) {
+    if (multinet_model_data_ == nullptr || !running_) {
+        return;
+    }
+
+    esp_mn_state_t mn_state;
+    // If input channels is 2, we need to fetch the left channel data
+    if (codec_->input_channels() == 2) {
+        auto mono_data = std::vector<int16_t>(data.size() / 2);
+        for (size_t i = 0, j = 0; i < mono_data.size(); ++i, j += 2) {
+            mono_data[i] = data[j];
+        }
+
+        StoreWakeWordData(mono_data);
+        mn_state = multinet_->detect(multinet_model_data_, const_cast<int16_t*>(mono_data.data()));
+    } else {
+        StoreWakeWordData(data);
+        mn_state = multinet_->detect(multinet_model_data_, const_cast<int16_t*>(data.data()));
+    }
+    
+    if (mn_state == ESP_MN_STATE_DETECTING) {
+        return;
+    } else if (mn_state == ESP_MN_STATE_DETECTED) {
+        esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_);
+        ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f", 
+                mn_result->command_id[0], mn_result->string, mn_result->prob[0]);
+        
+        if (mn_result->command_id[0] == 1) {
+            last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY;
+        }
+        running_ = false;
+        
+        if (wake_word_detected_callback_) {
+            wake_word_detected_callback_(last_detected_wake_word_);
+        }
+        multinet_->clean(multinet_model_data_);
+    } else if (mn_state == ESP_MN_STATE_TIMEOUT) {
+        ESP_LOGD(TAG, "Command word detection timeout, cleaning state");
+        multinet_->clean(multinet_model_data_);
+    }
+}
+
+size_t CustomWakeWord::GetFeedSize() {
+    if (multinet_model_data_ == nullptr) {
+        return 0;
+    }
+    return multinet_->get_samp_chunksize(multinet_model_data_);
+}
+
+void CustomWakeWord::StoreWakeWordData(const std::vector<int16_t>& data) {
+    // store audio data to wake_word_pcm_
+    wake_word_pcm_.push_back(data);
+    // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
+    while (wake_word_pcm_.size() > 2000 / 30) {
+        wake_word_pcm_.pop_front();
+    }
+}
+
+void CustomWakeWord::EncodeWakeWordData() {
+    const size_t stack_size = 4096 * 7;
+    wake_word_opus_.clear();
+    if (wake_word_encode_task_stack_ == nullptr) {
+        wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
+        assert(wake_word_encode_task_stack_ != nullptr);
+    }
+    if (wake_word_encode_task_buffer_ == nullptr) {
+        wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
+        assert(wake_word_encode_task_buffer_ != nullptr);
+    }
+
+    wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
+        auto this_ = (CustomWakeWord*)arg;
+        {
+            auto start_time = esp_timer_get_time();
+            auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
+            encoder->SetComplexity(0); // 0 is the fastest
+
+            int packets = 0;
+            for (auto& pcm: this_->wake_word_pcm_) {
+                encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
+                    std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
+                    this_->wake_word_opus_.emplace_back(std::move(opus));
+                    this_->wake_word_cv_.notify_all();
+                });
+                packets++;
+            }
+            this_->wake_word_pcm_.clear();
+
+            auto end_time = esp_timer_get_time();
+            ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000));
+
+            std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
+            this_->wake_word_opus_.push_back(std::vector<uint8_t>());
+            this_->wake_word_cv_.notify_all();
+        }
+        vTaskDelete(NULL);
+    }, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
+}
+
+bool CustomWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
+    std::unique_lock<std::mutex> lock(wake_word_mutex_);
+    wake_word_cv_.wait(lock, [this]() {
+        return !wake_word_opus_.empty();
+    });
+    opus.swap(wake_word_opus_.front());
+    wake_word_opus_.pop_front();
+    return !opus.empty();
+}