Upgrade Playlist Features
This commit is contained in:
@@ -1,208 +1,208 @@
|
||||
#include "afe_wake_word.h"
|
||||
#include "audio_service.h"
|
||||
|
||||
#include <esp_log.h>
|
||||
#include <sstream>
|
||||
|
||||
#define DETECTION_RUNNING_EVENT 1
|
||||
|
||||
#define TAG "AfeWakeWord"
|
||||
|
||||
AfeWakeWord::AfeWakeWord()
|
||||
: afe_data_(nullptr),
|
||||
wake_word_pcm_(),
|
||||
wake_word_opus_() {
|
||||
|
||||
event_group_ = xEventGroupCreate();
|
||||
}
|
||||
|
||||
AfeWakeWord::~AfeWakeWord() {
|
||||
if (afe_data_ != nullptr) {
|
||||
afe_iface_->destroy(afe_data_);
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_stack_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_stack_);
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_buffer_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
if (models_ != nullptr) {
|
||||
esp_srmodel_deinit(models_);
|
||||
}
|
||||
|
||||
vEventGroupDelete(event_group_);
|
||||
}
|
||||
|
||||
bool AfeWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
|
||||
codec_ = codec;
|
||||
int ref_num = codec_->input_reference() ? 1 : 0;
|
||||
|
||||
if (models_list == nullptr) {
|
||||
models_ = esp_srmodel_init("model");
|
||||
} else {
|
||||
models_ = models_list;
|
||||
}
|
||||
|
||||
if (models_ == nullptr || models_->num == -1) {
|
||||
ESP_LOGE(TAG, "Failed to initialize wakenet model");
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < models_->num; i++) {
|
||||
ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]);
|
||||
if (strstr(models_->model_name[i], ESP_WN_PREFIX) != NULL) {
|
||||
wakenet_model_ = models_->model_name[i];
|
||||
auto words = esp_srmodel_get_wake_words(models_, wakenet_model_);
|
||||
// split by ";" to get all wake words
|
||||
std::stringstream ss(words);
|
||||
std::string word;
|
||||
while (std::getline(ss, word, ';')) {
|
||||
wake_words_.push_back(word);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string input_format;
|
||||
for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
|
||||
input_format.push_back('M');
|
||||
}
|
||||
for (int i = 0; i < ref_num; i++) {
|
||||
input_format.push_back('R');
|
||||
}
|
||||
afe_config_t* afe_config = afe_config_init(input_format.c_str(), models_, AFE_TYPE_SR, AFE_MODE_HIGH_PERF);
|
||||
afe_config->aec_init = codec_->input_reference();
|
||||
afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF;
|
||||
afe_config->afe_perferred_core = 1;
|
||||
afe_config->afe_perferred_priority = 1;
|
||||
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
|
||||
|
||||
afe_iface_ = esp_afe_handle_from_config(afe_config);
|
||||
afe_data_ = afe_iface_->create_from_config(afe_config);
|
||||
|
||||
xTaskCreate([](void* arg) {
|
||||
auto this_ = (AfeWakeWord*)arg;
|
||||
this_->AudioDetectionTask();
|
||||
vTaskDelete(NULL);
|
||||
}, "audio_detection", 4096, this, 3, nullptr);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void AfeWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
|
||||
wake_word_detected_callback_ = callback;
|
||||
}
|
||||
|
||||
void AfeWakeWord::Start() {
|
||||
xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
|
||||
}
|
||||
|
||||
void AfeWakeWord::Stop() {
|
||||
xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
|
||||
if (afe_data_ != nullptr) {
|
||||
afe_iface_->reset_buffer(afe_data_);
|
||||
}
|
||||
}
|
||||
|
||||
void AfeWakeWord::Feed(const std::vector<int16_t>& data) {
|
||||
if (afe_data_ == nullptr) {
|
||||
return;
|
||||
}
|
||||
afe_iface_->feed(afe_data_, data.data());
|
||||
}
|
||||
|
||||
size_t AfeWakeWord::GetFeedSize() {
|
||||
if (afe_data_ == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return afe_iface_->get_feed_chunksize(afe_data_);
|
||||
}
|
||||
|
||||
void AfeWakeWord::AudioDetectionTask() {
|
||||
auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
|
||||
auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
|
||||
ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d",
|
||||
feed_size, fetch_size);
|
||||
|
||||
while (true) {
|
||||
xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY);
|
||||
|
||||
auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY);
|
||||
if (res == nullptr || res->ret_value == ESP_FAIL) {
|
||||
continue;;
|
||||
}
|
||||
|
||||
// Store the wake word data for voice recognition, like who is speaking
|
||||
StoreWakeWordData(res->data, res->data_size / sizeof(int16_t));
|
||||
|
||||
if (res->wakeup_state == WAKENET_DETECTED) {
|
||||
Stop();
|
||||
last_detected_wake_word_ = wake_words_[res->wakenet_model_index - 1];
|
||||
|
||||
if (wake_word_detected_callback_) {
|
||||
wake_word_detected_callback_(last_detected_wake_word_);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
|
||||
// store audio data to wake_word_pcm_
|
||||
wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
|
||||
// keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
|
||||
while (wake_word_pcm_.size() > 2000 / 30) {
|
||||
wake_word_pcm_.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
void AfeWakeWord::EncodeWakeWordData() {
|
||||
const size_t stack_size = 4096 * 7;
|
||||
wake_word_opus_.clear();
|
||||
if (wake_word_encode_task_stack_ == nullptr) {
|
||||
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
|
||||
assert(wake_word_encode_task_stack_ != nullptr);
|
||||
}
|
||||
if (wake_word_encode_task_buffer_ == nullptr) {
|
||||
wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
|
||||
assert(wake_word_encode_task_buffer_ != nullptr);
|
||||
}
|
||||
|
||||
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
|
||||
auto this_ = (AfeWakeWord*)arg;
|
||||
{
|
||||
auto start_time = esp_timer_get_time();
|
||||
auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
|
||||
encoder->SetComplexity(0); // 0 is the fastest
|
||||
|
||||
int packets = 0;
|
||||
for (auto& pcm: this_->wake_word_pcm_) {
|
||||
encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.emplace_back(std::move(opus));
|
||||
this_->wake_word_cv_.notify_all();
|
||||
});
|
||||
packets++;
|
||||
}
|
||||
this_->wake_word_pcm_.clear();
|
||||
|
||||
auto end_time = esp_timer_get_time();
|
||||
ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000));
|
||||
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
|
||||
this_->wake_word_cv_.notify_all();
|
||||
}
|
||||
vTaskDelete(NULL);
|
||||
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
bool AfeWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
|
||||
std::unique_lock<std::mutex> lock(wake_word_mutex_);
|
||||
wake_word_cv_.wait(lock, [this]() {
|
||||
return !wake_word_opus_.empty();
|
||||
});
|
||||
opus.swap(wake_word_opus_.front());
|
||||
wake_word_opus_.pop_front();
|
||||
return !opus.empty();
|
||||
}
|
||||
#include "afe_wake_word.h"
|
||||
#include "audio_service.h"
|
||||
|
||||
#include <esp_log.h>
|
||||
#include <sstream>
|
||||
|
||||
#define DETECTION_RUNNING_EVENT 1
|
||||
|
||||
#define TAG "AfeWakeWord"
|
||||
|
||||
AfeWakeWord::AfeWakeWord()
|
||||
: afe_data_(nullptr),
|
||||
wake_word_pcm_(),
|
||||
wake_word_opus_() {
|
||||
|
||||
event_group_ = xEventGroupCreate();
|
||||
}
|
||||
|
||||
AfeWakeWord::~AfeWakeWord() {
|
||||
if (afe_data_ != nullptr) {
|
||||
afe_iface_->destroy(afe_data_);
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_stack_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_stack_);
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_buffer_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
if (models_ != nullptr) {
|
||||
esp_srmodel_deinit(models_);
|
||||
}
|
||||
|
||||
vEventGroupDelete(event_group_);
|
||||
}
|
||||
|
||||
bool AfeWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
|
||||
codec_ = codec;
|
||||
int ref_num = codec_->input_reference() ? 1 : 0;
|
||||
|
||||
if (models_list == nullptr) {
|
||||
models_ = esp_srmodel_init("model");
|
||||
} else {
|
||||
models_ = models_list;
|
||||
}
|
||||
|
||||
if (models_ == nullptr || models_->num == -1) {
|
||||
ESP_LOGE(TAG, "Failed to initialize wakenet model");
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < models_->num; i++) {
|
||||
ESP_LOGI(TAG, "Model %d: %s", i, models_->model_name[i]);
|
||||
if (strstr(models_->model_name[i], ESP_WN_PREFIX) != NULL) {
|
||||
wakenet_model_ = models_->model_name[i];
|
||||
auto words = esp_srmodel_get_wake_words(models_, wakenet_model_);
|
||||
// split by ";" to get all wake words
|
||||
std::stringstream ss(words);
|
||||
std::string word;
|
||||
while (std::getline(ss, word, ';')) {
|
||||
wake_words_.push_back(word);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string input_format;
|
||||
for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
|
||||
input_format.push_back('M');
|
||||
}
|
||||
for (int i = 0; i < ref_num; i++) {
|
||||
input_format.push_back('R');
|
||||
}
|
||||
afe_config_t* afe_config = afe_config_init(input_format.c_str(), models_, AFE_TYPE_SR, AFE_MODE_HIGH_PERF);
|
||||
afe_config->aec_init = codec_->input_reference();
|
||||
afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF;
|
||||
afe_config->afe_perferred_core = 1;
|
||||
afe_config->afe_perferred_priority = 1;
|
||||
afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
|
||||
|
||||
afe_iface_ = esp_afe_handle_from_config(afe_config);
|
||||
afe_data_ = afe_iface_->create_from_config(afe_config);
|
||||
|
||||
xTaskCreate([](void* arg) {
|
||||
auto this_ = (AfeWakeWord*)arg;
|
||||
this_->AudioDetectionTask();
|
||||
vTaskDelete(NULL);
|
||||
}, "audio_detection", 4096, this, 3, nullptr);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void AfeWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
|
||||
wake_word_detected_callback_ = callback;
|
||||
}
|
||||
|
||||
void AfeWakeWord::Start() {
|
||||
xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
|
||||
}
|
||||
|
||||
void AfeWakeWord::Stop() {
|
||||
xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
|
||||
if (afe_data_ != nullptr) {
|
||||
afe_iface_->reset_buffer(afe_data_);
|
||||
}
|
||||
}
|
||||
|
||||
void AfeWakeWord::Feed(const std::vector<int16_t>& data) {
|
||||
if (afe_data_ == nullptr) {
|
||||
return;
|
||||
}
|
||||
afe_iface_->feed(afe_data_, data.data());
|
||||
}
|
||||
|
||||
size_t AfeWakeWord::GetFeedSize() {
|
||||
if (afe_data_ == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return afe_iface_->get_feed_chunksize(afe_data_);
|
||||
}
|
||||
|
||||
void AfeWakeWord::AudioDetectionTask() {
|
||||
auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
|
||||
auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
|
||||
ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d",
|
||||
feed_size, fetch_size);
|
||||
|
||||
while (true) {
|
||||
xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY);
|
||||
|
||||
auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY);
|
||||
if (res == nullptr || res->ret_value == ESP_FAIL) {
|
||||
continue;;
|
||||
}
|
||||
|
||||
// Store the wake word data for voice recognition, like who is speaking
|
||||
StoreWakeWordData(res->data, res->data_size / sizeof(int16_t));
|
||||
|
||||
if (res->wakeup_state == WAKENET_DETECTED) {
|
||||
Stop();
|
||||
last_detected_wake_word_ = wake_words_[res->wakenet_model_index - 1];
|
||||
|
||||
if (wake_word_detected_callback_) {
|
||||
wake_word_detected_callback_(last_detected_wake_word_);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
|
||||
// store audio data to wake_word_pcm_
|
||||
wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
|
||||
// keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
|
||||
while (wake_word_pcm_.size() > 2000 / 30) {
|
||||
wake_word_pcm_.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
void AfeWakeWord::EncodeWakeWordData() {
|
||||
const size_t stack_size = 4096 * 7;
|
||||
wake_word_opus_.clear();
|
||||
if (wake_word_encode_task_stack_ == nullptr) {
|
||||
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
|
||||
assert(wake_word_encode_task_stack_ != nullptr);
|
||||
}
|
||||
if (wake_word_encode_task_buffer_ == nullptr) {
|
||||
wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
|
||||
assert(wake_word_encode_task_buffer_ != nullptr);
|
||||
}
|
||||
|
||||
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
|
||||
auto this_ = (AfeWakeWord*)arg;
|
||||
{
|
||||
auto start_time = esp_timer_get_time();
|
||||
auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
|
||||
encoder->SetComplexity(0); // 0 is the fastest
|
||||
|
||||
int packets = 0;
|
||||
for (auto& pcm: this_->wake_word_pcm_) {
|
||||
encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.emplace_back(std::move(opus));
|
||||
this_->wake_word_cv_.notify_all();
|
||||
});
|
||||
packets++;
|
||||
}
|
||||
this_->wake_word_pcm_.clear();
|
||||
|
||||
auto end_time = esp_timer_get_time();
|
||||
ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000));
|
||||
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
|
||||
this_->wake_word_cv_.notify_all();
|
||||
}
|
||||
vTaskDelete(NULL);
|
||||
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
bool AfeWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
|
||||
std::unique_lock<std::mutex> lock(wake_word_mutex_);
|
||||
wake_word_cv_.wait(lock, [this]() {
|
||||
return !wake_word_opus_.empty();
|
||||
});
|
||||
opus.swap(wake_word_opus_.front());
|
||||
wake_word_opus_.pop_front();
|
||||
return !opus.empty();
|
||||
}
|
||||
|
||||
@@ -1,60 +1,60 @@
|
||||
#ifndef AFE_WAKE_WORD_H
|
||||
#define AFE_WAKE_WORD_H
|
||||
|
||||
#include <freertos/FreeRTOS.h>
|
||||
#include <freertos/task.h>
|
||||
#include <freertos/event_groups.h>
|
||||
|
||||
#include <esp_afe_sr_models.h>
|
||||
#include <esp_nsn_models.h>
|
||||
#include <model_path.h>
|
||||
|
||||
#include <deque>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
|
||||
#include "audio_codec.h"
|
||||
#include "wake_word.h"
|
||||
|
||||
class AfeWakeWord : public WakeWord {
|
||||
public:
|
||||
AfeWakeWord();
|
||||
~AfeWakeWord();
|
||||
|
||||
bool Initialize(AudioCodec* codec, srmodel_list_t* models_list);
|
||||
void Feed(const std::vector<int16_t>& data);
|
||||
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
|
||||
void Start();
|
||||
void Stop();
|
||||
size_t GetFeedSize();
|
||||
void EncodeWakeWordData();
|
||||
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
|
||||
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
|
||||
|
||||
private:
|
||||
srmodel_list_t *models_ = nullptr;
|
||||
esp_afe_sr_iface_t* afe_iface_ = nullptr;
|
||||
esp_afe_sr_data_t* afe_data_ = nullptr;
|
||||
char* wakenet_model_ = NULL;
|
||||
std::vector<std::string> wake_words_;
|
||||
EventGroupHandle_t event_group_;
|
||||
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
|
||||
AudioCodec* codec_ = nullptr;
|
||||
std::string last_detected_wake_word_;
|
||||
|
||||
TaskHandle_t wake_word_encode_task_ = nullptr;
|
||||
StaticTask_t* wake_word_encode_task_buffer_ = nullptr;
|
||||
StackType_t* wake_word_encode_task_stack_ = nullptr;
|
||||
std::deque<std::vector<int16_t>> wake_word_pcm_;
|
||||
std::deque<std::vector<uint8_t>> wake_word_opus_;
|
||||
std::mutex wake_word_mutex_;
|
||||
std::condition_variable wake_word_cv_;
|
||||
|
||||
void StoreWakeWordData(const int16_t* data, size_t size);
|
||||
void AudioDetectionTask();
|
||||
};
|
||||
|
||||
#endif
|
||||
#ifndef AFE_WAKE_WORD_H
|
||||
#define AFE_WAKE_WORD_H
|
||||
|
||||
#include <freertos/FreeRTOS.h>
|
||||
#include <freertos/task.h>
|
||||
#include <freertos/event_groups.h>
|
||||
|
||||
#include <esp_afe_sr_models.h>
|
||||
#include <esp_nsn_models.h>
|
||||
#include <model_path.h>
|
||||
|
||||
#include <deque>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
|
||||
#include "audio_codec.h"
|
||||
#include "wake_word.h"
|
||||
|
||||
class AfeWakeWord : public WakeWord {
|
||||
public:
|
||||
AfeWakeWord();
|
||||
~AfeWakeWord();
|
||||
|
||||
bool Initialize(AudioCodec* codec, srmodel_list_t* models_list);
|
||||
void Feed(const std::vector<int16_t>& data);
|
||||
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
|
||||
void Start();
|
||||
void Stop();
|
||||
size_t GetFeedSize();
|
||||
void EncodeWakeWordData();
|
||||
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
|
||||
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
|
||||
|
||||
private:
|
||||
srmodel_list_t *models_ = nullptr;
|
||||
esp_afe_sr_iface_t* afe_iface_ = nullptr;
|
||||
esp_afe_sr_data_t* afe_data_ = nullptr;
|
||||
char* wakenet_model_ = NULL;
|
||||
std::vector<std::string> wake_words_;
|
||||
EventGroupHandle_t event_group_;
|
||||
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
|
||||
AudioCodec* codec_ = nullptr;
|
||||
std::string last_detected_wake_word_;
|
||||
|
||||
TaskHandle_t wake_word_encode_task_ = nullptr;
|
||||
StaticTask_t* wake_word_encode_task_buffer_ = nullptr;
|
||||
StackType_t* wake_word_encode_task_stack_ = nullptr;
|
||||
std::deque<std::vector<int16_t>> wake_word_pcm_;
|
||||
std::deque<std::vector<uint8_t>> wake_word_opus_;
|
||||
std::mutex wake_word_mutex_;
|
||||
std::condition_variable wake_word_cv_;
|
||||
|
||||
void StoreWakeWordData(const int16_t* data, size_t size);
|
||||
void AudioDetectionTask();
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,190 +1,250 @@
|
||||
#include "custom_wake_word.h"
|
||||
#include "audio_service.h"
|
||||
#include "system_info.h"
|
||||
|
||||
#include <esp_log.h>
|
||||
#include "esp_mn_iface.h"
|
||||
#include "esp_mn_models.h"
|
||||
#include "esp_mn_speech_commands.h"
|
||||
|
||||
|
||||
#define TAG "CustomWakeWord"
|
||||
|
||||
|
||||
CustomWakeWord::CustomWakeWord()
|
||||
: wake_word_pcm_(), wake_word_opus_() {
|
||||
}
|
||||
|
||||
CustomWakeWord::~CustomWakeWord() {
|
||||
if (multinet_model_data_ != nullptr && multinet_ != nullptr) {
|
||||
multinet_->destroy(multinet_model_data_);
|
||||
multinet_model_data_ = nullptr;
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_stack_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_stack_);
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_buffer_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
if (models_ != nullptr) {
|
||||
esp_srmodel_deinit(models_);
|
||||
}
|
||||
}
|
||||
|
||||
bool CustomWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
|
||||
codec_ = codec;
|
||||
|
||||
if (models_list == nullptr) {
|
||||
models_ = esp_srmodel_init("model");
|
||||
} else {
|
||||
models_ = models_list;
|
||||
}
|
||||
|
||||
if (models_ == nullptr || models_->num == -1) {
|
||||
ESP_LOGE(TAG, "Failed to initialize wakenet model");
|
||||
return false;
|
||||
}
|
||||
|
||||
// 初始化 multinet (命令词识别)
|
||||
mn_name_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, ESP_MN_CHINESE);
|
||||
if (mn_name_ == nullptr) {
|
||||
ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr");
|
||||
ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word");
|
||||
return false;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "multinet: %s", mn_name_);
|
||||
multinet_ = esp_mn_handle_from_name(mn_name_);
|
||||
multinet_model_data_ = multinet_->create(mn_name_, 3000); // 3 秒超时
|
||||
multinet_->set_det_threshold(multinet_model_data_, CONFIG_CUSTOM_WAKE_WORD_THRESHOLD / 100.0f);
|
||||
esp_mn_commands_clear();
|
||||
esp_mn_commands_add(1, CONFIG_CUSTOM_WAKE_WORD);
|
||||
esp_mn_commands_update();
|
||||
|
||||
multinet_->print_active_speech_commands(multinet_model_data_);
|
||||
return true;
|
||||
}
|
||||
|
||||
void CustomWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
|
||||
wake_word_detected_callback_ = callback;
|
||||
}
|
||||
|
||||
void CustomWakeWord::Start() {
|
||||
running_ = true;
|
||||
}
|
||||
|
||||
void CustomWakeWord::Stop() {
|
||||
running_ = false;
|
||||
}
|
||||
|
||||
void CustomWakeWord::Feed(const std::vector<int16_t>& data) {
|
||||
if (multinet_model_data_ == nullptr || !running_) {
|
||||
return;
|
||||
}
|
||||
|
||||
esp_mn_state_t mn_state;
|
||||
// If input channels is 2, we need to fetch the left channel data
|
||||
if (codec_->input_channels() == 2) {
|
||||
auto mono_data = std::vector<int16_t>(data.size() / 2);
|
||||
for (size_t i = 0, j = 0; i < mono_data.size(); ++i, j += 2) {
|
||||
mono_data[i] = data[j];
|
||||
}
|
||||
|
||||
StoreWakeWordData(mono_data);
|
||||
mn_state = multinet_->detect(multinet_model_data_, const_cast<int16_t*>(mono_data.data()));
|
||||
} else {
|
||||
StoreWakeWordData(data);
|
||||
mn_state = multinet_->detect(multinet_model_data_, const_cast<int16_t*>(data.data()));
|
||||
}
|
||||
|
||||
if (mn_state == ESP_MN_STATE_DETECTING) {
|
||||
return;
|
||||
} else if (mn_state == ESP_MN_STATE_DETECTED) {
|
||||
esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_);
|
||||
ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f",
|
||||
mn_result->command_id[0], mn_result->string, mn_result->prob[0]);
|
||||
|
||||
if (mn_result->command_id[0] == 1) {
|
||||
last_detected_wake_word_ = CONFIG_CUSTOM_WAKE_WORD_DISPLAY;
|
||||
}
|
||||
running_ = false;
|
||||
|
||||
if (wake_word_detected_callback_) {
|
||||
wake_word_detected_callback_(last_detected_wake_word_);
|
||||
}
|
||||
multinet_->clean(multinet_model_data_);
|
||||
} else if (mn_state == ESP_MN_STATE_TIMEOUT) {
|
||||
ESP_LOGD(TAG, "Command word detection timeout, cleaning state");
|
||||
multinet_->clean(multinet_model_data_);
|
||||
}
|
||||
}
|
||||
|
||||
size_t CustomWakeWord::GetFeedSize() {
|
||||
if (multinet_model_data_ == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return multinet_->get_samp_chunksize(multinet_model_data_);
|
||||
}
|
||||
|
||||
void CustomWakeWord::StoreWakeWordData(const std::vector<int16_t>& data) {
|
||||
// store audio data to wake_word_pcm_
|
||||
wake_word_pcm_.push_back(data);
|
||||
// keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
|
||||
while (wake_word_pcm_.size() > 2000 / 30) {
|
||||
wake_word_pcm_.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
void CustomWakeWord::EncodeWakeWordData() {
|
||||
const size_t stack_size = 4096 * 7;
|
||||
wake_word_opus_.clear();
|
||||
if (wake_word_encode_task_stack_ == nullptr) {
|
||||
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
|
||||
assert(wake_word_encode_task_stack_ != nullptr);
|
||||
}
|
||||
if (wake_word_encode_task_buffer_ == nullptr) {
|
||||
wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
|
||||
assert(wake_word_encode_task_buffer_ != nullptr);
|
||||
}
|
||||
|
||||
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
|
||||
auto this_ = (CustomWakeWord*)arg;
|
||||
{
|
||||
auto start_time = esp_timer_get_time();
|
||||
auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
|
||||
encoder->SetComplexity(0); // 0 is the fastest
|
||||
|
||||
int packets = 0;
|
||||
for (auto& pcm: this_->wake_word_pcm_) {
|
||||
encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.emplace_back(std::move(opus));
|
||||
this_->wake_word_cv_.notify_all();
|
||||
});
|
||||
packets++;
|
||||
}
|
||||
this_->wake_word_pcm_.clear();
|
||||
|
||||
auto end_time = esp_timer_get_time();
|
||||
ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000));
|
||||
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
|
||||
this_->wake_word_cv_.notify_all();
|
||||
}
|
||||
vTaskDelete(NULL);
|
||||
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
bool CustomWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
|
||||
std::unique_lock<std::mutex> lock(wake_word_mutex_);
|
||||
wake_word_cv_.wait(lock, [this]() {
|
||||
return !wake_word_opus_.empty();
|
||||
});
|
||||
opus.swap(wake_word_opus_.front());
|
||||
wake_word_opus_.pop_front();
|
||||
return !opus.empty();
|
||||
}
|
||||
#include "custom_wake_word.h"
|
||||
#include "audio_service.h"
|
||||
#include "system_info.h"
|
||||
#include "assets.h"
|
||||
|
||||
#include <esp_log.h>
|
||||
#include <esp_mn_iface.h>
|
||||
#include <esp_mn_models.h>
|
||||
#include <esp_mn_speech_commands.h>
|
||||
#include <cJSON.h>
|
||||
|
||||
|
||||
#define TAG "CustomWakeWord"
|
||||
|
||||
|
||||
CustomWakeWord::CustomWakeWord()
|
||||
: wake_word_pcm_(), wake_word_opus_() {
|
||||
}
|
||||
|
||||
CustomWakeWord::~CustomWakeWord() {
|
||||
if (multinet_model_data_ != nullptr && multinet_ != nullptr) {
|
||||
multinet_->destroy(multinet_model_data_);
|
||||
multinet_model_data_ = nullptr;
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_stack_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_stack_);
|
||||
}
|
||||
|
||||
if (wake_word_encode_task_buffer_ != nullptr) {
|
||||
heap_caps_free(wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
if (models_ != nullptr) {
|
||||
esp_srmodel_deinit(models_);
|
||||
}
|
||||
}
|
||||
|
||||
void CustomWakeWord::ParseWakenetModelConfig() {
|
||||
// Read index.json
|
||||
auto& assets = Assets::GetInstance();
|
||||
void* ptr = nullptr;
|
||||
size_t size = 0;
|
||||
if (!assets.GetAssetData("index.json", ptr, size)) {
|
||||
ESP_LOGE(TAG, "Failed to read index.json");
|
||||
return;
|
||||
}
|
||||
cJSON* root = cJSON_ParseWithLength(static_cast<char*>(ptr), size);
|
||||
if (root == nullptr) {
|
||||
ESP_LOGE(TAG, "Failed to parse index.json");
|
||||
return;
|
||||
}
|
||||
cJSON* multinet_model = cJSON_GetObjectItem(root, "multinet_model");
|
||||
if (cJSON_IsObject(multinet_model)) {
|
||||
cJSON* language = cJSON_GetObjectItem(multinet_model, "language");
|
||||
cJSON* duration = cJSON_GetObjectItem(multinet_model, "duration");
|
||||
cJSON* threshold = cJSON_GetObjectItem(multinet_model, "threshold");
|
||||
cJSON* commands = cJSON_GetObjectItem(multinet_model, "commands");
|
||||
if (cJSON_IsString(language)) {
|
||||
language_ = language->valuestring;
|
||||
}
|
||||
if (cJSON_IsNumber(duration)) {
|
||||
duration_ = duration->valueint;
|
||||
}
|
||||
if (cJSON_IsNumber(threshold)) {
|
||||
threshold_ = threshold->valuedouble;
|
||||
}
|
||||
if (cJSON_IsArray(commands)) {
|
||||
for (int i = 0; i < cJSON_GetArraySize(commands); i++) {
|
||||
cJSON* command = cJSON_GetArrayItem(commands, i);
|
||||
if (cJSON_IsObject(command)) {
|
||||
cJSON* command_name = cJSON_GetObjectItem(command, "command");
|
||||
cJSON* text = cJSON_GetObjectItem(command, "text");
|
||||
cJSON* action = cJSON_GetObjectItem(command, "action");
|
||||
if (cJSON_IsString(command_name) && cJSON_IsString(text) && cJSON_IsString(action)) {
|
||||
commands_.push_back({command_name->valuestring, text->valuestring, action->valuestring});
|
||||
ESP_LOGI(TAG, "Command: %s, Text: %s, Action: %s", command_name->valuestring, text->valuestring, action->valuestring);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cJSON_Delete(root);
|
||||
}
|
||||
|
||||
|
||||
bool CustomWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
|
||||
codec_ = codec;
|
||||
commands_.clear();
|
||||
|
||||
if (models_list == nullptr) {
|
||||
language_ = "cn";
|
||||
models_ = esp_srmodel_init("model");
|
||||
#ifdef CONFIG_CUSTOM_WAKE_WORD
|
||||
threshold_ = CONFIG_CUSTOM_WAKE_WORD_THRESHOLD / 100.0f;
|
||||
commands_.push_back({CONFIG_CUSTOM_WAKE_WORD, CONFIG_CUSTOM_WAKE_WORD_DISPLAY, "wake"});
|
||||
#endif
|
||||
} else {
|
||||
models_ = models_list;
|
||||
ParseWakenetModelConfig();
|
||||
}
|
||||
|
||||
if (models_ == nullptr || models_->num == -1) {
|
||||
ESP_LOGE(TAG, "Failed to initialize wakenet model");
|
||||
return false;
|
||||
}
|
||||
|
||||
// 初始化 multinet (命令词识别)
|
||||
mn_name_ = esp_srmodel_filter(models_, ESP_MN_PREFIX, language_.c_str());
|
||||
if (mn_name_ == nullptr) {
|
||||
ESP_LOGE(TAG, "Failed to initialize multinet, mn_name is nullptr");
|
||||
ESP_LOGI(TAG, "Please refer to https://pcn7cs20v8cr.feishu.cn/wiki/CpQjwQsCJiQSWSkYEvrcxcbVnwh to add custom wake word");
|
||||
return false;
|
||||
}
|
||||
|
||||
multinet_ = esp_mn_handle_from_name(mn_name_);
|
||||
multinet_model_data_ = multinet_->create(mn_name_, duration_);
|
||||
multinet_->set_det_threshold(multinet_model_data_, threshold_);
|
||||
esp_mn_commands_clear();
|
||||
for (int i = 0; i < commands_.size(); i++) {
|
||||
esp_mn_commands_add(i + 1, commands_[i].command.c_str());
|
||||
}
|
||||
esp_mn_commands_update();
|
||||
|
||||
multinet_->print_active_speech_commands(multinet_model_data_);
|
||||
return true;
|
||||
}
|
||||
|
||||
void CustomWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
|
||||
wake_word_detected_callback_ = callback;
|
||||
}
|
||||
|
||||
void CustomWakeWord::Start() {
|
||||
running_ = true;
|
||||
}
|
||||
|
||||
void CustomWakeWord::Stop() {
|
||||
running_ = false;
|
||||
}
|
||||
|
||||
void CustomWakeWord::Feed(const std::vector<int16_t>& data) {
|
||||
if (multinet_model_data_ == nullptr || !running_) {
|
||||
return;
|
||||
}
|
||||
|
||||
esp_mn_state_t mn_state;
|
||||
// If input channels is 2, we need to fetch the left channel data
|
||||
if (codec_->input_channels() == 2) {
|
||||
auto mono_data = std::vector<int16_t>(data.size() / 2);
|
||||
for (size_t i = 0, j = 0; i < mono_data.size(); ++i, j += 2) {
|
||||
mono_data[i] = data[j];
|
||||
}
|
||||
|
||||
StoreWakeWordData(mono_data);
|
||||
mn_state = multinet_->detect(multinet_model_data_, const_cast<int16_t*>(mono_data.data()));
|
||||
} else {
|
||||
StoreWakeWordData(data);
|
||||
mn_state = multinet_->detect(multinet_model_data_, const_cast<int16_t*>(data.data()));
|
||||
}
|
||||
|
||||
if (mn_state == ESP_MN_STATE_DETECTING) {
|
||||
return;
|
||||
} else if (mn_state == ESP_MN_STATE_DETECTED) {
|
||||
esp_mn_results_t *mn_result = multinet_->get_results(multinet_model_data_);
|
||||
for (int i = 0; i < mn_result->num && running_; i++) {
|
||||
ESP_LOGI(TAG, "Custom wake word detected: command_id=%d, string=%s, prob=%f",
|
||||
mn_result->command_id[i], mn_result->string, mn_result->prob[i]);
|
||||
auto& command = commands_[mn_result->command_id[i] - 1];
|
||||
if (command.action == "wake") {
|
||||
last_detected_wake_word_ = command.text;
|
||||
running_ = false;
|
||||
|
||||
if (wake_word_detected_callback_) {
|
||||
wake_word_detected_callback_(last_detected_wake_word_);
|
||||
}
|
||||
}
|
||||
}
|
||||
multinet_->clean(multinet_model_data_);
|
||||
} else if (mn_state == ESP_MN_STATE_TIMEOUT) {
|
||||
ESP_LOGD(TAG, "Command word detection timeout, cleaning state");
|
||||
multinet_->clean(multinet_model_data_);
|
||||
}
|
||||
}
|
||||
|
||||
size_t CustomWakeWord::GetFeedSize() {
|
||||
if (multinet_model_data_ == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return multinet_->get_samp_chunksize(multinet_model_data_);
|
||||
}
|
||||
|
||||
void CustomWakeWord::StoreWakeWordData(const std::vector<int16_t>& data) {
|
||||
// store audio data to wake_word_pcm_
|
||||
wake_word_pcm_.push_back(data);
|
||||
// keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
|
||||
while (wake_word_pcm_.size() > 2000 / 30) {
|
||||
wake_word_pcm_.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
void CustomWakeWord::EncodeWakeWordData() {
|
||||
const size_t stack_size = 4096 * 7;
|
||||
wake_word_opus_.clear();
|
||||
if (wake_word_encode_task_stack_ == nullptr) {
|
||||
wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(stack_size, MALLOC_CAP_SPIRAM);
|
||||
assert(wake_word_encode_task_stack_ != nullptr);
|
||||
}
|
||||
if (wake_word_encode_task_buffer_ == nullptr) {
|
||||
wake_word_encode_task_buffer_ = (StaticTask_t*)heap_caps_malloc(sizeof(StaticTask_t), MALLOC_CAP_INTERNAL);
|
||||
assert(wake_word_encode_task_buffer_ != nullptr);
|
||||
}
|
||||
|
||||
wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
|
||||
auto this_ = (CustomWakeWord*)arg;
|
||||
{
|
||||
auto start_time = esp_timer_get_time();
|
||||
auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
|
||||
encoder->SetComplexity(0); // 0 is the fastest
|
||||
|
||||
int packets = 0;
|
||||
for (auto& pcm: this_->wake_word_pcm_) {
|
||||
encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.emplace_back(std::move(opus));
|
||||
this_->wake_word_cv_.notify_all();
|
||||
});
|
||||
packets++;
|
||||
}
|
||||
this_->wake_word_pcm_.clear();
|
||||
|
||||
auto end_time = esp_timer_get_time();
|
||||
ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000));
|
||||
|
||||
std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
|
||||
this_->wake_word_opus_.push_back(std::vector<uint8_t>());
|
||||
this_->wake_word_cv_.notify_all();
|
||||
}
|
||||
vTaskDelete(NULL);
|
||||
}, "encode_wake_word", stack_size, this, 2, wake_word_encode_task_stack_, wake_word_encode_task_buffer_);
|
||||
}
|
||||
|
||||
bool CustomWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
|
||||
std::unique_lock<std::mutex> lock(wake_word_mutex_);
|
||||
wake_word_cv_.wait(lock, [this]() {
|
||||
return !wake_word_opus_.empty();
|
||||
});
|
||||
opus.swap(wake_word_opus_.front());
|
||||
wake_word_opus_.pop_front();
|
||||
return !opus.empty();
|
||||
}
|
||||
|
||||
@@ -1,58 +1,69 @@
|
||||
#ifndef CUSTOM_WAKE_WORD_H
|
||||
#define CUSTOM_WAKE_WORD_H
|
||||
|
||||
#include <esp_attr.h>
|
||||
#include <esp_mn_iface.h>
|
||||
#include <esp_mn_models.h>
|
||||
#include <model_path.h>
|
||||
|
||||
#include <deque>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
#include <atomic>
|
||||
|
||||
#include "audio_codec.h"
|
||||
#include "wake_word.h"
|
||||
|
||||
class CustomWakeWord : public WakeWord {
|
||||
public:
|
||||
CustomWakeWord();
|
||||
~CustomWakeWord();
|
||||
|
||||
bool Initialize(AudioCodec* codec, srmodel_list_t* models_list);
|
||||
void Feed(const std::vector<int16_t>& data);
|
||||
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
|
||||
void Start();
|
||||
void Stop();
|
||||
size_t GetFeedSize();
|
||||
void EncodeWakeWordData();
|
||||
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
|
||||
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
|
||||
|
||||
private:
|
||||
// multinet 相关成员变量
|
||||
esp_mn_iface_t* multinet_ = nullptr;
|
||||
model_iface_data_t* multinet_model_data_ = nullptr;
|
||||
srmodel_list_t *models_ = nullptr;
|
||||
char* mn_name_ = nullptr;
|
||||
|
||||
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
|
||||
AudioCodec* codec_ = nullptr;
|
||||
std::string last_detected_wake_word_;
|
||||
std::atomic<bool> running_ = false;
|
||||
|
||||
TaskHandle_t wake_word_encode_task_ = nullptr;
|
||||
StaticTask_t* wake_word_encode_task_buffer_ = nullptr;
|
||||
StackType_t* wake_word_encode_task_stack_ = nullptr;
|
||||
std::deque<std::vector<int16_t>> wake_word_pcm_;
|
||||
std::deque<std::vector<uint8_t>> wake_word_opus_;
|
||||
std::mutex wake_word_mutex_;
|
||||
std::condition_variable wake_word_cv_;
|
||||
|
||||
void StoreWakeWordData(const std::vector<int16_t>& data);
|
||||
};
|
||||
|
||||
#endif
|
||||
#ifndef CUSTOM_WAKE_WORD_H
|
||||
#define CUSTOM_WAKE_WORD_H
|
||||
|
||||
#include <esp_attr.h>
|
||||
#include <esp_mn_iface.h>
|
||||
#include <esp_mn_models.h>
|
||||
#include <model_path.h>
|
||||
|
||||
#include <deque>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
#include <atomic>
|
||||
|
||||
#include "audio_codec.h"
|
||||
#include "wake_word.h"
|
||||
|
||||
class CustomWakeWord : public WakeWord {
|
||||
public:
|
||||
CustomWakeWord();
|
||||
~CustomWakeWord();
|
||||
|
||||
bool Initialize(AudioCodec* codec, srmodel_list_t* models_list);
|
||||
void Feed(const std::vector<int16_t>& data);
|
||||
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
|
||||
void Start();
|
||||
void Stop();
|
||||
size_t GetFeedSize();
|
||||
void EncodeWakeWordData();
|
||||
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
|
||||
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
|
||||
|
||||
private:
|
||||
struct Command {
|
||||
std::string command;
|
||||
std::string text;
|
||||
std::string action;
|
||||
};
|
||||
|
||||
// multinet 相关成员变量
|
||||
esp_mn_iface_t* multinet_ = nullptr;
|
||||
model_iface_data_t* multinet_model_data_ = nullptr;
|
||||
srmodel_list_t *models_ = nullptr;
|
||||
char* mn_name_ = nullptr;
|
||||
std::string language_ = "cn";
|
||||
int duration_ = 3000;
|
||||
float threshold_ = 0.2;
|
||||
std::deque<Command> commands_;
|
||||
|
||||
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
|
||||
AudioCodec* codec_ = nullptr;
|
||||
std::string last_detected_wake_word_;
|
||||
std::atomic<bool> running_ = false;
|
||||
|
||||
TaskHandle_t wake_word_encode_task_ = nullptr;
|
||||
StaticTask_t* wake_word_encode_task_buffer_ = nullptr;
|
||||
StackType_t* wake_word_encode_task_stack_ = nullptr;
|
||||
std::deque<std::vector<int16_t>> wake_word_pcm_;
|
||||
std::deque<std::vector<uint8_t>> wake_word_opus_;
|
||||
std::mutex wake_word_mutex_;
|
||||
std::condition_variable wake_word_cv_;
|
||||
|
||||
void StoreWakeWordData(const std::vector<int16_t>& data);
|
||||
void ParseWakenetModelConfig();
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,87 +1,87 @@
|
||||
#include "esp_wake_word.h"
|
||||
#include <esp_log.h>
|
||||
|
||||
|
||||
#define TAG "EspWakeWord"
|
||||
|
||||
EspWakeWord::EspWakeWord() {
|
||||
}
|
||||
|
||||
EspWakeWord::~EspWakeWord() {
|
||||
if (wakenet_data_ != nullptr) {
|
||||
wakenet_iface_->destroy(wakenet_data_);
|
||||
esp_srmodel_deinit(wakenet_model_);
|
||||
}
|
||||
}
|
||||
|
||||
bool EspWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
|
||||
codec_ = codec;
|
||||
|
||||
if (models_list == nullptr) {
|
||||
wakenet_model_ = esp_srmodel_init("model");
|
||||
} else {
|
||||
wakenet_model_ = models_list;
|
||||
}
|
||||
|
||||
if (wakenet_model_ == nullptr || wakenet_model_->num == -1) {
|
||||
ESP_LOGE(TAG, "Failed to initialize wakenet model");
|
||||
return false;
|
||||
}
|
||||
if(wakenet_model_->num > 1) {
|
||||
ESP_LOGW(TAG, "More than one model found, using the first one");
|
||||
} else if (wakenet_model_->num == 0) {
|
||||
ESP_LOGE(TAG, "No model found");
|
||||
return false;
|
||||
}
|
||||
char *model_name = wakenet_model_->model_name[0];
|
||||
wakenet_iface_ = (esp_wn_iface_t*)esp_wn_handle_from_name(model_name);
|
||||
wakenet_data_ = wakenet_iface_->create(model_name, DET_MODE_95);
|
||||
|
||||
int frequency = wakenet_iface_->get_samp_rate(wakenet_data_);
|
||||
int audio_chunksize = wakenet_iface_->get_samp_chunksize(wakenet_data_);
|
||||
ESP_LOGI(TAG, "Wake word(%s),freq: %d, chunksize: %d", model_name, frequency, audio_chunksize);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void EspWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
|
||||
wake_word_detected_callback_ = callback;
|
||||
}
|
||||
|
||||
void EspWakeWord::Start() {
|
||||
running_ = true;
|
||||
}
|
||||
|
||||
void EspWakeWord::Stop() {
|
||||
running_ = false;
|
||||
}
|
||||
|
||||
void EspWakeWord::Feed(const std::vector<int16_t>& data) {
|
||||
if (wakenet_data_ == nullptr || !running_) {
|
||||
return;
|
||||
}
|
||||
|
||||
int res = wakenet_iface_->detect(wakenet_data_, (int16_t *)data.data());
|
||||
if (res > 0) {
|
||||
last_detected_wake_word_ = wakenet_iface_->get_word_name(wakenet_data_, res);
|
||||
running_ = false;
|
||||
|
||||
if (wake_word_detected_callback_) {
|
||||
wake_word_detected_callback_(last_detected_wake_word_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t EspWakeWord::GetFeedSize() {
|
||||
if (wakenet_data_ == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return wakenet_iface_->get_samp_chunksize(wakenet_data_);
|
||||
}
|
||||
|
||||
void EspWakeWord::EncodeWakeWordData() {
|
||||
}
|
||||
|
||||
bool EspWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
|
||||
return false;
|
||||
}
|
||||
#include "esp_wake_word.h"
|
||||
#include <esp_log.h>
|
||||
|
||||
|
||||
#define TAG "EspWakeWord"
|
||||
|
||||
EspWakeWord::EspWakeWord() {
|
||||
}
|
||||
|
||||
EspWakeWord::~EspWakeWord() {
|
||||
if (wakenet_data_ != nullptr) {
|
||||
wakenet_iface_->destroy(wakenet_data_);
|
||||
esp_srmodel_deinit(wakenet_model_);
|
||||
}
|
||||
}
|
||||
|
||||
bool EspWakeWord::Initialize(AudioCodec* codec, srmodel_list_t* models_list) {
|
||||
codec_ = codec;
|
||||
|
||||
if (models_list == nullptr) {
|
||||
wakenet_model_ = esp_srmodel_init("model");
|
||||
} else {
|
||||
wakenet_model_ = models_list;
|
||||
}
|
||||
|
||||
if (wakenet_model_ == nullptr || wakenet_model_->num == -1) {
|
||||
ESP_LOGE(TAG, "Failed to initialize wakenet model");
|
||||
return false;
|
||||
}
|
||||
if(wakenet_model_->num > 1) {
|
||||
ESP_LOGW(TAG, "More than one model found, using the first one");
|
||||
} else if (wakenet_model_->num == 0) {
|
||||
ESP_LOGE(TAG, "No model found");
|
||||
return false;
|
||||
}
|
||||
char *model_name = wakenet_model_->model_name[0];
|
||||
wakenet_iface_ = (esp_wn_iface_t*)esp_wn_handle_from_name(model_name);
|
||||
wakenet_data_ = wakenet_iface_->create(model_name, DET_MODE_95);
|
||||
|
||||
int frequency = wakenet_iface_->get_samp_rate(wakenet_data_);
|
||||
int audio_chunksize = wakenet_iface_->get_samp_chunksize(wakenet_data_);
|
||||
ESP_LOGI(TAG, "Wake word(%s),freq: %d, chunksize: %d", model_name, frequency, audio_chunksize);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void EspWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
|
||||
wake_word_detected_callback_ = callback;
|
||||
}
|
||||
|
||||
void EspWakeWord::Start() {
|
||||
running_ = true;
|
||||
}
|
||||
|
||||
void EspWakeWord::Stop() {
|
||||
running_ = false;
|
||||
}
|
||||
|
||||
void EspWakeWord::Feed(const std::vector<int16_t>& data) {
|
||||
if (wakenet_data_ == nullptr || !running_) {
|
||||
return;
|
||||
}
|
||||
|
||||
int res = wakenet_iface_->detect(wakenet_data_, (int16_t *)data.data());
|
||||
if (res > 0) {
|
||||
last_detected_wake_word_ = wakenet_iface_->get_word_name(wakenet_data_, res);
|
||||
running_ = false;
|
||||
|
||||
if (wake_word_detected_callback_) {
|
||||
wake_word_detected_callback_(last_detected_wake_word_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t EspWakeWord::GetFeedSize() {
|
||||
if (wakenet_data_ == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
return wakenet_iface_->get_samp_chunksize(wakenet_data_);
|
||||
}
|
||||
|
||||
void EspWakeWord::EncodeWakeWordData() {
|
||||
}
|
||||
|
||||
bool EspWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1,42 +1,42 @@
|
||||
#ifndef ESP_WAKE_WORD_H
|
||||
#define ESP_WAKE_WORD_H
|
||||
|
||||
#include <esp_wn_iface.h>
|
||||
#include <esp_wn_models.h>
|
||||
#include <model_path.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <atomic>
|
||||
|
||||
#include "audio_codec.h"
|
||||
#include "wake_word.h"
|
||||
|
||||
class EspWakeWord : public WakeWord {
|
||||
public:
|
||||
EspWakeWord();
|
||||
~EspWakeWord();
|
||||
|
||||
bool Initialize(AudioCodec* codec, srmodel_list_t* models_list);
|
||||
void Feed(const std::vector<int16_t>& data);
|
||||
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
|
||||
void Start();
|
||||
void Stop();
|
||||
size_t GetFeedSize();
|
||||
void EncodeWakeWordData();
|
||||
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
|
||||
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
|
||||
|
||||
private:
|
||||
esp_wn_iface_t *wakenet_iface_ = nullptr;
|
||||
model_iface_data_t *wakenet_data_ = nullptr;
|
||||
srmodel_list_t *wakenet_model_ = nullptr;
|
||||
AudioCodec* codec_ = nullptr;
|
||||
std::atomic<bool> running_ = false;
|
||||
|
||||
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
|
||||
std::string last_detected_wake_word_;
|
||||
};
|
||||
|
||||
#endif
|
||||
#ifndef ESP_WAKE_WORD_H
|
||||
#define ESP_WAKE_WORD_H
|
||||
|
||||
#include <esp_wn_iface.h>
|
||||
#include <esp_wn_models.h>
|
||||
#include <model_path.h>
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <atomic>
|
||||
|
||||
#include "audio_codec.h"
|
||||
#include "wake_word.h"
|
||||
|
||||
class EspWakeWord : public WakeWord {
|
||||
public:
|
||||
EspWakeWord();
|
||||
~EspWakeWord();
|
||||
|
||||
bool Initialize(AudioCodec* codec, srmodel_list_t* models_list);
|
||||
void Feed(const std::vector<int16_t>& data);
|
||||
void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
|
||||
void Start();
|
||||
void Stop();
|
||||
size_t GetFeedSize();
|
||||
void EncodeWakeWordData();
|
||||
bool GetWakeWordOpus(std::vector<uint8_t>& opus);
|
||||
const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
|
||||
|
||||
private:
|
||||
esp_wn_iface_t *wakenet_iface_ = nullptr;
|
||||
model_iface_data_t *wakenet_data_ = nullptr;
|
||||
srmodel_list_t *wakenet_model_ = nullptr;
|
||||
AudioCodec* codec_ = nullptr;
|
||||
std::atomic<bool> running_ = false;
|
||||
|
||||
std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
|
||||
std::string last_detected_wake_word_;
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user