first commit

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/afe_audio_processor.cc

@@ -0,0 +1,161 @@
+#include "afe_audio_processor.h"
+#include <esp_log.h>
+
+#define PROCESSOR_RUNNING 0x01
+
+#define TAG "AfeAudioProcessor"
+
+AfeAudioProcessor::AfeAudioProcessor()
+    : afe_data_(nullptr) {
+    event_group_ = xEventGroupCreate();
+}
+
+void AfeAudioProcessor::Initialize(AudioCodec* codec) {
+    codec_ = codec;
+    int ref_num = codec_->input_reference() ? 1 : 0;
+
+    std::string input_format;
+    for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
+        input_format.push_back('M');
+    }
+    for (int i = 0; i < ref_num; i++) {
+        input_format.push_back('R');
+    }
+
+    srmodel_list_t *models = esp_srmodel_init("model");
+    char* ns_model_name = esp_srmodel_filter(models, ESP_NSNET_PREFIX, NULL);
+    char* vad_model_name = esp_srmodel_filter(models, ESP_VADN_PREFIX, NULL);
+    
+    afe_config_t* afe_config = afe_config_init(input_format.c_str(), NULL, AFE_TYPE_VC, AFE_MODE_HIGH_PERF);
+    afe_config->aec_mode = AEC_MODE_VOIP_HIGH_PERF;
+    afe_config->vad_mode = VAD_MODE_0;
+    afe_config->vad_min_noise_ms = 100;
+    if (vad_model_name != nullptr) {
+        afe_config->vad_model_name = vad_model_name;
+    }
+
+    if (ns_model_name != nullptr) {
+        afe_config->ns_init = true;
+        afe_config->ns_model_name = ns_model_name;
+        afe_config->afe_ns_mode = AFE_NS_MODE_NET;
+    } else {
+        afe_config->ns_init = false;
+    }
+
+    afe_config->afe_perferred_core = 1;
+    afe_config->afe_perferred_priority = 1;
+    afe_config->agc_init = false;
+    afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
+
+#ifdef CONFIG_USE_DEVICE_AEC
+    afe_config->aec_init = true;
+    afe_config->vad_init = false;
+#else
+    afe_config->aec_init = false;
+    afe_config->vad_init = true;
+#endif
+
+    afe_iface_ = esp_afe_handle_from_config(afe_config);
+    afe_data_ = afe_iface_->create_from_config(afe_config);
+    
+    xTaskCreate([](void* arg) {
+        auto this_ = (AfeAudioProcessor*)arg;
+        this_->AudioProcessorTask();
+        vTaskDelete(NULL);
+    }, "audio_communication", 4096, this, 3, NULL);
+}
+
+AfeAudioProcessor::~AfeAudioProcessor() {
+    if (afe_data_ != nullptr) {
+        afe_iface_->destroy(afe_data_);
+    }
+    vEventGroupDelete(event_group_);
+}
+
+size_t AfeAudioProcessor::GetFeedSize() {
+    if (afe_data_ == nullptr) {
+        return 0;
+    }
+    return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels();
+}
+
+void AfeAudioProcessor::Feed(const std::vector<int16_t>& data) {
+    if (afe_data_ == nullptr) {
+        return;
+    }
+    afe_iface_->feed(afe_data_, data.data());
+}
+
+void AfeAudioProcessor::Start() {
+    xEventGroupSetBits(event_group_, PROCESSOR_RUNNING);
+}
+
+void AfeAudioProcessor::Stop() {
+    xEventGroupClearBits(event_group_, PROCESSOR_RUNNING);
+    if (afe_data_ != nullptr) {
+        afe_iface_->reset_buffer(afe_data_);
+    }
+}
+
+bool AfeAudioProcessor::IsRunning() {
+    return xEventGroupGetBits(event_group_) & PROCESSOR_RUNNING;
+}
+
+void AfeAudioProcessor::OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) {
+    output_callback_ = callback;
+}
+
+void AfeAudioProcessor::OnVadStateChange(std::function<void(bool speaking)> callback) {
+    vad_state_change_callback_ = callback;
+}
+
+void AfeAudioProcessor::AudioProcessorTask() {
+    auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
+    auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
+    ESP_LOGI(TAG, "Audio communication task started, feed size: %d fetch size: %d",
+        feed_size, fetch_size);
+
+    while (true) {
+        xEventGroupWaitBits(event_group_, PROCESSOR_RUNNING, pdFALSE, pdTRUE, portMAX_DELAY);
+
+        auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY);
+        if ((xEventGroupGetBits(event_group_) & PROCESSOR_RUNNING) == 0) {
+            continue;
+        }
+        if (res == nullptr || res->ret_value == ESP_FAIL) {
+            if (res != nullptr) {
+                ESP_LOGI(TAG, "Error code: %d", res->ret_value);
+            }
+            continue;
+        }
+
+        // VAD state change
+        if (vad_state_change_callback_) {
+            if (res->vad_state == VAD_SPEECH && !is_speaking_) {
+                is_speaking_ = true;
+                vad_state_change_callback_(true);
+            } else if (res->vad_state == VAD_SILENCE && is_speaking_) {
+                is_speaking_ = false;
+                vad_state_change_callback_(false);
+            }
+        }
+
+        if (output_callback_) {
+            output_callback_(std::vector<int16_t>(res->data, res->data + res->data_size / sizeof(int16_t)));
+        }
+    }
+}
+
+void AfeAudioProcessor::EnableDeviceAec(bool enable) {
+    if (enable) {
+#if CONFIG_USE_DEVICE_AEC
+        afe_iface_->disable_vad(afe_data_);
+        afe_iface_->enable_aec(afe_data_);
+#else
+        ESP_LOGE(TAG, "Device AEC is not supported");
+#endif
+    } else {
+        afe_iface_->disable_aec(afe_data_);
+        afe_iface_->enable_vad(afe_data_);
+    }
+}

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/afe_audio_processor.h

@@ -0,0 +1,43 @@
+#ifndef AFE_AUDIO_PROCESSOR_H
+#define AFE_AUDIO_PROCESSOR_H
+
+#include <esp_afe_sr_models.h>
+#include <freertos/FreeRTOS.h>
+#include <freertos/task.h>
+#include <freertos/event_groups.h>
+
+#include <string>
+#include <vector>
+#include <functional>
+
+#include "audio_processor.h"
+#include "audio_codec.h"
+
+class AfeAudioProcessor : public AudioProcessor {
+public:
+    AfeAudioProcessor();
+    ~AfeAudioProcessor();
+
+    void Initialize(AudioCodec* codec) override;
+    void Feed(const std::vector<int16_t>& data) override;
+    void Start() override;
+    void Stop() override;
+    bool IsRunning() override;
+    void OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) override;
+    void OnVadStateChange(std::function<void(bool speaking)> callback) override;
+    size_t GetFeedSize() override;
+    void EnableDeviceAec(bool enable) override;
+
+private:
+    EventGroupHandle_t event_group_ = nullptr;
+    esp_afe_sr_iface_t* afe_iface_ = nullptr;
+    esp_afe_sr_data_t* afe_data_ = nullptr;
+    std::function<void(std::vector<int16_t>&& data)> output_callback_;
+    std::function<void(bool speaking)> vad_state_change_callback_;
+    AudioCodec* codec_ = nullptr;
+    bool is_speaking_ = false;
+
+    void AudioProcessorTask();
+};
+
+#endif 

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/afe_wake_word.cc

@@ -0,0 +1,192 @@
+#include "afe_wake_word.h"
+#include "application.h"
+
+#include <esp_log.h>
+#include <model_path.h>
+#include <arpa/inet.h>
+#include <sstream>
+
+#define DETECTION_RUNNING_EVENT 1
+
+#define TAG "AfeWakeWord"
+
+AfeWakeWord::AfeWakeWord()
+    : afe_data_(nullptr),
+      wake_word_pcm_(),
+      wake_word_opus_() {
+
+    event_group_ = xEventGroupCreate();
+}
+
+AfeWakeWord::~AfeWakeWord() {
+    if (afe_data_ != nullptr) {
+        afe_iface_->destroy(afe_data_);
+    }
+
+    if (wake_word_encode_task_stack_ != nullptr) {
+        heap_caps_free(wake_word_encode_task_stack_);
+    }
+
+    vEventGroupDelete(event_group_);
+}
+
+void AfeWakeWord::Initialize(AudioCodec* codec) {
+    codec_ = codec;
+    int ref_num = codec_->input_reference() ? 1 : 0;
+
+    srmodel_list_t *models = esp_srmodel_init("model");
+    if (models == nullptr || models->num == -1) {
+        ESP_LOGE(TAG, "Failed to initialize wakenet model");
+        return;
+    }
+    for (int i = 0; i < models->num; i++) {
+        ESP_LOGI(TAG, "Model %d: %s", i, models->model_name[i]);
+        if (strstr(models->model_name[i], ESP_WN_PREFIX) != NULL) {
+            wakenet_model_ = models->model_name[i];
+            auto words = esp_srmodel_get_wake_words(models, wakenet_model_);
+            // split by ";" to get all wake words
+            std::stringstream ss(words);
+            std::string word;
+            while (std::getline(ss, word, ';')) {
+                wake_words_.push_back(word);
+            }
+        }
+    }
+
+    std::string input_format;
+    for (int i = 0; i < codec_->input_channels() - ref_num; i++) {
+        input_format.push_back('M');
+    }
+    for (int i = 0; i < ref_num; i++) {
+        input_format.push_back('R');
+    }
+    afe_config_t* afe_config = afe_config_init(input_format.c_str(), models, AFE_TYPE_SR, AFE_MODE_HIGH_PERF);
+    afe_config->aec_init = codec_->input_reference();
+    afe_config->aec_mode = AEC_MODE_SR_HIGH_PERF;
+    afe_config->afe_perferred_core = 1;
+    afe_config->afe_perferred_priority = 1;
+    afe_config->memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM;
+    
+    afe_iface_ = esp_afe_handle_from_config(afe_config);
+    afe_data_ = afe_iface_->create_from_config(afe_config);
+
+    xTaskCreate([](void* arg) {
+        auto this_ = (AfeWakeWord*)arg;
+        this_->AudioDetectionTask();
+        vTaskDelete(NULL);
+    }, "audio_detection", 4096, this, 3, nullptr);
+}
+
+void AfeWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
+    wake_word_detected_callback_ = callback;
+}
+
+void AfeWakeWord::StartDetection() {
+    xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
+}
+
+void AfeWakeWord::StopDetection() {
+    xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
+    if (afe_data_ != nullptr) {
+        afe_iface_->reset_buffer(afe_data_);
+    }
+}
+
+bool AfeWakeWord::IsDetectionRunning() {
+    return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
+}
+
+void AfeWakeWord::Feed(const std::vector<int16_t>& data) {
+    if (afe_data_ == nullptr) {
+        return;
+    }
+    afe_iface_->feed(afe_data_, data.data());
+}
+
+size_t AfeWakeWord::GetFeedSize() {
+    if (afe_data_ == nullptr) {
+        return 0;
+    }
+    return afe_iface_->get_feed_chunksize(afe_data_) * codec_->input_channels();
+}
+
+void AfeWakeWord::AudioDetectionTask() {
+    auto fetch_size = afe_iface_->get_fetch_chunksize(afe_data_);
+    auto feed_size = afe_iface_->get_feed_chunksize(afe_data_);
+    ESP_LOGI(TAG, "Audio detection task started, feed size: %d fetch size: %d",
+        feed_size, fetch_size);
+
+    while (true) {
+        xEventGroupWaitBits(event_group_, DETECTION_RUNNING_EVENT, pdFALSE, pdTRUE, portMAX_DELAY);
+
+        auto res = afe_iface_->fetch_with_delay(afe_data_, portMAX_DELAY);
+        if (res == nullptr || res->ret_value == ESP_FAIL) {
+            continue;;
+        }
+
+        // Store the wake word data for voice recognition, like who is speaking
+        StoreWakeWordData(res->data, res->data_size / sizeof(int16_t));
+
+        if (res->wakeup_state == WAKENET_DETECTED) {
+            StopDetection();
+            last_detected_wake_word_ = wake_words_[res->wake_word_index - 1];
+
+            if (wake_word_detected_callback_) {
+                wake_word_detected_callback_(last_detected_wake_word_);
+            }
+        }
+    }
+}
+
+void AfeWakeWord::StoreWakeWordData(const int16_t* data, size_t samples) {
+    // store audio data to wake_word_pcm_
+    wake_word_pcm_.emplace_back(std::vector<int16_t>(data, data + samples));
+    // keep about 2 seconds of data, detect duration is 30ms (sample_rate == 16000, chunksize == 512)
+    while (wake_word_pcm_.size() > 2000 / 30) {
+        wake_word_pcm_.pop_front();
+    }
+}
+
+void AfeWakeWord::EncodeWakeWordData() {
+    wake_word_opus_.clear();
+    if (wake_word_encode_task_stack_ == nullptr) {
+        wake_word_encode_task_stack_ = (StackType_t*)heap_caps_malloc(4096 * 8, MALLOC_CAP_SPIRAM);
+    }
+    wake_word_encode_task_ = xTaskCreateStatic([](void* arg) {
+        auto this_ = (AfeWakeWord*)arg;
+        {
+            auto start_time = esp_timer_get_time();
+            auto encoder = std::make_unique<OpusEncoderWrapper>(16000, 1, OPUS_FRAME_DURATION_MS);
+            encoder->SetComplexity(0); // 0 is the fastest
+
+            int packets = 0;
+            for (auto& pcm: this_->wake_word_pcm_) {
+                encoder->Encode(std::move(pcm), [this_](std::vector<uint8_t>&& opus) {
+                    std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
+                    this_->wake_word_opus_.emplace_back(std::move(opus));
+                    this_->wake_word_cv_.notify_all();
+                });
+                packets++;
+            }
+            this_->wake_word_pcm_.clear();
+
+            auto end_time = esp_timer_get_time();
+            ESP_LOGI(TAG, "Encode wake word opus %d packets in %ld ms", packets, (long)((end_time - start_time) / 1000));
+
+            std::lock_guard<std::mutex> lock(this_->wake_word_mutex_);
+            this_->wake_word_opus_.push_back(std::vector<uint8_t>());
+            this_->wake_word_cv_.notify_all();
+        }
+        vTaskDelete(NULL);
+    }, "encode_detect_packets", 4096 * 8, this, 2, wake_word_encode_task_stack_, &wake_word_encode_task_buffer_);
+}
+
+bool AfeWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
+    std::unique_lock<std::mutex> lock(wake_word_mutex_);
+    wake_word_cv_.wait(lock, [this]() {
+        return !wake_word_opus_.empty();
+    });
+    opus.swap(wake_word_opus_.front());
+    wake_word_opus_.pop_front();
+    return !opus.empty();
+}

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/afe_wake_word.h

@@ -0,0 +1,59 @@
+#ifndef AFE_WAKE_WORD_H
+#define AFE_WAKE_WORD_H
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/task.h>
+#include <freertos/event_groups.h>
+
+#include <esp_afe_sr_models.h>
+#include <esp_nsn_models.h>
+
+#include <list>
+#include <string>
+#include <vector>
+#include <functional>
+#include <mutex>
+#include <condition_variable>
+
+#include "audio_codec.h"
+#include "wake_word.h"
+
+class AfeWakeWord : public WakeWord {
+public:
+    AfeWakeWord();
+    ~AfeWakeWord();
+
+    void Initialize(AudioCodec* codec);
+    void Feed(const std::vector<int16_t>& data);
+    void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
+    void StartDetection();
+    void StopDetection();
+    bool IsDetectionRunning();
+    size_t GetFeedSize();
+    void EncodeWakeWordData();
+    bool GetWakeWordOpus(std::vector<uint8_t>& opus);
+    const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
+
+private:
+    esp_afe_sr_iface_t* afe_iface_ = nullptr;
+    esp_afe_sr_data_t* afe_data_ = nullptr;
+    char* wakenet_model_ = NULL;
+    std::vector<std::string> wake_words_;
+    EventGroupHandle_t event_group_;
+    std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
+    AudioCodec* codec_ = nullptr;
+    std::string last_detected_wake_word_;
+
+    TaskHandle_t wake_word_encode_task_ = nullptr;
+    StaticTask_t wake_word_encode_task_buffer_;
+    StackType_t* wake_word_encode_task_stack_ = nullptr;
+    std::list<std::vector<int16_t>> wake_word_pcm_;
+    std::list<std::vector<uint8_t>> wake_word_opus_;
+    std::mutex wake_word_mutex_;
+    std::condition_variable wake_word_cv_;
+
+    void StoreWakeWordData(const int16_t* data, size_t size);
+    void AudioDetectionTask();
+};
+
+#endif

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/audio_debugger.cc

@@ -0,0 +1,68 @@
+#include "audio_debugger.h"
+#include "sdkconfig.h"
+
+#if CONFIG_USE_AUDIO_DEBUGGER
+#include <esp_log.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <errno.h>
+#include <cstring>
+#include <string>
+#endif
+
+#define TAG "AudioDebugger"
+
+
+AudioDebugger::AudioDebugger() {
+#if CONFIG_USE_AUDIO_DEBUGGER
+    udp_sockfd_ = socket(AF_INET, SOCK_DGRAM, 0);
+    if (udp_sockfd_ >= 0) {
+        // 解析配置的服务器地址 "IP:PORT"
+        std::string server_addr = CONFIG_AUDIO_DEBUG_UDP_SERVER;
+        size_t colon_pos = server_addr.find(':');
+        
+        if (colon_pos != std::string::npos) {
+            std::string ip = server_addr.substr(0, colon_pos);
+            int port = std::stoi(server_addr.substr(colon_pos + 1));
+            
+            memset(&udp_server_addr_, 0, sizeof(udp_server_addr_));
+            udp_server_addr_.sin_family = AF_INET;
+            udp_server_addr_.sin_port = htons(port);
+            inet_pton(AF_INET, ip.c_str(), &udp_server_addr_.sin_addr);
+            
+            ESP_LOGI(TAG, "Initialized server address: %s", CONFIG_AUDIO_DEBUG_UDP_SERVER);
+        } else {
+            ESP_LOGW(TAG, "Invalid server address: %s, should be IP:PORT", CONFIG_AUDIO_DEBUG_UDP_SERVER);
+            close(udp_sockfd_);
+            udp_sockfd_ = -1;
+        }
+    } else {
+        ESP_LOGW(TAG, "Failed to create UDP socket: %d", errno);
+    }
+#endif
+}
+
+AudioDebugger::~AudioDebugger() {
+#if CONFIG_USE_AUDIO_DEBUGGER
+    if (udp_sockfd_ >= 0) {
+        close(udp_sockfd_);
+        ESP_LOGI(TAG, "Closed UDP socket");
+    }
+#endif
+}
+
+void AudioDebugger::Feed(const std::vector<int16_t>& data) {
+#if CONFIG_USE_AUDIO_DEBUGGER
+    if (udp_sockfd_ >= 0) {
+        ssize_t sent = sendto(udp_sockfd_, data.data(), data.size() * sizeof(int16_t), 0,
+                             (struct sockaddr*)&udp_server_addr_, sizeof(udp_server_addr_));
+        if (sent < 0) {
+            ESP_LOGW(TAG, "Failed to send audio data to %s: %d", CONFIG_AUDIO_DEBUG_UDP_SERVER, errno);
+        } else {
+            ESP_LOGD(TAG, "Sent %d bytes audio data to %s", sent, CONFIG_AUDIO_DEBUG_UDP_SERVER);
+        }
+    }
+#endif
+}
+
+ 

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/audio_debugger.h

@@ -0,0 +1,22 @@
+#ifndef AUDIO_DEBUGGER_H
+#define AUDIO_DEBUGGER_H
+
+#include <vector>
+#include <cstdint>
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+class AudioDebugger {
+public:
+    AudioDebugger();
+    ~AudioDebugger();
+
+    void Feed(const std::vector<int16_t>& data);
+
+private:
+    int udp_sockfd_ = -1;
+    struct sockaddr_in udp_server_addr_;
+};
+
+#endif 

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/audio_processor.h

@@ -0,0 +1,25 @@
+#ifndef AUDIO_PROCESSOR_H
+#define AUDIO_PROCESSOR_H
+
+#include <string>
+#include <vector>
+#include <functional>
+
+#include "audio_codec.h"
+
+class AudioProcessor {
+public:
+    virtual ~AudioProcessor() = default;
+    
+    virtual void Initialize(AudioCodec* codec) = 0;
+    virtual void Feed(const std::vector<int16_t>& data) = 0;
+    virtual void Start() = 0;
+    virtual void Stop() = 0;
+    virtual bool IsRunning() = 0;
+    virtual void OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) = 0;
+    virtual void OnVadStateChange(std::function<void(bool speaking)> callback) = 0;
+    virtual size_t GetFeedSize() = 0;
+    virtual void EnableDeviceAec(bool enable) = 0;
+};
+
+#endif

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/esp_wake_word.cc

@@ -0,0 +1,89 @@
+#include "esp_wake_word.h"
+#include "application.h"
+
+#include <esp_log.h>
+#include <model_path.h>
+#include <arpa/inet.h>
+#include <sstream>
+
+#define DETECTION_RUNNING_EVENT 1
+
+#define TAG "EspWakeWord"
+
+EspWakeWord::EspWakeWord() {
+    event_group_ = xEventGroupCreate();
+}
+
+EspWakeWord::~EspWakeWord() {
+    if (wakenet_data_ != nullptr) {
+        wakenet_iface_->destroy(wakenet_data_);
+        esp_srmodel_deinit(wakenet_model_);
+    }
+
+    vEventGroupDelete(event_group_);
+}
+
+void EspWakeWord::Initialize(AudioCodec* codec) {
+    codec_ = codec;
+
+    wakenet_model_ = esp_srmodel_init("model");
+    if (wakenet_model_ == nullptr || wakenet_model_->num == -1) {
+        ESP_LOGE(TAG, "Failed to initialize wakenet model");
+        return;
+    }
+    if(wakenet_model_->num > 1) {
+        ESP_LOGW(TAG, "More than one model found, using the first one");
+    } else if (wakenet_model_->num == 0) {
+        ESP_LOGE(TAG, "No model found");
+        return;
+    }
+    char *model_name = wakenet_model_->model_name[0];
+    wakenet_iface_ = (esp_wn_iface_t*)esp_wn_handle_from_name(model_name);
+    wakenet_data_ = wakenet_iface_->create(model_name, DET_MODE_95);
+
+    int frequency = wakenet_iface_->get_samp_rate(wakenet_data_);
+    int audio_chunksize = wakenet_iface_->get_samp_chunksize(wakenet_data_);
+    ESP_LOGI(TAG, "Wake word(%s),freq: %d, chunksize: %d", model_name, frequency, audio_chunksize);
+}
+
+void EspWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
+    wake_word_detected_callback_ = callback;
+}
+
+void EspWakeWord::StartDetection() {
+    xEventGroupSetBits(event_group_, DETECTION_RUNNING_EVENT);
+}
+
+void EspWakeWord::StopDetection() {
+    xEventGroupClearBits(event_group_, DETECTION_RUNNING_EVENT);
+}
+
+bool EspWakeWord::IsDetectionRunning() {
+    return xEventGroupGetBits(event_group_) & DETECTION_RUNNING_EVENT;
+}
+
+void EspWakeWord::Feed(const std::vector<int16_t>& data) {
+    int res = wakenet_iface_->detect(wakenet_data_, (int16_t *)data.data());
+    if (res > 0) {
+        StopDetection();
+        last_detected_wake_word_ = wakenet_iface_->get_word_name(wakenet_data_, res);
+
+        if (wake_word_detected_callback_) {
+            wake_word_detected_callback_(last_detected_wake_word_);
+        }
+    }
+}
+
+size_t EspWakeWord::GetFeedSize() {
+    if (wakenet_data_ == nullptr) {
+        return 0;
+    }
+    return wakenet_iface_->get_samp_chunksize(wakenet_data_) * codec_->input_channels();
+}
+
+void EspWakeWord::EncodeWakeWordData() {
+}
+
+bool EspWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
+    return false;
+}

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/esp_wake_word.h

@@ -0,0 +1,49 @@
+#ifndef ESP_WAKE_WORD_H
+#define ESP_WAKE_WORD_H
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/task.h>
+#include <freertos/event_groups.h>
+
+#include <esp_wn_iface.h>
+#include <esp_wn_models.h>
+#include <model_path.h>
+
+#include <list>
+#include <string>
+#include <vector>
+#include <functional>
+#include <mutex>
+#include <condition_variable>
+
+#include "audio_codec.h"
+#include "wake_word.h"
+
+class EspWakeWord : public WakeWord {
+public:
+    EspWakeWord();
+    ~EspWakeWord();
+
+    void Initialize(AudioCodec* codec);
+    void Feed(const std::vector<int16_t>& data);
+    void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback);
+    void StartDetection();
+    void StopDetection();
+    bool IsDetectionRunning();
+    size_t GetFeedSize();
+    void EncodeWakeWordData();
+    bool GetWakeWordOpus(std::vector<uint8_t>& opus);
+    const std::string& GetLastDetectedWakeWord() const { return last_detected_wake_word_; }
+
+private:
+    esp_wn_iface_t *wakenet_iface_ = nullptr;
+    model_iface_data_t *wakenet_data_ = nullptr;
+    srmodel_list_t *wakenet_model_ = nullptr;
+    EventGroupHandle_t event_group_;
+    AudioCodec* codec_ = nullptr;
+
+    std::function<void(const std::string& wake_word)> wake_word_detected_callback_;
+    std::string last_detected_wake_word_;
+};
+
+#endif

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/no_audio_processor.cc

@@ -0,0 +1,50 @@
+#include "no_audio_processor.h"
+#include <esp_log.h>
+
+#define TAG "NoAudioProcessor"
+
+void NoAudioProcessor::Initialize(AudioCodec* codec) {
+    codec_ = codec;
+}
+
+void NoAudioProcessor::Feed(const std::vector<int16_t>& data) {
+    if (!is_running_ || !output_callback_) {
+        return;
+    }
+    // 直接将输入数据传递给输出回调
+    output_callback_(std::vector<int16_t>(data));
+}
+
+void NoAudioProcessor::Start() {
+    is_running_ = true;
+}
+
+void NoAudioProcessor::Stop() {
+    is_running_ = false;
+}
+
+bool NoAudioProcessor::IsRunning() {
+    return is_running_;
+}
+
+void NoAudioProcessor::OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) {
+    output_callback_ = callback;
+}
+
+void NoAudioProcessor::OnVadStateChange(std::function<void(bool speaking)> callback) {
+    vad_state_change_callback_ = callback;
+}
+
+size_t NoAudioProcessor::GetFeedSize() {
+    if (!codec_) {
+        return 0;
+    }
+    // 返回一个固定的帧大小，比如 30ms 的数据
+    return 30 * codec_->input_sample_rate() / 1000;
+}
+
+void NoAudioProcessor::EnableDeviceAec(bool enable) {
+    if (enable) {
+        ESP_LOGE(TAG, "Device AEC is not supported");
+    }
+}

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/no_audio_processor.h

@@ -0,0 +1,32 @@
+#ifndef DUMMY_AUDIO_PROCESSOR_H
+#define DUMMY_AUDIO_PROCESSOR_H
+
+#include <vector>
+#include <functional>
+
+#include "audio_processor.h"
+#include "audio_codec.h"
+
+class NoAudioProcessor : public AudioProcessor {
+public:
+    NoAudioProcessor() = default;
+    ~NoAudioProcessor() = default;
+
+    void Initialize(AudioCodec* codec) override;
+    void Feed(const std::vector<int16_t>& data) override;
+    void Start() override;
+    void Stop() override;
+    bool IsRunning() override;
+    void OnOutput(std::function<void(std::vector<int16_t>&& data)> callback) override;
+    void OnVadStateChange(std::function<void(bool speaking)> callback) override;
+    size_t GetFeedSize() override;
+    void EnableDeviceAec(bool enable) override;
+
+private:
+    AudioCodec* codec_ = nullptr;
+    std::function<void(std::vector<int16_t>&& data)> output_callback_;
+    std::function<void(bool speaking)> vad_state_change_callback_;
+    bool is_running_ = false;
+};
+
+#endif 

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/no_wake_word.cc

@@ -0,0 +1,45 @@
+#include "no_wake_word.h"
+#include <esp_log.h>
+
+#define TAG "NoWakeWord"
+
+void NoWakeWord::Initialize(AudioCodec* codec) {
+    codec_ = codec;
+}
+
+void NoWakeWord::Feed(const std::vector<int16_t>& data) {
+    // Do nothing - no wake word processing
+}
+
+void NoWakeWord::OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) {
+    // Do nothing - no wake word processing
+}
+
+void NoWakeWord::StartDetection() {
+    // Do nothing - no wake word processing
+}
+
+void NoWakeWord::StopDetection() {
+    // Do nothing - no wake word processing
+}
+
+bool NoWakeWord::IsDetectionRunning() {
+    return false;  // No wake word processing
+}
+
+size_t NoWakeWord::GetFeedSize() {
+    return 0;  // No specific feed size requirement
+}
+
+void NoWakeWord::EncodeWakeWordData() {
+    // Do nothing - no encoding needed
+}
+
+bool NoWakeWord::GetWakeWordOpus(std::vector<uint8_t>& opus) {
+    opus.clear();
+    return false;  // No opus data available
+}
+
+const std::string& NoWakeWord::GetLastDetectedWakeWord() const {
+    return last_detected_wake_word_;
+}

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/no_wake_word.h

@@ -0,0 +1,32 @@
+#ifndef NO_WAKE_WORD_H
+#define NO_WAKE_WORD_H
+
+#include <vector>
+#include <functional>
+#include <string>
+
+#include "wake_word.h"
+#include "audio_codec.h"
+
+class NoWakeWord : public WakeWord {
+public:
+    NoWakeWord() = default;
+    ~NoWakeWord() = default;
+
+    void Initialize(AudioCodec* codec) override;
+    void Feed(const std::vector<int16_t>& data) override;
+    void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) override;
+    void StartDetection() override;
+    void StopDetection() override;
+    bool IsDetectionRunning() override;
+    size_t GetFeedSize() override;
+    void EncodeWakeWordData() override;
+    bool GetWakeWordOpus(std::vector<uint8_t>& opus) override;
+    const std::string& GetLastDetectedWakeWord() const override;
+
+private:
+    AudioCodec* codec_ = nullptr;
+    std::string last_detected_wake_word_;
+};
+
+#endif 

1-Demo/IDF-DEMO/xiaozhi-esp32/main/audio_processing/wake_word.h

@@ -0,0 +1,26 @@
+#ifndef WAKE_WORD_H
+#define WAKE_WORD_H
+
+#include <string>
+#include <vector>
+#include <functional>
+
+#include "audio_codec.h"
+
+class WakeWord {
+public:
+    virtual ~WakeWord() = default;
+    
+    virtual void Initialize(AudioCodec* codec) = 0;
+    virtual void Feed(const std::vector<int16_t>& data) = 0;
+    virtual void OnWakeWordDetected(std::function<void(const std::string& wake_word)> callback) = 0;
+    virtual void StartDetection() = 0;
+    virtual void StopDetection() = 0;
+    virtual bool IsDetectionRunning() = 0;
+    virtual size_t GetFeedSize() = 0;
+    virtual void EncodeWakeWordData() = 0;
+    virtual bool GetWakeWordOpus(std::vector<uint8_t>& opus) = 0;
+    virtual const std::string& GetLastDetectedWakeWord() const = 0;
+};
+
+#endif