speaker_media_player.cpp

Go to the documentation of this file.

#include "speaker_media_player.h"
 
#ifdef USE_ESP32
 
#include "esphome/core/log.h"
 
#include "esphome/components/audio/audio.h"
#ifdef USE_OTA
#include "esphome/components/ota/ota_backend.h"
#endif
 
namespace esphome {
namespace speaker {
 
// Framework:
//  - Media player that can handle two streams: one for media and one for announcements
//    - Each stream has an individual speaker component for output
//  - Each stream is handled by an ``AudioPipeline`` object with two parts/tasks
//    - ``AudioReader`` handles reading from an HTTP source or from a PROGMEM flash set at compile time
//    - ``AudioDecoder`` handles decoding the audio file. All formats are limited to two channels and 16 bits per sample
//      - FLAC
//      - MP3 (based on the libhelix decoder)
//      - WAV
//    - Each task runs until it is done processing the file or it receives a stop command
//    - Inter-task communication uses a FreeRTOS Event Group
//    - The ``AudioPipeline`` sets up a ring buffer between the reader and decoder tasks. The decoder task outputs audio
//      directly to a speaker component.
//    - The pipelines internal state needs to be processed by regularly calling ``process_state``.
//  - Generic media player commands are received by the ``control`` function. The commands are added to the
//    ``media_control_command_queue_`` to be processed in the component's loop
//    - Local file play back is initiatied with ``play_file`` and adds it to the ``media_control_command_queue_``
//    - Starting a stream intializes the appropriate pipeline or stops it if it is already running
//    - Volume and mute commands are achieved by the ``mute``, ``unmute``, ``set_volume`` functions.
//      - Volume commands are ignored if the media control queue is full to avoid crashing with rapid volume
//        increases/decreases.
//      - These functions all send the appropriate information to the speakers to implement.
//    - Pausing is implemented in the decoder task and is also sent directly to the media speaker component to decrease
//      latency.
//  - The components main loop performs housekeeping:
//    - It reads the media control queue and processes it directly
//    - It determines the overall state of the media player by considering the state of each pipeline
//      - announcement playback takes highest priority
//    - Handles playlists and repeating by starting the appropriate file when a previous file is finished
//  - Logging only happens in the main loop task to reduce task stack memory usage.
 
static const uint32_t MEDIA_CONTROLS_QUEUE_LENGTH = 20;
 
static const UBaseType_t MEDIA_PIPELINE_TASK_PRIORITY = 1;
static const UBaseType_t ANNOUNCEMENT_PIPELINE_TASK_PRIORITY = 1;
 
static const char *const TAG = "speaker_media_player";
 
void SpeakerMediaPlayer::setup() {
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
  state = media_player::MEDIA_PLAYER_STATE_OFF;
#else
  state = media_player::MEDIA_PLAYER_STATE_IDLE;
#endif
 
  this->media_control_command_queue_ = xQueueCreate(MEDIA_CONTROLS_QUEUE_LENGTH, sizeof(MediaCallCommand));
 
  this->pref_ = this->make_entity_preference<VolumeRestoreState>();
 
  VolumeRestoreState volume_restore_state;
  if (this->pref_.load(&volume_restore_state)) {
    this->set_volume_(volume_restore_state.volume);
    this->set_mute_state_(volume_restore_state.is_muted);
  } else {
    this->set_volume_(this->volume_initial_);
    this->set_mute_state_(false);
  }
 
#ifdef USE_OTA_STATE_LISTENER
  ota::get_global_ota_callback()->add_global_state_listener(this);
#endif
 
  this->announcement_pipeline_ =
      make_unique<AudioPipeline>(this->announcement_speaker_, this->buffer_size_, this->task_stack_in_psram_, "ann",
                                 ANNOUNCEMENT_PIPELINE_TASK_PRIORITY);
 
  if (this->announcement_pipeline_ == nullptr) {
    ESP_LOGE(TAG, "Failed to create announcement pipeline");
    this->mark_failed();
  }
 
  if (!this->single_pipeline_()) {
    this->media_pipeline_ = make_unique<AudioPipeline>(this->media_speaker_, this->buffer_size_,
                                                       this->task_stack_in_psram_, "med", MEDIA_PIPELINE_TASK_PRIORITY);
 
    if (this->media_pipeline_ == nullptr) {
      ESP_LOGE(TAG, "Failed to create media pipeline");
      this->mark_failed();
    }
  }
 
  ESP_LOGI(TAG, "Set up speaker media player");
}
 
void SpeakerMediaPlayer::set_playlist_delay_ms(AudioPipelineType pipeline_type, uint32_t delay_ms) {
  switch (pipeline_type) {
    case AudioPipelineType::ANNOUNCEMENT:
      this->announcement_playlist_delay_ms_ = delay_ms;
      break;
    case AudioPipelineType::MEDIA:
      this->media_playlist_delay_ms_ = delay_ms;
      break;
  }
}
 
void SpeakerMediaPlayer::stop_and_unpause_media_() {
  this->media_pipeline_->stop();
  this->unpause_media_remaining_ = 3;
  this->set_interval("unpause_med", 50, [this]() {
    if (this->media_pipeline_state_ == AudioPipelineState::STOPPED) {
      this->cancel_interval("unpause_med");
      this->media_pipeline_->set_pause_state(false);
      this->is_paused_ = false;
    } else if (--this->unpause_media_remaining_ == 0) {
      this->cancel_interval("unpause_med");
    }
  });
}
 
void SpeakerMediaPlayer::watch_media_commands_() {
  if (!this->is_ready()) {
    return;
  }
 
  MediaCallCommand media_command;
 
  if (xQueueReceive(this->media_control_command_queue_, &media_command, 0) == pdTRUE) {
    bool enqueue = media_command.enqueue.has_value() && media_command.enqueue.value();
 
    if (media_command.url.has_value() || media_command.file.has_value()) {
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
      if (this->state == media_player::MEDIA_PLAYER_STATE_OFF) {
        this->state = media_player::MEDIA_PLAYER_STATE_ON;
        publish_state();
      }
#endif
      PlaylistItem playlist_item;
      if (media_command.url.has_value()) {
        playlist_item.url = *media_command.url.value();
        delete media_command.url.value();
      }
      if (media_command.file.has_value()) {
        playlist_item.file = media_command.file;
      }
 
      if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) {
        if (!enqueue) {
          // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause
          this->cancel_timeout("next_ann");
          this->announcement_playlist_.clear();
          if (media_command.file.has_value()) {
            this->announcement_pipeline_->start_file(playlist_item.file.value());
          } else if (media_command.url.has_value()) {
            this->announcement_pipeline_->start_url(playlist_item.url.value());
          }
          this->announcement_pipeline_->set_pause_state(false);
        }
        this->announcement_playlist_.push_back(playlist_item);
      } else {
        if (!enqueue) {
          // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause
          this->cancel_timeout("next_media");
          this->media_playlist_.clear();
          if (this->is_paused_) {
            // If paused, stop the media pipeline and unpause it after confirming its stopped. This avoids playing a
            // short segment of the paused file before starting the new one.
            this->stop_and_unpause_media_();
          } else {
            // Not paused, just directly start the file
            if (media_command.file.has_value()) {
              this->media_pipeline_->start_file(playlist_item.file.value());
            } else if (media_command.url.has_value()) {
              this->media_pipeline_->start_url(playlist_item.url.value());
            }
            this->media_pipeline_->set_pause_state(false);
            this->is_paused_ = false;
          }
        }
        this->media_playlist_.push_back(playlist_item);
      }
 
      return;  // Don't process the new file play command further
    }
 
    if (media_command.volume.has_value()) {
      this->set_volume_(media_command.volume.value());
      this->publish_state();
    }
 
    if (media_command.command.has_value()) {
      switch (media_command.command.value()) {
        case media_player::MEDIA_PLAYER_COMMAND_PLAY:
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
          if (this->state == media_player::MEDIA_PLAYER_STATE_OFF) {
            this->state = media_player::MEDIA_PLAYER_STATE_ON;
            publish_state();
          }
#endif
          if ((this->media_pipeline_ != nullptr) && (this->is_paused_)) {
            this->media_pipeline_->set_pause_state(false);
          }
          this->is_paused_ = false;
          break;
        case media_player::MEDIA_PLAYER_COMMAND_PAUSE:
          if ((this->media_pipeline_ != nullptr) && (!this->is_paused_)) {
            this->media_pipeline_->set_pause_state(true);
          }
          this->is_paused_ = true;
          break;
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
        case media_player::MEDIA_PLAYER_COMMAND_TURN_ON:
          if (this->state == media_player::MEDIA_PLAYER_STATE_OFF) {
            this->state = media_player::MEDIA_PLAYER_STATE_ON;
            this->publish_state();
          }
          break;
        case media_player::MEDIA_PLAYER_COMMAND_TURN_OFF:
          this->is_turn_off_ = true;
          // Intentional Fall-through
#endif
        case media_player::MEDIA_PLAYER_COMMAND_STOP:
          // Pipelines do not stop immediately after calling the stop command, so confirm its stopped before unpausing.
          // This avoids an audible short segment playing after receiving the stop command in a paused state.
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
          if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value()) ||
              (this->is_turn_off_ && this->announcement_pipeline_state_ != AudioPipelineState::STOPPED)) {
#else
          if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) {
#endif
            if (this->announcement_pipeline_ != nullptr) {
              this->cancel_timeout("next_ann");
              this->announcement_playlist_.clear();
              this->announcement_pipeline_->stop();
              this->unpause_announcement_remaining_ = 3;
              this->set_interval("unpause_ann", 50, [this]() {
                if (this->announcement_pipeline_state_ == AudioPipelineState::STOPPED) {
                  this->cancel_interval("unpause_ann");
                  this->announcement_pipeline_->set_pause_state(false);
                } else if (--this->unpause_announcement_remaining_ == 0) {
                  this->cancel_interval("unpause_ann");
                }
              });
            }
          } else {
            if (this->media_pipeline_ != nullptr) {
              this->cancel_timeout("next_media");
              this->media_playlist_.clear();
              this->stop_and_unpause_media_();
            }
          }
 
          break;
        case media_player::MEDIA_PLAYER_COMMAND_TOGGLE:
          if (this->media_pipeline_ != nullptr) {
            if (this->is_paused_) {
              this->media_pipeline_->set_pause_state(false);
              this->is_paused_ = false;
            } else {
              this->media_pipeline_->set_pause_state(true);
              this->is_paused_ = true;
            }
          }
          break;
        case media_player::MEDIA_PLAYER_COMMAND_MUTE: {
          this->set_mute_state_(true);
 
          this->publish_state();
          break;
        }
        case media_player::MEDIA_PLAYER_COMMAND_UNMUTE:
          this->set_mute_state_(false);
          this->publish_state();
          break;
        case media_player::MEDIA_PLAYER_COMMAND_VOLUME_UP:
          this->set_volume_(std::min(1.0f, this->volume + this->volume_increment_));
          this->publish_state();
          break;
        case media_player::MEDIA_PLAYER_COMMAND_VOLUME_DOWN:
          this->set_volume_(std::max(0.0f, this->volume - this->volume_increment_));
          this->publish_state();
          break;
        case media_player::MEDIA_PLAYER_COMMAND_REPEAT_ONE:
          if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) {
            this->announcement_repeat_one_ = true;
          } else {
            this->media_repeat_one_ = true;
          }
          break;
        case media_player::MEDIA_PLAYER_COMMAND_REPEAT_OFF:
          if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) {
            this->announcement_repeat_one_ = false;
          } else {
            this->media_repeat_one_ = false;
          }
          break;
        case media_player::MEDIA_PLAYER_COMMAND_CLEAR_PLAYLIST:
          if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) {
            if (this->announcement_playlist_.empty()) {
              this->announcement_playlist_.resize(1);
            }
          } else {
            if (this->media_playlist_.empty()) {
              this->media_playlist_.resize(1);
            }
          }
          break;
        default:
          break;
      }
    }
  }
}
 
#ifdef USE_OTA_STATE_LISTENER
void SpeakerMediaPlayer::on_ota_global_state(ota::OTAState state, float progress, uint8_t error,
                                             ota::OTAComponent *comp) {
  if (state == ota::OTA_STARTED) {
    if (this->media_pipeline_ != nullptr) {
      this->media_pipeline_->suspend_tasks();
    }
    if (this->announcement_pipeline_ != nullptr) {
      this->announcement_pipeline_->suspend_tasks();
    }
  } else if (state == ota::OTA_ERROR) {
    if (this->media_pipeline_ != nullptr) {
      this->media_pipeline_->resume_tasks();
    }
    if (this->announcement_pipeline_ != nullptr) {
      this->announcement_pipeline_->resume_tasks();
    }
  }
}
#endif
 
void SpeakerMediaPlayer::loop() {
  this->watch_media_commands_();
 
  // Determine state of the media player
  media_player::MediaPlayerState old_state = this->state;
 
  AudioPipelineState old_media_pipeline_state = this->media_pipeline_state_;
  if (this->media_pipeline_ != nullptr) {
    this->media_pipeline_state_ = this->media_pipeline_->process_state();
  }
 
  if (this->media_pipeline_state_ == AudioPipelineState::ERROR_READING) {
    ESP_LOGE(TAG, "The media pipeline's file reader encountered an error.");
  } else if (this->media_pipeline_state_ == AudioPipelineState::ERROR_DECODING) {
    ESP_LOGE(TAG, "The media pipeline's audio decoder encountered an error.");
  }
 
  AudioPipelineState old_announcement_pipeline_state = this->announcement_pipeline_state_;
  if (this->announcement_pipeline_ != nullptr) {
    this->announcement_pipeline_state_ = this->announcement_pipeline_->process_state();
  }
 
  if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_READING) {
    ESP_LOGE(TAG, "The announcement pipeline's file reader encountered an error.");
  } else if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_DECODING) {
    ESP_LOGE(TAG, "The announcement pipeline's audio decoder encountered an error.");
  }
 
  if (this->announcement_pipeline_state_ != AudioPipelineState::STOPPED) {
    this->state = media_player::MEDIA_PLAYER_STATE_ANNOUNCING;
  } else {
    if (!this->announcement_playlist_.empty()) {
      uint32_t timeout_ms = 0;
      if (old_announcement_pipeline_state == AudioPipelineState::PLAYING) {
        // Finished the current announcement file
        if (!this->announcement_repeat_one_) {
          //  Pop item off the playlist if repeat is disabled
          this->announcement_playlist_.pop_front();
        }
        // Only delay starting playback if moving on the next playlist item or repeating the current item
        timeout_ms = this->announcement_playlist_delay_ms_;
      }
 
      if (!this->announcement_playlist_.empty()) {
        // Start the next announcement file
        PlaylistItem playlist_item = this->announcement_playlist_.front();
        if (playlist_item.url.has_value()) {
          this->announcement_pipeline_->start_url(playlist_item.url.value());
        } else if (playlist_item.file.has_value()) {
          this->announcement_pipeline_->start_file(playlist_item.file.value());
        }
 
        if (timeout_ms > 0) {
          // Pause pipeline internally to facilitate the delay between items
          this->announcement_pipeline_->set_pause_state(true);
          // Internally unpause the pipeline after the delay between playlist items. Announcements do not follow the
          // media player's pause state.
          this->set_timeout("next_ann", timeout_ms, [this]() { this->announcement_pipeline_->set_pause_state(false); });
        }
      }
    } else {
      if (this->is_paused_) {
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
        if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) {
          this->state = media_player::MEDIA_PLAYER_STATE_PAUSED;
        }
#else
        this->state = media_player::MEDIA_PLAYER_STATE_PAUSED;
#endif
      } else if (this->media_pipeline_state_ == AudioPipelineState::PLAYING) {
        this->state = media_player::MEDIA_PLAYER_STATE_PLAYING;
      } else if (this->media_pipeline_state_ == AudioPipelineState::STOPPED) {
        if (!media_playlist_.empty()) {
          uint32_t timeout_ms = 0;
          if (old_media_pipeline_state == AudioPipelineState::PLAYING) {
            // Finished the current media file
            if (!this->media_repeat_one_) {
              // Pop item off the playlist if repeat is disabled
              this->media_playlist_.pop_front();
            }
            // Only delay starting playback if moving on the next playlist item or repeating the current item
            timeout_ms = this->media_playlist_delay_ms_;
          }
          if (!this->media_playlist_.empty()) {
            PlaylistItem playlist_item = this->media_playlist_.front();
            if (playlist_item.url.has_value()) {
              this->media_pipeline_->start_url(playlist_item.url.value());
            } else if (playlist_item.file.has_value()) {
              this->media_pipeline_->start_file(playlist_item.file.value());
            }
 
            if (timeout_ms > 0) {
              // Pause pipeline internally to facilitate the delay between items
              this->media_pipeline_->set_pause_state(true);
              // Internally unpause the pipeline after the delay between playlist items, if the media player state is
              // not paused.
              this->set_timeout("next_media", timeout_ms,
                                [this]() { this->media_pipeline_->set_pause_state(this->is_paused_); });
            }
          }
        } else {
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
          if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) {
            this->state = media_player::MEDIA_PLAYER_STATE_IDLE;
          }
#else
          this->state = media_player::MEDIA_PLAYER_STATE_IDLE;
#endif
        }
      }
    }
  }
 
  if (this->state != old_state) {
    this->publish_state();
    ESP_LOGD(TAG, "State changed to %s", media_player::media_player_state_to_string(this->state));
  }
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
  if (this->is_turn_off_ && (this->state == media_player::MEDIA_PLAYER_STATE_PAUSED ||
                             this->state == media_player::MEDIA_PLAYER_STATE_IDLE)) {
    this->is_turn_off_ = false;
    if (this->state == media_player::MEDIA_PLAYER_STATE_PAUSED) {
      this->state = media_player::MEDIA_PLAYER_STATE_IDLE;
      this->publish_state();
      ESP_LOGD(TAG, "State changed to %s", media_player::media_player_state_to_string(this->state));
    }
    this->state = media_player::MEDIA_PLAYER_STATE_OFF;
    this->publish_state();
    ESP_LOGD(TAG, "State changed to %s", media_player::media_player_state_to_string(this->state));
  }
#endif
}
 
void SpeakerMediaPlayer::play_file(audio::AudioFile *media_file, bool announcement, bool enqueue) {
  if (!this->is_ready()) {
    // Ignore any commands sent before the media player is setup
    return;
  }
 
  MediaCallCommand media_command;
 
  media_command.file = media_file;
  if (this->single_pipeline_() || announcement) {
    media_command.announce = true;
  } else {
    media_command.announce = false;
  }
  media_command.enqueue = enqueue;
  xQueueSend(this->media_control_command_queue_, &media_command, portMAX_DELAY);
}
 
void SpeakerMediaPlayer::control(const media_player::MediaPlayerCall &call) {
  if (!this->is_ready()) {
    // Ignore any commands sent before the media player is setup
    return;
  }
 
  MediaCallCommand media_command;
 
  auto ann = call.get_announcement();
  if (this->single_pipeline_() || (ann.has_value() && *ann)) {
    media_command.announce = true;
  } else {
    media_command.announce = false;
  }
 
  auto media_url = call.get_media_url();
  if (media_url.has_value()) {
    media_command.url =
        new std::string(*media_url);  // Must be manually deleted after receiving media_command from a queue
 
    auto cmd = call.get_command();
    if (cmd.has_value()) {
      if (*cmd == media_player::MEDIA_PLAYER_COMMAND_ENQUEUE) {
        media_command.enqueue = true;
      }
    }
 
    xQueueSend(this->media_control_command_queue_, &media_command, portMAX_DELAY);
    return;
  }
 
  auto vol = call.get_volume();
  if (vol.has_value()) {
    media_command.volume = vol;
    // Wait 0 ticks for queue to be free, volume sets aren't that important!
    xQueueSend(this->media_control_command_queue_, &media_command, 0);
    return;
  }
 
  auto cmd = call.get_command();
  if (cmd.has_value()) {
    media_command.command = cmd;
    TickType_t ticks_to_wait = portMAX_DELAY;
    if ((*cmd == media_player::MEDIA_PLAYER_COMMAND_VOLUME_UP) ||
        (*cmd == media_player::MEDIA_PLAYER_COMMAND_VOLUME_DOWN)) {
      ticks_to_wait = 0;  // Wait 0 ticks for queue to be free, volume sets aren't that important!
    }
    xQueueSend(this->media_control_command_queue_, &media_command, ticks_to_wait);
    return;
  }
}
 
media_player::MediaPlayerTraits SpeakerMediaPlayer::get_traits() {
  auto traits = media_player::MediaPlayerTraits();
  if (!this->single_pipeline_()) {
    traits.set_supports_pause(true);
  }
#ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
  traits.set_supports_turn_off_on(true);
#endif
 
  if (this->announcement_format_.has_value()) {
    traits.get_supported_formats().push_back(this->announcement_format_.value());
  }
  if (this->media_format_.has_value()) {
    traits.get_supported_formats().push_back(this->media_format_.value());
  } else if (this->single_pipeline_() && this->announcement_format_.has_value()) {
    // Only one pipeline is defined, so use the announcement format (if configured) for the default purpose
    media_player::MediaPlayerSupportedFormat media_format = this->announcement_format_.value();
    media_format.purpose = media_player::MediaPlayerFormatPurpose::PURPOSE_DEFAULT;
    traits.get_supported_formats().push_back(media_format);
  }
 
  return traits;
};
 
void SpeakerMediaPlayer::save_volume_restore_state_() {
  VolumeRestoreState volume_restore_state;
  volume_restore_state.volume = this->volume;
  volume_restore_state.is_muted = this->is_muted_;
  this->pref_.save(&volume_restore_state);
}
 
void SpeakerMediaPlayer::set_mute_state_(bool mute_state) {
  if (this->media_speaker_ != nullptr) {
    this->media_speaker_->set_mute_state(mute_state);
  }
  if (this->announcement_speaker_ != nullptr) {
    this->announcement_speaker_->set_mute_state(mute_state);
  }
 
  bool old_mute_state = this->is_muted_;
  this->is_muted_ = mute_state;
 
  this->save_volume_restore_state_();
 
  if (old_mute_state != mute_state) {
    if (mute_state) {
      this->defer([this]() { this->mute_trigger_.trigger(); });
    } else {
      this->defer([this]() { this->unmute_trigger_.trigger(); });
    }
  }
}
 
void SpeakerMediaPlayer::set_volume_(float volume, bool publish) {
  // Remap the volume to fit with in the configured limits
  float bounded_volume = remap<float, float>(volume, 0.0f, 1.0f, this->volume_min_, this->volume_max_);
 
  if (this->media_speaker_ != nullptr) {
    this->media_speaker_->set_volume(bounded_volume);
  }
 
  if (this->announcement_speaker_ != nullptr) {
    this->announcement_speaker_->set_volume(bounded_volume);
  }
 
  if (publish) {
    this->volume = volume;
    this->save_volume_restore_state_();
  }
 
  // Turn on the mute state if the volume is effectively zero, off otherwise
  if (volume < 0.001) {
    this->set_mute_state_(true);
  } else {
    this->set_mute_state_(false);
  }
 
  this->defer([this, volume]() { this->volume_trigger_.trigger(volume); });
}
 
}  // namespace speaker
}  // namespace esphome
 
#endif