diff --git a/src/app.rs b/src/app.rs index 2f8949ee..3bc20ceb 100644 --- a/src/app.rs +++ b/src/app.rs @@ -20,6 +20,7 @@ live_design! { use crate::shared::popup_list::PopupList; use crate::home::new_message_context_menu::*; use crate::shared::callout_tooltip::CalloutTooltip; + use crate::audio::audio_controller::AudioController; APP_TAB_COLOR = #344054 @@ -126,6 +127,7 @@ live_design! { {} } } + audio_controller = {} // Context menus should be shown above other UI elements, // but beneath the verification modal. @@ -174,6 +176,7 @@ impl LiveRegister for App { crate::home::live_design(cx); crate::profile::live_design(cx); crate::login::live_design(cx); + crate::audio::live_design(cx); } } diff --git a/src/audio/audio_controller.rs b/src/audio/audio_controller.rs new file mode 100644 index 00000000..b2a0f4a6 --- /dev/null +++ b/src/audio/audio_controller.rs @@ -0,0 +1,306 @@ +//! Audio controller, which just manages audio playback. + +use std::{collections::{hash_map::Entry, HashMap}, sync::{Arc, LazyLock, Mutex, RwLock}}; +use makepad_widgets::*; +use matrix_sdk_ui::timeline::TimelineEventItemId; + +use super::audio_message_ui::AudioMessageUIAction; + +type Audios = HashMap>, Arc>)>; + +pub static AUDIO_SET: LazyLock> = LazyLock::new(||{ + RwLock::new(HashMap::new()) +}); + +live_design! { + use link::theme::*; + use link::shaders::*; + use link::widgets::*; + + use crate::shared::styles::*; + use crate::shared::icon_button::*; + + // width & height is 0 because it is just a template. + pub AudioController = {{AudioController}} { + width: 0., height: 0., + visible: false, + } +} + +#[derive(Debug, Clone, DefaultNone)] +pub enum AudioControllerAction { + UiToPause(TimelineEventItemId), + None, +} + +#[derive(Debug, Clone, Default)] +pub struct Audio { + pub data: Arc<[u8]>, + pub channels: u16, + pub bit_depth: u16 +} + +#[derive(Debug, Clone, Default)] +pub enum Selected { + Playing(TimelineEventItemId, usize), + Paused(TimelineEventItemId, usize), + #[default] + None, +} + +#[derive(Live, LiveHook, Widget)] +pub struct AudioController { + #[deref] view: View, + #[rust] audio: Arc>, + #[rust] selected: Arc>, +} + +impl Drop for AudioController { + fn drop(&mut self) { + // todo!() + } +} + +impl Widget for AudioController { + fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) { + self.match_event(cx, event); + self.view.handle_event(cx, event, scope); + } + + fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep { + self.view.draw_walk(cx, scope, walk) + } +} + +impl MatchEvent for AudioController { + fn handle_startup(&mut self, cx: &mut Cx) { + let audio = self.audio.clone(); + let selected = self.selected.clone(); + cx.audio_output(0, move |_audio_info, output_buffer|{ + let mut selected_mg = selected.lock().unwrap(); + let audio_mg = audio.lock().unwrap(); + if let Selected::Playing(ref id, mut pos) = selected_mg.clone() { + let audio = audio_mg.clone(); + match (audio.channels, audio.bit_depth) { + (2, 16) => { + // stereo 16bit + output_buffer.zero(); + let (left, right) = output_buffer.stereo_mut(); + let mut i = 0; + while i < left.len() { + let left_i16 = i16::from_le_bytes([audio.data[pos], audio.data[pos + 1]]); + let right_i16 = i16::from_le_bytes([audio.data[pos + 2], audio.data[pos + 3]]); + + left[i] = left_i16 as f32 / i16::MAX as f32; + right[i] = right_i16 as f32 / i16::MAX as f32; + pos += 4; + i += 1; + *selected_mg = Selected::Playing(id.clone(), pos); + } + } + (2, 24) => { + // stereo 24bit + output_buffer.zero(); + let (left, right) = output_buffer.stereo_mut(); + let mut i = 0; + while i < left.len() { + let left_i32 = i32::from_le_bytes([0, audio.data[pos], audio.data[pos + 1], audio.data[pos + 2]]); + let right_i32 = i32::from_le_bytes([0, audio.data[pos + 3], audio.data[pos + 4], audio.data[pos + 5]]); + + left[i] = left_i32 as f32 / i32::MAX as f32; + right[i] = right_i32 as f32 / i32::MAX as f32; + pos += 6; + i += 1; + *selected_mg = Selected::Playing(id.clone(), pos); + } + } + (2, 32) => { + // stereo 24bit + output_buffer.zero(); + let (left, right) = output_buffer.stereo_mut(); + let mut i = 0; + while i < left.len() { + let left_i32 = i32::from_le_bytes([audio.data[pos], audio.data[pos + 1], audio.data[pos + 2], audio.data[pos + 3]]); + let right_i32 = i32::from_le_bytes([audio.data[pos + 4], audio.data[pos + 5], audio.data[pos + 6], audio.data[pos + 7]]); + + left[i] = left_i32 as f32 / i32::MAX as f32; + right[i] = right_i32 as f32 / i32::MAX as f32; + pos += 8; + i += 1; + *selected_mg = Selected::Playing(id.clone(), pos); + } + } + _ => { } + } + if pos > audio.data.len() { + *selected_mg = Selected::None; + } + } else { + output_buffer.zero(); + } + }); + } + + fn handle_audio_devices(&mut self, cx: &mut Cx, e: &AudioDevicesEvent) { + cx.use_audio_outputs(&e.default_output()) + } + fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { + for action in actions { + match action.downcast_ref() { + Some(AudioMessageUIAction::Play(new_id)) => { + let selected =self.selected.clone().lock().unwrap().clone(); + match selected { + Selected::Playing(current_id, current_pos) => { + if current_id != new_id.clone() { + cx.action(AudioControllerAction::UiToPause(current_id.clone())); + if let Some((_audio, old_pos, old_playing_status)) = AUDIO_SET.read().unwrap().get(¤t_id) { + *old_pos.lock().unwrap() = current_pos; + *old_playing_status.lock().unwrap() = false; + } + if let Some((audio, new_pos, new_playing_status)) = AUDIO_SET.read().unwrap().get(new_id) { + *self.audio.lock().unwrap() = audio.clone(); + let new_pos_mg = new_pos.lock().unwrap(); + *new_playing_status.lock().unwrap() = true; + *self.selected.lock().unwrap() = Selected::Playing(new_id.clone(), *new_pos_mg); + } + } + } + Selected::Paused(current_id, current_pos) => { + if ¤t_id == new_id { + *self.selected.lock().unwrap() = Selected::Playing(new_id.clone(), current_pos); + } else { + if let Some((_audio, old_pos, old_playing_status)) = AUDIO_SET.read().unwrap().get(¤t_id) { + *old_playing_status.lock().unwrap() = false; + *old_pos.lock().unwrap() = current_pos; + } + + if let Some((audio, new_pos, new_playing_status)) = AUDIO_SET.read().unwrap().get(new_id) { + *self.audio.lock().unwrap() = audio.clone(); + let new_pos = *new_pos.lock().unwrap(); + *new_playing_status.lock().unwrap() = true; + *self.selected.lock().unwrap() = Selected::Playing(new_id.clone(), new_pos); + } + } + } + Selected::None => { + if let Some((audio, _new_pos, new_playing_status)) = AUDIO_SET.read().unwrap().get(new_id) { + *new_playing_status.lock().unwrap() = true; + *self.audio.lock().unwrap() = audio.clone(); + *self.selected.lock().unwrap() = Selected::Playing(new_id.clone(), 44); + } + } + } + } + Some(AudioMessageUIAction::Pause(new_id)) => { + let selected =self.selected.clone().lock().unwrap().clone(); + if let Selected::Playing(current_id, current_pos) = selected { + if ¤t_id == new_id { + if let Some((_audio, old_pos, old_playing_status)) = AUDIO_SET.write().unwrap().get(¤t_id) { + *old_playing_status.lock().unwrap() = false; + *old_pos.lock().unwrap() = current_pos; + } + *self.selected.lock().unwrap() = Selected::Paused(new_id.clone(), current_pos); + } + } + } + Some(AudioMessageUIAction::Stop(new_id)) => { + let selected =self.selected.clone().lock().unwrap().clone(); + match selected { + Selected::Playing(current_id, _current_pos) => { + if ¤t_id == new_id { + if let Some((_audio, old_pos, old_playing_status)) = AUDIO_SET.write().unwrap().get(¤t_id) { + *old_playing_status.lock().unwrap() = false; + *old_pos.lock().unwrap() = 44; + } + *self.selected.lock().unwrap() = Selected::None; + } + } + Selected::Paused(current_id, _current_pos) => { + if ¤t_id == new_id { + if let Some((_audio, old_pos, old_playing_status)) = AUDIO_SET.write().unwrap().get(¤t_id) { + *old_playing_status.lock().unwrap() = false; + *old_pos.lock().unwrap() = 44; + } + *self.selected.lock().unwrap() = Selected::None; + } + } + _ => { } + } + } + _ => { } + } + } + } +} + +pub fn insert_new_audio_or_get(timeline_audio_event_item_id: &TimelineEventItemId, data: Arc<[u8]>, channels: u16, bit_depth: u16) -> (Audio, Arc>, Arc>) { + match AUDIO_SET.write().unwrap().entry(timeline_audio_event_item_id.clone()) { + Entry::Vacant(v) => { + let audio = Audio { + data, + channels, + bit_depth + }; + let pos = Arc::new(Mutex::new(44)); + let is_playing = Arc::new(Mutex::new(false)); + v.insert((audio, pos, is_playing)).clone() + } + Entry::Occupied(o) => { + o.get().clone() + } + } +} + + +pub fn parse_wav(data: &[u8]) -> Option<(u16, u16)> { + // Check that the data length is sufficient, at least 44 bytes are required (standard WAV file header size) + if data.len() < 44 { + log!("Insufficient data length"); + return None; + } + + // Check if the first 4 bytes are 'RIFF'. + if &data[0..4] != b"RIFF" { + log!("Not a `RIFF` file"); + return None; + } + + // Check if bytes 8-11 are 'WAVE'. + if &data[8..12] != b"WAVE" { + log!("Not a `WAVE` file"); + return None; + } + + // Check if bytes 12-15 are 'fmt'. + if &data[12..16] != b"fmt " { + log!("`fmt` block not found"); + return None; + } + + // Read the size of the fmt block (bytes 16-19) + let fmt_size = u32::from_le_bytes([data[16], data[17], data[18], data[19]]); + if fmt_size < 16 { + log!("`fmt` block size too small"); + return None; + } + + // Read the audio format (bytes 20-21) and make sure it's PCM (value 1) + let audio_format = u16::from_le_bytes([data[20], data[21]]); + if audio_format != 1 { + log!("Not a `PCM` file"); + return None; + } + + // Extract the number of channels (bytes 22-23) + let channels = u16::from_le_bytes([data[22], data[23]]); + + // Extract sampling rate (bytes 24-27) + // let sample_rate = u32::from_le_bytes([data[24], data[25], data[26], data[27]]); + + // Extract bit depth (bytes 34-35) + let bit_depth = u16::from_le_bytes([data[34], data[35]]); + + // Return the parsing result + // Some((channels, sample_rate, bits_per_sample)) + Some((channels, bit_depth)) +} diff --git a/src/audio/audio_message_ui.rs b/src/audio/audio_message_ui.rs new file mode 100644 index 00000000..35915c4a --- /dev/null +++ b/src/audio/audio_message_ui.rs @@ -0,0 +1,220 @@ +//! Audio message in timeline. +//! It just manages UI. +//! Audio playback is in `audio_controller.rs`. + +use makepad_widgets::*; +use matrix_sdk_ui::timeline::TimelineEventItemId; + +use super::audio_controller::AudioControllerAction; + +live_design! { + use link::theme::*; + use link::shaders::*; + use link::widgets::*; + + use crate::shared::styles::*; + use crate::shared::icon_button::*; + + pub AudioMessageUI = {{AudioMessageUI}} { + width: Fill, height: Fit, + flow: Down, + + fetching_info =