From e530f814e55025c92e47d806f50af08f7cd42d1c Mon Sep 17 00:00:00 2001 From: aaravlu Date: Mon, 17 Feb 2025 19:06:27 +0800 Subject: [PATCH 01/32] Remove dont support audio prompt --- src/home/room_screen.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index 00138b50..7b3f16c9 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -3630,7 +3630,7 @@ fn populate_audio_message_content( message_content_widget.show_html( cx, - format!("Audio: {filename}{mime}{duration}{size}{caption}
→ Audio playback not yet supported."), + format!("Audio: {filename}{mime}{duration}{size}{caption}
→"), ); true } @@ -4319,7 +4319,7 @@ impl Widget for Message { false } }; - + let message_view_area = self.view.area(); let hit = event.hits_with_mark_as_handled_fn( cx, From 3c26329581985bd6a9bbaa26dd16ee6628c8436f Mon Sep 17 00:00:00 2001 From: aaravlu Date: Mon, 17 Feb 2025 19:14:04 +0800 Subject: [PATCH 02/32] Remove TODO prompt & add basic --- src/home/room_screen.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index 7b3f16c9..e62a5be3 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -3242,6 +3242,7 @@ fn populate_message_view( cx, &item.html_or_plaintext(id!(content.message)), audio, + media_cache ); (item, false) } @@ -3602,7 +3603,9 @@ fn populate_audio_message_content( cx: &mut Cx, message_content_widget: &HtmlOrPlaintextRef, audio: &AudioMessageEventContent, + media_cache: &mut MediaCache ) -> bool { + let mut fully_drawn = false; // Display the file name, human-readable size, caption, and a button to download it. let filename = audio.filename(); let (duration, mime, size) = audio @@ -3625,14 +3628,30 @@ fn populate_audio_message_content( .map(|fb| format!("
{}", fb.body)) .or_else(|| audio.caption().map(|c| format!("
{c}"))) .unwrap_or_default(); - - // TODO: add an audio to play the audio file - message_content_widget.show_html( cx, format!("Audio: {filename}{mime}{duration}{size}{caption}
→"), ); - true + match audio.source.clone() { + MediaSource::Plain(mxc_uri) => { + match media_cache.try_get_media_or_fetch(mxc_uri, None) { + MediaCacheEntry::Requested => { + + }, + MediaCacheEntry::Loaded(data) => { + + }, + MediaCacheEntry::Failed => { + + } + } + } + MediaSource::Encrypted(e) => { + + } + } + + fully_drawn } From 6f1704a5f519e1416bc530c7f75758470339fdd3 Mon Sep 17 00:00:00 2001 From: aaravlu Date: Mon, 17 Feb 2025 19:15:20 +0800 Subject: [PATCH 03/32] Add svg file 'download' --- resources/icons/download.svg | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 resources/icons/download.svg diff --git a/resources/icons/download.svg b/resources/icons/download.svg new file mode 100644 index 00000000..3a3d31e4 --- /dev/null +++ b/resources/icons/download.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file From 7fffb2135ffd2adf0e4d6838567ca59b115cd0b7 Mon Sep 17 00:00:00 2001 From: aaravlu Date: Tue, 18 Feb 2025 11:25:39 +0800 Subject: [PATCH 04/32] Add audio player base --- resources/icons/pause.svg | 19 +++++++++++++++ resources/icons/play.svg | 17 +++++++++++++ src/home/room_screen.rs | 28 ++++++++++++++++++---- src/shared/audio_player.rs | 49 ++++++++++++++++++++++++++++++++++++++ src/shared/mod.rs | 2 ++ src/shared/styles.rs | 2 ++ 6 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 resources/icons/pause.svg create mode 100644 resources/icons/play.svg create mode 100644 src/shared/audio_player.rs diff --git a/resources/icons/pause.svg b/resources/icons/pause.svg new file mode 100644 index 00000000..b53765ee --- /dev/null +++ b/resources/icons/pause.svg @@ -0,0 +1,19 @@ + + + + + pause [#1006] + Created with Sketch. + + + + + + + + + + + + + \ No newline at end of file diff --git a/resources/icons/play.svg b/resources/icons/play.svg new file mode 100644 index 00000000..99e774c6 --- /dev/null +++ b/resources/icons/play.svg @@ -0,0 +1,17 @@ + + + + + play + Created with Sketch Beta. + + + + + + + + + + + \ No newline at end of file diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index e62a5be3..8e7a3917 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -50,6 +50,7 @@ live_design! { use crate::shared::search_bar::SearchBar; use crate::shared::avatar::Avatar; use crate::shared::text_or_image::TextOrImage; + use crate::shared::audio_player::AudioPlayer; use crate::shared::html_or_plaintext::*; use crate::shared::icon_button::*; use crate::home::room_read_receipt::*; @@ -433,6 +434,24 @@ live_design! { } } + AudioMessage = { + body = { + content = { + width: Fill, + height: Fit + padding: { left: 10.0 } + v = { + width: Fill, + height: Fit, + flow: Right, + reaction_list = { } + avatar_row = {} + audio_player = {} + } + } + } + } + // The view used for a condensed image message that came right after another message // from the same sender, and thus doesn't need to display the sender's profile again. // This excludes stickers and other animated GIFs, video clips, audio clips, etc. @@ -600,6 +619,7 @@ live_design! { Message = {} CondensedMessage = {} ImageMessage = {} + AudioMessage = {} CondensedImageMessage = {} SmallStateEvent = {} Empty = {} @@ -3605,7 +3625,7 @@ fn populate_audio_message_content( audio: &AudioMessageEventContent, media_cache: &mut MediaCache ) -> bool { - let mut fully_drawn = false; + let mut _fully_drawn = false; // Display the file name, human-readable size, caption, and a button to download it. let filename = audio.filename(); let (duration, mime, size) = audio @@ -3638,7 +3658,7 @@ fn populate_audio_message_content( MediaCacheEntry::Requested => { }, - MediaCacheEntry::Loaded(data) => { + MediaCacheEntry::Loaded(_data) => { }, MediaCacheEntry::Failed => { @@ -3646,12 +3666,12 @@ fn populate_audio_message_content( } } } - MediaSource::Encrypted(e) => { + MediaSource::Encrypted(_e) => { } } - fully_drawn + _fully_drawn } diff --git a/src/shared/audio_player.rs b/src/shared/audio_player.rs new file mode 100644 index 00000000..cc35ee53 --- /dev/null +++ b/src/shared/audio_player.rs @@ -0,0 +1,49 @@ +use makepad_widgets::*; + +live_design! { + use link::theme::*; + use link::shaders::*; + use link::widgets::*; + + use crate::shared::styles::*; + + pub AudioPlayer = {{AudioPlayer}} { + width: Fill, height: Fit, + flow: Overlay, + play_or_pause_button = { + width: Fit, + height: Fit, + margin: {left: 0, top: 4, bottom: 4, right: 4}, + padding: 8, + align: {x: 0.5, y: 0.5} + draw_icon: { + svg_file: (ICON_PLAY), + fn get_color(self) -> vec4 { + return #x888; + } + } + icon_walk: {width: 12, height: 12} + } + } +} + + +/// A view that holds an image or text content, and can switch between the two. +/// +/// This is useful for displaying alternate text when an image is not (yet) available +/// or fails to load. It can also be used to display a loading message while an image +/// is being fetched. +#[derive(Live, Widget, LiveHook)] +pub struct AudioPlayer { + #[deref] view: View, +} + +impl Widget for AudioPlayer { + fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) { + self.view.handle_event(cx, event, scope); + } + + fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep { + self.view.draw_walk(cx, scope, walk) + } +} diff --git a/src/shared/mod.rs b/src/shared/mod.rs index e517cb66..383288f6 100644 --- a/src/shared/mod.rs +++ b/src/shared/mod.rs @@ -12,6 +12,7 @@ pub mod text_or_image; pub mod typing_animation; pub mod popup_list; pub mod verification_badge; +pub mod audio_player; pub fn live_design(cx: &mut Cx) { // Order matters here, as some widget definitions depend on others. @@ -27,4 +28,5 @@ pub fn live_design(cx: &mut Cx) { popup_list::live_design(cx); verification_badge::live_design(cx); color_tooltip::live_design(cx); + audio_player::live_design(cx); } diff --git a/src/shared/styles.rs b/src/shared/styles.rs index ac2cba47..41bfc601 100644 --- a/src/shared/styles.rs +++ b/src/shared/styles.rs @@ -19,6 +19,8 @@ live_design! { pub ICON_SEND = dep("crate://self/resources/icon_send.svg") pub ICON_TRASH = dep("crate://self/resources/icons/trash.svg") pub ICON_VIEW_SOURCE = dep("crate://self/resources/icons/view_source.svg") + pub ICON_PLAY = dep("crate://self/resources/icons/play.svg") + pub ICON_PAUSE = dep("crate://self/resources/icons/pause.svg") pub TITLE_TEXT = { font_size: (13), From 7402dd48711aa4ed1f9af2386d3213d647cd72f4 Mon Sep 17 00:00:00 2001 From: aaravlu Date: Tue, 18 Feb 2025 15:30:26 +0800 Subject: [PATCH 05/32] Change message in timeline --- src/home/room_screen.rs | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index 8e7a3917..b62a956c 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -25,7 +25,7 @@ use crate::{ user_profile::{AvatarState, ShowUserProfileAction, UserProfile, UserProfileAndRoomId, UserProfilePaneInfo, UserProfileSlidingPaneRef, UserProfileSlidingPaneWidgetExt}, user_profile_cache, }, shared::{ - avatar::AvatarWidgetRefExt, html_or_plaintext::{HtmlOrPlaintextRef, HtmlOrPlaintextWidgetRefExt}, jump_to_bottom_button::{JumpToBottomButtonWidgetExt, UnreadMessageCount}, popup_list::enqueue_popup_notification, text_or_image::{TextOrImageRef, TextOrImageWidgetRefExt}, typing_animation::TypingAnimationWidgetExt + audio_player::{AudioPlayerRef, AudioPlayerWidgetRefExt}, avatar::AvatarWidgetRefExt, html_or_plaintext::{HtmlOrPlaintextRef, HtmlOrPlaintextWidgetRefExt}, jump_to_bottom_button::{JumpToBottomButtonWidgetExt, UnreadMessageCount}, popup_list::enqueue_popup_notification, text_or_image::{TextOrImageRef, TextOrImageWidgetRefExt}, typing_animation::TypingAnimationWidgetExt }, sliding_sync::{self, get_client, submit_async_request, take_timeline_endpoints, BackwardsPaginateUntilEventRequest, MatrixRequest, PaginationDirection, TimelineRequestSender, UserPowerLevels}, utils::{self, unix_time_millis_to_datetime, ImageFormat, MediaFormatConst, MEDIA_THUMBNAIL_FORMAT}, }; use crate::home::event_reaction_list::ReactionListWidgetRefExt; @@ -3249,18 +3249,14 @@ fn populate_message_view( } MessageOrStickerType::Audio(audio) => { has_html_body = audio.formatted.as_ref().is_some_and(|f| f.format == MessageFormat::Html); - let template = if use_compact_view { - live_id!(CondensedMessage) - } else { - live_id!(Message) - }; + let template = live_id!(AudioMessage); let (item, existed) = list.item_with_existed(cx, item_id, template); if existed && item_drawn_status.content_drawn { (item, true) } else { new_drawn_status.content_drawn = populate_audio_message_content( cx, - &item.html_or_plaintext(id!(content.message)), + &item.audio_player(id!(content.audio_player)), audio, media_cache ); @@ -3621,11 +3617,14 @@ fn populate_file_message_content( /// Returns whether the audio message content was fully drawn. fn populate_audio_message_content( cx: &mut Cx, - message_content_widget: &HtmlOrPlaintextRef, + audio_player: &AudioPlayerRef, audio: &AudioMessageEventContent, media_cache: &mut MediaCache ) -> bool { - let mut _fully_drawn = false; + if audio_player.is_empty() { + log!("Empty audio player"); + } + let mut fully_drawn = false; // Display the file name, human-readable size, caption, and a button to download it. let filename = audio.filename(); let (duration, mime, size) = audio @@ -3648,10 +3647,7 @@ fn populate_audio_message_content( .map(|fb| format!("
{}", fb.body)) .or_else(|| audio.caption().map(|c| format!("
{c}"))) .unwrap_or_default(); - message_content_widget.show_html( - cx, - format!("Audio: {filename}{mime}{duration}{size}{caption}
→"), - ); + audio_player.apply_over(cx, live! {}); match audio.source.clone() { MediaSource::Plain(mxc_uri) => { match media_cache.try_get_media_or_fetch(mxc_uri, None) { @@ -3659,7 +3655,7 @@ fn populate_audio_message_content( }, MediaCacheEntry::Loaded(_data) => { - + fully_drawn = true; }, MediaCacheEntry::Failed => { @@ -3671,7 +3667,7 @@ fn populate_audio_message_content( } } - _fully_drawn + fully_drawn } From acc3c3678474b49a9054af180bb95b3dcf3ac896 Mon Sep 17 00:00:00 2001 From: aaravlu Date: Tue, 18 Feb 2025 19:22:40 +0800 Subject: [PATCH 06/32] Finished Audio view base --- src/home/room_screen.rs | 46 ++++++++++++++++++++++++++++---------- src/shared/audio_player.rs | 31 ++++++++++++++----------- src/shared/styles.rs | 2 +- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index b62a956c..b91d67b8 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -371,7 +371,6 @@ live_design! { reaction_list = { } avatar_row = {} } - } } } @@ -412,16 +411,13 @@ live_design! { } } } - - // The view used for each static image-based message event in a room's timeline. - // This excludes stickers and other animated GIFs, video clips, audio clips, etc. - ImageMessage = { + AudioMessage = { body = { content = { width: Fill, height: Fit padding: { left: 10.0 } - message = { } + message = { } v = { width: Fill, height: Fit, @@ -430,28 +426,44 @@ live_design! { avatar_row = {} } } + } + } + CondensedAudioMessage = { + body = { + content = { + message = { } + { + width: Fill, + height: Fit + reaction_list = { } + avatar_row = {} + } + } } } - AudioMessage = { + // The view used for each static image-based message event in a room's timeline. + // This excludes stickers and other animated GIFs, video clips, audio clips, etc. + ImageMessage = { body = { content = { width: Fill, height: Fit padding: { left: 10.0 } + message = { } v = { width: Fill, height: Fit, flow: Right, reaction_list = { } avatar_row = {} - audio_player = {} } } } } + // The view used for a condensed image message that came right after another message // from the same sender, and thus doesn't need to display the sender's profile again. // This excludes stickers and other animated GIFs, video clips, audio clips, etc. @@ -618,9 +630,13 @@ live_design! { // Below, we must place all of the possible templates (views) that can be used in the portal list. Message = {} CondensedMessage = {} + ImageMessage = {} - AudioMessage = {} CondensedImageMessage = {} + + AudioMessage = {} + CondensedAudioMessage = {} + SmallStateEvent = {} Empty = {} DayDivider = {} @@ -3249,14 +3265,20 @@ fn populate_message_view( } MessageOrStickerType::Audio(audio) => { has_html_body = audio.formatted.as_ref().is_some_and(|f| f.format == MessageFormat::Html); - let template = live_id!(AudioMessage); + + let template = if use_compact_view { + live_id!(CondensedAudioMessage) + } else { + live_id!(AudioMessage) + }; + let (item, existed) = list.item_with_existed(cx, item_id, template); if existed && item_drawn_status.content_drawn { (item, true) } else { new_drawn_status.content_drawn = populate_audio_message_content( cx, - &item.audio_player(id!(content.audio_player)), + &item.audio_player(id!(content.message)), audio, media_cache ); @@ -3647,6 +3669,7 @@ fn populate_audio_message_content( .map(|fb| format!("
{}", fb.body)) .or_else(|| audio.caption().map(|c| format!("
{c}"))) .unwrap_or_default(); + audio_player.apply_over(cx, live! {}); match audio.source.clone() { MediaSource::Plain(mxc_uri) => { @@ -3663,7 +3686,6 @@ fn populate_audio_message_content( } } MediaSource::Encrypted(_e) => { - } } diff --git a/src/shared/audio_player.rs b/src/shared/audio_player.rs index cc35ee53..c94fbfc5 100644 --- a/src/shared/audio_player.rs +++ b/src/shared/audio_player.rs @@ -10,20 +10,25 @@ live_design! { pub AudioPlayer = {{AudioPlayer}} { width: Fill, height: Fit, flow: Overlay, - play_or_pause_button = { - width: Fit, - height: Fit, - margin: {left: 0, top: 4, bottom: 4, right: 4}, - padding: 8, - align: {x: 0.5, y: 0.5} - draw_icon: { - svg_file: (ICON_PLAY), - fn get_color(self) -> vec4 { - return #x888; - } - } - icon_walk: {width: 12, height: 12} + show_bg: true + draw_bg: { + color: #AA22AA } + + // play_or_pause_button = { + // width: Fit, + // height: Fit, + // margin: {left: 0, top: 4, bottom: 4, right: 4}, + // padding: 8, + // align: {x: 0.5, y: 0.5} + // draw_icon: { + // svg_file: (ICON_PLAY), + // fn get_color(self) -> vec4 { + // return #x888; + // } + // } + // icon_walk: {width: 12, height: 12} + // } } } diff --git a/src/shared/styles.rs b/src/shared/styles.rs index 41bfc601..8fc760f0 100644 --- a/src/shared/styles.rs +++ b/src/shared/styles.rs @@ -19,7 +19,7 @@ live_design! { pub ICON_SEND = dep("crate://self/resources/icon_send.svg") pub ICON_TRASH = dep("crate://self/resources/icons/trash.svg") pub ICON_VIEW_SOURCE = dep("crate://self/resources/icons/view_source.svg") - pub ICON_PLAY = dep("crate://self/resources/icons/play.svg") + pub ICON_PLAY = dep("crate://self/resources/icons/play.svg") pub ICON_PAUSE = dep("crate://self/resources/icons/pause.svg") pub TITLE_TEXT = { From 9e632c045e3b1f9d3685f8481607db59c4220eb0 Mon Sep 17 00:00:00 2001 From: aaravlu Date: Wed, 19 Feb 2025 18:17:59 +0800 Subject: [PATCH 07/32] audio playing issues --- Cargo.lock | 343 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + src/home/room_screen.rs | 5 +- src/media_cache.rs | 4 +- src/shared/audio_player.rs | 141 ++++++++++++--- src/shared/icon_button.rs | 20 +++ 6 files changed, 484 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8db5340e..7f1a6420 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,28 @@ dependencies = [ "memchr", ] +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.6.0", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-build" version = "0.1.0" @@ -385,6 +407,24 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -519,6 +559,8 @@ version = "1.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -528,6 +570,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -589,6 +640,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.20" @@ -629,6 +691,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "claxon" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bfbf56724aa9eca8afa4fcfadeb479e722935bb2a0900c2d37e0cc477af0688" + [[package]] name = "colorchoice" version = "1.0.3" @@ -687,6 +755,49 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce857aa0b77d77287acc1ac3e37a05a8c95a2af3647d23b15f263bdaeb7562b" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.14" @@ -787,6 +898,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-url" version = "0.3.1" @@ -945,6 +1062,15 @@ dependencies = [ "phf", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -1215,6 +1341,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "gloo-timers" version = "0.3.0" @@ -1312,6 +1444,12 @@ dependencies = [ "digest", ] +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.29.0" @@ -1778,6 +1916,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.72" @@ -1841,6 +1988,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lewton" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777b48df9aaab155475a83a7df3070395ea1ac6902f5cd062b8f2b028075c030" +dependencies = [ + "byteorder", + "ogg", + "tinyvec", +] + [[package]] name = "libc" version = "0.2.161" @@ -1921,6 +2079,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "macroific" version = "1.3.1" @@ -2520,6 +2687,12 @@ version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "515a63dc9666c865e848b043ab52fe9a5c713ae89cde4b5fbaae67cfd614b93a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2591,18 +2764,51 @@ dependencies = [ "libloading", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.6.0", + "jni-sys", + "log", + "ndk-sys", + "num_enum", + "thiserror", +] + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2619,6 +2825,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2638,6 +2855,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "objc-sys" version = "0.3.5" @@ -2704,6 +2942,38 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + +[[package]] +name = "ogg" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6951b4e8bf21c8193da321bcce9c9dd2e13c858fe078bf9054a288b419ae5d6e" +dependencies = [ + "byteorder", +] + [[package]] name = "ohos-sys" version = "0.2.2" @@ -3045,7 +3315,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", "socket2", "thiserror", @@ -3062,7 +3332,7 @@ dependencies = [ "bytes", "rand", "ring", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", "slab", "thiserror", @@ -3387,6 +3657,7 @@ dependencies = [ "robius-location", "robius-open", "robius-use-makepad", + "rodio", "serde", "serde_json", "tokio", @@ -3395,6 +3666,19 @@ dependencies = [ "url", ] +[[package]] +name = "rodio" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ceb6607dd738c99bc8cb28eff249b7cd5c8ec88b9db96c0608c1480d140fb1" +dependencies = [ + "claxon", + "cpal", + "hound", + "lewton", + "symphonia", +] + [[package]] name = "roxmltree" version = "0.20.0" @@ -3576,6 +3860,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.0.0" @@ -3945,6 +4235,55 @@ dependencies = [ "siphasher 1.0.1", ] +[[package]] +name = "symphonia" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "815c942ae7ee74737bb00f965fa5b5a2ac2ce7b6c01c0cc169bbeaf7abd5f5a9" +dependencies = [ + "lazy_static", + "symphonia-bundle-mp3", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c01c2aae70f0f1fb096b6f0ff112a930b1fb3626178fba3ae68b09dce71706d4" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "798306779e3dc7d5231bd5691f5a813496dc79d3f56bf82e25789f2094e022c3" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc622b9841a10089c5b18e99eb904f4341615d5aa55bbf4eedde1be721a4023c" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + [[package]] name = "syn" version = "2.0.87" diff --git a/Cargo.toml b/Cargo.toml index 08d9aaba..c8394700 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk", default-fe matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk", default-features = false, features = [ "rustls-tls" ] } rand = "0.8.5" rangemap = "1.5.0" +rodio = "0.20.1" serde = "1.0" serde_json = "1.0" tokio = { version = "1.33.0", features = ["macros", "rt-multi-thread"] } diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index b91d67b8..477e7be5 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -3643,6 +3643,7 @@ fn populate_audio_message_content( audio: &AudioMessageEventContent, media_cache: &mut MediaCache ) -> bool { + let _audio_player_uid = audio_player.widget_uid(); if audio_player.is_empty() { log!("Empty audio player"); } @@ -3670,14 +3671,14 @@ fn populate_audio_message_content( .or_else(|| audio.caption().map(|c| format!("
{c}"))) .unwrap_or_default(); - audio_player.apply_over(cx, live! {}); match audio.source.clone() { MediaSource::Plain(mxc_uri) => { match media_cache.try_get_media_or_fetch(mxc_uri, None) { MediaCacheEntry::Requested => { }, - MediaCacheEntry::Loaded(_data) => { + MediaCacheEntry::Loaded(data) => { + audio_player.set_data(data); fully_drawn = true; }, MediaCacheEntry::Failed => { diff --git a/src/media_cache.rs b/src/media_cache.rs index 9479d411..3902d374 100644 --- a/src/media_cache.rs +++ b/src/media_cache.rs @@ -5,7 +5,7 @@ use crate::{home::room_screen::TimelineUpdate, sliding_sync::{self, MatrixReques pub type MediaCacheEntryRef = Arc>; -/// An entry in the media cache. +/// An entry in the media cache. #[derive(Debug, Clone)] pub enum MediaCacheEntry { /// A request has been issued and we're waiting for it to complete. @@ -107,7 +107,7 @@ fn insert_into_cache>>( let new_value = match data { Ok(data) => { let data = data.into(); - + // debugging: dump out the media image to disk if false { if let MediaSource::Plain(mxc_uri) = _request.source { diff --git a/src/shared/audio_player.rs b/src/shared/audio_player.rs index c94fbfc5..bd340331 100644 --- a/src/shared/audio_player.rs +++ b/src/shared/audio_player.rs @@ -1,4 +1,7 @@ +use std::{collections::HashMap, io::Cursor, sync::Arc}; + use makepad_widgets::*; +use rodio::{cpal::Stream, Decoder, OutputStream, OutputStreamHandle, Sink}; live_design! { use link::theme::*; @@ -6,45 +9,76 @@ live_design! { use link::widgets::*; use crate::shared::styles::*; + use crate::shared::icon_button::*; pub AudioPlayer = {{AudioPlayer}} { width: Fill, height: Fit, - flow: Overlay, - show_bg: true - draw_bg: { - color: #AA22AA + flow: Down, + spacing: 20, + + audio_info =