From 5a7f77bc92c2179281531f55fc7be630976d88c8 Mon Sep 17 00:00:00 2001 From: Marco Romano Date: Thu, 19 Oct 2023 13:32:43 +0200 Subject: [PATCH] Extract more content from audio messages. (#1607) `TimelineItemAudioContent`: - Use `java.time.Duration` instead of milliseconds. This will ease up things in the future because currently milliseconds are sent over the wire but in the future seconds will be sent (as per the stable MSC). Using `Duration` will allow our downstream code to be independent of what's passed over the wire. - Rename `audioSource` property to `mediaSource` to better match its type. `AudioMessageType`: - Add and populate new fields `details` and `isVoiceMessage` to be used by voice messages. --- .../messages/impl/MessagesFlowNode.kt | 2 +- .../TimelineItemContentMessageFactory.kt | 7 +++-- .../model/event/TimelineItemAudioContent.kt | 5 ++-- .../event/TimelineItemAudioContentProvider.kt | 5 ++-- .../DefaultRoomLastMessageFormatterTest.kt | 2 +- .../matrix/api/media/AudioDetails.kt | 24 +++++++++++++++ .../api/timeline/item/event/MessageType.kt | 5 +++- .../matrix/impl/media/AudioDetails.kt | 30 +++++++++++++++++++ .../timeline/item/event/EventMessageMapper.kt | 8 ++++- 9 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt create mode 100644 libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt index fcb2e7e5e8..21e384906e 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt @@ -238,7 +238,7 @@ class MessagesFlowNode @AssistedInject constructor( backstack.push(navTarget) } is TimelineItemAudioContent -> { - val mediaSource = event.content.audioSource + val mediaSource = event.content.mediaSource val navTarget = NavTarget.MediaViewer( mediaInfo = MediaInfo( name = event.content.body, diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt index ae2ea4f350..323f110f47 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt @@ -40,6 +40,7 @@ import io.element.android.libraries.matrix.api.timeline.item.event.NoticeMessage import io.element.android.libraries.matrix.api.timeline.item.event.TextMessageType import io.element.android.libraries.matrix.api.timeline.item.event.UnknownMessageType import io.element.android.libraries.matrix.api.timeline.item.event.VideoMessageType +import java.time.Duration import javax.inject.Inject class TimelineItemContentMessageFactory @Inject constructor( @@ -103,11 +104,11 @@ class TimelineItemContentMessageFactory @Inject constructor( } is AudioMessageType -> TimelineItemAudioContent( body = messageType.body, - audioSource = messageType.source, - duration = messageType.info?.duration?.toMillis() ?: 0L, + mediaSource = messageType.source, + duration = messageType.info?.duration ?: Duration.ZERO, mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream, formattedFileSize = fileSizeFormatter.format(messageType.info?.size ?: 0), - fileExtension = fileExtensionExtractor.extractFromName(messageType.body) + fileExtension = fileExtensionExtractor.extractFromName(messageType.body), ) is FileMessageType -> { val fileExtension = fileExtensionExtractor.extractFromName(messageType.body) diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContent.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContent.kt index 485b863170..9d9a41e0e3 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContent.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContent.kt @@ -18,11 +18,12 @@ package io.element.android.features.messages.impl.timeline.model.event import io.element.android.features.messages.impl.media.helper.formatFileExtensionAndSize import io.element.android.libraries.matrix.api.media.MediaSource +import java.time.Duration data class TimelineItemAudioContent( val body: String, - val duration: Long, - val audioSource: MediaSource, + val duration: Duration, + val mediaSource: MediaSource, val mimeType: String, val formattedFileSize: String, val fileExtension: String, diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContentProvider.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContentProvider.kt index ed424781f8..06cb53b6fe 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContentProvider.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/model/event/TimelineItemAudioContentProvider.kt @@ -19,6 +19,7 @@ package io.element.android.features.messages.impl.timeline.model.event import androidx.compose.ui.tooling.preview.PreviewParameterProvider import io.element.android.libraries.core.mimetype.MimeTypes import io.element.android.libraries.matrix.api.media.MediaSource +import java.time.Duration open class TimelineItemAudioContentProvider : PreviewParameterProvider { override val values: Sequence @@ -34,6 +35,6 @@ fun aTimelineItemAudioContent(fileName: String = "A sound.mp3") = TimelineItemAu mimeType = MimeTypes.Pdf, formattedFileSize = "100kB", fileExtension = "mp3", - duration = 100, - audioSource = MediaSource(""), + duration = Duration.ofMillis(100), + mediaSource = MediaSource(""), ) diff --git a/libraries/eventformatter/impl/src/test/kotlin/io/element/android/libraries/eventformatter/impl/DefaultRoomLastMessageFormatterTest.kt b/libraries/eventformatter/impl/src/test/kotlin/io/element/android/libraries/eventformatter/impl/DefaultRoomLastMessageFormatterTest.kt index 4c26fcb3c7..50d313f132 100644 --- a/libraries/eventformatter/impl/src/test/kotlin/io/element/android/libraries/eventformatter/impl/DefaultRoomLastMessageFormatterTest.kt +++ b/libraries/eventformatter/impl/src/test/kotlin/io/element/android/libraries/eventformatter/impl/DefaultRoomLastMessageFormatterTest.kt @@ -161,7 +161,7 @@ class DefaultRoomLastMessageFormatterTest { val sharedContentMessagesTypes = arrayOf( TextMessageType(body, null), VideoMessageType(body, MediaSource("url"), null), - AudioMessageType(body, MediaSource("url"), null), + AudioMessageType(body, MediaSource("url"), null, null, false), ImageMessageType(body, MediaSource("url"), null), FileMessageType(body, MediaSource("url"), null), LocationMessageType(body, "geo:1,2", null), diff --git a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt new file mode 100644 index 0000000000..f8cd2d3fb4 --- /dev/null +++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.matrix.api.media + +import java.time.Duration + +data class AudioDetails( + val duration: Duration, + val waveform: List, +) diff --git a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/MessageType.kt b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/MessageType.kt index dc06d5c94a..ba6eeca819 100644 --- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/MessageType.kt +++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/timeline/item/event/MessageType.kt @@ -16,6 +16,7 @@ package io.element.android.libraries.matrix.api.timeline.item.event +import io.element.android.libraries.matrix.api.media.AudioDetails import io.element.android.libraries.matrix.api.media.AudioInfo import io.element.android.libraries.matrix.api.media.FileInfo import io.element.android.libraries.matrix.api.media.ImageInfo @@ -46,7 +47,9 @@ data class LocationMessageType( data class AudioMessageType( val body: String, val source: MediaSource, - val info: AudioInfo? + val info: AudioInfo?, + val details: AudioDetails?, + val isVoiceMessage: Boolean, ) : MessageType data class VideoMessageType( diff --git a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt new file mode 100644 index 0000000000..c3fa11e40c --- /dev/null +++ b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.matrix.impl.media + +import io.element.android.libraries.matrix.api.media.AudioDetails +import org.matrix.rustcomponents.sdk.UnstableAudioDetailsContent as RustAudioDetails + +fun RustAudioDetails.map(): AudioDetails = AudioDetails( + duration = duration, + waveform = waveform.map { it.toInt() }, +) + +fun AudioDetails.map(): RustAudioDetails = RustAudioDetails( + duration = duration, + waveform = waveform.map { it.toUShort() } +) diff --git a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/timeline/item/event/EventMessageMapper.kt b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/timeline/item/event/EventMessageMapper.kt index 0a59cfddab..18d2e1bdeb 100644 --- a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/timeline/item/event/EventMessageMapper.kt +++ b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/timeline/item/event/EventMessageMapper.kt @@ -75,7 +75,13 @@ class EventMessageMapper { fun mapMessageType(type: RustMessageType?) = when (type) { is RustMessageType.Audio -> { - AudioMessageType(type.content.body, type.content.source.map(), type.content.info?.map()) + AudioMessageType( + body = type.content.body, + source = type.content.source.map(), + info = type.content.info?.map(), + details = type.content.audio?.map(), + isVoiceMessage = type.content.voice != null, + ) } is RustMessageType.File -> { FileMessageType(type.content.body, type.content.source.map(), type.content.info?.map())