From 0e4141863bfd8e662de510e757f7b3710d165f2d Mon Sep 17 00:00:00 2001 From: Marco Romano Date: Fri, 27 Oct 2023 14:49:58 +0200 Subject: [PATCH] Always treat waveform as List (#1663) [MSC3246](https://github.com/matrix-org/matrix-spec-proposals/pull/3246) specifies the waveform as a list of ints because: > Because floating point numbers are not allowed in Matrix events Though DSP on audio data is almost always done using their floating point representation. This PR brings the float<->int rescaling in the `matrix` module so that the application code can always work with float waveform samples. --- .../TimelineItemContentMessageFactory.kt | 3 +- .../impl/voicemessages/WaveformUtils.kt | 28 ------------------- .../composer/VoiceMessageComposerPresenter.kt | 3 +- .../matrix/api/media/AudioDetails.kt | 2 +- .../libraries/matrix/api/room/MatrixRoom.kt | 2 +- .../matrix/impl/media/AudioDetails.kt | 20 +++++++++++-- .../matrix/impl/room/RustMatrixRoom.kt | 5 ++-- .../matrix/test/room/FakeMatrixRoom.kt | 2 +- .../libraries/mediaupload/api/MediaSender.kt | 2 +- .../mediaupload/api/MediaUploadInfo.kt | 2 +- 10 files changed, 28 insertions(+), 41 deletions(-) delete mode 100644 features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/WaveformUtils.kt diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt index 3262653134..4f78c38393 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/factories/event/TimelineItemContentMessageFactory.kt @@ -29,7 +29,6 @@ import io.element.android.features.messages.impl.timeline.model.event.TimelineIt import io.element.android.features.messages.impl.timeline.model.event.TimelineItemVoiceContent import io.element.android.features.messages.impl.timeline.util.FileExtensionExtractor import io.element.android.features.messages.impl.timeline.util.toHtmlDocument -import io.element.android.features.messages.impl.voicemessages.fromMSC3246range import io.element.android.libraries.androidutils.filesize.FileSizeFormatter import io.element.android.libraries.core.mimetype.MimeTypes import io.element.android.libraries.featureflag.api.FeatureFlagService @@ -118,7 +117,7 @@ class TimelineItemContentMessageFactory @Inject constructor( mediaSource = messageType.source, duration = messageType.info?.duration ?: Duration.ZERO, mimeType = messageType.info?.mimetype ?: MimeTypes.OctetStream, - waveform = messageType.details?.waveform?.fromMSC3246range()?.toImmutableList() ?: persistentListOf(), + waveform = messageType.details?.waveform?.toImmutableList() ?: persistentListOf(), ) else -> TimelineItemAudioContent( body = messageType.body, diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/WaveformUtils.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/WaveformUtils.kt deleted file mode 100644 index 83926f0f22..0000000000 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/WaveformUtils.kt +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.element.android.features.messages.impl.voicemessages - -/** - * Resizes the given [0;1024] int list as per unstable MSC3246 spec - * to a [0;1] range float list to be used for waveform rendering. - */ -internal fun List.fromMSC3246range(): List = map { it / 1024f } - -/** - * Resizes the given [0;1] float list to [0;1024] int list as per unstable MSC3246 spec. - */ -internal fun List.toMSC3246range(): List = map { (it * 1024).toInt() } diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/composer/VoiceMessageComposerPresenter.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/composer/VoiceMessageComposerPresenter.kt index f3f86a7bf9..557b744d3b 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/composer/VoiceMessageComposerPresenter.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/composer/VoiceMessageComposerPresenter.kt @@ -28,7 +28,6 @@ import androidx.compose.runtime.setValue import androidx.core.net.toUri import androidx.lifecycle.Lifecycle import io.element.android.features.messages.impl.voicemessages.VoiceMessageException -import io.element.android.features.messages.impl.voicemessages.toMSC3246range import io.element.android.libraries.architecture.Presenter import io.element.android.libraries.di.RoomScope import io.element.android.libraries.di.SingleIn @@ -219,7 +218,7 @@ class VoiceMessageComposerPresenter @Inject constructor( val result = mediaSender.sendVoiceMessage( uri = file.toUri(), mimeType = mimeType, - waveForm = waveform.toMSC3246range(), + waveForm = waveform, ) if (result.isFailure) { diff --git a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt index f8cd2d3fb4..059369c5da 100644 --- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt +++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/media/AudioDetails.kt @@ -20,5 +20,5 @@ import java.time.Duration data class AudioDetails( val duration: Duration, - val waveform: List, + val waveform: List, ) diff --git a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/room/MatrixRoom.kt b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/room/MatrixRoom.kt index 23db1ef00f..1f2de7720f 100644 --- a/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/room/MatrixRoom.kt +++ b/libraries/matrix/api/src/main/kotlin/io/element/android/libraries/matrix/api/room/MatrixRoom.kt @@ -195,7 +195,7 @@ interface MatrixRoom : Closeable { suspend fun sendVoiceMessage( file: File, audioInfo: AudioInfo, - waveform: List, + waveform: List, progressCallback: ProgressCallback? ): Result diff --git a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt index c3fa11e40c..5bca137a85 100644 --- a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt +++ b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/media/AudioDetails.kt @@ -21,10 +21,26 @@ import org.matrix.rustcomponents.sdk.UnstableAudioDetailsContent as RustAudioDet fun RustAudioDetails.map(): AudioDetails = AudioDetails( duration = duration, - waveform = waveform.map { it.toInt() }, + waveform = waveform.fromMSC3246range(), ) fun AudioDetails.map(): RustAudioDetails = RustAudioDetails( duration = duration, - waveform = waveform.map { it.toUShort() } + waveform = waveform.toMSC3246range() ) + +/** + * Resizes the given [0;1024] int list as per unstable MSC3246 spec + * to a [0;1] float list to be used for waveform rendering. + * + * https://github.com/matrix-org/matrix-spec-proposals/blob/travis/msc/audio-waveform/proposals/3246-audio-waveform.md + */ +internal fun List.fromMSC3246range(): List = map { it.toInt() / 1024f } + +/** + * Resizes the given [0;1] float list as per unstable MSC3246 spec + * to a [0;1024] int list to be used for waveform rendering. + * + * https://github.com/matrix-org/matrix-spec-proposals/blob/travis/msc/audio-waveform/proposals/3246-audio-waveform.md + */ +internal fun List.toMSC3246range(): List = map { (it * 1024).toInt().toUShort() } diff --git a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/room/RustMatrixRoom.kt b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/room/RustMatrixRoom.kt index 44f095a38c..fa7330b8eb 100644 --- a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/room/RustMatrixRoom.kt +++ b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/room/RustMatrixRoom.kt @@ -46,6 +46,7 @@ import io.element.android.libraries.matrix.api.widget.MatrixWidgetSettings import io.element.android.libraries.matrix.impl.core.toProgressWatcher import io.element.android.libraries.matrix.impl.media.MediaUploadHandlerImpl import io.element.android.libraries.matrix.impl.media.map +import io.element.android.libraries.matrix.impl.media.toMSC3246range import io.element.android.libraries.matrix.impl.notificationsettings.RustNotificationSettingsService import io.element.android.libraries.matrix.impl.poll.toInner import io.element.android.libraries.matrix.impl.room.location.toInner @@ -499,13 +500,13 @@ class RustMatrixRoom( override suspend fun sendVoiceMessage( file: File, audioInfo: AudioInfo, - waveform: List, + waveform: List, progressCallback: ProgressCallback?, ): Result = sendAttachment(listOf(file)) { innerRoom.sendVoiceMessage( url = file.path, audioInfo = audioInfo.map(), - waveform = waveform.map { it.toUShort() }, + waveform = waveform.toMSC3246range(), progressWatcher = progressCallback?.toProgressWatcher(), ) } diff --git a/libraries/matrix/test/src/main/kotlin/io/element/android/libraries/matrix/test/room/FakeMatrixRoom.kt b/libraries/matrix/test/src/main/kotlin/io/element/android/libraries/matrix/test/room/FakeMatrixRoom.kt index a2adf0b0bb..244a769895 100644 --- a/libraries/matrix/test/src/main/kotlin/io/element/android/libraries/matrix/test/room/FakeMatrixRoom.kt +++ b/libraries/matrix/test/src/main/kotlin/io/element/android/libraries/matrix/test/room/FakeMatrixRoom.kt @@ -387,7 +387,7 @@ class FakeMatrixRoom( override suspend fun sendVoiceMessage( file: File, audioInfo: AudioInfo, - waveform: List, + waveform: List, progressCallback: ProgressCallback? ): Result = fakeSendMedia(progressCallback) diff --git a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt index dde62e7513..dd51198e59 100644 --- a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt +++ b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt @@ -55,7 +55,7 @@ class MediaSender @Inject constructor( suspend fun sendVoiceMessage( uri: Uri, mimeType: String, - waveForm: List, + waveForm: List, progressCallback: ProgressCallback? = null ): Result { return preProcessor diff --git a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt index e1debf6bda..39c978f625 100644 --- a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt +++ b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt @@ -29,6 +29,6 @@ sealed interface MediaUploadInfo { data class Image(override val file: File, val imageInfo: ImageInfo, val thumbnailFile: File) : MediaUploadInfo data class Video(override val file: File, val videoInfo: VideoInfo, val thumbnailFile: File) : MediaUploadInfo data class Audio(override val file: File, val audioInfo: AudioInfo) : MediaUploadInfo - data class VoiceMessage(override val file: File, val audioInfo: AudioInfo, val waveform: List) : MediaUploadInfo + data class VoiceMessage(override val file: File, val audioInfo: AudioInfo, val waveform: List) : MediaUploadInfo data class AnyFile(override val file: File, val fileInfo: FileInfo) : MediaUploadInfo }