From b476654489487da0b3543655945a256112ff58da Mon Sep 17 00:00:00 2001 From: jonnyandrew Date: Mon, 23 Oct 2023 18:28:00 +0100 Subject: [PATCH] Record and send voice messages (#1596) --------- Co-authored-by: ElementBot --- changelog.d/1596.feature | 1 + features/messages/impl/build.gradle.kts | 2 + .../impl/src/main/AndroidManifest.xml | 20 ++ .../messages/impl/MessagesPresenter.kt | 3 + .../features/messages/impl/MessagesState.kt | 1 + .../messages/impl/MessagesStateProvider.kt | 1 + .../features/messages/impl/MessagesView.kt | 19 + .../messagecomposer/MessageComposerView.kt | 9 +- .../VoiceMessageComposerEvents.kt | 5 + .../VoiceMessageComposerPresenter.kt | 154 ++++++++- .../VoiceMessageComposerState.kt | 1 + .../VoiceMessageComposerStateProvider.kt | 3 +- .../voicemessages/VoiceMessageException.kt | 26 ++ .../VoiceMessagePermissionRationaleDialog.kt | 37 ++ .../messages/MessagesPresenterTest.kt | 17 +- .../VoiceMessageComposerPresenterTest.kt | 325 +++++++++++++++++- gradle/libs.versions.toml | 2 + .../android/libraries/core/hash/Hash.kt | 35 ++ .../libraries/mediaupload/api/MediaSender.kt | 52 ++- .../mediaupload/api/MediaUploadInfo.kt | 1 + .../mediaupload/AndroidMediaPreProcessor.kt | 1 + .../mediaupload/test/FakeMediaPreProcessor.kt | 18 + .../api/PermissionsStateProvider.kt | 5 +- .../libraries/textcomposer/TextComposer.kt | 43 ++- ...dingProgress.kt => VoiceMessagePreview.kt} | 21 +- .../components/VoiceMessageRecording.kt | 102 ++++++ .../textcomposer/model/VoiceMessageState.kt | 6 +- libraries/voicerecorder/api/build.gradle.kts | 32 ++ .../voicerecorder/api/VoiceRecorder.kt | 55 +++ .../voicerecorder/api/VoiceRecorderState.kt | 44 +++ libraries/voicerecorder/impl/build.gradle.kts | 48 +++ .../voicerecorder/impl/VoiceRecorderImpl.kt | 132 +++++++ .../impl/audio/AndroidAudioReader.kt | 139 ++++++++ .../voicerecorder/impl/audio/Audio.kt | 28 ++ .../voicerecorder/impl/audio/AudioConfig.kt | 35 ++ .../impl/audio/AudioLevelCalculator.kt | 28 ++ .../voicerecorder/impl/audio/AudioReader.kt | 37 ++ .../impl/audio/DecibelAudioLevelCalculator.kt | 49 +++ .../impl/audio/DefaultEncoder.kt | 63 ++++ .../voicerecorder/impl/audio/Encoder.kt | 28 ++ .../voicerecorder/impl/audio/SampleRate.kt | 24 ++ .../impl/di/VoiceRecorderModule.kt | 58 ++++ .../impl/file/DefaultVoiceFileManager.kt | 49 +++ .../impl/file/VoiceFileConfig.kt | 30 ++ .../impl/file/VoiceFileManager.kt | 25 ++ .../impl/VoiceRecorderImplTest.kt | 134 ++++++++ .../audio/DecibelAudioLevelCalculatorTest.kt | 46 +++ .../test/FakeAudioLevelCalculator.kt | 26 ++ .../voicerecorder/test/FakeAudioReader.kt | 49 +++ .../test/FakeAudioRecorderFactory.kt | 30 ++ .../voicerecorder/test/FakeEncoder.kt | 40 +++ .../voicerecorder/test/FakeFileSystem.kt | 43 +++ .../test/FakeVoiceFileManager.kt | 37 ++ libraries/voicerecorder/test/build.gradle.kts | 30 ++ .../voicerecorder/test/FakeVoiceRecorder.kt | 74 ++++ .../kotlin/extension/DependencyHandleScope.kt | 1 + ...ViewVoice-D-5_5_null_0,NEXUS_5,1.0,en].png | 4 +- ...ViewVoice-N-5_6_null_0,NEXUS_5,1.0,en].png | 4 +- ...gProgress-D-11_11_null,NEXUS_5,1.0,en].png | 3 - ...gProgress-N-11_12_null,NEXUS_5,1.0,en].png | 3 - ...ndButton-D-11_11_null,NEXUS_5,1.0,en].png} | 0 ...ndButton-N-11_12_null,NEXUS_5,1.0,en].png} | 0 ...rmatting-D-12_12_null,NEXUS_5,1.0,en].png} | 0 ...rmatting-N-12_13_null,NEXUS_5,1.0,en].png} | 0 ...gePreview-D-13_13_null,NEXUS_5,1.0,en].png | 3 + ...gePreview-N-13_14_null,NEXUS_5,1.0,en].png | 3 + ...Recording-D-14_14_null,NEXUS_5,1.0,en].png | 3 + ...Recording-N-14_15_null,NEXUS_5,1.0,en].png | 3 + 68 files changed, 2271 insertions(+), 79 deletions(-) create mode 100644 changelog.d/1596.feature create mode 100644 features/messages/impl/src/main/AndroidManifest.xml create mode 100644 features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageException.kt create mode 100644 features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessagePermissionRationaleDialog.kt create mode 100644 libraries/core/src/main/kotlin/io/element/android/libraries/core/hash/Hash.kt rename libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/{RecordingProgress.kt => VoiceMessagePreview.kt} (75%) create mode 100644 libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessageRecording.kt create mode 100644 libraries/voicerecorder/api/build.gradle.kts create mode 100644 libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorder.kt create mode 100644 libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorderState.kt create mode 100644 libraries/voicerecorder/impl/build.gradle.kts create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImpl.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AndroidAudioReader.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Audio.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioConfig.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioLevelCalculator.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioReader.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculator.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DefaultEncoder.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Encoder.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/SampleRate.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/di/VoiceRecorderModule.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/DefaultVoiceFileManager.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileConfig.kt create mode 100644 libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileManager.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImplTest.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculatorTest.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioLevelCalculator.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioReader.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioRecorderFactory.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeEncoder.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeFileSystem.kt create mode 100644 libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeVoiceFileManager.kt create mode 100644 libraries/voicerecorder/test/build.gradle.kts create mode 100644 libraries/voicerecorder/test/src/main/kotlin/io/element/android/libraries/voicerecorder/test/FakeVoiceRecorder.kt delete mode 100644 tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_RecordingProgress-D-11_11_null,NEXUS_5,1.0,en].png delete mode 100644 tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_RecordingProgress-N-11_12_null,NEXUS_5,1.0,en].png rename tests/uitests/src/test/snapshots/images/{ui_S_t[l.textcomposer.components_null_SendButton-D-12_12_null,NEXUS_5,1.0,en].png => ui_S_t[l.textcomposer.components_null_SendButton-D-11_11_null,NEXUS_5,1.0,en].png} (100%) rename tests/uitests/src/test/snapshots/images/{ui_S_t[l.textcomposer.components_null_SendButton-N-12_13_null,NEXUS_5,1.0,en].png => ui_S_t[l.textcomposer.components_null_SendButton-N-11_12_null,NEXUS_5,1.0,en].png} (100%) rename tests/uitests/src/test/snapshots/images/{ui_S_t[l.textcomposer.components_null_TextFormatting-D-13_13_null,NEXUS_5,1.0,en].png => ui_S_t[l.textcomposer.components_null_TextFormatting-D-12_12_null,NEXUS_5,1.0,en].png} (100%) rename tests/uitests/src/test/snapshots/images/{ui_S_t[l.textcomposer.components_null_TextFormatting-N-13_14_null,NEXUS_5,1.0,en].png => ui_S_t[l.textcomposer.components_null_TextFormatting-N-12_13_null,NEXUS_5,1.0,en].png} (100%) create mode 100644 tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_VoiceMessagePreview-D-13_13_null,NEXUS_5,1.0,en].png create mode 100644 tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_VoiceMessagePreview-N-13_14_null,NEXUS_5,1.0,en].png create mode 100644 tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_VoiceMessageRecording-D-14_14_null,NEXUS_5,1.0,en].png create mode 100644 tests/uitests/src/test/snapshots/images/ui_S_t[l.textcomposer.components_null_VoiceMessageRecording-N-14_15_null,NEXUS_5,1.0,en].png diff --git a/changelog.d/1596.feature b/changelog.d/1596.feature new file mode 100644 index 0000000000..5108d6008b --- /dev/null +++ b/changelog.d/1596.feature @@ -0,0 +1 @@ +Record and send voice messages \ No newline at end of file diff --git a/features/messages/impl/build.gradle.kts b/features/messages/impl/build.gradle.kts index 956b80949a..8ee3adb13c 100644 --- a/features/messages/impl/build.gradle.kts +++ b/features/messages/impl/build.gradle.kts @@ -51,6 +51,7 @@ dependencies { implementation(projects.libraries.mediaupload.api) implementation(projects.libraries.permissions.api) implementation(projects.libraries.preferences.api) + implementation(projects.libraries.voicerecorder.api) implementation(projects.features.networkmonitor.api) implementation(projects.services.analytics.api) implementation(libs.coil.compose) @@ -80,6 +81,7 @@ dependencies { testImplementation(projects.libraries.permissions.test) testImplementation(projects.libraries.preferences.test) testImplementation(projects.libraries.textcomposer.test) + testImplementation(projects.libraries.voicerecorder.test) testImplementation(libs.test.mockk) ksp(libs.showkase.processor) diff --git a/features/messages/impl/src/main/AndroidManifest.xml b/features/messages/impl/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..a00e8e1873 --- /dev/null +++ b/features/messages/impl/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesPresenter.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesPresenter.kt index 6125e920b8..cf15db5bd9 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesPresenter.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesPresenter.kt @@ -63,6 +63,7 @@ import io.element.android.libraries.androidutils.clipboard.ClipboardHelper import io.element.android.libraries.architecture.Async import io.element.android.libraries.architecture.Presenter import io.element.android.libraries.core.coroutine.CoroutineDispatchers +import io.element.android.libraries.core.meta.BuildMeta import io.element.android.libraries.designsystem.components.avatar.AvatarData import io.element.android.libraries.designsystem.components.avatar.AvatarSize import io.element.android.libraries.designsystem.utils.snackbar.SnackbarDispatcher @@ -101,6 +102,7 @@ class MessagesPresenter @AssistedInject constructor( private val preferencesStore: PreferencesStore, private val featureFlagsService: FeatureFlagService, @Assisted private val navigator: MessagesNavigator, + private val buildMeta: BuildMeta, ) : Presenter { @AssistedFactory @@ -203,6 +205,7 @@ class MessagesPresenter @AssistedInject constructor( enableTextFormatting = enableTextFormatting, enableVoiceMessages = enableVoiceMessages, enableInRoomCalls = enableInRoomCalls, + appName = buildMeta.applicationName, eventSink = { handleEvents(it) } ) } diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesState.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesState.kt index 3a0585f390..81feec4b63 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesState.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesState.kt @@ -50,5 +50,6 @@ data class MessagesState( val enableTextFormatting: Boolean, val enableVoiceMessages: Boolean, val enableInRoomCalls: Boolean, + val appName: String, val eventSink: (MessagesEvents) -> Unit ) diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesStateProvider.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesStateProvider.kt index 249c4b487e..ae279f6d49 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesStateProvider.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesStateProvider.kt @@ -86,5 +86,6 @@ fun aMessagesState() = MessagesState( enableTextFormatting = true, enableVoiceMessages = true, enableInRoomCalls = true, + appName = "Element", eventSink = {} ) diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesView.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesView.kt index 5a7168e7ce..6dd91aa01c 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesView.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesView.kt @@ -63,6 +63,8 @@ import io.element.android.features.messages.impl.timeline.components.reactionsum import io.element.android.features.messages.impl.timeline.components.retrysendmenu.RetrySendMenuEvents import io.element.android.features.messages.impl.timeline.components.retrysendmenu.RetrySendMessageMenu import io.element.android.features.messages.impl.timeline.model.TimelineItem +import io.element.android.features.messages.impl.voicemessages.VoiceMessageComposerEvents +import io.element.android.features.messages.impl.voicemessages.VoiceMessagePermissionRationaleDialog import io.element.android.features.networkmonitor.api.ui.ConnectivityIndicatorView import io.element.android.libraries.androidutils.ui.hideKeyboard import io.element.android.libraries.designsystem.atomic.molecules.IconTitlePlaceholdersRowMolecule @@ -83,6 +85,7 @@ import io.element.android.libraries.designsystem.theme.components.Text import io.element.android.libraries.designsystem.theme.components.TopAppBar import io.element.android.libraries.designsystem.utils.CommonDrawables import io.element.android.libraries.designsystem.utils.LogCompositions +import io.element.android.libraries.designsystem.utils.OnLifecycleEvent import io.element.android.libraries.designsystem.utils.snackbar.SnackbarHost import io.element.android.libraries.designsystem.utils.snackbar.rememberSnackbarHostState import io.element.android.libraries.matrix.api.core.UserId @@ -107,6 +110,10 @@ fun MessagesView( ) { LogCompositions(tag = "MessagesScreen", msg = "Root") + OnLifecycleEvent { _, event -> + state.voiceMessageComposerState.eventSink(VoiceMessageComposerEvents.LifecycleEvent(event)) + } + AttachmentStateView( state = state.composerState.attachmentsState, onPreviewAttachments = onPreviewAttachments, @@ -306,6 +313,18 @@ private fun MessagesViewContent( enableTextFormatting = state.enableTextFormatting, ) + if (state.enableVoiceMessages && state.voiceMessageComposerState.showPermissionRationaleDialog) { + VoiceMessagePermissionRationaleDialog( + onContinue = { + state.voiceMessageComposerState.eventSink(VoiceMessageComposerEvents.AcceptPermissionRationale) + }, + onDismiss = { + state.voiceMessageComposerState.eventSink(VoiceMessageComposerEvents.DismissPermissionsRationale) + }, + appName = state.appName + ) + } + ExpandableBottomSheetScaffold( sheetDragHandle = if (state.composerState.showTextFormatting) { @Composable { BottomSheetDragHandle() } diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/messagecomposer/MessageComposerView.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/messagecomposer/MessageComposerView.kt index 150ae23f9b..8f3899f139 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/messagecomposer/MessageComposerView.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/messagecomposer/MessageComposerView.kt @@ -71,10 +71,14 @@ internal fun MessageComposerView( } } - fun onVoiceRecordButtonEvent(press: PressEvent) { + val onVoiceRecordButtonEvent = { press: PressEvent -> voiceMessageState.eventSink(VoiceMessageComposerEvents.RecordButtonEvent(press)) } + fun onSendVoiceMessage() { + voiceMessageState.eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + } + TextComposer( modifier = modifier, state = state.richTextEditorState, @@ -89,7 +93,8 @@ internal fun MessageComposerView( onDismissTextFormatting = ::onDismissTextFormatting, enableTextFormatting = enableTextFormatting, enableVoiceMessages = enableVoiceMessages, - onVoiceRecordButtonEvent = ::onVoiceRecordButtonEvent, + onVoiceRecordButtonEvent = onVoiceRecordButtonEvent, + onSendVoiceMessage = ::onSendVoiceMessage, onError = ::onError, ) } diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerEvents.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerEvents.kt index 7d6803fc41..42ba0d1d07 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerEvents.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerEvents.kt @@ -16,10 +16,15 @@ package io.element.android.features.messages.impl.voicemessages +import androidx.lifecycle.Lifecycle import io.element.android.libraries.textcomposer.model.PressEvent sealed interface VoiceMessageComposerEvents { data class RecordButtonEvent( val pressEvent: PressEvent ): VoiceMessageComposerEvents + data object SendVoiceMessage: VoiceMessageComposerEvents + data object AcceptPermissionRationale: VoiceMessageComposerEvents + data object DismissPermissionsRationale: VoiceMessageComposerEvents + data class LifecycleEvent(val event: Lifecycle.Event): VoiceMessageComposerEvents } diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerPresenter.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerPresenter.kt index 106125934b..78e7d0ceb8 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerPresenter.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerPresenter.kt @@ -16,49 +16,171 @@ package io.element.android.features.messages.impl.voicemessages +import android.Manifest import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue +import androidx.core.net.toUri +import androidx.lifecycle.Lifecycle import io.element.android.libraries.architecture.Presenter import io.element.android.libraries.di.RoomScope import io.element.android.libraries.di.SingleIn +import io.element.android.libraries.mediaupload.api.MediaSender +import io.element.android.libraries.permissions.api.PermissionsEvents +import io.element.android.libraries.permissions.api.PermissionsPresenter import io.element.android.libraries.textcomposer.model.PressEvent import io.element.android.libraries.textcomposer.model.VoiceMessageState +import io.element.android.libraries.voicerecorder.api.VoiceRecorder +import io.element.android.libraries.voicerecorder.api.VoiceRecorderState +import io.element.android.services.analytics.api.AnalyticsService +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import timber.log.Timber +import java.io.File import javax.inject.Inject @SingleIn(RoomScope::class) -class VoiceMessageComposerPresenter @Inject constructor() : Presenter { +class VoiceMessageComposerPresenter @Inject constructor( + private val appCoroutineScope: CoroutineScope, + private val voiceRecorder: VoiceRecorder, + private val analyticsService: AnalyticsService, + private val mediaSender: MediaSender, + permissionsPresenterFactory: PermissionsPresenter.Factory +) : Presenter { + private val permissionsPresenter = permissionsPresenterFactory.create(Manifest.permission.RECORD_AUDIO) + @Composable override fun present(): VoiceMessageComposerState { - var voiceMessageState by remember { mutableStateOf(VoiceMessageState.Idle) } + val localCoroutineScope = rememberCoroutineScope() + val recorderState by voiceRecorder.state.collectAsState(initial = VoiceRecorderState.Idle) - fun onRecordButtonPress(event: VoiceMessageComposerEvents.RecordButtonEvent) = when(event.pressEvent) { - PressEvent.PressStart -> { - // TODO start the recording - voiceMessageState = VoiceMessageState.Recording - } - PressEvent.LongPressEnd -> { - // TODO finish the recording - voiceMessageState = VoiceMessageState.Idle + val permissionState = permissionsPresenter.present() + var isSending by remember { mutableStateOf(false) } + + val onLifecycleEvent = { event: Lifecycle.Event -> + when (event) { + Lifecycle.Event.ON_PAUSE -> { + appCoroutineScope.finishRecording() + } + Lifecycle.Event.ON_DESTROY -> { + appCoroutineScope.cancelRecording() + } + else -> {} } - PressEvent.Tapped -> { - // TODO discard the recording and show the 'hold to record' tooltip - voiceMessageState = VoiceMessageState.Idle + } + + val onRecordButtonPress = { event: VoiceMessageComposerEvents.RecordButtonEvent -> + val permissionGranted = permissionState.permissionGranted + when (event.pressEvent) { + PressEvent.PressStart -> { + Timber.v("Voice message record button pressed") + when { + permissionGranted -> { + localCoroutineScope.startRecording() + } + else -> { + Timber.i("Voice message permission needed") + permissionState.eventSink(PermissionsEvents.RequestPermissions) + } + } + } + PressEvent.LongPressEnd -> { + Timber.v("Voice message record button released") + localCoroutineScope.finishRecording() + } + PressEvent.Tapped -> { + Timber.v("Voice message record button tapped") + localCoroutineScope.cancelRecording() + } } } + val onAcceptPermissionsRationale = { + permissionState.eventSink(PermissionsEvents.OpenSystemSettingAndCloseDialog) + } + + val onDismissPermissionsRationale = { + permissionState.eventSink(PermissionsEvents.CloseDialog) + } + + val onSendButtonPress = lambda@{ + val finishedState = recorderState as? VoiceRecorderState.Finished + if (finishedState == null) { + val exception = VoiceMessageException.FileException("No file to send") + analyticsService.trackError(exception) + Timber.e(exception) + return@lambda + } + if (isSending) { + return@lambda + } + isSending = true + appCoroutineScope.sendMessage( + file = finishedState.file, + mimeType = finishedState.mimeType, + ).invokeOnCompletion { + isSending = false + } + } - fun handleEvents(event: VoiceMessageComposerEvents) { + val handleEvents: (VoiceMessageComposerEvents) -> Unit = { event -> when (event) { is VoiceMessageComposerEvents.RecordButtonEvent -> onRecordButtonPress(event) + is VoiceMessageComposerEvents.SendVoiceMessage -> localCoroutineScope.launch { + onSendButtonPress() + } + VoiceMessageComposerEvents.DismissPermissionsRationale -> onDismissPermissionsRationale() + VoiceMessageComposerEvents.AcceptPermissionRationale -> onAcceptPermissionsRationale() + is VoiceMessageComposerEvents.LifecycleEvent -> onLifecycleEvent(event.event) } } return VoiceMessageComposerState( - voiceMessageState = voiceMessageState, - eventSink = { handleEvents(it) } + voiceMessageState = when (val state = recorderState) { + is VoiceRecorderState.Recording -> VoiceMessageState.Recording(level = state.level) + is VoiceRecorderState.Finished -> VoiceMessageState.Preview + else -> VoiceMessageState.Idle + }, + showPermissionRationaleDialog = permissionState.showDialog, + eventSink = handleEvents, ) } + + private fun CoroutineScope.startRecording() = launch { + try { + voiceRecorder.startRecord() + } catch (e: SecurityException) { + Timber.e(e, "Voice message error") + analyticsService.trackError(VoiceMessageException.PermissionMissing("Expected permission to record but none", e)) + } + } + + private fun CoroutineScope.finishRecording() = launch { + voiceRecorder.stopRecord() + } + + private fun CoroutineScope.cancelRecording() = launch { + voiceRecorder.stopRecord(cancelled = true) + } + + private fun CoroutineScope.sendMessage( + file: File, mimeType: String, + ) = launch { + val result = mediaSender.sendVoiceMessage( + uri = file.toUri(), + mimeType = mimeType, + waveForm = emptyList(), // TODO generate waveform + ) + + if (result.isFailure) { + Timber.e(result.exceptionOrNull(), "Voice message error") + return@launch + } + + voiceRecorder.deleteRecording() + } } diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerState.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerState.kt index bacbe76324..8f0ab827b5 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerState.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerState.kt @@ -22,6 +22,7 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState @Stable data class VoiceMessageComposerState( val voiceMessageState: VoiceMessageState, + val showPermissionRationaleDialog: Boolean, val eventSink: (VoiceMessageComposerEvents) -> Unit, ) diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerStateProvider.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerStateProvider.kt index 63b59596c0..1a904beee3 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerStateProvider.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageComposerStateProvider.kt @@ -22,7 +22,7 @@ import io.element.android.libraries.textcomposer.model.VoiceMessageState internal open class VoiceMessageComposerStateProvider : PreviewParameterProvider { override val values: Sequence get() = sequenceOf( - aVoiceMessageComposerState(voiceMessageState = VoiceMessageState.Recording), + aVoiceMessageComposerState(voiceMessageState = VoiceMessageState.Recording(level = 0.5)), ) } @@ -30,5 +30,6 @@ internal fun aVoiceMessageComposerState( voiceMessageState: VoiceMessageState = VoiceMessageState.Idle, ) = VoiceMessageComposerState( voiceMessageState = voiceMessageState, + showPermissionRationaleDialog = false, eventSink = {}, ) diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageException.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageException.kt new file mode 100644 index 0000000000..2020b687ae --- /dev/null +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessageException.kt @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.features.messages.impl.voicemessages + +internal sealed class VoiceMessageException : Exception() { + data class FileException( + override val message: String?, override val cause: Throwable? = null + ) : VoiceMessageException() + data class PermissionMissing( + override val message: String?, override val cause: Throwable? + ) : VoiceMessageException() +} diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessagePermissionRationaleDialog.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessagePermissionRationaleDialog.kt new file mode 100644 index 0000000000..19b7f7cb46 --- /dev/null +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/voicemessages/VoiceMessagePermissionRationaleDialog.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.features.messages.impl.voicemessages + +import androidx.compose.runtime.Composable +import androidx.compose.ui.res.stringResource +import io.element.android.libraries.designsystem.components.dialogs.ConfirmationDialog +import io.element.android.libraries.ui.strings.CommonStrings + +@Composable +internal fun VoiceMessagePermissionRationaleDialog( + onContinue: () -> Unit, + onDismiss: () -> Unit, + appName: String, +) { + ConfirmationDialog( + content = stringResource(CommonStrings.error_missing_microphone_voice_rationale_android, appName), + onSubmitClicked = onContinue, + onDismiss = onDismiss, + submitText = stringResource(CommonStrings.action_continue), + cancelText = stringResource(CommonStrings.action_cancel), + ) +} diff --git a/features/messages/impl/src/test/kotlin/io/element/android/features/messages/MessagesPresenterTest.kt b/features/messages/impl/src/test/kotlin/io/element/android/features/messages/MessagesPresenterTest.kt index b02afd90fb..b0958a5d2d 100644 --- a/features/messages/impl/src/test/kotlin/io/element/android/features/messages/MessagesPresenterTest.kt +++ b/features/messages/impl/src/test/kotlin/io/element/android/features/messages/MessagesPresenterTest.kt @@ -66,6 +66,7 @@ import io.element.android.libraries.matrix.test.AN_EVENT_ID import io.element.android.libraries.matrix.test.A_ROOM_ID import io.element.android.libraries.matrix.test.A_SESSION_ID import io.element.android.libraries.matrix.test.A_SESSION_ID_2 +import io.element.android.libraries.matrix.test.core.aBuildMeta import io.element.android.libraries.matrix.test.room.FakeMatrixRoom import io.element.android.libraries.matrix.test.room.aRoomMember import io.element.android.libraries.mediapickers.test.FakePickerProvider @@ -75,6 +76,7 @@ import io.element.android.libraries.permissions.api.PermissionsPresenter import io.element.android.libraries.permissions.test.FakePermissionsPresenter import io.element.android.libraries.permissions.test.FakePermissionsPresenterFactory import io.element.android.libraries.textcomposer.model.MessageComposerMode +import io.element.android.libraries.voicerecorder.test.FakeVoiceRecorder import io.element.android.services.analytics.test.FakeAnalyticsService import io.element.android.tests.testutils.WarmUpRule import io.element.android.tests.testutils.consumeItemsUntilPredicate @@ -607,20 +609,28 @@ class MessagesPresenterTest { analyticsService: FakeAnalyticsService = FakeAnalyticsService(), permissionsPresenter: PermissionsPresenter = FakePermissionsPresenter(), ): MessagesPresenter { + val mediaSender = MediaSender(FakeMediaPreProcessor(), matrixRoom) + val permissionsPresenterFactory = FakePermissionsPresenterFactory(permissionsPresenter) val messageComposerPresenter = MessageComposerPresenter( appCoroutineScope = this, room = matrixRoom, mediaPickerProvider = FakePickerProvider(), featureFlagService = FakeFeatureFlagService(mapOf(FeatureFlags.NotificationSettings.key to true)), localMediaFactory = FakeLocalMediaFactory(mockMediaUrl), - mediaSender = MediaSender(FakeMediaPreProcessor(), matrixRoom), + mediaSender = mediaSender, snackbarDispatcher = SnackbarDispatcher(), analyticsService = analyticsService, messageComposerContext = MessageComposerContextImpl(), richTextEditorStateFactory = TestRichTextEditorStateFactory(), - permissionsPresenterFactory = FakePermissionsPresenterFactory(permissionsPresenter), + permissionsPresenterFactory = permissionsPresenterFactory, + ) + val voiceMessageComposerPresenter = VoiceMessageComposerPresenter( + this, + FakeVoiceRecorder(), + analyticsService, + mediaSender, + permissionsPresenterFactory, ) - val voiceMessageComposerPresenter = VoiceMessageComposerPresenter() val timelinePresenter = TimelinePresenter( timelineItemsFactory = aTimelineItemsFactory(), room = matrixRoom, @@ -649,6 +659,7 @@ class MessagesPresenterTest { clipboardHelper = clipboardHelper, preferencesStore = preferencesStore, featureFlagsService = FakeFeatureFlagService(), + buildMeta = aBuildMeta(), dispatchers = coroutineDispatchers, ) } diff --git a/features/messages/impl/src/test/kotlin/io/element/android/features/messages/voicemessages/VoiceMessageComposerPresenterTest.kt b/features/messages/impl/src/test/kotlin/io/element/android/features/messages/voicemessages/VoiceMessageComposerPresenterTest.kt index 008226bf05..d1ee074e46 100644 --- a/features/messages/impl/src/test/kotlin/io/element/android/features/messages/voicemessages/VoiceMessageComposerPresenterTest.kt +++ b/features/messages/impl/src/test/kotlin/io/element/android/features/messages/voicemessages/VoiceMessageComposerPresenterTest.kt @@ -18,16 +18,31 @@ package io.element.android.features.messages.voicemessages +import android.Manifest +import androidx.lifecycle.Lifecycle import app.cash.molecule.RecompositionMode import app.cash.molecule.moleculeFlow +import app.cash.turbine.TurbineTestContext import app.cash.turbine.test import com.google.common.truth.Truth.assertThat import io.element.android.features.messages.impl.voicemessages.VoiceMessageComposerEvents import io.element.android.features.messages.impl.voicemessages.VoiceMessageComposerPresenter +import io.element.android.features.messages.impl.voicemessages.VoiceMessageComposerState +import io.element.android.features.messages.impl.voicemessages.VoiceMessageException +import io.element.android.libraries.matrix.test.room.FakeMatrixRoom +import io.element.android.libraries.mediaupload.api.MediaSender +import io.element.android.libraries.mediaupload.test.FakeMediaPreProcessor +import io.element.android.libraries.permissions.api.PermissionsPresenter +import io.element.android.libraries.permissions.api.aPermissionsState +import io.element.android.libraries.permissions.test.FakePermissionsPresenter +import io.element.android.libraries.permissions.test.FakePermissionsPresenterFactory import io.element.android.libraries.textcomposer.model.PressEvent import io.element.android.libraries.textcomposer.model.VoiceMessageState +import io.element.android.libraries.voicerecorder.test.FakeVoiceRecorder +import io.element.android.services.analytics.test.FakeAnalyticsService import io.element.android.tests.testutils.WarmUpRule import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.TestScope import kotlinx.coroutines.test.runTest import org.junit.Rule import org.junit.Test @@ -37,53 +52,349 @@ class VoiceMessageComposerPresenterTest { @get:Rule val warmUpRule = WarmUpRule() + private val voiceRecorder = FakeVoiceRecorder() + private val analyticsService = FakeAnalyticsService() + private val matrixRoom = FakeMatrixRoom() + private val mediaPreProcessor = FakeMediaPreProcessor().apply { givenAudioResult() } + private val mediaSender = MediaSender(mediaPreProcessor, matrixRoom) + @Test fun `present - initial state`() = runTest { - val presenter = createPresenter() + val presenter = createVoiceMessageComposerPresenter() moleculeFlow(RecompositionMode.Immediate) { presenter.present() }.test { val initialState = awaitItem() assertThat(initialState.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + + testPauseAndDestroy(initialState) } } @Test fun `present - recording state`() = runTest { - val presenter = createPresenter() + val presenter = createVoiceMessageComposerPresenter() moleculeFlow(RecompositionMode.Immediate) { presenter.present() }.test { awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) - assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Recording) + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Recording(0.2)) + + testPauseAndDestroy(finalState) } } @Test fun `present - abort recording`() = runTest { - val presenter = createPresenter() + val presenter = createVoiceMessageComposerPresenter() moleculeFlow(RecompositionMode.Immediate) { presenter.present() }.test { awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.Tapped)) - assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Idle) + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + + testPauseAndDestroy(finalState) } } @Test fun `present - finish recording`() = runTest { - val presenter = createPresenter() + val presenter = createVoiceMessageComposerPresenter() + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.LongPressEnd)) + + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Preview) + + testPauseAndDestroy(finalState) + } + } + + @Test + fun `present - send recording`() = runTest { + val presenter = createVoiceMessageComposerPresenter() moleculeFlow(RecompositionMode.Immediate) { presenter.present() }.test { awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.LongPressEnd)) + awaitItem().eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + assertThat(matrixRoom.sendMediaCount).isEqualTo(1) + + testPauseAndDestroy(finalState) + } + } + + @Test + fun `present - send recording before previous completed, waits`() = runTest { + val presenter = createVoiceMessageComposerPresenter() + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.LongPressEnd)) + awaitItem().run { + eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + } + + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + assertThat(matrixRoom.sendMediaCount).isEqualTo(1) + + testPauseAndDestroy(finalState) + } + } + + @Test + fun `present - send failures aren't tracked`() = runTest { + // Let sending fail due to media preprocessing error + mediaPreProcessor.givenResult(Result.failure(Exception())) + val presenter = createVoiceMessageComposerPresenter() + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.LongPressEnd)) + + val finalState = awaitItem().apply { + assertThat(voiceMessageState).isEqualTo(VoiceMessageState.Preview) + eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + } + assertThat(matrixRoom.sendMediaCount).isEqualTo(0) + assertThat(analyticsService.trackedErrors).hasSize(0) + + testPauseAndDestroy(finalState) + } + } + + @Test + fun `present - send failures can be retried`() = runTest { + // Let sending fail due to media preprocessing error + val presenter = createVoiceMessageComposerPresenter() + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + mediaPreProcessor.givenResult(Result.failure(Exception())) + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.LongPressEnd)) + val previewState = awaitItem() + + previewState.eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + + ensureAllEventsConsumed() + assertThat(previewState.voiceMessageState).isEqualTo(VoiceMessageState.Preview) + assertThat(matrixRoom.sendMediaCount).isEqualTo(0) + + mediaPreProcessor.givenAudioResult() + previewState.eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + assertThat(matrixRoom.sendMediaCount).isEqualTo(1) + + testPauseAndDestroy(finalState) + } + } + + @Test + fun `present - send error - missing recording is tracked`() = runTest { + val presenter = createVoiceMessageComposerPresenter() + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + val initialState = awaitItem() + // Send the message before recording anything + initialState.eventSink(VoiceMessageComposerEvents.SendVoiceMessage) + assertThat(initialState.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + assertThat(matrixRoom.sendMediaCount).isEqualTo(0) + assertThat(analyticsService.trackedErrors).hasSize(1) + + testPauseAndDestroy(initialState) + } + } + + @Test + fun `present - record error - security exceptions are tracked`() = runTest { + val exception = SecurityException("") + voiceRecorder.givenThrowsSecurityException(exception) + val presenter = createVoiceMessageComposerPresenter() + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + val initialState = awaitItem() + initialState.eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + + assertThat(matrixRoom.sendMediaCount).isEqualTo(0) + assertThat(analyticsService.trackedErrors).containsExactly( + VoiceMessageException.PermissionMissing(message = "Expected permission to record but none", cause = exception) + ) + + testPauseAndDestroy(initialState) + } + } + + @Test + fun `present - permission accepted first time`() = runTest { + val permissionsPresenter = createFakePermissionsPresenter( + recordPermissionGranted = false, + ) + val presenter = createVoiceMessageComposerPresenter( + permissionsPresenter = permissionsPresenter, + ) + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + val initialState = awaitItem() + initialState.eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Idle) + + initialState.eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.LongPressEnd)) + permissionsPresenter.setPermissionGranted() + + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Recording(0.2)) + + testPauseAndDestroy(finalState) } } - private fun createPresenter() = VoiceMessageComposerPresenter() + + @Test + fun `present - permission denied previously`() = runTest { + val permissionsPresenter = createFakePermissionsPresenter( + recordPermissionGranted = false, + ) + val presenter = createVoiceMessageComposerPresenter( + permissionsPresenter = permissionsPresenter, + ) + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + + // See the dialog and accept it + awaitItem().also { + assertThat(it.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + assertThat(it.showPermissionRationaleDialog).isTrue() + it.eventSink(VoiceMessageComposerEvents.AcceptPermissionRationale) + } + + // Dialog is hidden, user accepts permissions + assertThat(awaitItem().showPermissionRationaleDialog).isFalse() + + permissionsPresenter.setPermissionGranted() + + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + val finalState = awaitItem() + assertThat(finalState.voiceMessageState).isEqualTo(VoiceMessageState.Recording(0.2)) + + testPauseAndDestroy(finalState) + } + } + + @Test + fun `present - permission rationale dismissed`() = runTest { + val permissionsPresenter = createFakePermissionsPresenter( + recordPermissionGranted = false, + ) + val presenter = createVoiceMessageComposerPresenter( + permissionsPresenter = permissionsPresenter, + ) + moleculeFlow(RecompositionMode.Immediate) { + presenter.present() + }.test { + awaitItem().eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + + // See the dialog and accept it + awaitItem().also { + assertThat(it.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + assertThat(it.showPermissionRationaleDialog).isTrue() + it.eventSink(VoiceMessageComposerEvents.DismissPermissionsRationale) + } + + // Dialog is hidden, user tries to record again + awaitItem().also { + assertThat(it.showPermissionRationaleDialog).isFalse() + it.eventSink(VoiceMessageComposerEvents.RecordButtonEvent(PressEvent.PressStart)) + } + + // Dialog is shown once again + val finalState = awaitItem().also { + assertThat(it.voiceMessageState).isEqualTo(VoiceMessageState.Idle) + assertThat(it.showPermissionRationaleDialog).isTrue() + } + + testPauseAndDestroy(finalState) + } + } + + private suspend fun TurbineTestContext.testPauseAndDestroy( + mostRecentState: VoiceMessageComposerState, + ) { + mostRecentState.eventSink( + VoiceMessageComposerEvents.LifecycleEvent(event = Lifecycle.Event.ON_PAUSE) + ) + + val onPauseState = when (mostRecentState.voiceMessageState) { + VoiceMessageState.Idle, + VoiceMessageState.Preview -> { + mostRecentState + } + is VoiceMessageState.Recording -> { + awaitItem().also { + assertThat(it.voiceMessageState).isEqualTo(VoiceMessageState.Preview) + } + } + } + + onPauseState.eventSink( + VoiceMessageComposerEvents.LifecycleEvent(event = Lifecycle.Event.ON_DESTROY) + ) + + when (onPauseState.voiceMessageState) { + VoiceMessageState.Idle -> + ensureAllEventsConsumed() + is VoiceMessageState.Recording, + VoiceMessageState.Preview -> + assertThat(awaitItem().voiceMessageState).isEqualTo(VoiceMessageState.Idle) + } + } + + private fun TestScope.createVoiceMessageComposerPresenter( + permissionsPresenter: PermissionsPresenter = createFakePermissionsPresenter(), + ): VoiceMessageComposerPresenter { + return VoiceMessageComposerPresenter( + this, + voiceRecorder, + analyticsService, + mediaSender, + FakePermissionsPresenterFactory(permissionsPresenter), + ) + } + + private fun createFakePermissionsPresenter( + recordPermissionGranted: Boolean = true, + recordPermissionShowDialog: Boolean = false, + ): FakePermissionsPresenter { + val initialPermissionState = aPermissionsState( + showDialog = recordPermissionShowDialog, + permission = Manifest.permission.RECORD_AUDIO, + permissionGranted = recordPermissionGranted, + ) + return FakePermissionsPresenter( + initialState = initialPermissionState + ) + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1c86738fa0..7c3afb48dc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -75,6 +75,7 @@ google_firebase_bom = "com.google.firebase:firebase-bom:32.4.0" # AndroidX androidx_core = { module = "androidx.core:core", version.ref = "core" } androidx_corektx = { module = "androidx.core:core-ktx", version.ref = "core" } +androidx_annotationjvm = { module = "androidx.annotation:annotation-jvm", version = "1.7.0" } androidx_datastore_preferences = { module = "androidx.datastore:datastore-preferences", version.ref = "datastore" } androidx_datastore_datastore = { module = "androidx.datastore:datastore", version.ref = "datastore" } androidx_exifinterface = "androidx.exifinterface:exifinterface:1.3.6" @@ -164,6 +165,7 @@ statemachine = "com.freeletics.flowredux:compose:1.2.0" maplibre = "org.maplibre.gl:android-sdk:10.2.0" maplibre_ktx = "org.maplibre.gl:android-sdk-ktx-v7:2.0.1" maplibre_annotation = "org.maplibre.gl:android-plugin-annotation-v9:2.0.1" +opusencoder = "io.element.android:opusencoder:1.1.0" # Analytics posthog = "com.posthog.android:posthog:2.0.3" diff --git a/libraries/core/src/main/kotlin/io/element/android/libraries/core/hash/Hash.kt b/libraries/core/src/main/kotlin/io/element/android/libraries/core/hash/Hash.kt new file mode 100644 index 0000000000..760431a7be --- /dev/null +++ b/libraries/core/src/main/kotlin/io/element/android/libraries/core/hash/Hash.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.core.hash + +import java.security.MessageDigest +import java.util.Locale + +/** + * Compute a Hash of a String, using md5 algorithm. + */ +fun String.md5() = try { + val digest = MessageDigest.getInstance("md5") + val locale = Locale.ROOT + digest.update(toByteArray()) + digest.digest() + .joinToString("") { String.format(locale, "%02X", it) } + .lowercase(locale) +} catch (exc: Exception) { + // Should not happen, but just in case + hashCode().toString() +} diff --git a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt index 899e92efc5..dde62e7513 100644 --- a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt +++ b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaSender.kt @@ -50,17 +50,44 @@ class MediaSender @Inject constructor( .flatMapCatching { info -> room.sendMedia(info, progressCallback) } - .onFailure { error -> - val job = ongoingUploadJobs.remove(Job) - if (error !is CancellationException) { - job?.cancel() - } - } - .onSuccess { - ongoingUploadJobs.remove(Job) + .handleSendResult() + } + suspend fun sendVoiceMessage( + uri: Uri, + mimeType: String, + waveForm: List, + progressCallback: ProgressCallback? = null + ): Result { + return preProcessor + .process( + uri = uri, + mimeType = mimeType, + deleteOriginal = true, + compressIfPossible = false + ) + .flatMapCatching { info -> + val audioInfo = (info as MediaUploadInfo.Audio).audioInfo + val newInfo = MediaUploadInfo.VoiceMessage( + file = info.file, + audioInfo = audioInfo, + waveform = waveForm, + ) + room.sendMedia(newInfo, progressCallback) } + .handleSendResult() } + private fun Result.handleSendResult() = this + .onFailure { error -> + val job = ongoingUploadJobs.remove(Job) + if (error !is CancellationException) { + job?.cancel() + } + } + .onSuccess { + ongoingUploadJobs.remove(Job) + } + private suspend fun MatrixRoom.sendMedia( uploadInfo: MediaUploadInfo, progressCallback: ProgressCallback?, @@ -90,7 +117,14 @@ class MediaSender @Inject constructor( progressCallback = progressCallback ) } - + is MediaUploadInfo.VoiceMessage -> { + sendVoiceMessage( + file = uploadInfo.file, + audioInfo = uploadInfo.audioInfo, + waveform = uploadInfo.waveform, + progressCallback = progressCallback + ) + } is MediaUploadInfo.AnyFile -> { sendFile( file = uploadInfo.file, diff --git a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt index 51f6372b23..e1debf6bda 100644 --- a/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt +++ b/libraries/mediaupload/api/src/main/kotlin/io/element/android/libraries/mediaupload/api/MediaUploadInfo.kt @@ -29,5 +29,6 @@ sealed interface MediaUploadInfo { data class Image(override val file: File, val imageInfo: ImageInfo, val thumbnailFile: File) : MediaUploadInfo data class Video(override val file: File, val videoInfo: VideoInfo, val thumbnailFile: File) : MediaUploadInfo data class Audio(override val file: File, val audioInfo: AudioInfo) : MediaUploadInfo + data class VoiceMessage(override val file: File, val audioInfo: AudioInfo, val waveform: List) : MediaUploadInfo data class AnyFile(override val file: File, val fileInfo: FileInfo) : MediaUploadInfo } diff --git a/libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/AndroidMediaPreProcessor.kt b/libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/AndroidMediaPreProcessor.kt index cd968530f8..205be3b241 100644 --- a/libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/AndroidMediaPreProcessor.kt +++ b/libraries/mediaupload/impl/src/main/kotlin/io/element/android/libraries/mediaupload/AndroidMediaPreProcessor.kt @@ -118,6 +118,7 @@ class AndroidMediaPreProcessor @Inject constructor( is MediaUploadInfo.Audio -> copy(file = renamedFile) is MediaUploadInfo.Image -> copy(file = renamedFile) is MediaUploadInfo.Video -> copy(file = renamedFile) + is MediaUploadInfo.VoiceMessage -> copy(file = renamedFile) } } diff --git a/libraries/mediaupload/test/src/main/kotlin/io/element/android/libraries/mediaupload/test/FakeMediaPreProcessor.kt b/libraries/mediaupload/test/src/main/kotlin/io/element/android/libraries/mediaupload/test/FakeMediaPreProcessor.kt index d94414d2d7..8e7e71e8fb 100644 --- a/libraries/mediaupload/test/src/main/kotlin/io/element/android/libraries/mediaupload/test/FakeMediaPreProcessor.kt +++ b/libraries/mediaupload/test/src/main/kotlin/io/element/android/libraries/mediaupload/test/FakeMediaPreProcessor.kt @@ -17,11 +17,14 @@ package io.element.android.libraries.mediaupload.test import android.net.Uri +import io.element.android.libraries.matrix.api.media.AudioInfo import io.element.android.libraries.matrix.api.media.FileInfo import io.element.android.libraries.mediaupload.api.MediaPreProcessor import io.element.android.libraries.mediaupload.api.MediaUploadInfo import io.element.android.tests.testutils.simulateLongTask import java.io.File +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toJavaDuration class FakeMediaPreProcessor : MediaPreProcessor { @@ -53,4 +56,19 @@ class FakeMediaPreProcessor : MediaPreProcessor { fun givenResult(value: Result) { this.result = value } + + fun givenAudioResult() { + givenResult( + Result.success( + MediaUploadInfo.Audio( + file = File("audio.ogg"), + audioInfo = AudioInfo( + duration = 1000.seconds.toJavaDuration(), + size = 1000, + mimetype = "audio/ogg", + ), + ) + ) + ) + } } diff --git a/libraries/permissions/api/src/main/kotlin/io/element/android/libraries/permissions/api/PermissionsStateProvider.kt b/libraries/permissions/api/src/main/kotlin/io/element/android/libraries/permissions/api/PermissionsStateProvider.kt index cc59d96b44..19797b9075 100644 --- a/libraries/permissions/api/src/main/kotlin/io/element/android/libraries/permissions/api/PermissionsStateProvider.kt +++ b/libraries/permissions/api/src/main/kotlin/io/element/android/libraries/permissions/api/PermissionsStateProvider.kt @@ -31,10 +31,11 @@ open class PermissionsStateProvider : PreviewParameterProvider fun aPermissionsState( showDialog: Boolean, - permission: String = Manifest.permission.POST_NOTIFICATIONS + permission: String = Manifest.permission.POST_NOTIFICATIONS, + permissionGranted: Boolean = false, ) = PermissionsState( permission = permission, - permissionGranted = false, + permissionGranted = permissionGranted, shouldShowRationale = false, showDialog = showDialog, permissionAlreadyAsked = false, diff --git a/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/TextComposer.kt b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/TextComposer.kt index 911cebb142..7924077394 100644 --- a/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/TextComposer.kt +++ b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/TextComposer.kt @@ -64,7 +64,8 @@ import io.element.android.libraries.testtags.testTag import io.element.android.libraries.textcomposer.components.ComposerOptionsButton import io.element.android.libraries.textcomposer.components.DismissTextFormattingButton import io.element.android.libraries.textcomposer.components.RecordButton -import io.element.android.libraries.textcomposer.components.RecordingProgress +import io.element.android.libraries.textcomposer.components.VoiceMessagePreview +import io.element.android.libraries.textcomposer.components.VoiceMessageRecording import io.element.android.libraries.textcomposer.components.SendButton import io.element.android.libraries.textcomposer.components.TextFormatting import io.element.android.libraries.textcomposer.components.textInputRoundedCornerShape @@ -95,6 +96,7 @@ fun TextComposer( onAddAttachment: () -> Unit = {}, onDismissTextFormatting: () -> Unit = {}, onVoiceRecordButtonEvent: (PressEvent) -> Unit = {}, + onSendVoiceMessage: () -> Unit = {}, onError: (Throwable) -> Unit = {}, ) { val onSendClicked = { @@ -137,24 +139,39 @@ fun TextComposer( composerMode = composerMode, ) } - val recordButton = @Composable { + val recordVoiceButton = @Composable { RecordButton( onPressStart = { onVoiceRecordButtonEvent(PressEvent.PressStart) }, onLongPressEnd = { onVoiceRecordButtonEvent(PressEvent.LongPressEnd) }, onTap = { onVoiceRecordButtonEvent(PressEvent.Tapped) }, ) } + val sendVoiceButton = @Composable { + SendButton( + canSendMessage = voiceMessageState is VoiceMessageState.Preview, + onClick = { onSendVoiceMessage() }, + composerMode = composerMode, + ) + } val textFormattingOptions = @Composable { TextFormatting(state = state) } - val sendOrRecordButton = if (canSendMessage || !enableVoiceMessages) { - sendButton - } else { - recordButton + val sendOrRecordButton = when { + enableVoiceMessages && !canSendMessage -> + when (voiceMessageState) { + is VoiceMessageState.Preview -> sendVoiceButton + else -> recordVoiceButton + } + else -> + sendButton } - val recordingProgress = @Composable { - RecordingProgress() + val voiceRecording = @Composable { + if (voiceMessageState is VoiceMessageState.Recording) { + VoiceMessageRecording(voiceMessageState.level) + } else if (voiceMessageState is VoiceMessageState.Preview) { + VoiceMessagePreview() + } } if (showTextFormatting) { @@ -170,11 +187,12 @@ fun TextComposer( } else { StandardLayout( voiceMessageState = voiceMessageState, + enableVoiceMessages = enableVoiceMessages, modifier = layoutModifier, composerOptionsButton = composerOptionsButton, textInput = textInput, endButton = sendOrRecordButton, - recordingProgress = recordingProgress, + voiceRecording = voiceRecording, ) } @@ -190,9 +208,10 @@ fun TextComposer( @Composable private fun StandardLayout( voiceMessageState: VoiceMessageState, + enableVoiceMessages: Boolean, textInput: @Composable () -> Unit, composerOptionsButton: @Composable () -> Unit, - recordingProgress: @Composable () -> Unit, + voiceRecording: @Composable () -> Unit, endButton: @Composable () -> Unit, modifier: Modifier = Modifier, ) { @@ -200,13 +219,13 @@ private fun StandardLayout( modifier = modifier, verticalAlignment = Alignment.Bottom, ) { - if (voiceMessageState is VoiceMessageState.Recording) { + if (enableVoiceMessages && voiceMessageState !is VoiceMessageState.Idle) { Box( modifier = Modifier .padding(start = 16.dp, bottom = 8.dp, top = 8.dp) .weight(1f) ) { - recordingProgress() + voiceRecording() } } else { Box( diff --git a/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/RecordingProgress.kt b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessagePreview.kt similarity index 75% rename from libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/RecordingProgress.kt rename to libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessagePreview.kt index 2fc0420e05..351293a329 100644 --- a/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/RecordingProgress.kt +++ b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessagePreview.kt @@ -17,14 +17,10 @@ package io.element.android.libraries.textcomposer.components import androidx.compose.foundation.background -import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.heightIn import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.size -import androidx.compose.foundation.shape.CircleShape import androidx.compose.material3.MaterialTheme import androidx.compose.runtime.Composable import androidx.compose.ui.Alignment @@ -36,7 +32,7 @@ import io.element.android.libraries.designsystem.theme.components.Text import io.element.android.libraries.theme.ElementTheme @Composable -internal fun RecordingProgress( +internal fun VoiceMessagePreview( modifier: Modifier = Modifier, ) { Row( @@ -50,16 +46,9 @@ internal fun RecordingProgress( .heightIn(26.dp), verticalAlignment = Alignment.CenterVertically, ) { - Box( - modifier = Modifier - .size(8.dp) - .background(color = ElementTheme.colors.textCriticalPrimary, shape = CircleShape) - ) - Spacer(Modifier.size(8.dp)) - - // TODO Replace with timer UI + // TODO Replace with recording preview UI Text( - text = "Recording...", // Not localized because it is a placeholder + text = "Finished recording", // Not localized because it is a placeholder color = ElementTheme.colors.textSecondary, style = ElementTheme.typography.fontBodySmMedium ) @@ -68,6 +57,6 @@ internal fun RecordingProgress( @PreviewsDayNight @Composable -internal fun RecordingProgressPreview() = ElementPreview { - RecordingProgress() +internal fun VoiceMessagePreviewPreview() = ElementPreview { + VoiceMessagePreview() } diff --git a/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessageRecording.kt b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessageRecording.kt new file mode 100644 index 0000000000..24703a579c --- /dev/null +++ b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/components/VoiceMessageRecording.kt @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.textcomposer.components + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import io.element.android.libraries.designsystem.preview.ElementPreview +import io.element.android.libraries.designsystem.preview.PreviewsDayNight +import io.element.android.libraries.designsystem.theme.components.Text +import io.element.android.libraries.theme.ElementTheme + +@Composable +internal fun VoiceMessageRecording( + level: Double, + modifier: Modifier = Modifier, +) { + Row( + modifier = modifier + .fillMaxWidth() + .background( + color = ElementTheme.colors.bgSubtleSecondary, + shape = MaterialTheme.shapes.medium, + ) + .padding(start = 12.dp, end = 20.dp, top = 8.dp, bottom = 8.dp) + .heightIn(26.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Box( + modifier = Modifier + .size(8.dp) + .background(color = ElementTheme.colors.textCriticalPrimary, shape = CircleShape) + ) + Spacer(Modifier.size(8.dp)) + + // TODO Replace with timer UI + Text( + text = "Recording...", // Not localized because it is a placeholder + color = ElementTheme.colors.textSecondary, + style = ElementTheme.typography.fontBodySmMedium + ) + + Spacer(Modifier.size(20.dp)) + + // TODO Replace with waveform UI + DebugAudioLevel( + modifier = Modifier.weight(1f), level = level + ) + } +} + +@Composable +private fun DebugAudioLevel( + level: Double, + modifier: Modifier = Modifier, +) { + Box( + modifier = modifier + .height(26.dp) + ) { + Box( + modifier = Modifier + .align(Alignment.CenterEnd) + .fillMaxWidth(level.toFloat()) + .background(ElementTheme.colors.iconQuaternary, shape = MaterialTheme.shapes.small) + .fillMaxHeight() + ) + } +} + +@PreviewsDayNight +@Composable +internal fun VoiceMessageRecordingPreview() = ElementPreview { + VoiceMessageRecording(0.5) +} diff --git a/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/model/VoiceMessageState.kt b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/model/VoiceMessageState.kt index d376c4ee70..835000478a 100644 --- a/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/model/VoiceMessageState.kt +++ b/libraries/textcomposer/impl/src/main/kotlin/io/element/android/libraries/textcomposer/model/VoiceMessageState.kt @@ -18,5 +18,9 @@ package io.element.android.libraries.textcomposer.model sealed class VoiceMessageState { data object Idle: VoiceMessageState() - data object Recording: VoiceMessageState() + + data object Preview: VoiceMessageState() + data class Recording( + val level: Double, + ): VoiceMessageState() } diff --git a/libraries/voicerecorder/api/build.gradle.kts b/libraries/voicerecorder/api/build.gradle.kts new file mode 100644 index 0000000000..bed69b7d28 --- /dev/null +++ b/libraries/voicerecorder/api/build.gradle.kts @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + id("io.element.android-library") + alias(libs.plugins.anvil) +} + +android { + namespace = "io.element.android.libraries.voicerecorder.api" +} + +anvil { + generateDaggerFactories.set(true) +} + +dependencies { + implementation(libs.androidx.annotationjvm) + implementation(libs.coroutines.core) +} diff --git a/libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorder.kt b/libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorder.kt new file mode 100644 index 0000000000..77465ddeea --- /dev/null +++ b/libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorder.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.api + +import android.Manifest +import androidx.annotation.RequiresPermission +import kotlinx.coroutines.flow.StateFlow + +/** + * Audio recorder which records audio to opus/ogg files. + */ +interface VoiceRecorder { + /** + * Start a recording. + * + * Call [stopRecord] to stop the recording and release resources. + */ + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + suspend fun startRecord() + + /** + * Stop the current recording. + * + * Call [deleteRecording] to delete any recorded audio. + * + * @param cancelled If true, the recording is deleted. + */ + suspend fun stopRecord( + cancelled: Boolean = false + ) + + /** + * Stop the current recording and delete the output file. + */ + suspend fun deleteRecording() + + /** + * The current state of the recorder. + */ + val state: StateFlow +} diff --git a/libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorderState.kt b/libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorderState.kt new file mode 100644 index 0000000000..8d531c3565 --- /dev/null +++ b/libraries/voicerecorder/api/src/main/kotlin/io/element/android/libraries/voicerecorder/api/VoiceRecorderState.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.api + +import java.io.File + +sealed class VoiceRecorderState { + /** + * The recorder is idle and not recording. + */ + data object Idle : VoiceRecorderState() + + /** + * The recorder is currently recording. + * + * @property level The current audio level of the recording as a fraction of 1. + */ + data class Recording(val level: Double) : VoiceRecorderState() + + /** + * The recorder has finished recording. + * + * @property file The recorded file. + * @property mimeType The mime type of the file. + */ + data class Finished( + val file: File, + val mimeType: String, + ) : VoiceRecorderState() +} diff --git a/libraries/voicerecorder/impl/build.gradle.kts b/libraries/voicerecorder/impl/build.gradle.kts new file mode 100644 index 0000000000..6ebfb28997 --- /dev/null +++ b/libraries/voicerecorder/impl/build.gradle.kts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { + id("io.element.android-library") + alias(libs.plugins.anvil) +} + +android { + namespace = "io.element.android.libraries.voicerecorder.impl" +} + +anvil { + generateDaggerFactories.set(true) +} + +dependencies { + api(projects.libraries.voicerecorder.api) + api(libs.opusencoder) + + implementation(libs.dagger) + implementation(projects.libraries.matrix.api) + implementation(projects.libraries.core) + implementation(projects.libraries.di) + + implementation(libs.androidx.annotationjvm) + implementation(libs.coroutines.core) + + testImplementation(projects.tests.testutils) + testImplementation(libs.test.junit) + testImplementation(libs.test.truth) + testImplementation(libs.test.mockk) + testImplementation(libs.test.turbine) + testImplementation(libs.coroutines.core) + testImplementation(libs.coroutines.test) +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImpl.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImpl.kt new file mode 100644 index 0000000000..ef91118371 --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImpl.kt @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl + +import android.Manifest +import androidx.annotation.RequiresPermission +import com.squareup.anvil.annotations.ContributesBinding +import io.element.android.libraries.core.coroutine.CoroutineDispatchers +import io.element.android.libraries.core.coroutine.childScope +import io.element.android.libraries.di.RoomScope +import io.element.android.libraries.di.SingleIn +import io.element.android.libraries.voicerecorder.api.VoiceRecorder +import io.element.android.libraries.voicerecorder.api.VoiceRecorderState +import io.element.android.libraries.voicerecorder.impl.audio.Audio +import io.element.android.libraries.voicerecorder.impl.audio.AudioConfig +import io.element.android.libraries.voicerecorder.impl.audio.AudioLevelCalculator +import io.element.android.libraries.voicerecorder.impl.audio.AudioReader +import io.element.android.libraries.voicerecorder.impl.audio.Encoder +import io.element.android.libraries.voicerecorder.impl.file.VoiceFileConfig +import io.element.android.libraries.voicerecorder.impl.file.VoiceFileManager +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.launch +import timber.log.Timber +import java.io.File +import java.util.UUID +import javax.inject.Inject + +@SingleIn(RoomScope::class) +@ContributesBinding(RoomScope::class) +class VoiceRecorderImpl @Inject constructor( + private val dispatchers: CoroutineDispatchers, + private val audioReaderFactory: AudioReader.Factory, + private val encoder: Encoder, + private val fileManager: VoiceFileManager, + private val config: AudioConfig, + private val fileConfig: VoiceFileConfig, + private val audioLevelCalculator: AudioLevelCalculator, + appCoroutineScope: CoroutineScope, +) : VoiceRecorder { + private val voiceCoroutineScope by lazy { + appCoroutineScope.childScope(dispatchers.io, "VoiceRecorder-${UUID.randomUUID()}") + } + + private var outputFile: File? = null + private var audioReader: AudioReader? = null + private var recordingJob: Job? = null + + private val _state = MutableStateFlow(VoiceRecorderState.Idle) + override val state: StateFlow = _state + + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override suspend fun startRecord() { + Timber.i("Voice recorder started recording") + outputFile = fileManager.createFile() + .also(encoder::init) + + val audioRecorder = audioReaderFactory.create(config, dispatchers).also { audioReader = it } + + recordingJob = voiceCoroutineScope.launch { + audioRecorder.record { audio -> + when (audio) { + is Audio.Data -> { + val audioLevel = audioLevelCalculator.calculateAudioLevel(audio.buffer) + _state.emit(VoiceRecorderState.Recording(audioLevel)) + encoder.encode(audio.buffer, audio.readSize) + } + is Audio.Error -> { + Timber.e("Voice message error: code=${audio.audioRecordErrorCode}") + _state.emit(VoiceRecorderState.Recording(0.0)) + } + } + } + } + } + + /** + * Stop the current recording. + * + * Call [deleteRecording] to delete any recorded audio. + */ + override suspend fun stopRecord( + cancelled: Boolean + ) { + recordingJob?.cancel()?.also { + Timber.i("Voice recorder stopped recording") + } + recordingJob = null + + audioReader?.stop() + audioReader = null + encoder.release() + + if (cancelled) { + deleteRecording() + } + + _state.emit( + when (val file = outputFile) { + null -> VoiceRecorderState.Idle + else -> VoiceRecorderState.Finished(file, fileConfig.mimeType) + } + ) + } + + /** + * Stop the current recording and delete the output file. + */ + override suspend fun deleteRecording() { + outputFile?.let(fileManager::deleteFile)?.also { + Timber.i("Voice recorder deleted recording") + } + outputFile = null + _state.emit(VoiceRecorderState.Idle) + } +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AndroidAudioReader.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AndroidAudioReader.kt new file mode 100644 index 0000000000..a2342f3c2f --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AndroidAudioReader.kt @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import android.Manifest +import android.media.AudioRecord +import android.media.audiofx.AutomaticGainControl +import android.media.audiofx.NoiseSuppressor +import androidx.annotation.RequiresPermission +import com.squareup.anvil.annotations.ContributesBinding +import io.element.android.libraries.core.coroutine.CoroutineDispatchers +import io.element.android.libraries.core.data.tryOrNull +import io.element.android.libraries.di.RoomScope +import kotlinx.coroutines.isActive +import kotlinx.coroutines.withContext + +class AndroidAudioReader +@RequiresPermission(Manifest.permission.RECORD_AUDIO) private constructor( + private val config: AudioConfig, + private val dispatchers: CoroutineDispatchers, +) : AudioReader { + private val audioRecord: AudioRecord + private var noiseSuppressor: NoiseSuppressor? = null + private var automaticGainControl: AutomaticGainControl? = null + private val outputBuffer: ShortArray + + init { + outputBuffer = createOutputBuffer(config.sampleRate) + audioRecord = AudioRecord.Builder().setAudioSource(config.source).setAudioFormat(config.format).setBufferSizeInBytes(outputBuffer.sizeInBytes()).build() + noiseSuppressor = requestNoiseSuppressor(audioRecord) + automaticGainControl = requestAutomaticGainControl(audioRecord) + } + + /** + * Record audio data continuously. + * + * @param onAudio callback when audio is read. + */ + override suspend fun record( + onAudio: suspend (Audio) -> Unit, + ) { + audioRecord.startRecording() + withContext(dispatchers.io) { + while (isActive) { + if (audioRecord.recordingState != AudioRecord.RECORDSTATE_RECORDING) { + break + } + onAudio(read()) + } + } + } + + private fun read(): Audio { + val result = audioRecord.read(outputBuffer, 0, outputBuffer.size) + + if (isAudioRecordErrorResult(result)) { + return Audio.Error(result) + } + + return Audio.Data( + result, + outputBuffer, + ) + } + + override fun stop() { + if (audioRecord.state == AudioRecord.STATE_INITIALIZED) { + audioRecord.stop() + } + audioRecord.release() + + noiseSuppressor?.release() + noiseSuppressor = null + + automaticGainControl?.release() + automaticGainControl = null + } + + private fun createOutputBuffer(sampleRate: SampleRate): ShortArray { + val bufferSizeInShorts = AudioRecord.getMinBufferSize( + sampleRate.hz, + config.format.channelMask, + config.format.encoding + ) + return ShortArray(bufferSizeInShorts) + } + + private fun requestNoiseSuppressor(audioRecord: AudioRecord): NoiseSuppressor? { + if (!NoiseSuppressor.isAvailable()) { + return null + } + + return tryOrNull { + NoiseSuppressor.create(audioRecord.audioSessionId).apply { + enabled = true + } + } + } + + private fun requestAutomaticGainControl(audioRecord: AudioRecord): AutomaticGainControl? { + if (!AutomaticGainControl.isAvailable()) { + return null + } + + return tryOrNull { + AutomaticGainControl.create(audioRecord.audioSessionId).apply { + enabled = true + } + } + } + + @ContributesBinding(RoomScope::class) + companion object Factory : AudioReader.Factory { + @RequiresPermission(Manifest.permission.RECORD_AUDIO) + override fun create(config: AudioConfig, dispatchers: CoroutineDispatchers): AndroidAudioReader { + return AndroidAudioReader(config, dispatchers) + } + } +} + +private fun isAudioRecordErrorResult(result: Int): Boolean { + return result < 0 +} + +private fun ShortArray.sizeInBytes(): Int = size * Short.SIZE_BYTES diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Audio.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Audio.kt new file mode 100644 index 0000000000..3e51d615f4 --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Audio.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +sealed class Audio { + class Data( + val readSize: Int, + val buffer: ShortArray, + ) : Audio() + + data class Error( + val audioRecordErrorCode: Int + ) : Audio() +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioConfig.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioConfig.kt new file mode 100644 index 0000000000..6ff912c2ae --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioConfig.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import android.media.AudioFormat +import android.media.MediaRecorder.AudioSource + +/** + * Audio configuration for voice recording. + * + * @property source the audio source to use, see constants in [AudioSource] + * @property format the audio format to use, see [AudioFormat] + * @property sampleRate the sample rate to use. Ensure this matches the value set in [format]. + * @property bitRate the bitrate in bps + */ +data class AudioConfig( + val source: Int, + val format: AudioFormat, + val sampleRate: SampleRate, + val bitRate: Int, +) diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioLevelCalculator.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioLevelCalculator.kt new file mode 100644 index 0000000000..554b6ba4b1 --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioLevelCalculator.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +interface AudioLevelCalculator { + /** + * Calculate the audio level of the audio buffer. + * + * @param buffer The audio buffer containing raw audio data. + * + * @return A value between 0 and 1. + */ + fun calculateAudioLevel(buffer: ShortArray): Double +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioReader.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioReader.kt new file mode 100644 index 0000000000..230c9533fd --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/AudioReader.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import io.element.android.libraries.core.coroutine.CoroutineDispatchers + +interface AudioReader { + /** + * Record audio data continuously. + * + * @param onAudio callback when audio is read. + */ + suspend fun record( + onAudio: suspend (Audio) -> Unit, + ) + + fun stop() + + interface Factory { + fun create(config: AudioConfig, dispatchers: CoroutineDispatchers): AudioReader + } + +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculator.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculator.kt new file mode 100644 index 0000000000..8a16acf83b --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculator.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import com.squareup.anvil.annotations.ContributesBinding +import io.element.android.libraries.di.RoomScope +import javax.inject.Inject +import kotlin.math.log10 +import kotlin.math.min +import kotlin.math.sqrt + +@ContributesBinding(RoomScope::class) +class DecibelAudioLevelCalculator @Inject constructor() : AudioLevelCalculator { + companion object { + private const val REFERENCE_DB = 50.0 // Reference dB for normal conversation + } + + override fun calculateAudioLevel(buffer: ShortArray): Double { + val rms = buffer.rootMeanSquare() + + // Convert to decibels and clip + val db = 20 * log10(rms / REFERENCE_DB) + val clipped = min(db, REFERENCE_DB) + + // Scale to the range [0.0, 1.0] + return clipped / REFERENCE_DB + } + + private fun ShortArray.rootMeanSquare(): Double { + // Use Double to avoid overflow + val sumOfSquares: Double = sumOf { it.toDouble() * it.toDouble() } + val avgSquare = sumOfSquares / size.toDouble() + return sqrt(avgSquare) + } +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DefaultEncoder.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DefaultEncoder.kt new file mode 100644 index 0000000000..a888824fe5 --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DefaultEncoder.kt @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import com.squareup.anvil.annotations.ContributesBinding +import io.element.android.libraries.di.RoomScope +import io.element.android.opusencoder.OggOpusEncoder +import timber.log.Timber +import java.io.File +import javax.inject.Inject +import javax.inject.Provider + +/** + * Safe wrapper for OggOpusEncoder. + */ +@ContributesBinding(RoomScope::class) +class DefaultEncoder @Inject constructor( + private val encoderProvider: Provider, + config: AudioConfig, +) : Encoder { + private val bitRate = config.bitRate +private val sampleRate = config.sampleRate.asEncoderModel() + + private var encoder: OggOpusEncoder? = null + override fun init( + file: File, + ) { + encoder?.release() + encoder = encoderProvider.get().apply { + init(file.absolutePath, sampleRate) + setBitrate(bitRate) + // TODO check encoder application: 2048 (voice, default is typically 2049 as audio) + } + } + + override fun encode( + buffer: ShortArray, + readSize: Int, + ) { + encoder?.encode(buffer, readSize) + ?: Timber.w("Can't encode when encoder not initialized") + } + + override fun release() { + encoder?.release() + ?: Timber.w("Can't release encoder that is not initialized") + encoder = null + } +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Encoder.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Encoder.kt new file mode 100644 index 0000000000..67685635aa --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/Encoder.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import java.io.File + +interface Encoder { + + fun init(file: File) + + fun encode(buffer: ShortArray, readSize: Int) + + fun release() +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/SampleRate.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/SampleRate.kt new file mode 100644 index 0000000000..b392b6e19f --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/audio/SampleRate.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import io.element.android.opusencoder.configuration.SampleRate as LibOpusOggSampleRate + +data object SampleRate { + const val hz = 48_000 + fun asEncoderModel() = LibOpusOggSampleRate.Rate48kHz +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/di/VoiceRecorderModule.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/di/VoiceRecorderModule.kt new file mode 100644 index 0000000000..b21ab48ac3 --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/di/VoiceRecorderModule.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.di + +import android.media.AudioFormat +import android.media.MediaRecorder +import com.squareup.anvil.annotations.ContributesTo +import dagger.Module +import dagger.Provides +import io.element.android.libraries.di.RoomScope +import io.element.android.libraries.voicerecorder.impl.audio.AudioConfig +import io.element.android.libraries.voicerecorder.impl.audio.SampleRate +import io.element.android.libraries.voicerecorder.impl.file.VoiceFileConfig +import io.element.android.opusencoder.OggOpusEncoder + +@Module +@ContributesTo(RoomScope::class) +object VoiceRecorderModule { + @Provides + fun provideAudioConfig(): AudioConfig { + val sampleRate = SampleRate + return AudioConfig( + format = AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(sampleRate.hz) + .setChannelMask(AudioFormat.CHANNEL_IN_MONO) + .build(), + bitRate = 24_000, // 24 kbps + sampleRate = sampleRate, + source = MediaRecorder.AudioSource.MIC, + ) + } + + @Provides + fun provideVoiceFileConfig(): VoiceFileConfig = + VoiceFileConfig( + cacheSubdir = "voice_recordings", + fileExt = "ogg", + mimeType = "audio/ogg", + ) + + @Provides + fun provideOggOpusEncoder(): OggOpusEncoder = OggOpusEncoder.create() +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/DefaultVoiceFileManager.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/DefaultVoiceFileManager.kt new file mode 100644 index 0000000000..07ef54991f --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/DefaultVoiceFileManager.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.file + +import com.squareup.anvil.annotations.ContributesBinding +import io.element.android.libraries.core.hash.md5 +import io.element.android.libraries.di.CacheDirectory +import io.element.android.libraries.di.RoomScope +import io.element.android.libraries.matrix.api.core.RoomId +import io.element.android.libraries.matrix.api.room.MatrixRoom +import java.io.File +import java.util.UUID +import javax.inject.Inject + +@ContributesBinding(RoomScope::class) +class DefaultVoiceFileManager @Inject constructor( + @CacheDirectory private val cacheDir: File, + private val config: VoiceFileConfig, + room: MatrixRoom, +) : VoiceFileManager { + + private val roomId: RoomId = room.roomId + + override fun createFile(): File { + val fileName = "${UUID.randomUUID()}.${config.fileExt}" + val outputDirectory = File(cacheDir, config.cacheSubdir) + val roomDir = File(outputDirectory, roomId.value.md5()) + .apply(File::mkdirs) + return File(roomDir, fileName) + } + + override fun deleteFile(file: File) { + file.delete() + } +} diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileConfig.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileConfig.kt new file mode 100644 index 0000000000..a7b1f4607d --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileConfig.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.file + +/** + * File configuration for voice recording. + * + * @property cacheSubdir the subdirectory in the cache dir to use. + * @property fileExt the file extension for audio files. + * @property mimeType the mime type of audio files. + */ +data class VoiceFileConfig( + val cacheSubdir: String, + val fileExt: String, + val mimeType: String, +) diff --git a/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileManager.kt b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileManager.kt new file mode 100644 index 0000000000..77e85b910e --- /dev/null +++ b/libraries/voicerecorder/impl/src/main/kotlin/io/element/android/libraries/voicerecorder/impl/file/VoiceFileManager.kt @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.file + +import java.io.File + +interface VoiceFileManager { + fun createFile(): File + + fun deleteFile(file: File) +} diff --git a/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImplTest.kt b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImplTest.kt new file mode 100644 index 0000000000..847e1c514f --- /dev/null +++ b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/VoiceRecorderImplTest.kt @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl + +import android.media.AudioFormat +import android.media.MediaRecorder +import app.cash.turbine.test +import com.google.common.truth.Truth.assertThat +import io.element.android.libraries.voicerecorder.api.VoiceRecorderState +import io.element.android.libraries.voicerecorder.impl.audio.Audio +import io.element.android.libraries.voicerecorder.impl.audio.AudioConfig +import io.element.android.libraries.voicerecorder.impl.audio.SampleRate +import io.element.android.libraries.voicerecorder.impl.di.VoiceRecorderModule +import io.element.android.libraries.voicerecorder.test.FakeAudioLevelCalculator +import io.element.android.libraries.voicerecorder.test.FakeAudioRecorderFactory +import io.element.android.libraries.voicerecorder.test.FakeEncoder +import io.element.android.libraries.voicerecorder.test.FakeFileSystem +import io.element.android.libraries.voicerecorder.test.FakeVoiceFileManager +import io.element.android.tests.testutils.testCoroutineDispatchers +import io.mockk.mockk +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.runTest +import org.junit.BeforeClass +import org.junit.Test +import java.io.File + +class VoiceRecorderImplTest { + private val fakeFileSystem = FakeFileSystem() + + @Test + fun `it emits the initial state`() = runTest { + val voiceRecorder = createVoiceRecorder() + voiceRecorder.state.test { + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle) + } + } + + @Test + fun `when recording, it emits the recording state`() = runTest { + val voiceRecorder = createVoiceRecorder() + voiceRecorder.state.test { + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle) + + voiceRecorder.startRecord() + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(1.0)) + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(0.0)) + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Recording(1.0)) + } + } + + @Test + fun `when stopped, it provides a file`() = runTest { + val voiceRecorder = createVoiceRecorder() + voiceRecorder.state.test { + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle) + + voiceRecorder.startRecord() + skipItems(3) + voiceRecorder.stopRecord() + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Finished(File(FILE_PATH), "audio/ogg")) + assertThat(fakeFileSystem.files[File(FILE_PATH)]).isEqualTo(ENCODED_DATA) + } + } + + @Test + fun `when cancelled, it deletes the file`() = runTest { + val voiceRecorder = createVoiceRecorder() + voiceRecorder.state.test { + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle) + + voiceRecorder.startRecord() + skipItems(3) + voiceRecorder.stopRecord(cancelled = true) + assertThat(awaitItem()).isEqualTo(VoiceRecorderState.Idle) + assertThat(fakeFileSystem.files[File(FILE_PATH)]).isNull() + } + } + + private fun TestScope.createVoiceRecorder(): VoiceRecorderImpl { + val fileConfig = VoiceRecorderModule.provideVoiceFileConfig() + return VoiceRecorderImpl( + dispatchers = testCoroutineDispatchers(), + audioReaderFactory = FakeAudioRecorderFactory( + audio = AUDIO, + ), + encoder = FakeEncoder(fakeFileSystem), + config = AudioConfig( + format = AUDIO_FORMAT, + bitRate = 24_000, // 24 kbps + sampleRate = SampleRate, + source = MediaRecorder.AudioSource.MIC, + ), + fileConfig = fileConfig, + fileManager = FakeVoiceFileManager(fakeFileSystem, fileConfig, FILE_ID), + audioLevelCalculator = FakeAudioLevelCalculator(), + appCoroutineScope = backgroundScope, + ) + } + + companion object { + const val FILE_ID: String = "recording" + const val FILE_PATH = "voice_recordings/${FILE_ID}.ogg" + private lateinit var AUDIO_FORMAT: AudioFormat + + // FakeEncoder doesn't actually encode, it just writes the data to the file + private const val ENCODED_DATA = "[32767, 32767, 32767][32767, 32767, 32767]" + private const val MAX_AMP = Short.MAX_VALUE + private val AUDIO = listOf( + Audio.Data(3, shortArrayOf(MAX_AMP, MAX_AMP, MAX_AMP)), + Audio.Error(-1), + Audio.Data(3, shortArrayOf(MAX_AMP, MAX_AMP, MAX_AMP)), + ) + + @BeforeClass + @JvmStatic + fun initAudioFormat() { + AUDIO_FORMAT = mockk() + } + } +} diff --git a/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculatorTest.kt b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculatorTest.kt new file mode 100644 index 0000000000..8ffbf1ef8e --- /dev/null +++ b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/impl/audio/DecibelAudioLevelCalculatorTest.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.impl.audio + +import org.junit.Test + +class DecibelAudioLevelCalculatorTest { + + @Test + fun `given max values, it returns values within range`() { + val calculator = DecibelAudioLevelCalculator() + val buffer = ShortArray(100) { Short.MAX_VALUE } + val level = calculator.calculateAudioLevel(buffer) + assert(level in 0.0..1.0) + } + + @Test + fun `given mixed values, it returns values within range`() { + val calculator = DecibelAudioLevelCalculator() + val buffer = shortArrayOf(Short.MAX_VALUE, Short.MIN_VALUE, -1, 1) + val level = calculator.calculateAudioLevel(buffer) + assert(level in 0.0..1.0) + } + + @Test + fun `given min values, it returns values within range`() { + val calculator = DecibelAudioLevelCalculator() + val buffer = ShortArray(100) { Short.MIN_VALUE } + val level = calculator.calculateAudioLevel(buffer) + assert(level in 0.0..1.0) + } +} diff --git a/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioLevelCalculator.kt b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioLevelCalculator.kt new file mode 100644 index 0000000000..1615067f6c --- /dev/null +++ b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioLevelCalculator.kt @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.test + +import io.element.android.libraries.voicerecorder.impl.audio.AudioLevelCalculator +import kotlin.math.abs + +class FakeAudioLevelCalculator: AudioLevelCalculator { + override fun calculateAudioLevel(buffer: ShortArray): Double { + return buffer.map { abs(it.toDouble()) }.average() / Short.MAX_VALUE + } +} diff --git a/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioReader.kt b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioReader.kt new file mode 100644 index 0000000000..71fd2df041 --- /dev/null +++ b/libraries/voicerecorder/impl/src/test/kotlin/io/element/android/libraries/voicerecorder/test/FakeAudioReader.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.element.android.libraries.voicerecorder.test + +import io.element.android.libraries.core.coroutine.CoroutineDispatchers +import io.element.android.libraries.voicerecorder.impl.audio.Audio +import io.element.android.libraries.voicerecorder.impl.audio.AudioReader +import kotlinx.coroutines.isActive +import kotlinx.coroutines.withContext +import kotlinx.coroutines.yield + +class FakeAudioReader( + private val dispatchers: CoroutineDispatchers, + private val audio: List