From e817afef840cd6643e47613e67a686c901f5801c Mon Sep 17 00:00:00 2001 From: ganfra Date: Thu, 15 Jun 2023 12:00:40 +0200 Subject: [PATCH] Media: introduce a Kind.File so we don't use In-memory bytearray in timeline --- .../impl/media/viewer/MediaViewerView.kt | 6 +- .../components/event/TimelineItemImageView.kt | 2 +- .../components/event/TimelineItemVideoView.kt | 2 +- .../matrix/ui/media/CoilMediaFetcher.kt | 95 ++++++++++++++----- .../matrix/ui/media/MediaRequestData.kt | 15 ++- 5 files changed, 93 insertions(+), 27 deletions(-) diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/media/viewer/MediaViewerView.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/media/viewer/MediaViewerView.kt index afbb9bb331..9cc1ed69fd 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/media/viewer/MediaViewerView.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/media/viewer/MediaViewerView.kt @@ -54,9 +54,11 @@ import androidx.compose.ui.unit.dp import coil.compose.AsyncImage import io.element.android.features.messages.impl.media.local.LocalMedia import io.element.android.features.messages.impl.media.local.LocalMediaView +import io.element.android.features.messages.impl.media.local.MediaInfo import io.element.android.features.messages.impl.media.local.rememberLocalMediaViewState import io.element.android.libraries.architecture.Async import io.element.android.libraries.architecture.isLoading +import io.element.android.libraries.core.mimetype.MimeTypes import io.element.android.libraries.designsystem.components.button.BackButton import io.element.android.libraries.designsystem.components.dialogs.RetryDialog import io.element.android.libraries.designsystem.preview.ElementPreviewDark @@ -140,6 +142,7 @@ fun MediaViewerView( mediaInfo = state.mediaInfo, ) ThumbnailView( + mediaInfo = state.mediaInfo, thumbnailSource = state.thumbnailSource, showThumbnail = showThumbnail, ) @@ -211,6 +214,7 @@ private fun MediaViewerTopBar( private fun ThumbnailView( thumbnailSource: MediaSource?, showThumbnail: Boolean, + mediaInfo: MediaInfo, ) { AnimatedVisibility( visible = showThumbnail, @@ -223,7 +227,7 @@ private fun ThumbnailView( ) { val mediaRequestData = MediaRequestData( source = thumbnailSource, - kind = MediaRequestData.Kind.Content + kind = MediaRequestData.Kind.File(mediaInfo.name, mediaInfo.mimeType) ) AsyncImage( modifier = Modifier.fillMaxSize(), diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemImageView.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemImageView.kt index 566e899a36..6b176604f4 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemImageView.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemImageView.kt @@ -43,7 +43,7 @@ fun TimelineItemImageView( modifier = modifier ) { BlurHashAsyncImage( - model = MediaRequestData(content.mediaSource, MediaRequestData.Kind.Content), + model = MediaRequestData(content.mediaSource, MediaRequestData.Kind.File(content.body, content.mimeType)), blurHash = content.blurhash, modifier = Modifier.fillMaxSize(), contentScale = ContentScale.Fit, diff --git a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemVideoView.kt b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemVideoView.kt index aa024e1033..f3bd4129d1 100644 --- a/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemVideoView.kt +++ b/features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemVideoView.kt @@ -49,7 +49,7 @@ fun TimelineItemVideoView( contentAlignment = Alignment.Center, ) { BlurHashAsyncImage( - model = MediaRequestData(content.thumbnailSource, MediaRequestData.Kind.Content), + model = MediaRequestData(content.thumbnailSource, MediaRequestData.Kind.File(content.body, content.mimeType)), blurHash = content.blurHash, modifier = Modifier.fillMaxSize(), contentScale = ContentScale.Fit, diff --git a/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/CoilMediaFetcher.kt b/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/CoilMediaFetcher.kt index d638db2902..8b421c6c28 100644 --- a/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/CoilMediaFetcher.kt +++ b/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/CoilMediaFetcher.kt @@ -17,42 +17,93 @@ package io.element.android.libraries.matrix.ui.media import coil.ImageLoader +import coil.decode.DataSource +import coil.decode.ImageSource import coil.fetch.FetchResult import coil.fetch.Fetcher +import coil.fetch.SourceResult import coil.request.Options import io.element.android.libraries.designsystem.components.avatar.AvatarData import io.element.android.libraries.matrix.api.MatrixClient import io.element.android.libraries.matrix.api.media.MatrixMediaLoader +import io.element.android.libraries.matrix.api.media.MediaSource +import io.element.android.libraries.matrix.api.media.toFile +import okio.Buffer +import okio.Path.Companion.toOkioPath +import timber.log.Timber import java.nio.ByteBuffer internal class CoilMediaFetcher( private val mediaLoader: MatrixMediaLoader, private val mediaData: MediaRequestData?, - private val options: Options, - private val imageLoader: ImageLoader + private val options: Options ) : Fetcher { override suspend fun fetch(): FetchResult? { - return loadMedia() - .map { data -> - val byteBuffer = ByteBuffer.wrap(data) - imageLoader.components.newFetcher(byteBuffer, options, imageLoader)?.first?.fetch() - }.getOrThrow() + if (mediaData?.source == null) return null + return when (mediaData.kind) { + is MediaRequestData.Kind.Content -> fetchContent(mediaData.source, options) + is MediaRequestData.Kind.Thumbnail -> fetchThumbnail(mediaData.source, mediaData.kind, options) + is MediaRequestData.Kind.File -> fetchFile(mediaData.source, mediaData.kind) + } } - private suspend fun loadMedia(): Result { - if (mediaData?.source == null) return Result.failure(IllegalStateException("No media data to fetch.")) - return when (mediaData.kind) { - is MediaRequestData.Kind.Content -> mediaLoader.loadMediaContent(source = mediaData.source) - is MediaRequestData.Kind.Thumbnail -> mediaLoader.loadMediaThumbnail( - source = mediaData.source, - width = mediaData.kind.width, - height = mediaData.kind.height - ) + /** + * This method is here to avoid using [MatrixMediaLoader.loadMediaContent] as too many ByteArray allocations will flood the memory and cause lots of GC. + * The MediaFile will be closed (and so destroyed from disk) when the image source is closed. + * + */ + private suspend fun fetchFile(mediaSource: MediaSource, kind: MediaRequestData.Kind.File): FetchResult? { + return mediaLoader.downloadMediaFile(mediaSource, kind.mimeType, kind.body) + .map { mediaFile -> + val file = mediaFile.toFile() + SourceResult( + source = ImageSource(file = file.toOkioPath(), closeable = mediaFile), + mimeType = null, + dataSource = DataSource.DISK + ) + } + .onFailure { + Timber.e(it) + } + .getOrNull() + } + + private suspend fun fetchContent(mediaSource: MediaSource, options: Options): FetchResult? { + return mediaLoader.loadMediaContent( + source = mediaSource, + ).map { byteArray -> + byteArray.asSourceResult(options) + }.getOrNull() + } + + private suspend fun fetchThumbnail(mediaSource: MediaSource, kind: MediaRequestData.Kind.Thumbnail, options: Options): FetchResult? { + return mediaLoader.loadMediaThumbnail( + source = mediaSource, + width = kind.width, + height = kind.height + ).map { byteArray -> + byteArray.asSourceResult(options) + }.getOrNull() + } + + private fun ByteArray.asSourceResult(options: Options): SourceResult { + val byteBuffer = ByteBuffer.wrap(this) + val bufferedSource = try { + Buffer().apply { write(byteBuffer) } + } finally { + byteBuffer.position(0) } + return SourceResult( + source = ImageSource(bufferedSource, options.context), + mimeType = null, + dataSource = DataSource.MEMORY + ) } - class MediaRequestDataFactory(private val client: MatrixClient) : + class MediaRequestDataFactory( + private val client: MatrixClient + ) : Fetcher.Factory { override fun create( data: MediaRequestData, @@ -62,13 +113,14 @@ internal class CoilMediaFetcher( return CoilMediaFetcher( mediaLoader = client.mediaLoader, mediaData = data, - options = options, - imageLoader = imageLoader + options = options ) } } - class AvatarFactory(private val client: MatrixClient) : + class AvatarFactory( + private val client: MatrixClient + ) : Fetcher.Factory { override fun create( @@ -79,8 +131,7 @@ internal class CoilMediaFetcher( return CoilMediaFetcher( mediaLoader = client.mediaLoader, mediaData = data.toMediaRequestData(), - options = options, - imageLoader = imageLoader + options = options ) } } diff --git a/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt b/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt index 02a7ed4e8c..f2593766bc 100644 --- a/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt +++ b/libraries/matrixui/src/main/kotlin/io/element/android/libraries/matrix/ui/media/MediaRequestData.kt @@ -18,17 +18,28 @@ package io.element.android.libraries.matrix.ui.media import io.element.android.libraries.matrix.api.media.MediaSource +/** + * Can be use with [coil.compose.AsyncImage] to load a [MediaSource]. + * This will go internally through our [CoilMediaFetcher]. + * + * Example of usage: + * AsyncImage( + * model = MediaRequestData(mediaSource, MediaRequestData.Kind.Content), + * contentScale = ContentScale.Fit, + * ) + * + */ data class MediaRequestData( val source: MediaSource?, val kind: Kind ) { sealed interface Kind { + object Content : Kind + data class File(val body: String?, val mimeType: String) : Kind data class Thumbnail(val width: Long, val height: Long) : Kind { constructor(size: Long) : this(size, size) } - - object Content : Kind } }